diff --git a/.github/ISSUE_TEMPLATE/company-public-audit-request.yml b/.github/ISSUE_TEMPLATE/company-public-audit-request.yml new file mode 100644 index 00000000..1b832e80 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/company-public-audit-request.yml @@ -0,0 +1,106 @@ +name: Company public audit request +description: Request onboarding to the bounded LiminalQA self-service public quality audit +title: "[Company audit] " +body: + - type: markdown + attributes: + value: | + Use this form only for public HTTPS pages owned by your company or explicitly authorized for review. + + Do not include passwords, tokens, API keys, cookies, private URLs, customer/account data, or security-vulnerability details. The self-service workflow is limited to passive public quality and accessibility evidence. + + - type: input + id: company + attributes: + label: Company or project name + placeholder: Example Company + validations: + required: true + + - type: input + id: repository + attributes: + label: Company GitHub repository + description: Repository that will call the reusable workflow or host the audit contract + placeholder: https://github.com/example/company-web + validations: + required: true + + - type: textarea + id: origins + attributes: + label: Authorized public origins + description: One exact public HTTPS origin per line. Do not include paths, credentials, tokens, or private hosts. + placeholder: | + https://www.example.com + https://docs.example.com + validations: + required: true + + - type: textarea + id: routes + attributes: + label: Initial public routes + description: List up to eight public routes and their purpose. Query keys must be declared separately and must not contain sensitive values. + placeholder: | + / — marketing homepage + /pricing — pricing information + /docs — public documentation + validations: + required: true + + - type: dropdown + id: cadence + attributes: + label: Intended cadence + options: + - One-time baseline + - Manual on demand + - Weekly + - Before releases + - On pull requests + validations: + required: true + + - type: dropdown + id: gate + attributes: + label: Initial gate mode + description: New integrations should normally begin with evidence-only mode. + options: + - never — evidence only + - high — fail only on HIGH aggregate severity + - any-signal — fail on any WARN + validations: + required: true + + - type: textarea + id: goals + attributes: + label: Audit goals + description: Describe the public user journeys, accessibility expectations, and quality risks you want the evidence to cover. + validations: + required: true + + - type: checkboxes + id: authorization + attributes: + label: Authorization and safety confirmation + options: + - label: I am authorized to request passive quality testing for the listed public origins. + required: true + - label: I understand the workflow does not authenticate, submit forms, publish content, call private APIs, perform financial actions, fuzz, exploit, or load test. + required: true + - label: I will not place credentials, secrets, customer data, or private vulnerability details in the issue or audit contract. + required: true + - label: I understand automated signals require human review and are not a security or compliance certification. + required: true + + - type: input + id: contact + attributes: + label: Public contact or GitHub handle + description: Optional public contact for onboarding questions. Do not provide private credentials or account information. + placeholder: "@company-engineering" + validations: + required: false diff --git a/.github/workflows/company-public-audit-engine-ci.yml b/.github/workflows/company-public-audit-engine-ci.yml new file mode 100644 index 00000000..05b0d58f --- /dev/null +++ b/.github/workflows/company-public-audit-engine-ci.yml @@ -0,0 +1,103 @@ +name: Company Public Audit Engine CI + +on: + push: + branches: + - agent/company-self-service-audit-v0-1 + paths: + - .github/workflows/company-public-audit.yml + - .github/workflows/company-public-audit-engine-ci.yml + - audits/templates/company-public-audit.example.json + - scripts/company_public_audit_engine.py + - scripts/company_public_browser_probe.mjs + - tests/test_company_public_audit_engine.py + - docs/COMPANY_SELF_SERVICE_AUDIT.md + - docs/examples/company-audit-caller.yml + pull_request: + paths: + - .github/workflows/company-public-audit.yml + - .github/workflows/company-public-audit-engine-ci.yml + - audits/templates/company-public-audit.example.json + - scripts/company_public_audit_engine.py + - scripts/company_public_browser_probe.mjs + - tests/test_company_public_audit_engine.py + - docs/COMPANY_SELF_SERVICE_AUDIT.md + - docs/examples/company-audit-caller.yml + +permissions: + contents: read + +jobs: + contract-tests: + name: Validate schema, workflow, scripts, and fail-closed tests + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout exact revision + uses: actions/checkout@v4 + with: + persist-credentials: false + show-progress: false + + - name: Validate GitHub Actions workflows with pinned actionlint + id: actionlint + shell: bash + run: | + set -euo pipefail + go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 + set +e + "$(go env GOPATH)/bin/actionlint" \ + -ignore 'SC2016|SC2129|SC2155' \ + .github/workflows/company-public-audit.yml \ + .github/workflows/company-public-audit-engine-ci.yml \ + > actionlint.txt 2>&1 + code=$? + set -e + cat actionlint.txt + { + echo "### actionlint" + echo '```text' + cat actionlint.txt + echo '```' + } >> "${GITHUB_STEP_SUMMARY}" + exit "${code}" + + - name: Upload actionlint diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: company-audit-actionlint-${{ github.run_id }}-${{ github.run_attempt }} + path: actionlint.txt + if-no-files-found: warn + retention-days: 7 + + - name: Validate Python, Node, and JSON syntax + run: | + set -euo pipefail + python3 -m py_compile scripts/company_public_audit_engine.py + node --check scripts/company_public_browser_probe.mjs + python3 -m json.tool audits/templates/company-public-audit.example.json >/dev/null + + - name: Run fail-closed regression suite + run: python3 -m unittest -v tests/test_company_public_audit_engine.py + + - name: Validate example contract and matrix + run: | + set -euo pipefail + python3 scripts/company_public_audit_engine.py validate \ + --config audits/templates/company-public-audit.example.json \ + --output /tmp/company-audit-config.json + matrix="$(python3 scripts/company_public_audit_engine.py matrix \ + --config /tmp/company-audit-config.json)" + test "$(jq -r '.include | length' <<<"${matrix}")" -eq 2 + test "$(jq -r '.max_parallel' <<<"${matrix}")" -eq 2 + + reusable-workflow-smoke: + name: Run example.com through reusable pipeline + needs: contract-tests + uses: ./.github/workflows/company-public-audit.yml + with: + config_path: audits/templates/company-public-audit.example.json + engine_ref: ${{ github.event.pull_request.head.sha || github.sha }} + retention_days: 7 + fail_on: never diff --git a/.github/workflows/company-public-audit.yml b/.github/workflows/company-public-audit.yml new file mode 100644 index 00000000..0d161656 --- /dev/null +++ b/.github/workflows/company-public-audit.yml @@ -0,0 +1,519 @@ +name: LiminalQA Company Public Audit + +on: + workflow_call: + inputs: + config_path: + description: Path to the audit JSON contract in the caller repository + required: true + type: string + engine_ref: + description: Exact LiminalQAengineer SHA or release tag; use the same ref as the called workflow + required: false + default: main + type: string + retention_days: + description: Evidence artifact retention in days + required: false + default: 30 + type: number + fail_on: + description: never, high, or any-signal + required: false + default: never + type: string + outputs: + verdict: + description: PASS or WARN + value: ${{ jobs.aggregate.outputs.verdict }} + severity: + description: NONE, LOW, MEDIUM, or HIGH + value: ${{ jobs.aggregate.outputs.severity }} + artifact_name: + description: Aggregate evidence artifact name + value: ${{ jobs.aggregate.outputs.artifact_name }} + result_sha256: + description: SHA-256 of the aggregate machine-readable result + value: ${{ jobs.aggregate.outputs.result_sha256 }} + workflow_dispatch: + inputs: + config_path: + description: Path to a committed audit JSON contract + required: true + default: audits/templates/company-public-audit.example.json + type: string + engine_ref: + description: Exact LiminalQAengineer SHA, release tag, or branch + required: true + default: main + type: string + retention_days: + description: Evidence artifact retention in days + required: true + default: 30 + type: number + fail_on: + description: Gate behavior + required: true + default: never + type: choice + options: + - never + - high + - any-signal + +permissions: + contents: read + +concurrency: + group: liminalqa-company-audit-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: false + +env: + ENGINE_REPOSITORY: safal207/LiminalQAengineer + NODE_OPTIONS: --max-old-space-size=4096 + NPM_CONFIG_AUDIT: "false" + NPM_CONFIG_FUND: "false" + +jobs: + prepare: + name: Validate exact audit contract + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.contract.outputs.matrix }} + max_parallel: ${{ steps.contract.outputs.max_parallel }} + expected_cells: ${{ steps.contract.outputs.expected_cells }} + config_sha256: ${{ steps.contract.outputs.config_sha256 }} + engine_sha: ${{ steps.engine.outputs.engine_sha }} + caller_sha: ${{ steps.caller.outputs.caller_sha }} + caller_repository: ${{ steps.caller.outputs.caller_repository }} + + steps: + - name: Validate workflow inputs + shell: bash + env: + FAIL_ON: ${{ inputs.fail_on }} + RETENTION_DAYS: ${{ inputs.retention_days }} + run: | + set -euo pipefail + case "${FAIL_ON}" in + never|high|any-signal) ;; + *) echo "Unsupported fail_on: ${FAIL_ON}" >&2; exit 2 ;; + esac + test "${RETENTION_DAYS}" -ge 1 + test "${RETENTION_DAYS}" -le 90 + + - name: Checkout exact caller repository + uses: actions/checkout@v4 + with: + path: caller + persist-credentials: false + + - name: Record exact caller identity + id: caller + shell: bash + run: | + set -euo pipefail + caller_sha="$(git -C caller rev-parse HEAD)" + test "${caller_sha}" = "${GITHUB_SHA}" + echo "caller_sha=${caller_sha}" >> "${GITHUB_OUTPUT}" + echo "caller_repository=${GITHUB_REPOSITORY}" >> "${GITHUB_OUTPUT}" + + - name: Checkout pinned LiminalQA engine + uses: actions/checkout@v4 + with: + repository: ${{ env.ENGINE_REPOSITORY }} + ref: ${{ inputs.engine_ref }} + path: engine + persist-credentials: false + + - name: Record exact engine revision + id: engine + shell: bash + run: | + set -euo pipefail + engine_sha="$(git -C engine rev-parse HEAD)" + echo "engine_sha=${engine_sha}" >> "${GITHUB_OUTPUT}" + printf 'Engine repository: `%s`\n\nEngine SHA: `%s`\n' \ + "${ENGINE_REPOSITORY}" "${engine_sha}" >> "${GITHUB_STEP_SUMMARY}" + + - name: Resolve config path inside caller checkout + id: config_path + shell: bash + env: + CONFIG_PATH: ${{ inputs.config_path }} + run: | + set -euo pipefail + python3 - <<'PY' + import os + from pathlib import Path + + root = Path("caller").resolve() + candidate = (root / os.environ["CONFIG_PATH"]).resolve() + if candidate != root and root not in candidate.parents: + raise SystemExit("config_path escapes the caller checkout") + if not candidate.is_file(): + raise SystemExit(f"config file does not exist: {candidate}") + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as handle: + handle.write(f"path={candidate}\n") + PY + + - name: Validate fail-closed contract and build matrix + id: contract + shell: bash + env: + CONFIG_FILE: ${{ steps.config_path.outputs.path }} + run: | + set -euo pipefail + mkdir -p prepared + python3 -m py_compile engine/scripts/company_public_audit_engine.py + node --check engine/scripts/company_public_browser_probe.mjs + validation="$(python3 engine/scripts/company_public_audit_engine.py validate \ + --config "${CONFIG_FILE}" \ + --output prepared/validated-config.json)" + matrix_payload="$(python3 engine/scripts/company_public_audit_engine.py matrix \ + --config prepared/validated-config.json)" + config_sha="$(jq -r '.config_sha256' <<<"${validation}")" + test "${config_sha}" = "$(jq -r '.config_sha256' <<<"${matrix_payload}")" + echo "matrix=$(jq -c '{include:.include}' <<<"${matrix_payload}")" >> "${GITHUB_OUTPUT}" + echo "max_parallel=$(jq -r '.max_parallel' <<<"${matrix_payload}")" >> "${GITHUB_OUTPUT}" + echo "expected_cells=$(jq -r '.include | length' <<<"${matrix_payload}")" >> "${GITHUB_OUTPUT}" + echo "config_sha256=${config_sha}" >> "${GITHUB_OUTPUT}" + jq '{company, targets, profiles, settings, category_thresholds, boundaries}' \ + prepared/validated-config.json >> "${GITHUB_STEP_SUMMARY}" + + - name: Upload validated contract + uses: actions/upload-artifact@v4 + with: + name: liminalqa-contract-${{ github.run_id }}-${{ github.run_attempt }} + path: prepared/validated-config.json + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + audit: + name: Audit · ${{ matrix.target_id }} · ${{ matrix.profile }} + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.prepare.outputs.max_parallel) }} + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + + steps: + - name: Checkout exact caller revision + uses: actions/checkout@v4 + with: + ref: ${{ needs.prepare.outputs.caller_sha }} + path: caller + persist-credentials: false + + - name: Checkout exact LiminalQA engine revision + uses: actions/checkout@v4 + with: + repository: ${{ env.ENGINE_REPOSITORY }} + ref: ${{ needs.prepare.outputs.engine_sha }} + path: engine + persist-credentials: false + + - name: Resolve exact caller config path + id: config_path + shell: bash + env: + CONFIG_PATH: ${{ inputs.config_path }} + run: | + set -euo pipefail + python3 - <<'PY' + import os + from pathlib import Path + + root = Path("caller").resolve() + candidate = (root / os.environ["CONFIG_PATH"]).resolve() + if candidate != root and root not in candidate.parents: + raise SystemExit("config_path escapes the caller checkout") + if not candidate.is_file(): + raise SystemExit(f"config file does not exist: {candidate}") + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as handle: + handle.write(f"path={candidate}\n") + PY + + - name: Revalidate exact contract and matrix cell + shell: bash + env: + CONFIG_FILE: ${{ steps.config_path.outputs.path }} + EXPECTED_CONFIG_SHA: ${{ needs.prepare.outputs.config_sha256 }} + TARGET_ID: ${{ matrix.target_id }} + TARGET_URL: ${{ matrix.target_url }} + PROFILE: ${{ matrix.profile }} + run: | + set -euo pipefail + mkdir -p prepared + validation="$(python3 engine/scripts/company_public_audit_engine.py validate \ + --config "${CONFIG_FILE}" \ + --output prepared/validated-config.json)" + test "$(jq -r '.config_sha256' <<<"${validation}")" = "${EXPECTED_CONFIG_SHA}" + jq -e \ + --arg id "${TARGET_ID}" \ + --arg url "${TARGET_URL}" \ + --arg profile "${PROFILE}" \ + '(.targets[] | select(.id == $id and .url == $url)) and (.profiles | index($profile) != null)' \ + prepared/validated-config.json >/dev/null + + - name: Install pinned browser tooling + run: npm install --no-save --package-lock=false puppeteer-core@24.16.0 + + - name: Locate Chrome and validate runtime + id: runtime + shell: bash + run: | + set -euo pipefail + python3 --version + node --version + python3 -m py_compile engine/scripts/company_public_audit_engine.py + node --check engine/scripts/company_public_browser_probe.mjs + chrome="$(command -v google-chrome-stable || command -v google-chrome || command -v chromium || command -v chromium-browser || true)" + test -n "${chrome}" + "${chrome}" --version + echo "chrome=${chrome}" >> "${GITHUB_OUTPUT}" + + - name: Collect passive browser and keyboard evidence + shell: bash + run: | + set -euo pipefail + output="reports/${{ matrix.cell_id }}" + rm -rf "${output}" + mkdir -p "${output}" + node engine/scripts/company_public_browser_probe.mjs \ + --config prepared/validated-config.json \ + --target-id "${{ matrix.target_id }}" \ + --profile "${{ matrix.profile }}" \ + --chrome "${{ steps.runtime.outputs.chrome }}" \ + --output-dir "${output}" + cat "${output}/browser-summary.md" >> "${GITHUB_STEP_SUMMARY}" + + - name: Collect pinned Lighthouse evidence + shell: bash + env: + AUDIT_TARGET_URL: ${{ matrix.target_url }} + AUDIT_PROFILE: ${{ matrix.profile }} + AUDIT_OUTPUT: reports/${{ matrix.cell_id }}/raw-lighthouse + run: | + set -euo pipefail + mkdir -p "${AUDIT_OUTPUT}" + export LIGHTHOUSE_RUNS="$(jq -r '.settings.lighthouse_runs' prepared/validated-config.json)" + cat > lighthouserc.company.cjs <<'EOF' + const desktop = process.env.AUDIT_PROFILE === "desktop"; + module.exports = { + ci: { + collect: { + url: [process.env.AUDIT_TARGET_URL], + numberOfRuns: Number(process.env.LIGHTHOUSE_RUNS), + settings: { + onlyCategories: ["performance", "accessibility", "best-practices", "seo"], + ...(desktop ? { preset: "desktop" } : {}), + }, + }, + upload: { + target: "filesystem", + outputDir: process.env.AUDIT_OUTPUT, + }, + }, + }; + EOF + npx --yes @lhci/cli@0.15.1 autorun --config=lighthouserc.company.cjs + python3 engine/scripts/company_public_audit_engine.py summarize-lighthouse \ + --config prepared/validated-config.json \ + --target-id "${{ matrix.target_id }}" \ + --profile "${{ matrix.profile }}" \ + --input-dir "${AUDIT_OUTPUT}" \ + --output-dir "reports/${{ matrix.cell_id }}" + cat "reports/${{ matrix.cell_id }}/lighthouse-summary.md" >> "${GITHUB_STEP_SUMMARY}" + + - name: Record exact-attempt provenance and hashes + shell: bash + run: | + set -euo pipefail + output="reports/${{ matrix.cell_id }}" + jq -n \ + --arg run_id "${{ github.run_id }}" \ + --arg run_attempt "${{ github.run_attempt }}" \ + --arg caller_repository "${{ needs.prepare.outputs.caller_repository }}" \ + --arg caller_sha "${{ needs.prepare.outputs.caller_sha }}" \ + --arg engine_repository "${ENGINE_REPOSITORY}" \ + --arg engine_sha "${{ needs.prepare.outputs.engine_sha }}" \ + --arg config_sha256 "${{ needs.prepare.outputs.config_sha256 }}" \ + --arg cell_id "${{ matrix.cell_id }}" \ + --arg target_id "${{ matrix.target_id }}" \ + --arg target_url "${{ matrix.target_url }}" \ + --arg target_kind "${{ matrix.target_kind }}" \ + --arg profile "${{ matrix.profile }}" \ + --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{ + run_id:$run_id, + run_attempt:$run_attempt, + caller_repository:$caller_repository, + caller_sha:$caller_sha, + engine_repository:$engine_repository, + engine_sha:$engine_sha, + config_sha256:$config_sha256, + cell_id:$cell_id, + target_id:$target_id, + target_url:$target_url, + target_kind:$target_kind, + profile:$profile, + generated_at:$generated_at + }' > "${output}/exact-attempt.json" + find "${output}" -type f ! -name SHA256SUMS.txt -print0 \ + | sort -z \ + | xargs -0 sha256sum > "${output}/SHA256SUMS.txt" + + - name: Upload exact cell evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: liminalqa-cell-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.cell_id }} + path: reports/${{ matrix.cell_id }}/ + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + aggregate: + name: Aggregate exact company evidence + needs: + - prepare + - audit + if: always() && needs.prepare.result == 'success' && needs.audit.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 12 + outputs: + verdict: ${{ steps.outputs.outputs.verdict }} + severity: ${{ steps.outputs.outputs.severity }} + result_sha256: ${{ steps.outputs.outputs.result_sha256 }} + artifact_name: ${{ steps.outputs.outputs.artifact_name }} + + steps: + - name: Checkout exact caller revision + uses: actions/checkout@v4 + with: + ref: ${{ needs.prepare.outputs.caller_sha }} + path: caller + persist-credentials: false + + - name: Checkout exact LiminalQA engine revision + uses: actions/checkout@v4 + with: + repository: ${{ env.ENGINE_REPOSITORY }} + ref: ${{ needs.prepare.outputs.engine_sha }} + path: engine + persist-credentials: false + + - name: Resolve exact caller config path + id: config_path + shell: bash + env: + CONFIG_PATH: ${{ inputs.config_path }} + run: | + set -euo pipefail + python3 - <<'PY' + import os + from pathlib import Path + + root = Path("caller").resolve() + candidate = (root / os.environ["CONFIG_PATH"]).resolve() + if candidate != root and root not in candidate.parents: + raise SystemExit("config_path escapes the caller checkout") + if not candidate.is_file(): + raise SystemExit(f"config file does not exist: {candidate}") + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as handle: + handle.write(f"path={candidate}\n") + PY + + - name: Revalidate exact contract + shell: bash + env: + CONFIG_FILE: ${{ steps.config_path.outputs.path }} + EXPECTED_CONFIG_SHA: ${{ needs.prepare.outputs.config_sha256 }} + run: | + set -euo pipefail + mkdir -p prepared + validation="$(python3 engine/scripts/company_public_audit_engine.py validate \ + --config "${CONFIG_FILE}" \ + --output prepared/validated-config.json)" + test "$(jq -r '.config_sha256' <<<"${validation}")" = "${EXPECTED_CONFIG_SHA}" + + - name: Download only this run attempt's cell artifacts + uses: actions/download-artifact@v4 + with: + pattern: liminalqa-cell-${{ github.run_id }}-${{ github.run_attempt }}-* + path: downloaded-cells + + - name: Enforce complete exact-attempt set and aggregate + shell: bash + run: | + set -euo pipefail + manifest_count="$(find downloaded-cells -name exact-attempt.json -type f | wc -l)" + test "${manifest_count}" -eq "${{ needs.prepare.outputs.expected_cells }}" + rm -rf reports/company-audit + python3 engine/scripts/company_public_audit_engine.py aggregate \ + --config prepared/validated-config.json \ + --input-dir downloaded-cells \ + --output-dir reports/company-audit \ + --run-id "${{ github.run_id }}" \ + --run-attempt "${{ github.run_attempt }}" \ + --caller-repository "${{ needs.prepare.outputs.caller_repository }}" \ + --caller-sha "${{ needs.prepare.outputs.caller_sha }}" \ + --engine-sha "${{ needs.prepare.outputs.engine_sha }}" + jq -n \ + --arg run_id "${{ github.run_id }}" \ + --arg run_attempt "${{ github.run_attempt }}" \ + --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{run_id:$run_id,run_attempt:$run_attempt,generated_at:$generated_at}' \ + > reports/company-audit/aggregate-exact-attempt.json + find reports/company-audit -type f ! -name SHA256SUMS.txt -print0 \ + | sort -z \ + | xargs -0 sha256sum > reports/company-audit/SHA256SUMS.txt + cat reports/company-audit/company-audit-summary.md >> "${GITHUB_STEP_SUMMARY}" + + - name: Expose reusable workflow outputs + id: outputs + shell: bash + run: | + set -euo pipefail + output_json="reports/company-audit/workflow-outputs.json" + artifact_name="liminalqa-company-audit-${{ github.run_id }}-${{ github.run_attempt }}" + echo "verdict=$(jq -r '.verdict' "${output_json}")" >> "${GITHUB_OUTPUT}" + echo "severity=$(jq -r '.severity' "${output_json}")" >> "${GITHUB_OUTPUT}" + echo "result_sha256=$(jq -r '.result_sha256' "${output_json}")" >> "${GITHUB_OUTPUT}" + echo "artifact_name=${artifact_name}" >> "${GITHUB_OUTPUT}" + + - name: Upload aggregate evidence packet + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.outputs.outputs.artifact_name }} + path: reports/company-audit/ + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} + + - name: Apply optional quality gate + shell: bash + env: + FAIL_ON: ${{ inputs.fail_on }} + VERDICT: ${{ steps.outputs.outputs.verdict }} + SEVERITY: ${{ steps.outputs.outputs.severity }} + run: | + set -euo pipefail + case "${FAIL_ON}" in + never) + exit 0 + ;; + high) + test "${SEVERITY}" != "HIGH" + ;; + any-signal) + test "${VERDICT}" = "PASS" + ;; + *) + echo "Unsupported fail_on: ${FAIL_ON}" >&2 + exit 2 + ;; + esac diff --git a/audits/company-self-service/smoke-result-2026-07-20.json b/audits/company-self-service/smoke-result-2026-07-20.json new file mode 100644 index 00000000..3a884854 --- /dev/null +++ b/audits/company-self-service/smoke-result-2026-07-20.json @@ -0,0 +1,126 @@ +{ + "schema_version": "liminalqa-company-self-service-smoke-v1", + "observed_at": "2026-07-20T18:21:22Z", + "workflow": { + "name": "Company Public Audit Engine CI", + "run_id": 29766920933, + "run_attempt": 1, + "engine_sha": "b45239e0f284d3205bdc6f3a77de5649bb6fadc5", + "caller_repository": "safal207/LiminalQAengineer", + "caller_sha": "c33ae65b7ca4e26b07195b9e7f01d48d01119e73", + "config_sha256": "3539e76d19f9a93428242e5d8ed4cb6d5bee2779b303e205420d45b6f836e26f" + }, + "validation": { + "actionlint": "PASS", + "python_node_json_syntax": "PASS", + "fail_closed_unit_tests": "PASS", + "example_contract_and_matrix": "PASS", + "reusable_prepare": "PASS", + "desktop_browser_and_lighthouse": "PASS", + "mobile_browser_and_lighthouse": "PASS", + "exact_attempt_aggregation": "PASS", + "artifact_upload": "PASS", + "quality_gate_never": "PASS", + "expected_cells": 2, + "accepted_manifests": 2 + }, + "aggregate": { + "verdict": "WARN", + "severity": "LOW", + "reason": "The example.com SEO score of 80 was below the example contract threshold of 85 in both profiles; no browser accessibility signal was observed.", + "result_sha256": "905ae9bbd3c022e96a4b75bf37c5ce90a9a033db505d320a6bd0aa944e603377", + "summary_sha256": "5725b93f8378b683715897b9a065bc8a52e9f0cea400936cd21d07e97bb5c4e2", + "evidence_index_sha256": "67ea9b2ad6f49eddc0014e6aad240ce8658e49f56a9e05f4471fa69311158950", + "workflow_outputs_sha256": "af06931e0fdbe7bad3db4f24050c7c5c34af1191a13d5f26a8cec6459662a9ae", + "aggregate_exact_attempt_sha256": "9747b837d07d949f4ede0fd112bae2bcb2e1df464d617d3cfbaf0059651db79a" + }, + "cells": [ + { + "cell_id": "home-desktop", + "http_status": 200, + "category_scores": { + "performance": 100, + "accessibility": 100, + "best_practices": 96, + "seo": 80 + }, + "browser_signals": { + "keyboard_focus_gap": false, + "unnamed_sequential_controls": 0, + "nested_interactive_controls": 0, + "unnamed_accessibility_controls": 0 + }, + "browser_result_sha256": "9416e2aebbaa7a4d42a1bb1dd87cb8472493e4f60d04b64d8444383ab033d2b8", + "lighthouse_summary_sha256": "a88ae84227e6c4723cf8827303094fdd7d07d7931d84dba1fc62708fb80f25d9", + "exact_attempt_manifest_sha256": "00f2a8a9276f41c822083a748471bca70a983799cbb91633288bb75134f523ba" + }, + { + "cell_id": "home-mobile", + "http_status": 200, + "category_scores": { + "performance": 100, + "accessibility": 100, + "best_practices": 96, + "seo": 80 + }, + "browser_signals": { + "keyboard_focus_gap": false, + "unnamed_sequential_controls": 0, + "nested_interactive_controls": 0, + "unnamed_accessibility_controls": 0 + }, + "browser_result_sha256": "77ae022cec75748962a42dbf7ce4dd680e9dda6aee1c438aba22bacc1e6a89fd", + "lighthouse_summary_sha256": "fe848d5b08560ea71a169ccda52ff2b634a9e415edbd666aa4c32bf9457aa64b", + "exact_attempt_manifest_sha256": "b1720a3af14fd84d6e74e02e5cad7e90856d9b155acff11224e2bff0d285bb45" + } + ], + "artifacts": { + "aggregate": { + "id": 8471326847, + "name": "liminalqa-company-audit-29766920933-1", + "digest": "sha256:0f08e1b07a8163f12ef0568651831b6cd1a2e777fdc67ad1ad5b9250e452c70c" + }, + "desktop_cell": { + "id": 8471266753, + "name": "liminalqa-cell-29766920933-1-home-desktop", + "digest": "sha256:f907d0e662e3b000670044aef14e9ae5d736a5c5e6cd0ed008ff77232308b171" + }, + "mobile_cell": { + "id": 8471247582, + "name": "liminalqa-cell-29766920933-1-home-mobile", + "digest": "sha256:30db3f008a15cc981bb5746f59b927d32db7617a63c81199b4dd1e57ae2255c4" + }, + "validated_contract": { + "id": 8471215620, + "name": "liminalqa-contract-29766920933-1", + "digest": "sha256:b9b98a1c5a0ba85b74d5bcabdbc9db3675e3fcfb4c5d4dc481e4c6e2c7df0aca" + }, + "actionlint": { + "id": 8471208472, + "name": "company-audit-actionlint-29766920933-1", + "digest": "sha256:076e49166fe92d80b0c0260db901099b973c223895b82f07f743de1cfac6fe35" + } + }, + "boundaries": { + "public_pages_only": true, + "authenticated_testing": false, + "account_access": false, + "credentials_or_secrets": false, + "direct_api_testing": false, + "form_submission": false, + "financial_operations": false, + "fuzzing": false, + "load_testing": false, + "active_security_testing": false, + "server_state_change": false, + "vulnerability_claim": false + }, + "authority": { + "mode": "evidence_only", + "ownership": false, + "approval": false, + "external_submission": false, + "deployment": false, + "merge": false + } +} diff --git a/audits/templates/company-public-audit.example.json b/audits/templates/company-public-audit.example.json new file mode 100644 index 00000000..bf60f594 --- /dev/null +++ b/audits/templates/company-public-audit.example.json @@ -0,0 +1,58 @@ +{ + "schema_version": "liminalqa-company-public-audit-v1", + "company": { + "name": "Example Company", + "audit_name": "Public website quality baseline" + }, + "allowed_origins": [ + "https://example.com" + ], + "allowed_query_keys": [], + "targets": [ + { + "id": "home", + "url": "https://example.com/", + "kind": "marketing" + } + ], + "profiles": [ + "desktop", + "mobile" + ], + "settings": { + "settle_ms": 3000, + "keyboard_tab_steps": 20, + "lighthouse_runs": 1, + "max_parallel": 2, + "retain_body_sample": false + }, + "category_thresholds": { + "performance": 0.65, + "accessibility": 0.85, + "best-practices": 0.85, + "seo": 0.85 + }, + "boundaries": { + "public_pages_only": true, + "natural_navigation_only": true, + "passive_browser_observation": true, + "keyboard_navigation_only": true, + "authenticated_testing": false, + "account_access": false, + "credentials_or_secrets": false, + "direct_api_testing": false, + "form_submission": false, + "publishing": false, + "financial_operations": false, + "fuzzing": false, + "load_testing": false, + "active_security_testing": false, + "server_state_change": false, + "vulnerability_claim": false + }, + "notes": [ + "Replace example.com with public HTTPS origins owned or explicitly authorized by the company.", + "The pipeline produces quality and accessibility evidence, not a penetration test or compliance certification.", + "No credentials, cookies, custom headers, form actions, direct application API calls, or state-changing instructions are accepted by the schema." + ] +} diff --git a/docs/COMPANY_SELF_SERVICE_AUDIT.md b/docs/COMPANY_SELF_SERVICE_AUDIT.md new file mode 100644 index 00000000..5b2e37d2 --- /dev/null +++ b/docs/COMPANY_SELF_SERVICE_AUDIT.md @@ -0,0 +1,316 @@ +# LiminalQA company self-service public audit + +This capability lets a company run a bounded public website quality and accessibility audit from its own GitHub repository while reusing the centrally maintained LiminalQA workflow and evidence engine. + +The company keeps control of: + +- the exact URLs under review; +- the caller repository and caller commit; +- the exact LiminalQA engine SHA or release tag; +- the schedule and optional quality gate; +- artifact retention. + +The pipeline keeps the audit bounded and reproducible: + +```text +company-owned JSON contract +→ fail-closed validation +→ target × desktop/mobile matrix +→ passive browser observation +→ keyboard/accessibility evidence +→ pinned Lighthouse runs +→ exact-attempt manifests +→ SHA-256 evidence index +→ aggregate PASS/WARN packet +``` + +## What it collects + +For every configured target and browser profile: + +- HTTP navigation status, final URL, and redirect chain; +- full-page screenshot; +- title, language, headings, landmarks, and structural counts; +- visible and sequentially focusable interactive controls; +- keyboard Tab trace and first-focus state; +- unnamed accessibility-tree controls; +- unnamed sequential controls; +- nested interactive controls; +- duplicate IDs, missing image alternatives, and unlabeled visible inputs; +- sanitized console signatures and failed-request metadata; +- navigation/resource timing totals; +- one to three pinned Lighthouse runs; +- category scores and core metrics; +- exact caller SHA, engine SHA, run ID, run attempt, config hash, and file hashes. + +The aggregate artifact contains: + +```text +company-audit-result.json +company-audit-summary.md +evidence-index.md +workflow-outputs.json +aggregate-exact-attempt.json +SHA256SUMS.txt +``` + +Each matrix-cell artifact also contains its browser result, screenshot, raw Lighthouse reports, summaries, exact-attempt manifest, and SHA-256 manifest. + +## What it deliberately cannot do + +The JSON schema contains no fields for: + +- usernames, passwords, tokens, API keys, cookies, or custom headers; +- authentication or account access; +- custom JavaScript or browser-script injection; +- form submission, publishing, or state-changing actions; +- direct application API testing; +- orders, transfers, payments, or financial operations; +- fuzzing, enumeration, exploitation, or load testing; +- private, local, loopback, or custom-port targets. + +Every contract must preserve the complete boundary block. Weakening or deleting one boundary causes validation to fail before browser execution. + +This is a public quality and accessibility evidence workflow. It is not a penetration test, vulnerability report, legal/compliance certification, or substitute for manual assistive-technology testing. + +## Option A — call the workflow from the company repository + +Create a workflow such as `.github/workflows/public-quality-audit.yml`: + +```yaml +name: Company Public Quality Audit + +on: + workflow_dispatch: + schedule: + - cron: "0 6 * * 1" + +permissions: + contents: read + +jobs: + public-audit: + uses: safal207/LiminalQAengineer/.github/workflows/company-public-audit.yml@PINNED_LIMINALQA_SHA + with: + config_path: .github/liminalqa/public-audit.json + engine_ref: PINNED_LIMINALQA_SHA + retention_days: 30 + fail_on: never +``` + +Use the same reviewed SHA or release tag for both: + +- the reusable workflow reference after `@`; +- `engine_ref`. + +This prevents the workflow definition and its scripts from silently coming from different revisions. A full example with outputs is stored at: + +```text +docs/examples/company-audit-caller.yml +``` + +GitHub permits a repository to call a reusable workflow from a public repository when the caller's Actions policy allows public actions and reusable workflows. Pinning a commit SHA provides the strongest stability and supply-chain boundary. + +## Option B — fork LiminalQAengineer + +A company may also fork this repository, add its contract, and run: + +```text +Actions +→ LiminalQA Company Public Audit +→ Run workflow +``` + +Provide: + +- the committed config path; +- an exact engine SHA/tag or the reviewed fork branch; +- retention days; +- gate mode. + +The workflow uses `workflow_dispatch` and the same evidence engine as external callers. + +## Contract format + +Start from: + +```text +audits/templates/company-public-audit.example.json +``` + +Minimal example: + +```json +{ + "schema_version": "liminalqa-company-public-audit-v1", + "company": { + "name": "Example Company", + "audit_name": "Public website quality baseline" + }, + "allowed_origins": [ + "https://www.example.com" + ], + "allowed_query_keys": [], + "targets": [ + { + "id": "home", + "url": "https://www.example.com/", + "kind": "marketing" + } + ], + "profiles": [ + "desktop", + "mobile" + ], + "settings": { + "settle_ms": 3000, + "keyboard_tab_steps": 20, + "lighthouse_runs": 1, + "max_parallel": 2, + "retain_body_sample": false + }, + "category_thresholds": { + "performance": 0.65, + "accessibility": 0.85, + "best-practices": 0.85, + "seo": 0.85 + }, + "boundaries": { + "public_pages_only": true, + "natural_navigation_only": true, + "passive_browser_observation": true, + "keyboard_navigation_only": true, + "authenticated_testing": false, + "account_access": false, + "credentials_or_secrets": false, + "direct_api_testing": false, + "form_submission": false, + "publishing": false, + "financial_operations": false, + "fuzzing": false, + "load_testing": false, + "active_security_testing": false, + "server_state_change": false, + "vulnerability_claim": false + }, + "notes": [] +} +``` + +### Origins and targets + +- `allowed_origins` accepts 1–8 exact public HTTPS origins. +- Localhost, private/reserved IP addresses, credentials, and custom ports are rejected. +- `targets` accepts 1–8 pages. +- IDs must be lowercase slugs and unique. +- A target's origin must appear in `allowed_origins`. +- URL fragments are rejected. + +### Query strings + +Queries are rejected unless each query key is explicitly listed in `allowed_query_keys`. + +Sensitive-looking keys such as `token`, `secret`, `auth`, `session`, `password`, `key`, `email`, `phone`, or `account` are rejected even when listed. + +Example: + +```json +{ + "allowed_query_keys": ["symbol"], + "targets": [ + { + "id": "btc-chart", + "url": "https://charts.example.com/chart?symbol=BTCUSD", + "kind": "chart" + } + ] +} +``` + +### Profiles + +Supported profiles: + +- `desktop`; +- `mobile`. + +The matrix contains one evidence cell per target/profile combination. + +### Settings + +| Setting | Range | Meaning | +|---|---:|---| +| `settle_ms` | 0–15000 | Additional wait after DOMContentLoaded | +| `keyboard_tab_steps` | 0–40 | Passive Tab presses used for focus evidence | +| `lighthouse_runs` | 1–3 | Exact Lighthouse runs per cell; median scores are used | +| `max_parallel` | 1–4 | Maximum concurrent matrix jobs | +| `retain_body_sample` | boolean | Retain up to 2,000 public visible-text characters; default should remain false | + +## Quality gate modes + +`fail_on` controls whether evidence only warns or can fail the caller workflow: + +| Value | Behavior | +|---|---| +| `never` | Always upload evidence; do not fail from quality signals | +| `high` | Fail only when aggregate severity is `HIGH` | +| `any-signal` | Fail whenever aggregate verdict is `WARN` | + +The recommended onboarding mode is `never`. Review several runs before enabling a gate because public networks, regional variants, consent layers, and third-party resources can affect laboratory results. + +## Reusable outputs + +A caller receives: + +- `verdict` — `PASS` or `WARN`; +- `severity` — `NONE`, `LOW`, `MEDIUM`, or `HIGH`; +- `artifact_name` — aggregate artifact name; +- `result_sha256` — SHA-256 of `company-audit-result.json`. + +These outputs can feed a later company-owned approval or reporting job. LiminalQA itself does not merge, deploy, file an external ticket, or claim ownership of remediation. + +## Decision boundary + +The pipeline promotes only bounded automated signals: + +```text +navigation failure or HTTP >= 400 +→ HIGH quality warning + +sequential focusables + zero Tab targets +→ keyboard focus warning + +unnamed sequential/accessibility controls +or nested interactive controls +→ accessibility warning + +Lighthouse median below configured threshold +→ Lighthouse quality warning +``` + +A workflow success state alone is never treated as evidence. Each conclusion is tied to exact result content, screenshots, raw reports, manifests, and hashes. + +Automated warnings require human review before they are described as confirmed product defects. Root cause, user impact, severity, and remediation ownership remain human decisions. + +## Versioning recommendation + +Before external use: + +1. review and merge the engine; +2. create a signed or protected release tag such as `company-audit-v1`; +3. publish the tag's commit SHA in this document; +4. have callers pin the SHA or tag in both workflow locations; +5. make incompatible schema changes under a new schema/workflow version. + +## Files in this capability + +```text +.github/workflows/company-public-audit.yml +.github/workflows/company-public-audit-engine-ci.yml +audits/templates/company-public-audit.example.json +scripts/company_public_audit_engine.py +scripts/company_public_browser_probe.mjs +tests/test_company_public_audit_engine.py +docs/examples/company-audit-caller.yml +docs/COMPANY_SELF_SERVICE_AUDIT.md +``` diff --git a/docs/audits/COMPANY_SELF_SERVICE_AUDIT_SMOKE_2026-07-20.md b/docs/audits/COMPANY_SELF_SERVICE_AUDIT_SMOKE_2026-07-20.md new file mode 100644 index 00000000..0fba9588 --- /dev/null +++ b/docs/audits/COMPANY_SELF_SERVICE_AUDIT_SMOKE_2026-07-20.md @@ -0,0 +1,149 @@ +# Company self-service audit smoke — 2026-07-20 + +## Verdict + +The reusable LiminalQA company audit pipeline completed an end-to-end public smoke test successfully. + +```text +company-owned contract +→ fail-closed validation +→ exact caller and engine revisions +→ desktop/mobile matrix +→ passive browser and keyboard observation +→ pinned Lighthouse evidence +→ exact-attempt manifests +→ 2/2 manifest aggregation +→ reusable outputs and evidence artifact +``` + +Workflow execution status: **PASS**. + +The audited example portfolio returned an aggregate quality verdict of `WARN / LOW` because `example.com` scored `80` for SEO against the example threshold of `85`. This is expected evidence behavior, not a workflow failure. Browser accessibility signals were all zero. + +## Exact execution + +```text +workflow: Company Public Audit Engine CI +run: 29766920933 +attempt: 1 +engine SHA: b45239e0f284d3205bdc6f3a77de5649bb6fadc5 +caller repository: safal207/LiminalQAengineer +caller SHA: c33ae65b7ca4e26b07195b9e7f01d48d01119e73 +config SHA-256: 3539e76d19f9a93428242e5d8ed4cb6d5bee2779b303e205420d45b6f836e26f +``` + +## Passed gates + +- `actionlint` with pinned `v1.7.7`; +- Python, Node and JSON syntax; +- fail-closed contract unit tests; +- deterministic example contract and matrix; +- reusable-workflow contract preparation; +- desktop browser and Lighthouse cell; +- mobile browser and Lighthouse cell; +- exact run/attempt manifest enforcement; +- aggregate report and SHA-256 generation; +- aggregate artifact upload; +- evidence-only `fail_on: never` gate. + +## Cell results + +| Cell | HTTP | Performance | Accessibility | Best Practices | SEO | Browser accessibility signals | +|---|---:|---:|---:|---:|---:|---| +| `home-desktop` | 200 | 100 | 100 | 96 | 80 | none | +| `home-mobile` | 200 | 100 | 100 | 96 | 80 | none | + +Both cells reported: + +```text +keyboard_focus_gap: false +unnamed_sequential_controls: 0 +nested_interactive_controls: 0 +unnamed_accessibility_controls: 0 +``` + +## Aggregate artifact + +```text +artifact ID: 8471326847 +artifact: liminalqa-company-audit-29766920933-1 +artifact digest: sha256:0f08e1b07a8163f12ef0568651831b6cd1a2e777fdc67ad1ad5b9250e452c70c +``` + +Aggregate content hashes: + +```text +company-audit-result.json: +905ae9bbd3c022e96a4b75bf37c5ce90a9a033db505d320a6bd0aa944e603377 + +company-audit-summary.md: +5725b93f8378b683715897b9a065bc8a52e9f0cea400936cd21d07e97bb5c4e2 + +evidence-index.md: +67ea9b2ad6f49eddc0014e6aad240ce8658e49f56a9e05f4471fa69311158950 + +workflow-outputs.json: +af06931e0fdbe7bad3db4f24050c7c5c34af1191a13d5f26a8cec6459662a9ae + +aggregate-exact-attempt.json: +9747b837d07d949f4ede0fd112bae2bcb2e1df464d617d3cfbaf0059651db79a +``` + +## Per-cell evidence + +### Desktop + +```text +artifact ID: 8471266753 +artifact digest: sha256:f907d0e662e3b000670044aef14e9ae5d736a5c5e6cd0ed008ff77232308b171 +browser result: 9416e2aebbaa7a4d42a1bb1dd87cb8472493e4f60d04b64d8444383ab033d2b8 +Lighthouse summary: a88ae84227e6c4723cf8827303094fdd7d07d7931d84dba1fc62708fb80f25d9 +exact-attempt manifest: 00f2a8a9276f41c822083a748471bca70a983799cbb91633288bb75134f523ba +``` + +### Mobile + +```text +artifact ID: 8471247582 +artifact digest: sha256:30db3f008a15cc981bb5746f59b927d32db7617a63c81199b4dd1e57ae2255c4 +browser result: 77ae022cec75748962a42dbf7ce4dd680e9dda6aee1c438aba22bacc1e6a89fd +Lighthouse summary: fe848d5b08560ea71a169ccda52ff2b634a9e415edbd666aa4c32bf9457aa64b +exact-attempt manifest: b1720a3af14fd84d6e74e02e5cad7e90856d9b155acff11224e2bff0d285bb45 +``` + +## Contract and lint evidence + +```text +validated contract artifact ID: 8471215620 +validated contract digest: sha256:b9b98a1c5a0ba85b74d5bcabdbc9db3675e3fcfb4c5d4dc481e4c6e2c7df0aca + +actionlint artifact ID: 8471208472 +actionlint artifact digest: sha256:076e49166fe92d80b0c0260db901099b973c223895b82f07f743de1cfac6fe35 +``` + +## How a company calls it + +The company commits its bounded JSON contract and adds a small caller workflow: + +```yaml +jobs: + public-audit: + uses: safal207/LiminalQAengineer/.github/workflows/company-public-audit.yml@PINNED_LIMINALQA_SHA + with: + config_path: .github/liminalqa/public-audit.json + engine_ref: PINNED_LIMINALQA_SHA + retention_days: 30 + fail_on: never +``` + +The same reviewed SHA or release tag should be used in both positions so the workflow definition and engine scripts cannot silently come from different revisions. + +## Safety boundary + +The successful smoke used a public HTTPS route only. The engine does not accept credentials, cookies, custom request headers, authentication, account access, JavaScript injection, form submission, direct application APIs, publishing, financial operations, fuzzing, exploitation or load testing. + +The result is quality and accessibility evidence. It is not a penetration test, vulnerability report, compliance certification or automatic defect judgment. + +## Authority + +The pipeline produces evidence and optional quality gates only. It grants no ownership, approval, external-submission, deployment or merge authority. diff --git a/docs/examples/company-audit-caller.yml b/docs/examples/company-audit-caller.yml new file mode 100644 index 00000000..d45db5e3 --- /dev/null +++ b/docs/examples/company-audit-caller.yml @@ -0,0 +1,30 @@ +name: Company Public Quality Audit + +on: + workflow_dispatch: + schedule: + - cron: "0 6 * * 1" + +permissions: + contents: read + +jobs: + public-audit: + # Replace PINNED_LIMINALQA_SHA with a reviewed release SHA or tag. + uses: safal207/LiminalQAengineer/.github/workflows/company-public-audit.yml@PINNED_LIMINALQA_SHA + with: + config_path: .github/liminalqa/public-audit.json + engine_ref: PINNED_LIMINALQA_SHA + retention_days: 30 + fail_on: never + + consume-result: + needs: public-audit + runs-on: ubuntu-latest + steps: + - name: Print reusable workflow outputs + run: | + echo "Verdict: ${{ needs.public-audit.outputs.verdict }}" + echo "Severity: ${{ needs.public-audit.outputs.severity }}" + echo "Artifact: ${{ needs.public-audit.outputs.artifact_name }}" + echo "Result SHA-256: ${{ needs.public-audit.outputs.result_sha256 }}" diff --git a/scripts/company_public_audit_engine.py b/scripts/company_public_audit_engine.py new file mode 100644 index 00000000..079265ce --- /dev/null +++ b/scripts/company_public_audit_engine.py @@ -0,0 +1,994 @@ +#!/usr/bin/env python3 +"""Validate, summarize, and aggregate self-service public company audits. + +This engine is deliberately fail-closed. It accepts only allowlisted public HTTPS +pages and quality/accessibility observation settings. It has no schema fields for +credentials, cookies, custom headers, JavaScript injection, form submission, +direct application APIs, financial operations, fuzzing, or load testing. +""" + +from __future__ import annotations + +import argparse +import hashlib +import ipaddress +import json +import re +import statistics +import sys +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +SCHEMA_VERSION = "liminalqa-company-public-audit-v1" +RESULT_SCHEMA_VERSION = "liminalqa-company-public-audit-result-v1" +LIGHTHOUSE_SCHEMA_VERSION = "liminalqa-company-lighthouse-summary-v1" +CATEGORIES = ("performance", "accessibility", "best-practices", "seo") +PROFILES = {"desktop", "mobile"} +SENSITIVE_QUERY_KEYS = re.compile( + r"(?:token|secret|key|auth|session|password|passwd|credential|cookie|email|phone|account|user_id|userid)", + re.IGNORECASE, +) +ID_PATTERN = re.compile(r"^[a-z][a-z0-9-]{0,47}$") +KIND_PATTERN = re.compile(r"^[a-z][a-z0-9-]{0,47}$") + +REQUIRED_BOUNDARIES: dict[str, bool] = { + "public_pages_only": True, + "natural_navigation_only": True, + "passive_browser_observation": True, + "keyboard_navigation_only": True, + "authenticated_testing": False, + "account_access": False, + "credentials_or_secrets": False, + "direct_api_testing": False, + "form_submission": False, + "publishing": False, + "financial_operations": False, + "fuzzing": False, + "load_testing": False, + "active_security_testing": False, + "server_state_change": False, + "vulnerability_claim": False, +} + +TOP_LEVEL_KEYS = { + "schema_version", + "company", + "allowed_origins", + "allowed_query_keys", + "targets", + "profiles", + "settings", + "category_thresholds", + "boundaries", + "notes", +} +COMPANY_KEYS = {"name", "audit_name"} +SETTINGS_KEYS = { + "settle_ms", + "keyboard_tab_steps", + "lighthouse_runs", + "max_parallel", + "retain_body_sample", +} +TARGET_KEYS = {"id", "url", "kind"} + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_text(value: str) -> str: + return sha256_bytes(value.encode("utf-8")) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def load_json(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise ValueError(f"Expected a JSON object in {path}") + return value + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def require_exact_keys(value: dict[str, Any], allowed: set[str], label: str) -> None: + extra = sorted(set(value) - allowed) + if extra: + raise ValueError(f"Unsupported {label} keys: {', '.join(extra)}") + + +def require_string(value: Any, label: str, minimum: int = 1, maximum: int = 200) -> str: + if not isinstance(value, str): + raise ValueError(f"{label} must be a string") + normalized = value.strip() + if not minimum <= len(normalized) <= maximum: + raise ValueError(f"{label} length must be between {minimum} and {maximum}") + return normalized + + +def reject_non_public_hostname(hostname: str) -> None: + lowered = hostname.lower().rstrip(".") + if lowered in {"localhost", "localhost.localdomain"} or lowered.endswith(".local"): + raise ValueError(f"Local hostname is not allowed: {hostname}") + try: + address = ipaddress.ip_address(lowered) + except ValueError: + return + if not address.is_global: + raise ValueError(f"Non-public IP target is not allowed: {hostname}") + + +def normalize_origin(raw_origin: str) -> str: + parsed = urlsplit(require_string(raw_origin, "allowed origin", 8, 300)) + if parsed.scheme != "https": + raise ValueError("Only HTTPS origins are allowed") + if parsed.username or parsed.password or parsed.port: + raise ValueError("Credentials and custom ports are not allowed in origins") + if not parsed.hostname: + raise ValueError("Origin hostname is required") + reject_non_public_hostname(parsed.hostname) + if parsed.query or parsed.fragment: + raise ValueError("Origin query strings and fragments are not allowed") + if parsed.path not in {"", "/"}: + raise ValueError("allowed_origins must contain origins, not paths") + return f"https://{parsed.hostname.lower()}" + + +def normalize_target_url( + raw_url: str, + allowed_origins: set[str], + allowed_query_keys: set[str], +) -> str: + parsed = urlsplit(require_string(raw_url, "target URL", 8, 2000)) + if parsed.scheme != "https": + raise ValueError("Only HTTPS target URLs are allowed") + if parsed.username or parsed.password or parsed.port: + raise ValueError("Credentials and custom ports are not allowed in targets") + if not parsed.hostname: + raise ValueError("Target hostname is required") + reject_non_public_hostname(parsed.hostname) + if parsed.fragment: + raise ValueError("URL fragments are not allowed") + origin = f"https://{parsed.hostname.lower()}" + if origin not in allowed_origins: + raise ValueError(f"Target origin is outside allowed_origins: {origin}") + + query_pairs = parse_qsl(parsed.query, keep_blank_values=True) + normalized_pairs: list[tuple[str, str]] = [] + if len(query_pairs) > 8: + raise ValueError("A target may contain at most 8 query parameters") + for key, value in query_pairs: + if key not in allowed_query_keys: + raise ValueError(f"Query key is not allowlisted: {key}") + if SENSITIVE_QUERY_KEYS.search(key): + raise ValueError(f"Sensitive-looking query key is forbidden: {key}") + if len(value) > 200: + raise ValueError(f"Query value for {key} is too long") + normalized_pairs.append((key, value)) + + path = parsed.path or "/" + if not path.startswith("/") or len(path) > 1000: + raise ValueError("Target path is invalid") + return urlunsplit(("https", parsed.hostname.lower(), path, urlencode(normalized_pairs, doseq=True), "")) + + +def validate_config(raw: dict[str, Any]) -> dict[str, Any]: + require_exact_keys(raw, TOP_LEVEL_KEYS, "top-level") + if raw.get("schema_version") != SCHEMA_VERSION: + raise ValueError(f"schema_version must equal {SCHEMA_VERSION}") + + company = raw.get("company") + if not isinstance(company, dict): + raise ValueError("company must be an object") + require_exact_keys(company, COMPANY_KEYS, "company") + normalized_company = { + "name": require_string(company.get("name"), "company.name", 1, 120), + "audit_name": require_string(company.get("audit_name"), "company.audit_name", 1, 160), + } + + origins_raw = raw.get("allowed_origins") + if not isinstance(origins_raw, list) or not 1 <= len(origins_raw) <= 8: + raise ValueError("allowed_origins must contain 1 to 8 origins") + origins = [normalize_origin(value) for value in origins_raw] + if len(origins) != len(set(origins)): + raise ValueError("allowed_origins must be unique") + origin_set = set(origins) + + query_keys_raw = raw.get("allowed_query_keys") + if not isinstance(query_keys_raw, list) or len(query_keys_raw) > 20: + raise ValueError("allowed_query_keys must be a list with at most 20 entries") + query_keys: list[str] = [] + for value in query_keys_raw: + key = require_string(value, "allowed query key", 1, 64) + if not re.fullmatch(r"[A-Za-z0-9_.-]+", key): + raise ValueError(f"Invalid query key: {key}") + if SENSITIVE_QUERY_KEYS.search(key): + raise ValueError(f"Sensitive-looking query key is forbidden: {key}") + query_keys.append(key) + if len(query_keys) != len(set(query_keys)): + raise ValueError("allowed_query_keys must be unique") + query_key_set = set(query_keys) + + targets_raw = raw.get("targets") + if not isinstance(targets_raw, list) or not 1 <= len(targets_raw) <= 8: + raise ValueError("targets must contain 1 to 8 public pages") + targets: list[dict[str, str]] = [] + target_ids: set[str] = set() + for index, target in enumerate(targets_raw): + if not isinstance(target, dict): + raise ValueError(f"targets[{index}] must be an object") + require_exact_keys(target, TARGET_KEYS, f"targets[{index}]") + target_id = require_string(target.get("id"), f"targets[{index}].id", 1, 48) + if not ID_PATTERN.fullmatch(target_id): + raise ValueError(f"Invalid target id: {target_id}") + if target_id in target_ids: + raise ValueError(f"Duplicate target id: {target_id}") + target_ids.add(target_id) + kind = require_string(target.get("kind"), f"targets[{index}].kind", 1, 48) + if not KIND_PATTERN.fullmatch(kind): + raise ValueError(f"Invalid target kind: {kind}") + targets.append( + { + "id": target_id, + "url": normalize_target_url(target.get("url"), origin_set, query_key_set), + "kind": kind, + } + ) + + profiles_raw = raw.get("profiles") + if not isinstance(profiles_raw, list) or not 1 <= len(profiles_raw) <= 2: + raise ValueError("profiles must contain desktop, mobile, or both") + profiles = [require_string(value, "profile", 1, 20) for value in profiles_raw] + if len(profiles) != len(set(profiles)) or not set(profiles).issubset(PROFILES): + raise ValueError("profiles must be unique values from: desktop, mobile") + + settings = raw.get("settings") + if not isinstance(settings, dict): + raise ValueError("settings must be an object") + require_exact_keys(settings, SETTINGS_KEYS, "settings") + settle_ms = settings.get("settle_ms") + keyboard_steps = settings.get("keyboard_tab_steps") + lighthouse_runs = settings.get("lighthouse_runs") + max_parallel = settings.get("max_parallel") + retain_body_sample = settings.get("retain_body_sample") + if not isinstance(settle_ms, int) or not 0 <= settle_ms <= 15000: + raise ValueError("settings.settle_ms must be an integer from 0 to 15000") + if not isinstance(keyboard_steps, int) or not 0 <= keyboard_steps <= 40: + raise ValueError("settings.keyboard_tab_steps must be an integer from 0 to 40") + if not isinstance(lighthouse_runs, int) or not 1 <= lighthouse_runs <= 3: + raise ValueError("settings.lighthouse_runs must be an integer from 1 to 3") + if not isinstance(max_parallel, int) or not 1 <= max_parallel <= 4: + raise ValueError("settings.max_parallel must be an integer from 1 to 4") + if not isinstance(retain_body_sample, bool): + raise ValueError("settings.retain_body_sample must be boolean") + + thresholds_raw = raw.get("category_thresholds") + if not isinstance(thresholds_raw, dict) or set(thresholds_raw) != set(CATEGORIES): + raise ValueError(f"category_thresholds must contain exactly: {', '.join(CATEGORIES)}") + thresholds: dict[str, float] = {} + for category in CATEGORIES: + value = thresholds_raw.get(category) + if not isinstance(value, (int, float)) or not 0 <= float(value) <= 1: + raise ValueError(f"Threshold for {category} must be between 0 and 1") + thresholds[category] = round(float(value), 3) + + boundaries = raw.get("boundaries") + if not isinstance(boundaries, dict): + raise ValueError("boundaries must be an object") + if set(boundaries) != set(REQUIRED_BOUNDARIES): + missing = sorted(set(REQUIRED_BOUNDARIES) - set(boundaries)) + extra = sorted(set(boundaries) - set(REQUIRED_BOUNDARIES)) + raise ValueError(f"Boundary keys mismatch; missing={missing}, extra={extra}") + for key, expected in REQUIRED_BOUNDARIES.items(): + if boundaries.get(key) is not expected: + raise ValueError(f"Boundary {key} must be {str(expected).lower()}") + + notes_raw = raw.get("notes") + if not isinstance(notes_raw, list) or len(notes_raw) > 20: + raise ValueError("notes must be a list with at most 20 entries") + notes = [require_string(value, "note", 1, 500) for value in notes_raw] + + return { + "schema_version": SCHEMA_VERSION, + "company": normalized_company, + "allowed_origins": origins, + "allowed_query_keys": query_keys, + "targets": targets, + "profiles": profiles, + "settings": { + "settle_ms": settle_ms, + "keyboard_tab_steps": keyboard_steps, + "lighthouse_runs": lighthouse_runs, + "max_parallel": max_parallel, + "retain_body_sample": retain_body_sample, + }, + "category_thresholds": thresholds, + "boundaries": dict(REQUIRED_BOUNDARIES), + "notes": notes, + } + + +def load_validated_config(path: Path) -> tuple[dict[str, Any], str]: + config = validate_config(load_json(path)) + return config, sha256_text(canonical_json(config)) + + +def build_matrix(config: dict[str, Any]) -> dict[str, Any]: + include = [] + target_by_id = {target["id"]: target for target in config["targets"]} + for target_id in target_by_id: + target = target_by_id[target_id] + for profile in config["profiles"]: + include.append( + { + "target_id": target["id"], + "target_url": target["url"], + "target_kind": target["kind"], + "profile": profile, + "cell_id": f"{target['id']}-{profile}", + } + ) + return {"include": include} + + +def percentile_median(values: Iterable[float | int | None]) -> float | None: + present = [float(value) for value in values if isinstance(value, (int, float))] + if not present: + return None + return round(float(statistics.median(present)), 3) + + +def find_lighthouse_reports(input_dir: Path) -> list[tuple[Path, dict[str, Any]]]: + reports: list[tuple[Path, dict[str, Any]]] = [] + for candidate in sorted(input_dir.rglob("*.json")): + try: + value = load_json(candidate) + except (OSError, ValueError, json.JSONDecodeError): + continue + if isinstance(value.get("categories"), dict) and isinstance(value.get("audits"), dict): + reports.append((candidate, value)) + if not reports: + raise ValueError(f"No Lighthouse reports found in {input_dir}") + return reports + + +def category_score(report: dict[str, Any], category: str) -> int: + raw = report.get("categories", {}).get(category, {}) + score = raw.get("score") if isinstance(raw, dict) else None + if not isinstance(score, (int, float)): + return 0 + return round(max(0.0, min(1.0, float(score))) * 100) + + +def audit_numeric(report: dict[str, Any], audit_id: str) -> float | None: + raw = report.get("audits", {}).get(audit_id, {}) + value = raw.get("numericValue") if isinstance(raw, dict) else None + return float(value) if isinstance(value, (int, float)) else None + + +def summarize_findings(reports: list[dict[str, Any]], limit: int = 12) -> list[dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + ignored = {"notApplicable", "manual", "informative"} + for report in reports: + audits = report.get("audits", {}) + if not isinstance(audits, dict): + continue + for audit_id, raw in audits.items(): + if not isinstance(raw, dict) or raw.get("scoreDisplayMode") in ignored: + continue + score = raw.get("score") + if not isinstance(score, (int, float)) or score >= 1: + continue + details = raw.get("details") if isinstance(raw.get("details"), dict) else {} + grouped[audit_id].append( + { + "score": float(score), + "title": raw.get("title") or audit_id, + "display_value": raw.get("displayValue"), + "savings_ms": details.get("overallSavingsMs") + if isinstance(details.get("overallSavingsMs"), (int, float)) + else None, + } + ) + findings = [] + for audit_id, values in grouped.items(): + findings.append( + { + "audit_id": audit_id, + "title": values[0]["title"], + "failed_run_count": len(values), + "median_score": round(statistics.median(item["score"] for item in values), 3), + "max_savings_ms": round( + max((item["savings_ms"] or 0) for item in values), 1 + ), + "display_value": next( + (item["display_value"] for item in values if item["display_value"]), None + ), + } + ) + findings.sort( + key=lambda item: ( + -item["failed_run_count"], + item["median_score"], + -item["max_savings_ms"], + item["audit_id"], + ) + ) + return findings[:limit] + + +def summarize_lighthouse( + config: dict[str, Any], + config_hash: str, + target_id: str, + profile: str, + input_dir: Path, +) -> dict[str, Any]: + targets = {target["id"]: target for target in config["targets"]} + if target_id not in targets: + raise ValueError(f"Unknown target id: {target_id}") + if profile not in config["profiles"]: + raise ValueError(f"Profile is outside config: {profile}") + report_pairs = find_lighthouse_reports(input_dir) + reports = [report for _, report in report_pairs] + expected_runs = config["settings"]["lighthouse_runs"] + if len(reports) != expected_runs: + raise ValueError(f"Expected {expected_runs} Lighthouse reports, found {len(reports)}") + + thresholds = config["category_thresholds"] + categories: dict[str, Any] = {} + failed_categories: list[str] = [] + for category in CATEGORIES: + scores = [category_score(report, category) for report in reports] + median_score = round(statistics.median(scores)) + threshold = round(thresholds[category] * 100) + status = "PASS" if median_score >= threshold else "WARN" + if status == "WARN": + failed_categories.append(category) + categories[category] = { + "scores": scores, + "median_score": median_score, + "threshold": threshold, + "status": status, + } + + metrics = { + "first_contentful_paint_ms": percentile_median( + audit_numeric(report, "first-contentful-paint") for report in reports + ), + "largest_contentful_paint_ms": percentile_median( + audit_numeric(report, "largest-contentful-paint") for report in reports + ), + "total_blocking_time_ms": percentile_median( + audit_numeric(report, "total-blocking-time") for report in reports + ), + "cumulative_layout_shift": percentile_median( + audit_numeric(report, "cumulative-layout-shift") for report in reports + ), + "speed_index_ms": percentile_median( + audit_numeric(report, "speed-index") for report in reports + ), + } + largest_gap = max( + (value["threshold"] - value["median_score"] for value in categories.values()), + default=0, + ) + severity = "HIGH" if largest_gap >= 30 else "MEDIUM" if largest_gap >= 15 else "LOW" + target = targets[target_id] + raw_reports = [ + { + "file": str(path.relative_to(input_dir)), + "sha256": sha256_file(path), + "requested_url": report.get("requestedUrl"), + "final_url": report.get("finalUrl"), + "fetch_time": report.get("fetchTime"), + "lighthouse_version": report.get("lighthouseVersion"), + } + for path, report in report_pairs + ] + return { + "schema_version": LIGHTHOUSE_SCHEMA_VERSION, + "generated_at": now_iso(), + "company": config["company"], + "target": target, + "profile": profile, + "config_sha256": config_hash, + "run_count": len(reports), + "verdict": "PASS" if not failed_categories else "WARN", + "severity": severity, + "categories": categories, + "core_metrics": metrics, + "top_findings": summarize_findings(reports), + "raw_reports": raw_reports, + "boundaries": { + "public_web_quality_only": True, + "security_vulnerability_claim": False, + "compliance_certification": False, + }, + } + + +def render_lighthouse_markdown(summary: dict[str, Any]) -> str: + target = summary["target"] + lines = [ + f"# LiminalQA · {summary['company']['name']} · {target['id']} · {summary['profile']}", + "", + f"**Verdict:** {summary['verdict']} ", + f"**Severity:** {summary['severity']} ", + f"**Target:** `{target['url']}` ", + f"**Lighthouse runs:** {summary['run_count']}", + "", + "## Category scores", + "", + "| Category | Scores | Median | Threshold | Status |", + "|---|---|---:|---:|---|", + ] + for category, value in summary["categories"].items(): + scores = ", ".join(str(score) for score in value["scores"]) + lines.append( + f"| {category} | {scores} | {value['median_score']} | {value['threshold']} | {value['status']} |" + ) + lines.extend(["", "## Core metrics", ""]) + for key, value in summary["core_metrics"].items(): + lines.append(f"- **{key}:** {value if value is not None else 'n/a'}") + lines.extend(["", "## Recurring findings", ""]) + for finding in summary["top_findings"][:8]: + lines.append( + f"- **{finding['title']}** — failed {finding['failed_run_count']}/{summary['run_count']} runs; median score {finding['median_score']}" + ) + if not summary["top_findings"]: + lines.append("No scored Lighthouse findings were returned.") + lines.extend( + [ + "", + "## Evidence boundary", + "", + "> Passive public web quality evidence only. This is not a penetration test,", + "> security vulnerability report, or compliance certification.", + "", + ] + ) + return "\n".join(lines) + + +def severity_rank(value: str) -> int: + return {"NONE": 0, "LOW": 1, "MEDIUM": 2, "HIGH": 3}.get(value, 0) + + +def classify_cell(browser: dict[str, Any], lighthouse: dict[str, Any]) -> dict[str, Any]: + reasons: list[str] = [] + severity = "NONE" + navigation = browser.get("navigation", {}) + status = navigation.get("status") + if navigation.get("error") or not isinstance(status, int) or status >= 400: + reasons.append("navigation_failed_or_non_success") + severity = "HIGH" + + signals = browser.get("signals", {}) + accessibility_signals = { + "keyboard_focus_gap": signals.get("keyboard_focus_gap", False), + "unnamed_sequential_controls": int(signals.get("unnamed_sequential_controls", 0) or 0), + "nested_interactive_controls": int(signals.get("nested_interactive_controls", 0) or 0), + "unnamed_accessibility_controls": int(signals.get("unnamed_accessibility_controls", 0) or 0), + } + if accessibility_signals["keyboard_focus_gap"]: + reasons.append("keyboard_focus_gap") + if severity_rank(severity) < severity_rank("MEDIUM"): + severity = "MEDIUM" + for key in ( + "unnamed_sequential_controls", + "nested_interactive_controls", + "unnamed_accessibility_controls", + ): + if accessibility_signals[key] > 0: + reasons.append(key) + if severity_rank(severity) < severity_rank("MEDIUM"): + severity = "MEDIUM" + + if lighthouse.get("verdict") == "WARN": + reasons.append("lighthouse_threshold_warning") + lighthouse_severity = lighthouse.get("severity", "LOW") + if severity_rank(lighthouse_severity) > severity_rank(severity): + severity = lighthouse_severity + + if int(browser.get("console", {}).get("error_count", 0) or 0) > 0: + reasons.append("console_errors_observed") + if severity_rank(severity) < severity_rank("LOW"): + severity = "LOW" + + return { + "verdict": "PASS" if not reasons else "WARN", + "severity": severity, + "reasons": reasons, + "accessibility_signals": accessibility_signals, + } + + +def parse_sha256_manifest(path: Path) -> list[dict[str, str]]: + entries = [] + for line in path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped: + continue + parts = stripped.split(maxsplit=1) + if len(parts) != 2 or not re.fullmatch(r"[0-9a-f]{64}", parts[0]): + raise ValueError(f"Invalid SHA256 manifest line in {path}: {line}") + entries.append({"sha256": parts[0], "file": parts[1].lstrip("* ")}) + return entries + + +def aggregate_results( + config: dict[str, Any], + config_hash: str, + input_dir: Path, + run_id: str, + run_attempt: str, + caller_repository: str, + caller_sha: str, + engine_sha: str, +) -> dict[str, Any]: + expected_cells = { + f"{target['id']}-{profile}" + for target in config["targets"] + for profile in config["profiles"] + } + manifests = sorted(input_dir.rglob("exact-attempt.json")) + if len(manifests) != len(expected_cells): + raise ValueError( + f"Expected {len(expected_cells)} exact-attempt manifests, found {len(manifests)}" + ) + + cells: list[dict[str, Any]] = [] + seen: set[str] = set() + for manifest_path in manifests: + artifact_dir = manifest_path.parent + manifest = load_json(manifest_path) + required = { + "run_id": run_id, + "run_attempt": run_attempt, + "caller_repository": caller_repository, + "caller_sha": caller_sha, + "engine_sha": engine_sha, + "config_sha256": config_hash, + } + for key, expected in required.items(): + if str(manifest.get(key)) != str(expected): + raise ValueError( + f"Manifest mismatch for {key} in {manifest_path}: {manifest.get(key)!r} != {expected!r}" + ) + cell_id = require_string(manifest.get("cell_id"), "manifest.cell_id", 1, 100) + if cell_id not in expected_cells: + raise ValueError(f"Unexpected cell id: {cell_id}") + if cell_id in seen: + raise ValueError(f"Duplicate cell id: {cell_id}") + seen.add(cell_id) + + browser_path = artifact_dir / "browser-result.json" + lighthouse_path = artifact_dir / "lighthouse-summary.json" + sums_path = artifact_dir / "SHA256SUMS.txt" + for required_path in (browser_path, lighthouse_path, sums_path): + if not required_path.is_file(): + raise ValueError(f"Missing evidence file: {required_path}") + browser = load_json(browser_path) + lighthouse = load_json(lighthouse_path) + classification = classify_cell(browser, lighthouse) + target_id = manifest["target_id"] + profile = manifest["profile"] + cells.append( + { + "cell_id": cell_id, + "target_id": target_id, + "profile": profile, + "target_url": manifest["target_url"], + "classification": classification, + "navigation": browser.get("navigation"), + "browser_signals": browser.get("signals"), + "lighthouse": { + "verdict": lighthouse.get("verdict"), + "severity": lighthouse.get("severity"), + "categories": lighthouse.get("categories"), + "core_metrics": lighthouse.get("core_metrics"), + "top_findings": lighthouse.get("top_findings", [])[:8], + }, + "evidence": { + "artifact_directory": artifact_dir.name, + "manifest_sha256": sha256_file(manifest_path), + "browser_result_sha256": sha256_file(browser_path), + "lighthouse_summary_sha256": sha256_file(lighthouse_path), + "sha256_manifest_sha256": sha256_file(sums_path), + "files": parse_sha256_manifest(sums_path), + }, + } + ) + + missing = expected_cells - seen + if missing: + raise ValueError(f"Missing cells: {', '.join(sorted(missing))}") + cells.sort(key=lambda cell: (cell["target_id"], cell["profile"])) + + warning_cells = [cell for cell in cells if cell["classification"]["verdict"] == "WARN"] + severity = max( + (cell["classification"]["severity"] for cell in cells), + key=severity_rank, + default="NONE", + ) + reason_counts = Counter( + reason for cell in cells for reason in cell["classification"]["reasons"] + ) + recurring_lighthouse = Counter( + finding["audit_id"] + for cell in cells + for finding in cell["lighthouse"]["top_findings"] + ) + + return { + "schema_version": RESULT_SCHEMA_VERSION, + "generated_at": now_iso(), + "company": config["company"], + "verdict": "PASS" if not warning_cells else "WARN", + "severity": severity, + "summary": { + "target_count": len(config["targets"]), + "profile_count": len(config["profiles"]), + "cell_count": len(cells), + "pass_cells": len(cells) - len(warning_cells), + "warning_cells": len(warning_cells), + "reason_counts": dict(sorted(reason_counts.items())), + "recurring_lighthouse_findings": [ + {"audit_id": audit_id, "cell_count": count} + for audit_id, count in recurring_lighthouse.most_common(15) + ], + }, + "provenance": { + "run_id": run_id, + "run_attempt": run_attempt, + "caller_repository": caller_repository, + "caller_sha": caller_sha, + "engine_repository": "safal207/LiminalQAengineer", + "engine_sha": engine_sha, + "config_sha256": config_hash, + }, + "coordinate_model": { + "O": "allowlisted public URL + browser profile + viewport + unauthenticated state + observation time", + "N": "passive browser, keyboard observer, and Lighthouse quality sensor", + "X": "company -> route -> component -> quality/accessibility signal", + "Y": "loading -> rendered -> accessible -> focusable or degraded", + "Z": "desktop/mobile profile", + "T": "navigation -> settle -> keyboard trace -> Lighthouse capture -> aggregation", + }, + "cells": cells, + "boundaries": config["boundaries"], + "limitations": [ + "The pipeline observes only the allowlisted public URLs in the supplied contract.", + "Lighthouse scores and automated DOM signals are triage evidence, not final proof of a product defect.", + "The result is not a penetration test, security vulnerability report, or compliance certification.", + "No credentials, accounts, forms, direct application APIs, state changes, fuzzing, or load testing are supported.", + ], + "authority": { + "mode": "evidence_only", + "grants": { + "ownership": False, + "approval": False, + "external_submission": False, + "deployment": False, + "merge": False, + }, + }, + } + + +def markdown_escape(value: Any) -> str: + return str(value).replace("|", "\\|").replace("\n", " ") + + +def render_portfolio_markdown(result: dict[str, Any]) -> str: + lines = [ + f"# LiminalQA self-service audit · {result['company']['name']}", + "", + f"**Audit:** {result['company']['audit_name']} ", + f"**Verdict:** {result['verdict']} ", + f"**Highest quality severity:** {result['severity']} ", + f"**Run:** `{result['provenance']['run_id']}` attempt `{result['provenance']['run_attempt']}`", + "", + "## Portfolio", + "", + "| Target | Profile | HTTP | Lighthouse | Perf | A11y | Best | SEO | Browser reasons |", + "|---|---|---:|---|---:|---:|---:|---:|---|", + ] + for cell in result["cells"]: + categories = cell["lighthouse"]["categories"] or {} + score = lambda name: categories.get(name, {}).get("median_score", "n/a") + status = (cell.get("navigation") or {}).get("status", "n/a") + reasons = ", ".join(cell["classification"]["reasons"]) or "—" + lines.append( + "| {target} | {profile} | {http} | {lh} | {perf} | {a11y} | {best} | {seo} | {reasons} |".format( + target=markdown_escape(cell["target_id"]), + profile=markdown_escape(cell["profile"]), + http=status, + lh=cell["lighthouse"]["verdict"], + perf=score("performance"), + a11y=score("accessibility"), + best=score("best-practices"), + seo=score("seo"), + reasons=markdown_escape(reasons), + ) + ) + lines.extend(["", "## Recurring signals", ""]) + if result["summary"]["reason_counts"]: + for reason, count in result["summary"]["reason_counts"].items(): + lines.append(f"- **{reason}:** {count} cells") + else: + lines.append("No automated warning signals were produced.") + lines.extend(["", "## Evidence provenance", ""]) + for key, value in result["provenance"].items(): + lines.append(f"- **{key}:** `{value}`") + lines.extend( + [ + "", + "## Boundary", + "", + "> Public, allowlisted, passive quality and accessibility evidence only.", + "> Not a penetration test, vulnerability report, or compliance certification.", + "", + ] + ) + return "\n".join(lines) + + +def render_evidence_index(result: dict[str, Any]) -> str: + lines = [ + f"# Evidence index · {result['company']['name']}", + "", + "| Cell | Browser result SHA-256 | Lighthouse summary SHA-256 | Manifest SHA-256 |", + "|---|---|---|---|", + ] + for cell in result["cells"]: + evidence = cell["evidence"] + lines.append( + f"| {cell['cell_id']} | `{evidence['browser_result_sha256']}` | `{evidence['lighthouse_summary_sha256']}` | `{evidence['manifest_sha256']}` |" + ) + lines.extend( + [ + "", + "A workflow success state is not treated as proof by itself. The evidence consists of", + "the exact manifests, result content, screenshots, raw Lighthouse reports, and hashes.", + "", + ] + ) + return "\n".join(lines) + + +def command_validate(args: argparse.Namespace) -> int: + config, config_hash = load_validated_config(Path(args.config)) + if args.output: + write_json(Path(args.output), config) + print(json.dumps({"config_sha256": config_hash, "company": config["company"]})) + return 0 + + +def command_matrix(args: argparse.Namespace) -> int: + config, config_hash = load_validated_config(Path(args.config)) + value = build_matrix(config) + value["config_sha256"] = config_hash + value["max_parallel"] = config["settings"]["max_parallel"] + print(json.dumps(value, separators=(",", ":"))) + return 0 + + +def command_summarize_lighthouse(args: argparse.Namespace) -> int: + config, config_hash = load_validated_config(Path(args.config)) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + summary = summarize_lighthouse( + config, + config_hash, + args.target_id, + args.profile, + Path(args.input_dir), + ) + write_json(output_dir / "lighthouse-summary.json", summary) + (output_dir / "lighthouse-summary.md").write_text( + render_lighthouse_markdown(summary), encoding="utf-8" + ) + return 0 + + +def command_aggregate(args: argparse.Namespace) -> int: + config, config_hash = load_validated_config(Path(args.config)) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + result = aggregate_results( + config, + config_hash, + Path(args.input_dir), + str(args.run_id), + str(args.run_attempt), + args.caller_repository, + args.caller_sha, + args.engine_sha, + ) + result_path = output_dir / "company-audit-result.json" + summary_path = output_dir / "company-audit-summary.md" + index_path = output_dir / "evidence-index.md" + write_json(result_path, result) + summary_path.write_text(render_portfolio_markdown(result), encoding="utf-8") + index_path.write_text(render_evidence_index(result), encoding="utf-8") + outputs = { + "verdict": result["verdict"], + "severity": result["severity"], + "result_sha256": sha256_file(result_path), + "summary_sha256": sha256_file(summary_path), + "evidence_index_sha256": sha256_file(index_path), + } + write_json(output_dir / "workflow-outputs.json", outputs) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser("validate") + validate.add_argument("--config", required=True) + validate.add_argument("--output") + validate.set_defaults(func=command_validate) + + matrix = subparsers.add_parser("matrix") + matrix.add_argument("--config", required=True) + matrix.set_defaults(func=command_matrix) + + lighthouse = subparsers.add_parser("summarize-lighthouse") + lighthouse.add_argument("--config", required=True) + lighthouse.add_argument("--target-id", required=True) + lighthouse.add_argument("--profile", required=True) + lighthouse.add_argument("--input-dir", required=True) + lighthouse.add_argument("--output-dir", required=True) + lighthouse.set_defaults(func=command_summarize_lighthouse) + + aggregate = subparsers.add_parser("aggregate") + aggregate.add_argument("--config", required=True) + aggregate.add_argument("--input-dir", required=True) + aggregate.add_argument("--output-dir", required=True) + aggregate.add_argument("--run-id", required=True) + aggregate.add_argument("--run-attempt", required=True) + aggregate.add_argument("--caller-repository", required=True) + aggregate.add_argument("--caller-sha", required=True) + aggregate.add_argument("--engine-sha", required=True) + aggregate.set_defaults(func=command_aggregate) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + try: + return int(args.func(args)) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/company_public_browser_probe.mjs b/scripts/company_public_browser_probe.mjs new file mode 100644 index 00000000..231ffd75 --- /dev/null +++ b/scripts/company_public_browser_probe.mjs @@ -0,0 +1,526 @@ +#!/usr/bin/env node + +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import puppeteer from "puppeteer-core"; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const sha256 = (value) => crypto.createHash("sha256").update(value).digest("hex"); + +const PROFILE_DEFINITIONS = { + desktop: { + userAgent: + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + viewport: { + width: 1440, + height: 1000, + deviceScaleFactor: 1, + isMobile: false, + hasTouch: false, + }, + }, + mobile: { + userAgent: + "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36", + viewport: { + width: 412, + height: 915, + deviceScaleFactor: 2.625, + isMobile: true, + hasTouch: true, + }, + }, +}; + +function parseArgs(argv) { + const args = {}; + for (let index = 0; index < argv.length; index += 2) { + const key = argv[index]; + const value = argv[index + 1]; + if (!key?.startsWith("--") || value === undefined) { + throw new Error(`Invalid argument near ${key ?? ""}`); + } + args[key.slice(2)] = value; + } + return args; +} + +function normalizeText(value) { + return String(value || "").replace(/\s+/g, " ").trim(); +} + +function sanitizeUrl(rawValue) { + try { + const url = new URL(rawValue); + return `${url.protocol}//${url.host}${url.pathname}`; + } catch { + return String(rawValue || "").slice(0, 500); + } +} + +function sanitizeText(rawValue) { + let value = String(rawValue || ""); + value = value.replace(/https:\/\/[^\s"'<>]+/gi, (candidate) => sanitizeUrl(candidate)); + value = value.replace( + /\b(?:bearer\s+)?[A-Za-z0-9_-]{24,}\.[A-Za-z0-9_.-]{10,}\b/gi, + "[REDACTED_TOKEN]", + ); + value = value.replace( + /\b(?:token|secret|api[_-]?key|password|session)\s*[:=]\s*[^\s,;]+/gi, + "$1=[REDACTED]", + ); + return normalizeText(value).slice(0, 800); +} + +function walkAccessibility(node, output = { nodes: 0, unnamedInteractive: 0, names: [] }) { + if (!node) return output; + output.nodes += 1; + const role = String(node.role || "").toLowerCase(); + const interactiveRoles = new Set([ + "button", + "link", + "checkbox", + "radio", + "textbox", + "combobox", + "menuitem", + "tab", + "slider", + "switch", + "spinbutton", + ]); + const name = normalizeText(node.name); + if (interactiveRoles.has(role) && !name) output.unnamedInteractive += 1; + if (name && output.names.length < 100) output.names.push({ role, name: name.slice(0, 160) }); + for (const child of node.children || []) walkAccessibility(child, output); + return output; +} + +async function activeElementState(page) { + return page.evaluate(() => { + const element = document.activeElement; + if (!element || element === document.body || element === document.documentElement) return null; + const normalize = (value) => String(value || "").replace(/\s+/g, " ").trim(); + const rectangle = element.getBoundingClientRect(); + const style = getComputedStyle(element); + let href = null; + if (element instanceof HTMLAnchorElement && element.href) { + try { + const url = new URL(element.href); + href = `${url.protocol}//${url.host}${url.pathname}`; + } catch { + href = null; + } + } + return { + tag: element.tagName.toLowerCase(), + role: element.getAttribute("role"), + id: element.id || null, + test_id: + element.getAttribute("data-testid") || + element.getAttribute("data-qa-id") || + element.getAttribute("data-test") || + null, + name: normalize( + element.getAttribute("aria-label") || + element.getAttribute("title") || + element.getAttribute("alt") || + element.textContent, + ).slice(0, 160), + href, + tab_index: element.tabIndex, + visible: + rectangle.width > 0 && + rectangle.height > 0 && + style.display !== "none" && + style.visibility !== "hidden" && + Number(style.opacity || 1) > 0, + focus_indicator: { + outline_style: style.outlineStyle, + outline_width: style.outlineWidth, + box_shadow: style.boxShadow.slice(0, 240), + }, + }; + }); +} + +async function keyboardTrace(page, steps) { + await page + .evaluate(() => { + if (document.activeElement instanceof HTMLElement) document.activeElement.blur(); + if (document.body instanceof HTMLElement) { + document.body.tabIndex = -1; + document.body.focus({ preventScroll: true }); + document.body.removeAttribute("tabindex"); + } + }) + .catch(() => null); + + const trace = []; + for (let index = 0; index < steps; index += 1) { + await page.keyboard.press("Tab"); + await sleep(70); + trace.push(await activeElementState(page)); + } + const nonNull = trace.filter(Boolean); + const unique = [ + ...new Map( + nonNull.map((entry) => [ + `${entry.tag}|${entry.role || ""}|${entry.id || ""}|${entry.test_id || ""}|${entry.name}|${entry.href || ""}`, + entry, + ]), + ).values(), + ]; + return { + attempted_steps: steps, + non_null_steps: nonNull.length, + unique_focus_targets: unique.length, + first_focus: nonNull[0] || null, + skip_link_reached: nonNull.some((entry) => /skip( to)? (main|content)/i.test(entry.name || "")), + unique_targets: unique, + trace, + }; +} + +async function inspectDom(page, retainBodySample) { + return page.evaluate((retainSample) => { + const normalize = (value) => String(value || "").replace(/\s+/g, " ").trim(); + const visible = (element) => { + const rectangle = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return ( + rectangle.width > 0 && + rectangle.height > 0 && + style.display !== "none" && + style.visibility !== "hidden" && + Number(style.opacity || 1) > 0 + ); + }; + const accessibleName = (element) => + normalize( + element.getAttribute("aria-label") || + element.getAttribute("title") || + element.getAttribute("alt") || + element.textContent, + ); + const descriptor = (element) => ({ + tag: element.tagName.toLowerCase(), + role: element.getAttribute("role"), + id: element.id || null, + test_id: + element.getAttribute("data-testid") || + element.getAttribute("data-qa-id") || + element.getAttribute("data-test") || + null, + tab_index: element.tabIndex, + disabled: Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"), + name: accessibleName(element).slice(0, 160), + }); + + const bodyText = normalize(document.body?.innerText || ""); + const interactiveSelector = + "a[href], button, input, select, textarea, [role='button'], [role='link'], [role='tab'], [role='menuitem'], [role='checkbox'], [role='radio'], [role='switch'], [tabindex]"; + const interactives = [...document.querySelectorAll(interactiveSelector)].filter(visible); + const enabled = interactives.filter( + (element) => !element.disabled && element.getAttribute("aria-disabled") !== "true", + ); + const sequential = enabled.filter((element) => element.tabIndex >= 0); + const unnamedSequential = sequential.filter((element) => !accessibleName(element)); + + const nestedSelector = + "button button, button a[href], a[href] button, [role='button'] button, button [role='button'], [role='link'] button, button [role='link']"; + const nested = [...document.querySelectorAll(nestedSelector)].filter(visible); + + const ids = [...document.querySelectorAll("[id]")].map((element) => element.id).filter(Boolean); + const duplicateIds = [...new Set(ids.filter((id, index) => ids.indexOf(id) !== index))]; + const images = [...document.querySelectorAll("img")].filter(visible); + const inputs = [...document.querySelectorAll("input, select, textarea")].filter(visible); + const unlabeledInputs = inputs.filter((element) => { + const explicitLabel = element.id + ? document.querySelector(`label[for="${CSS.escape(element.id)}"]`) + : null; + return !accessibleName(element) && !explicitLabel; + }); + const headings = [...document.querySelectorAll("h1,h2,h3,h4,h5,h6")] + .filter(visible) + .map((element) => ({ + level: Number(element.tagName.slice(1)), + text: normalize(element.textContent).slice(0, 240), + })) + .slice(0, 80); + + return { + title: document.title, + html_lang: document.documentElement.lang || null, + body_text_sha256_input: bodyText, + body_text_length: bodyText.length, + body_text_sample: retainSample ? bodyText.slice(0, 2000) : null, + headings, + landmarks: { + main: document.querySelectorAll("main, [role='main']").length, + navigation: document.querySelectorAll("nav, [role='navigation']").length, + banner: document.querySelectorAll("header, [role='banner']").length, + contentinfo: document.querySelectorAll("footer, [role='contentinfo']").length, + }, + visible_counts: { + interactive: interactives.length, + enabled_interactive: enabled.length, + sequential_focusable: sequential.length, + buttons: [...document.querySelectorAll("button, [role='button']")].filter(visible).length, + links: [...document.querySelectorAll("a[href], [role='link']")].filter(visible).length, + inputs: inputs.length, + images: images.length, + canvas: [...document.querySelectorAll("canvas")].filter(visible).length, + svg: [...document.querySelectorAll("svg")].filter(visible).length, + video: [...document.querySelectorAll("video")].filter(visible).length, + iframe: document.querySelectorAll("iframe").length, + }, + unnamed_sequential_controls: unnamedSequential.slice(0, 50).map(descriptor), + nested_interactive_controls: nested.slice(0, 50).map((element) => ({ + child: descriptor(element), + parent: element.parentElement ? descriptor(element.parentElement) : null, + })), + duplicate_ids: duplicateIds.slice(0, 100), + missing_alt_visible_images: images.filter((image) => !normalize(image.getAttribute("alt"))).length, + unlabeled_visible_inputs: unlabeledInputs.slice(0, 50).map(descriptor), + forms_present: document.querySelectorAll("form").length, + }; + }, retainBodySample); +} + +async function observe() { + const args = parseArgs(process.argv.slice(2)); + const configPath = args.config; + const targetId = args["target-id"]; + const profileId = args.profile; + const chromePath = args.chrome; + const outputDir = args["output-dir"]; + if (!configPath || !targetId || !profileId || !chromePath || !outputDir) { + throw new Error("Required: --config --target-id --profile --chrome --output-dir"); + } + + const config = JSON.parse(await fs.readFile(configPath, "utf8")); + const target = config.targets.find((candidate) => candidate.id === targetId); + if (!target) throw new Error(`Unknown target id: ${targetId}`); + if (!config.profiles.includes(profileId) || !PROFILE_DEFINITIONS[profileId]) { + throw new Error(`Profile is outside the validated contract: ${profileId}`); + } + const profile = PROFILE_DEFINITIONS[profileId]; + await fs.rm(outputDir, { recursive: true, force: true }); + await fs.mkdir(outputDir, { recursive: true }); + + const browser = await puppeteer.launch({ + executablePath: chromePath, + headless: true, + args: [ + "--no-sandbox", + "--disable-dev-shm-usage", + "--disable-background-networking", + "--disable-component-update", + "--disable-domain-reliability", + ], + }); + + const page = await browser.newPage(); + page.setDefaultNavigationTimeout(90000); + await page.setUserAgent(profile.userAgent); + await page.setViewport(profile.viewport); + + const consoleEntries = []; + const failedRequests = []; + const responseStats = []; + page.on("console", (message) => { + if (consoleEntries.length >= 200) return; + consoleEntries.push({ type: message.type(), text: sanitizeText(message.text()) }); + }); + page.on("requestfailed", (request) => { + if (failedRequests.length >= 100) return; + failedRequests.push({ + url: sanitizeUrl(request.url()), + method: request.method(), + resource_type: request.resourceType(), + error: sanitizeText(request.failure()?.errorText || "unknown"), + }); + }); + page.on("response", (response) => { + if (responseStats.length >= 2500) return; + responseStats.push({ + url: sanitizeUrl(response.url()), + status: response.status(), + resource_type: response.request().resourceType(), + }); + }); + + const startedAt = Date.now(); + let navigationResponse = null; + let navigationError = null; + try { + navigationResponse = await page.goto(target.url, { + waitUntil: "domcontentloaded", + timeout: 90000, + }); + } catch (error) { + navigationError = sanitizeText(error?.message || error); + } + await sleep(config.settings.settle_ms); + const settledAt = Date.now(); + + const dom = await inspectDom(page, config.settings.retain_body_sample); + const bodyText = dom.body_text_sha256_input; + delete dom.body_text_sha256_input; + dom.body_text_sha256 = sha256(bodyText); + + const accessibilityTree = await page.accessibility.snapshot({ interestingOnly: false }).catch(() => null); + const accessibility = walkAccessibility(accessibilityTree); + const keyboard = await keyboardTrace(page, config.settings.keyboard_tab_steps).catch(() => ({ + attempted_steps: config.settings.keyboard_tab_steps, + non_null_steps: 0, + unique_focus_targets: 0, + first_focus: null, + skip_link_reached: false, + unique_targets: [], + trace: [], + })); + + const performance = await page + .evaluate(() => { + const navigation = performance.getEntriesByType("navigation")[0]; + const resources = performance.getEntriesByType("resource"); + const byType = {}; + let transferSize = 0; + let encodedBodySize = 0; + for (const entry of resources) { + const type = entry.initiatorType || "other"; + byType[type] = (byType[type] || 0) + 1; + transferSize += Number(entry.transferSize || 0); + encodedBodySize += Number(entry.encodedBodySize || 0); + } + return { + navigation: navigation + ? { + type: navigation.type, + response_start_ms: navigation.responseStart, + response_end_ms: navigation.responseEnd, + dom_content_loaded_ms: navigation.domContentLoadedEventEnd, + load_event_ms: navigation.loadEventEnd, + transfer_size: navigation.transferSize, + } + : null, + resource_count: resources.length, + resource_count_by_initiator: byType, + resource_transfer_size: transferSize, + resource_encoded_body_size: encodedBodySize, + }; + }) + .catch(() => null); + + const redirectChain = navigationResponse + ? navigationResponse + .request() + .redirectChain() + .map((request) => sanitizeUrl(request.url())) + : []; + const screenshotName = "screenshot.png"; + await page + .screenshot({ path: path.join(outputDir, screenshotName), fullPage: true }) + .catch(() => null); + + const result = { + schema_version: "liminalqa-company-browser-result-v1", + observed_at: new Date().toISOString(), + company: config.company, + target, + profile: profileId, + viewport: profile.viewport, + navigation: { + requested_url: target.url, + final_url: sanitizeUrl(page.url()), + status: navigationResponse?.status() ?? null, + error: navigationError, + redirect_chain: redirectChain, + started_at: new Date(startedAt).toISOString(), + settled_at: new Date(settledAt).toISOString(), + wall_time_ms: settledAt - startedAt, + }, + dom, + accessibility: { + node_count: accessibility.nodes, + unnamed_interactive_count: accessibility.unnamedInteractive, + named_node_sample: accessibility.names, + }, + keyboard, + console: { + error_count: consoleEntries.filter((entry) => entry.type === "error").length, + warning_count: consoleEntries.filter((entry) => entry.type === "warning").length, + signatures: consoleEntries, + }, + network: { + response_count: responseStats.length, + status_4xx_count: responseStats.filter((entry) => entry.status >= 400 && entry.status < 500).length, + status_5xx_count: responseStats.filter((entry) => entry.status >= 500).length, + failed_request_count: failedRequests.length, + failed_requests: failedRequests, + response_status_counts: responseStats.reduce((counts, entry) => { + const key = String(entry.status); + counts[key] = (counts[key] || 0) + 1; + return counts; + }, {}), + }, + performance, + signals: { + sequential_focusable_count: dom.visible_counts.sequential_focusable, + keyboard_unique_focus_targets: keyboard.unique_focus_targets, + keyboard_focus_gap: + config.settings.keyboard_tab_steps > 0 && + dom.visible_counts.sequential_focusable > 0 && + keyboard.unique_focus_targets === 0, + unnamed_sequential_controls: dom.unnamed_sequential_controls.length, + nested_interactive_controls: dom.nested_interactive_controls.length, + unnamed_accessibility_controls: accessibility.unnamedInteractive, + duplicate_ids: dom.duplicate_ids.length, + missing_alt_visible_images: dom.missing_alt_visible_images, + unlabeled_visible_inputs: dom.unlabeled_visible_inputs.length, + }, + screenshot: screenshotName, + boundaries: config.boundaries, + authority: { + mode: "evidence_only", + grants: { + ownership: false, + approval: false, + external_submission: false, + deployment: false, + merge: false, + }, + }, + }; + + const resultText = `${JSON.stringify(result, null, 2)}\n`; + await fs.writeFile(path.join(outputDir, "browser-result.json"), resultText); + const summary = + `# Browser evidence · ${config.company.name} · ${target.id} · ${profileId}\n\n` + + `- HTTP: ${result.navigation.status ?? "n/a"}\n` + + `- Final URL: ${result.navigation.final_url}\n` + + `- Sequential focusables: ${result.signals.sequential_focusable_count}\n` + + `- Unique Tab targets: ${result.signals.keyboard_unique_focus_targets}\n` + + `- Unnamed sequential controls: ${result.signals.unnamed_sequential_controls}\n` + + `- Nested interactive controls: ${result.signals.nested_interactive_controls}\n` + + `- Accessibility-tree unnamed controls: ${result.signals.unnamed_accessibility_controls}\n` + + `- Console errors: ${result.console.error_count}\n\n` + + `Public passive observation only; no authentication, forms, direct APIs, state changes, fuzzing, or load testing.\n`; + await fs.writeFile(path.join(outputDir, "browser-summary.md"), summary); + await fs.writeFile( + path.join(outputDir, "BROWSER_SHA256SUMS.txt"), + `${sha256(resultText)} browser-result.json\n${sha256(summary)} browser-summary.md\n`, + ); + + await page.close(); + await browser.close(); +} + +observe().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/tests/test_company_public_audit_engine.py b/tests/test_company_public_audit_engine.py new file mode 100644 index 00000000..1ed71e58 --- /dev/null +++ b/tests/test_company_public_audit_engine.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +ENGINE_PATH = ROOT / "scripts" / "company_public_audit_engine.py" +EXAMPLE_PATH = ROOT / "audits" / "templates" / "company-public-audit.example.json" + +spec = importlib.util.spec_from_file_location("company_public_audit_engine", ENGINE_PATH) +assert spec and spec.loader +engine = importlib.util.module_from_spec(spec) +spec.loader.exec_module(engine) + + +class CompanyPublicAuditContractTests(unittest.TestCase): + def setUp(self) -> None: + self.example = json.loads(EXAMPLE_PATH.read_text(encoding="utf-8")) + + def test_example_contract_is_valid_and_deterministic(self) -> None: + first = engine.validate_config(copy.deepcopy(self.example)) + second = engine.validate_config(copy.deepcopy(self.example)) + self.assertEqual(first, second) + self.assertEqual(first["company"]["name"], "Example Company") + self.assertEqual(first["targets"][0]["url"], "https://example.com/") + self.assertEqual(first["boundaries"], engine.REQUIRED_BOUNDARIES) + self.assertEqual( + engine.sha256_text(engine.canonical_json(first)), + engine.sha256_text(engine.canonical_json(second)), + ) + + def test_matrix_contains_every_target_profile_cell(self) -> None: + validated = engine.validate_config(copy.deepcopy(self.example)) + matrix = engine.build_matrix(validated) + self.assertEqual( + matrix, + { + "include": [ + { + "target_id": "home", + "target_url": "https://example.com/", + "target_kind": "marketing", + "profile": "desktop", + "cell_id": "home-desktop", + }, + { + "target_id": "home", + "target_url": "https://example.com/", + "target_kind": "marketing", + "profile": "mobile", + "cell_id": "home-mobile", + }, + ] + }, + ) + + def test_unknown_top_level_field_is_rejected(self) -> None: + value = copy.deepcopy(self.example) + value["custom_javascript"] = "fetch('/private')" + with self.assertRaisesRegex(ValueError, "Unsupported top-level keys"): + engine.validate_config(value) + + def test_credentials_and_custom_ports_are_rejected(self) -> None: + for url in ( + "https://user:password@example.com/", + "https://example.com:8443/", + ): + value = copy.deepcopy(self.example) + value["targets"][0]["url"] = url + with self.subTest(url=url), self.assertRaises(ValueError): + engine.validate_config(value) + + def test_private_and_local_network_targets_are_rejected(self) -> None: + for origin in ( + "https://localhost", + "https://127.0.0.1", + "https://10.0.0.1", + "https://192.168.1.20", + "https://[::1]", + ): + value = copy.deepcopy(self.example) + value["allowed_origins"] = [origin] + value["targets"][0]["url"] = f"{origin}/" + with self.subTest(origin=origin), self.assertRaises(ValueError): + engine.validate_config(value) + + def test_query_parameters_require_explicit_non_sensitive_allowlist(self) -> None: + value = copy.deepcopy(self.example) + value["targets"][0]["url"] = "https://example.com/chart?symbol=BTCUSD" + with self.assertRaisesRegex(ValueError, "Query key is not allowlisted"): + engine.validate_config(value) + + value["allowed_query_keys"] = ["symbol"] + validated = engine.validate_config(value) + self.assertEqual(validated["targets"][0]["url"], "https://example.com/chart?symbol=BTCUSD") + + sensitive = copy.deepcopy(self.example) + sensitive["allowed_query_keys"] = ["access_token"] + sensitive["targets"][0]["url"] = "https://example.com/?access_token=abc" + with self.assertRaisesRegex(ValueError, "Sensitive-looking query key"): + engine.validate_config(sensitive) + + def test_origin_escape_is_rejected(self) -> None: + value = copy.deepcopy(self.example) + value["targets"][0]["url"] = "https://other.example/" + with self.assertRaisesRegex(ValueError, "outside allowed_origins"): + engine.validate_config(value) + + def test_boundary_weakening_is_rejected(self) -> None: + value = copy.deepcopy(self.example) + value["boundaries"]["authenticated_testing"] = True + with self.assertRaisesRegex(ValueError, "authenticated_testing must be false"): + engine.validate_config(value) + + value = copy.deepcopy(self.example) + del value["boundaries"]["load_testing"] + with self.assertRaisesRegex(ValueError, "Boundary keys mismatch"): + engine.validate_config(value) + + def test_target_and_profile_limits_are_enforced(self) -> None: + value = copy.deepcopy(self.example) + value["targets"] = [ + {"id": f"page-{index}", "url": f"https://example.com/{index}", "kind": "page"} + for index in range(9) + ] + with self.assertRaisesRegex(ValueError, "1 to 8"): + engine.validate_config(value) + + value = copy.deepcopy(self.example) + value["profiles"] = ["desktop", "tablet"] + with self.assertRaisesRegex(ValueError, "desktop, mobile"): + engine.validate_config(value) + + def test_lighthouse_summary_uses_all_exact_runs(self) -> None: + validated = engine.validate_config(copy.deepcopy(self.example)) + config_hash = engine.sha256_text(engine.canonical_json(validated)) + report = { + "requestedUrl": "https://example.com/", + "finalUrl": "https://example.com/", + "fetchTime": "2026-07-20T00:00:00Z", + "lighthouseVersion": "test", + "categories": { + "performance": {"score": 0.5}, + "accessibility": {"score": 0.9}, + "best-practices": {"score": 0.8}, + "seo": {"score": 1.0}, + }, + "audits": { + "largest-contentful-paint": {"numericValue": 5000}, + "total-blocking-time": {"numericValue": 300}, + "unused-javascript": { + "score": 0.4, + "scoreDisplayMode": "numeric", + "title": "Reduce unused JavaScript", + "details": {"overallSavingsMs": 800}, + }, + }, + } + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "lhr-1.json").write_text(json.dumps(report), encoding="utf-8") + summary = engine.summarize_lighthouse( + validated, + config_hash, + "home", + "desktop", + root, + ) + self.assertEqual(summary["verdict"], "WARN") + self.assertEqual(summary["categories"]["performance"]["median_score"], 50) + self.assertEqual(summary["core_metrics"]["largest_contentful_paint_ms"], 5000.0) + self.assertEqual(summary["top_findings"][0]["audit_id"], "unused-javascript") + + def test_cell_classification_separates_navigation_and_quality_signals(self) -> None: + browser = { + "navigation": {"status": 200, "error": None}, + "signals": { + "keyboard_focus_gap": True, + "unnamed_sequential_controls": 1, + "nested_interactive_controls": 0, + "unnamed_accessibility_controls": 0, + }, + "console": {"error_count": 0}, + } + lighthouse = {"verdict": "PASS", "severity": "NONE"} + result = engine.classify_cell(browser, lighthouse) + self.assertEqual(result["verdict"], "WARN") + self.assertEqual(result["severity"], "MEDIUM") + self.assertIn("keyboard_focus_gap", result["reasons"]) + + browser["navigation"] = {"status": 500, "error": None} + result = engine.classify_cell(browser, lighthouse) + self.assertEqual(result["severity"], "HIGH") + self.assertIn("navigation_failed_or_non_success", result["reasons"]) + + +if __name__ == "__main__": + unittest.main()