diff --git a/.gitignore b/.gitignore index 7a76012..319cb44 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,12 @@ node_modules/ site_front_2/flaskapp/static/dist/ +#test runner +results/ +.test_run/ +test-runner/runner_config.json +test-runner/test_cases.json + # py site_front_2/_playground/ @@ -42,4 +48,4 @@ docker-emlo/csv_import_files docker-emlo/solr-conf/solr/home/**/core.properties # ENV folder -site-front-2 +site-front-2 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5ab3471..55c1c6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,6 @@ volumes: redis: + test_run_state: # persists .test_run/ results across restarts services: web: @@ -15,6 +16,27 @@ services: - 5000:5000 command: bash -c "/bin/docker-entrypoint.sh" + # ── Test Runner ─────────────────────────────────────────────────────────── + # Self-contained service. All files live in ./test-runner/. + # Proxied via the web container — users access it at localhost:5000/test-run. + # Start independently: docker compose up test-runner + # Remove from stack: comment out this block + TEST_RUNNER_URL in web above + test-runner: + build: + context: ./test-runner + dockerfile: Dockerfile + restart: unless-stopped + expose: + - "8085" + ports: + - 8085:8085 # internal only — not exposed to host + volumes: + - test_run_state:/app/.test_run + environment: + DEBUG: "false" + # ───────────────────────────────────────────────────────────────────────── + + solr: build: ./docker-emlo/solr-conf restart: always diff --git a/requirements.txt b/requirements.txt index 9f1e5b8..f1cf865 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ Flask==3.1.2 python-dotenv==1.1.1 requests==2.32.5 -werkzeug==3.1.3 +werkzeug==3.1.3 \ No newline at end of file diff --git a/run.py b/run.py index 0654937..cd5f9a0 100644 --- a/run.py +++ b/run.py @@ -12,9 +12,11 @@ from views.solr import solr_bp from views.shortURL import shortURL_bp from views.redirectUUID import redirect_uuid_bp +from views.test_runner import test_runner_bp from config import Config import os + def create_app(): app = Flask(__name__) app.config.from_object(Config) @@ -32,14 +34,18 @@ def create_app(): app.register_blueprint(solr_bp) app.register_blueprint(shortURL_bp) app.register_blueprint(redirect_uuid_bp) + app.register_blueprint(test_runner_bp) return app + app = create_app() + def main(): debug_mode = os.getenv('DEBUG', 'false').lower() == 'true' app.run(host='0.0.0.0', port=int(app.config['PORT']), debug=debug_mode) + if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/test-runner/Dockerfile b/test-runner/Dockerfile new file mode 100644 index 0000000..67c1107 --- /dev/null +++ b/test-runner/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Install Chromium and ALL its system dependencies in one shot +RUN playwright install --with-deps chromium + +COPY . . + +EXPOSE 8085 + +CMD ["python", "app.py"] diff --git a/test-runner/Readme.md b/test-runner/Readme.md new file mode 100644 index 0000000..02cae58 --- /dev/null +++ b/test-runner/Readme.md @@ -0,0 +1,103 @@ +# Test Runner + +A visual regression testing tool that compares two versions of your site by loading the same pages on both environments, taking screenshots, and highlighting pixel-level differences. Great for catching unexpected UI changes before they reach users. + +When you run a test, the tool opens both sites in a headless browser, captures screenshots at the same viewport size, and overlays them to produce a diff image. Each test gets a match percentage — if it drops below your threshold, the test fails. + +Results stream to the UI in real time so you can watch tests pass or fail as they happen. + +--- + +## Setup + +**1. Rename the template config files** + +```bash +cp runner_config.template.json runner_config.json +cp test_cases.template.json test_cases.json +``` + +These files tell the runner which sites to compare and which pages to test. Without them the runner won't start. + +**2. Start the service** + +```bash +docker compose up test-runner +``` + +Visit `http:///test-run` — you should see the test runner UI. + +--- + +## Settings + +Open the **Settings** tab before your first run. This is where you configure the two environments you want to compare. + +**Sites** +Enter a name and base URL for each site. The base URL is the root of the site — page paths get appended to it when each test runs. For example if Site A is `https://staging.example.com` and you test the path `/about`, the runner will load `https://staging.example.com/about`. + +**Authentication** +If either site sits behind HTTP basic auth, enable it and enter the credentials. This is per-site so you can have auth on one and not the other. + +**Runner behaviour** + +- **Diff threshold** — how strict the pixel comparison is. `0.1` means a 10% colour difference per pixel is acceptable before it counts as a changed pixel. Lower = stricter. +- **Max diff pixels** — how many changed pixels are allowed before a test is marked as failed. `0` means any difference fails the test. +- **Page timeout** — how long to wait for a page to fully load before giving up. Default is 2 minutes — increase this for slow pages. +- **Viewport** — the browser window size used for screenshots. Both sites are screenshotted at the same size so the comparison is fair. +- **Capture all screenshots** — by default only diff images are saved. Turn this on to also save the individual screenshots for passing tests. +- **Fail fast** — stop the entire run as soon as one test fails instead of continuing through the rest. +- **Clean old results** — delete previous screenshots and diffs before each run so results don't accumulate. Leave this on unless you need to keep historical images. + +Click **Save Settings** when done. Settings are written to `runner_config.json`. + +--- + +## Test Cases + +Open the **Test Cases** tab to manage the pages you want to compare. + +Each test case has: +- **ID** — a unique identifier, auto-generated when you add a new test +- **Test name** — a human-readable label shown in the results sidebar (e.g. `Homepage`, `Search Results`) +- **URI** — the page path to test, starting with `/` (e.g. `/en/`, `/collections/all`) +- **Description** — optional notes about what the page is or what to look out for + +Click **+ Add Test** to add a row, fill in the name and URI, then click **Save All**. Test cases are written to `test_cases.json`. + +> You can edit test cases between runs but not while a run is in progress. + +--- + +## Running tests + +Hit **Execute Tests** on the Run tab. The runner will work through your test cases one by one — you'll see each test move from queued → running → passed or failed in the sidebar as it completes. + +Click any test in the sidebar to see: +- **Match percentage** — how similar the two screenshots are +- **Diff image** — a visual overlay showing exactly what changed (red pixels = differences) +- **Screenshots** — side by side images of both sites (only shown if Capture all screenshots is on, or if the test failed) +- **Load times** — how long each site took to load the page +- **Error details** — if a test couldn't run, the reason is shown here + +Once all tests complete, a summary bar appears at the top showing totals for passed, failed, and errors. + +--- + +## Troubleshooting + +**Tests are stuck in queue after a run** +The runner crashed before it could start. Check the Console Output panel at the bottom of the Run tab — the error message will tell you what went wrong. Most commonly it's a missing or invalid `runner_config.json`. + +**A test shows execution error** +The runner couldn't load the page — usually a network issue, bad URL, or the page timing out. Check the error message in the test detail panel and make sure the URL is reachable from inside Docker. + +**Settings or test cases aren't saving** +Make sure `runner_config.json` and `test_cases.json` exist in the `test-runner/` folder. If you skipped Step 1 of setup, go back and create them from the templates. + +**The service won't start** +```bash +docker compose build test-runner +docker compose up test-runner +``` +A rebuild is usually needed after the first install or if dependencies have changed. \ No newline at end of file diff --git a/test-runner/app.py b/test-runner/app.py new file mode 100644 index 0000000..6f01b60 --- /dev/null +++ b/test-runner/app.py @@ -0,0 +1,30 @@ +""" +Standalone Flask app for the test-runner service. +Runs internally on port 5001. Users access it via the main app at: + http://localhost:5000/test-run +""" +from flask import Flask, send_from_directory +from test_run import test_run_bp +import os + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +def create_app(): + app = Flask(__name__) + app.register_blueprint(test_run_bp) + + @app.route("/") + def landing(): + return send_from_directory(BASE_DIR, "landing.html") + + @app.errorhandler(404) + def not_found(e): + return send_from_directory(BASE_DIR, "404.html"), 404 + + return app + +app = create_app() + +if __name__ == "__main__": + debug_mode = os.getenv("DEBUG", "false").lower() == "true" + app.run(host="0.0.0.0", port=8085, debug=debug_mode) diff --git a/test-runner/error.html b/test-runner/error.html new file mode 100644 index 0000000..853d88d --- /dev/null +++ b/test-runner/error.html @@ -0,0 +1,89 @@ + + + + +404 — Not Found + + + + + + + + + +
+
+
404
+ +
+
+ Page not found +
+ +

Nothing here

+

The page you're looking for doesn't exist in this service. This runner only handles test-related routes.

+ +
+ Requested + — +
+ +
+ ← Home + Open Test Runner +
+
+
+ + + + + + \ No newline at end of file diff --git a/test-runner/index.html b/test-runner/index.html new file mode 100644 index 0000000..d164152 --- /dev/null +++ b/test-runner/index.html @@ -0,0 +1,998 @@ + + + + +Visual Regression — Test Runner + + + + + + + + + + +
+
+
+

Visual Regression Tests

+

Compare site layouts side by side.

+
+ +
+
+
+
+
+
+ Tests + 0 +
+ +
+
+
Click Execute Tests to run your tests.
Results will appear here.
+
+
+ +
+ + +
+
+

Test Cases

Add, edit or remove the URLs you want to compare.

+
+ + +
+
+
+ ⚠ A run is in progress — test cases are read-only until it finishes. +
+
+ + + + + + + + + + + +
IDTest NameURIDescription
Loading…
+
+
+ + +
+
+

Settings

Configure sites, authentication and runner behaviour. Saved to runner_config.json.

+
+
+ ⚠ A run is in progress — settings are read-only until it finishes. +
+
+ + +
+
Sites
+
+
+

Site A

+
+
+
+
+

Site B

+
+
+
+
+
+ + +
+
Authentication
+
+
+

Site A Auth

+
+ + +
+
+
+
+
+

Site B Auth

+
+ + +
+
+
+
+
+
+ + +
+
Runner Configuration
+
+
+

Behaviour

+
+ + +
+
+ + +
+
+ + +
+
+
+

Comparison & Timing

+
+ + + Per-pixel colour tolerance +
+
+ + + Allowed differing pixels before marking failed +
+
+ + + Max wait per page — 120000 = 2 min. Increase for slow pages. +
+
+ +
+ + × + +
+ Width × Height in pixels +
+
+
+
+ +
+ + ✓ Saved +
+
+
+ + + + \ No newline at end of file diff --git a/test-runner/landing.html b/test-runner/landing.html new file mode 100644 index 0000000..b9b7eab --- /dev/null +++ b/test-runner/landing.html @@ -0,0 +1,145 @@ + + + + +Visual Regression — Test Runner + + + + + + + + + +
+
+
+ Test Runner Service +
+

Visual regression
testing, simplified

+

Compare pages across two environments side by side. Catch visual regressions before they reach your users — with pixel-level diff reports and real-time streaming results.

+ + Open Test Runner + + + + +
+ +
+
+
🔍
+

Pixel-level Diffs

+

Compare screenshots between two environments and highlight exactly what changed, down to the pixel.

+
+
+
⚡
+

Real-time Streaming

+

Watch tests run live with server-sent events. No polling, no waiting — results appear as they happen.

+
+
+
⚙️
+

Fully Configurable

+

Set base URLs, auth credentials, diff thresholds, viewport sizes and more from the settings panel.

+
+
+ +
+
How it works
+
+
+
1
+
+

Configure your environments

+

Set the base URLs for both sites you want to compare, along with any auth credentials required to access them.

+
+
+
+
2
+
+

Add your test cases

+

Define the pages you want to test using /relative/paths. Give each test a name and it'll be tracked across runs.

+
+
+
+
3
+
+

Run and review

+

Hit Run Tests and watch results stream in live. Click any test to see screenshots, diffs, load times and match percentage.

+
+
+
+
4
+
+

Investigate failures

+

Any test below your diff threshold is flagged as failed. The diff image highlights exactly what changed between the two environments.

+
+
+
+
+ + + + + \ No newline at end of file diff --git a/test-runner/requirements.txt b/test-runner/requirements.txt new file mode 100644 index 0000000..9a73bd1 --- /dev/null +++ b/test-runner/requirements.txt @@ -0,0 +1,4 @@ +flask>=2.3.0 +playwright>=1.40.0 +Pillow>=10.0.0 +pixelmatch>=0.3.0 \ No newline at end of file diff --git a/test-runner/runner.py b/test-runner/runner.py new file mode 100644 index 0000000..9484a18 --- /dev/null +++ b/test-runner/runner.py @@ -0,0 +1,499 @@ +import os +import sys +import json +import time +import shutil +from io import BytesIO +from datetime import datetime, timezone +from pathlib import Path + +from playwright.sync_api import sync_playwright, Page +from PIL import Image +from pixelmatch.contrib.PIL import pixelmatch + + +# ===================== CONFIG ===================== + +SCHEMA_VERSION = "1.3" + +BASE_DIR = Path(__file__).resolve().parent + +CONFIG_FILE = BASE_DIR / "runner_config.json" +TESTS_FILE = BASE_DIR / "test_cases.json" +SCREENSHOT_DIR = BASE_DIR / ".test_run/results" + + +# ===================== HELPERS ===================== + +def fatal(msg: str): + print(f"\n❌ {msg}", flush=True) + sys.exit(1) + + +def emit_result(result: dict): + """ + Print a structured result line immediately after each test completes. + Prefixed with ##RESULT## so test_run.py can parse it unambiguously. + flush=True is critical — without it Python buffers stdout until process exit. + """ + print(f"##RESULT## {json.dumps(result)}", flush=True) + + +# ===================== LOAD CONFIG ===================== + +if not CONFIG_FILE.exists(): + fatal(f"runner_config.json not found at {CONFIG_FILE}") + +try: + with open(CONFIG_FILE) as f: + CFG = json.load(f) +except json.JSONDecodeError as exc: + fatal(f"runner_config.json is not valid JSON: {exc}") + +sites = CFG.get("sites", {}) +CL_CFG = sites.get("cl", {}) +OX_CFG = sites.get("ox", {}) +CL_NAME = CL_CFG.get("name", "Site A") +OX_NAME = OX_CFG.get("name", "Site B") +CL_BASE_URL = CL_CFG.get("base_url", "").rstrip("/") +OX_BASE_URL = OX_CFG.get("base_url", "").rstrip("/") + +auth_cfg = CFG.get("auth", {}) +CL_AUTH = auth_cfg.get("cl", {}) +OX_AUTH = auth_cfg.get("ox", {}) + +run_cfg = CFG.get("config", {}) +VIEWPORT = { + "width": run_cfg.get("viewport_width", 1440), + "height": run_cfg.get("viewport_height", 900), +} +DIFF_THRESHOLD = run_cfg.get("diff_threshold", 0.1) +MAX_DIFF_PIXELS = run_cfg.get("max_diff_pixels", 0) +FAIL_FAST = run_cfg.get("fail_fast", False) +CLEAN_OLD_RESULTS = run_cfg.get("clean_old_results", True) +CAPTURE_ALL_SCREENSHOTS = run_cfg.get("capture_all_screenshots", False) +# Timeout in ms for page load + wait_for_content — default 120s to handle slow pages +PAGE_TIMEOUT_MS = int(run_cfg.get("page_timeout_ms", 120_000)) + + +# ===================== VALIDATION ===================== + +if not CL_BASE_URL or not OX_BASE_URL: + fatal("Missing base_url for cl or ox in runner_config.json → sites section") + +if not TESTS_FILE.exists(): + fatal(f"test_cases.json not found at {TESTS_FILE}") + + +# ===================== STORAGE SETUP ===================== + +if CLEAN_OLD_RESULTS and SCREENSHOT_DIR.exists(): + shutil.rmtree(SCREENSHOT_DIR) + +SCREENSHOT_DIR.mkdir(parents=True, exist_ok=True) + + +# ===================== LOAD TEST CASES ===================== + +try: + with open(TESTS_FILE) as f: + TESTS = json.load(f) +except json.JSONDecodeError as exc: + fatal(f"test_cases.json is not valid JSON: {exc}") + +if not TESTS: + fatal("No test cases found in test_cases.json") + + +# ===================== HELPERS ===================== + +def build_context_args(site_auth: dict) -> dict: + """Build Playwright browser context kwargs, adding HTTP auth if required.""" + args = {"viewport": VIEWPORT} + if site_auth.get("required") and site_auth.get("username"): + args["http_credentials"] = { + "username": site_auth["username"], + "password": site_auth.get("password", ""), + } + return args + + +def wait_for_content(page: Page, timeout: int = 30000, idle_for: float = 1.5): + """ + Wait until the page content is truly ready — no spinners, no pending requests. + + Steps: + 1. wait for networkidle (Playwright built-in — no requests for 500ms) + 2. wait for common loading indicators to disappear + 3. poll until no pending XHR/fetch for `idle_for` consecutive seconds + + This handles 'Loading please wait…' overlays and async data fetching. + """ + # Step 1 — networkidle: Playwright waits until no network requests for 500ms + try: + page.wait_for_load_state("networkidle", timeout=timeout) + except Exception: + pass # timeout is acceptable — we continue with the other checks + + # Step 2 — wait for common loading selectors to disappear + LOADING_SELECTORS = [ + "[class*='loading']", + "[class*='spinner']", + "[class*='loader']", + "[id*='loading']", + "[aria-label*='Loading']", + "[aria-busy='true']", + ] + selector = ", ".join(LOADING_SELECTORS) + deadline = time.time() + 10 + while time.time() < deadline: + try: + still_loading = page.evaluate(f""" + () => {{ + const els = document.querySelectorAll('{selector}'); + return Array.from(els).some(el => {{ + const s = window.getComputedStyle(el); + return s.display !== 'none' + && s.visibility !== 'hidden' + && s.opacity !== '0'; + }}); + }} + """) + if not still_loading: + break + except Exception: + break + time.sleep(0.3) + + # Step 3 — inject request counter and wait until idle + try: + page.evaluate(""" + () => { + if (window.__pendingRequests === undefined) { + window.__pendingRequests = 0; + const _fetch = window.fetch; + window.fetch = function(...args) { + window.__pendingRequests++; + return _fetch.apply(this, args) + .finally(() => window.__pendingRequests--); + }; + const _open = XMLHttpRequest.prototype.open; + XMLHttpRequest.prototype.open = function(...args) { + window.__pendingRequests++; + this.addEventListener('loadend', + () => window.__pendingRequests--); + return _open.apply(this, args); + }; + } + } + """) + deadline = time.time() + 10 + idle_since = None + while time.time() < deadline: + pending = page.evaluate("() => window.__pendingRequests || 0") + if pending == 0: + if idle_since is None: + idle_since = time.time() + elif time.time() - idle_since >= idle_for: + break + else: + idle_since = None + time.sleep(0.2) + except Exception: + pass + + +def get_performance_timings(page: Page) -> dict: + """ + Extract detailed performance timings from the Navigation Timing API. + Returns a flat dict of all timing breakdowns in milliseconds. + """ + try: + return page.evaluate(""" + () => { + const e = performance.getEntriesByType('navigation')[0]; + if (!e) return {}; + const r = (a, b) => Math.max(0, Math.round(a - b)); + return { + // Network + dns_ms: r(e.domainLookupEnd, e.domainLookupStart), + tcp_ms: r(e.connectEnd, e.connectStart), + tls_ms: e.secureConnectionStart > 0 + ? r(e.requestStart, e.secureConnectionStart) + : 0, + ttfb_ms: r(e.responseStart, e.requestStart), + download_ms: r(e.responseEnd, e.responseStart), + network_ms: r(e.responseEnd, e.fetchStart), + // Rendering + dom_parse_ms: r(e.domInteractive, e.responseEnd), + dom_content_ms: r(e.domContentLoadedEventEnd, e.responseEnd), + render_ms: r(e.domComplete, e.domInteractive), + // Totals + load_ms: Math.round(e.loadEventEnd), + total_ms: r(e.loadEventEnd, e.fetchStart), + }; + } + """) or {} + except Exception: + return {} + + +def capture_viewport(page: Page) -> bytes: + """Screenshot of exactly the configured viewport — no full-page scroll.""" + return page.screenshot( + full_page=False, + clip={ + "x": 0, "y": 0, + "width": VIEWPORT["width"], + "height": VIEWPORT["height"], + }, + ) + + +def save_image(bytes_data: bytes, path: Path): + Image.open(BytesIO(bytes_data)).save(path) + + +def normalize_id(value: str) -> str: + return value.strip().lower().replace(" ", "_").replace("/", "_") + + +# ===================== MAIN RUNNER ===================== + +def main(): + run_id = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + run_started_at = datetime.now(timezone.utc) + + results = [] + failed_count = 0 + + print(f"\n🚀 UI Layout Test Run: {run_id}", flush=True) + print(f" {CL_NAME}: {CL_BASE_URL}", flush=True) + print(f" {OX_NAME}: {OX_BASE_URL}", flush=True) + print(f"📄 Loaded {len(TESTS)} test cases", flush=True) + + with sync_playwright() as p: + browser = p.chromium.launch() + + # Separate contexts per site so each gets its own session and auth + cl_context = browser.new_context(**build_context_args(CL_AUTH)) + ox_context = browser.new_context(**build_context_args(OX_AUTH)) + + cl_page = cl_context.new_page() + ox_page = ox_context.new_page() + + browser_info = { + "name": browser.browser_type.name, + "version": browser.version, + "headless": True, + "viewport": VIEWPORT, + "user_agent": cl_page.evaluate("() => navigator.userAgent"), + } + + try: + for _, test in TESTS.items(): + uri = test.get("uri", "") + test_name = test.get("test_name", "") or test.get("id", "unknown") + test_id = test.get("id") or normalize_id(test_name) + desc = test.get("desc", "") + + cl_site_url = CL_BASE_URL + uri + ox_site_url = OX_BASE_URL + uri + + print(f"\n▶ Executing test: {test_name}", flush=True) + if desc: + print(f" ↳ {desc}", flush=True) + + start_time = time.time() + + status = "passed" + execution_error = None + matching_percentage = None + cl_timings = {} + ox_timings = {} + artifacts = { + "cl_site_image": None, + "ox_site_image": None, + "diff_image": None, + } + + try: + print(f" → Loading {CL_NAME}", flush=True) + cl_page.goto(cl_site_url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT_MS) + wait_for_content(cl_page, timeout=PAGE_TIMEOUT_MS) + cl_timings = get_performance_timings(cl_page) + cl_img_bytes = capture_viewport(cl_page) + + print(f" → Loading {OX_NAME}", flush=True) + ox_page.goto(ox_site_url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT_MS) + wait_for_content(ox_page, timeout=PAGE_TIMEOUT_MS) + ox_timings = get_performance_timings(ox_page) + ox_img_bytes = capture_viewport(ox_page) + + if CAPTURE_ALL_SCREENSHOTS: + cl_name = f"{test_id}_cl_site.png" + ox_name = f"{test_id}_ox_site.png" + save_image(cl_img_bytes, SCREENSHOT_DIR / cl_name) + save_image(ox_img_bytes, SCREENSHOT_DIR / ox_name) + artifacts["cl_site_image"] = cl_name + artifacts["ox_site_image"] = ox_name + + img_cl = Image.open(BytesIO(cl_img_bytes)) + img_ox = Image.open(BytesIO(ox_img_bytes)) + + # Ensure same dimensions before diffing + if img_cl.size != img_ox.size: + img_ox = img_ox.resize(img_cl.size, Image.LANCZOS) + + diff_img = Image.new("RGBA", img_cl.size) + diff_pixels = pixelmatch( + img_cl, img_ox, diff_img, + threshold=DIFF_THRESHOLD, + ) + + total_pixels = img_cl.size[0] * img_cl.size[1] + matching_percentage = round( + ((total_pixels - diff_pixels) / total_pixels) * 100, 2 + ) + + diff_name = f"diff_{test_id}.png" + diff_img.save(SCREENSHOT_DIR / diff_name) + artifacts["diff_image"] = diff_name + + if diff_pixels > MAX_DIFF_PIXELS: + status = "failed" + failed_count += 1 + print(f" ❌ FAILED ({matching_percentage}% match)", flush=True) + else: + print(" ✅ PASSED", flush=True) + + except Exception as e: + status = "execution-error" + execution_error = str(e) + failed_count += 1 + print(f" ❌ EXECUTION ERROR: {e}", flush=True) + if FAIL_FAST: + raise + + duration_ms = int((time.time() - start_time) * 1000) + + result = { + "id": test_id, + "test_name": test_name, + "uri": uri, + "desc": desc, + "cl_site_url": cl_site_url, + "ox_site_url": ox_site_url, + # Flat fields for backwards compat with UI sidebar + "cl_load_time_ms": cl_timings.get("load_ms"), + "ox_load_time_ms": ox_timings.get("load_ms"), + # Full breakdowns shown in detail panel + "cl_timings": cl_timings, + "ox_timings": ox_timings, + "status": status, + "execution_error": execution_error, + "matching_percentage": matching_percentage, + "duration_ms": duration_ms, + "artifacts": artifacts, + } + + results.append(result) + emit_result(result) # real-time UI update via ##RESULT## + + if status != "passed" and FAIL_FAST: + break + + finally: + # Mark any tests that never ran as execution-error so the UI + # doesn't leave them stuck in queued state + completed_ids = {r["id"] for r in results} + for _, test in TESTS.items(): + tid = test.get("id") or normalize_id(test.get("test_name", "")) + name = test.get("test_name", tid) + if tid not in completed_ids: + skipped = { + "id": tid, + "test_name": name, + "uri": test.get("uri", ""), + "desc": test.get("desc", ""), + "cl_site_url": CL_BASE_URL + test.get("uri", ""), + "ox_site_url": OX_BASE_URL + test.get("uri", ""), + "cl_load_time_ms": None, + "ox_load_time_ms": None, + "cl_timings": {}, + "ox_timings": {}, + "status": "execution-error", + "execution_error": "Test did not run — script exited early", + "matching_percentage": None, + "duration_ms": 0, + "artifacts": { + "cl_site_image": None, + "ox_site_image": None, + "diff_image": None, + }, + } + results.append(skipped) + emit_result(skipped) + + cl_context.close() + ox_context.close() + browser.close() + + run_completed_at = datetime.now(timezone.utc) + run_duration = int((run_completed_at - run_started_at).total_seconds() * 1000) + + # ===================== FINAL REPORT ===================== + + total = len(results) + passed = sum(1 for r in results if r["status"] == "passed") + failed = sum(1 for r in results if r["status"] == "failed") + exec_err = sum(1 for r in results if r["status"] == "execution-error") + + report = { + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "run_started_at": run_started_at.isoformat(), + "run_completed_at": run_completed_at.isoformat(), + "run_duration": run_duration, + "config": { + "fail_fast": FAIL_FAST, + "clean_old_results": CLEAN_OLD_RESULTS, + "capture_all_screenshots": CAPTURE_ALL_SCREENSHOTS, + "viewport": VIEWPORT, + "diff_threshold": DIFF_THRESHOLD, + "max_diff_pixels": MAX_DIFF_PIXELS, + "browser": browser_info, + "sites": { + "cl": {"name": CL_NAME, "base_url": CL_BASE_URL}, + "ox": {"name": OX_NAME, "base_url": OX_BASE_URL}, + }, + }, + "summary": { + "total": total, + "passed": passed, + "failed": failed, + "execution_error": exec_err, + "pass_percentage": round((passed / total) * 100, 2) if total else 0.0, + "fail_percentage": round((failed / total) * 100, 2) if total else 0.0, + "execution_error_percentage": round((exec_err / total) * 100, 2) if total else 0.0, + }, + "tests": results, + } + + report_path = SCREENSHOT_DIR / f"reports_{run_id}.json" + try: + SCREENSHOT_DIR.mkdir(parents=True, exist_ok=True) + tmp_report = report_path.with_suffix(".tmp") + tmp_report.write_text(json.dumps(report, indent=2)) + tmp_report.replace(report_path) + print(f"\n📄 Report generated: {report_path}", flush=True) + except Exception as exc: + print(f"\n❌ Failed to write report: {exc}", flush=True) + + sys.exit(1 if failed_count > 0 else 0) + + +# ===================== ENTRY POINT ===================== + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test-runner/runner_config.template.json b/test-runner/runner_config.template.json new file mode 100644 index 0000000..d272491 --- /dev/null +++ b/test-runner/runner_config.template.json @@ -0,0 +1,34 @@ +{ + "sites": { + "cl": { + "name": "site_a_name", + "base_url": "site_a_url" + }, + "ox": { + "name": "site_b_name", + "base_url": "site_b_url" + } + }, + "auth": { + "cl": { + "required": false, + "username": "site_a_username_only_true", + "password": "site_a_pass_only_true" + }, + "ox": { + "required": false, + "username": "site_b_username_only_true", + "password": "site_b_pass_only_true" + } + }, + "config": { + "capture_all_screenshots": true, + "fail_fast": false, + "clean_old_results": true, + "diff_threshold": 0.1, + "max_diff_pixels": 0, + "viewport_width": 1440, + "viewport_height": 900, + "page_timeout_ms": 120000 + } +} \ No newline at end of file diff --git a/test-runner/test_cases.template.json b/test-runner/test_cases.template.json new file mode 100644 index 0000000..4d302ad --- /dev/null +++ b/test-runner/test_cases.template.json @@ -0,0 +1,102 @@ +{ + "test_1": { + "test_name": "Home page", + "uri": "/", + "desc": "Testing home page layout" + }, + "test_10": { + "test_name": "Browse works for year 1611", + "uri": "/browse/works?year=1611", + "desc": "" + }, + "test_11": { + "test_name": "Quick search", + "uri": "/forms/quick?everything=godalming&search_type=quick", + "desc": "Quick search godalming" + }, + "test_12": { + "test_name": "Search for Newton", + "uri": "/forms/advanced?people=newton", + "desc": "Searching for newton in all people" + }, + "test_13": { + "test_name": "Work profile", + "uri": "/profile/work/8a0b8091-ea30-444d-a9b3-d45029b56d3c", + "desc": "Work profile template check" + }, + "test_14": { + "test_name": "Person profile", + "uri": "/profile/person/6bd60310-988c-47b3-9a91-17af575047f4", + "desc": "Person profile layout check" + }, + "test_15": { + "test_name": "Location profile", + "uri": "/profile/location/8d126056-cd1b-44e6-ac87-b59cbeb718b6", + "desc": "Location profile layout check" + }, + "test_16": { + "test_name": "Organizations profile", + "uri": "/profile/person/476425ae-34b3-421f-b1c7-e9e7fd990241", + "desc": "Organizations profile layout check" + }, + "test_17": { + "test_name": "Repository profile", + "uri": "/profile/repository/44e4cc09-01f3-463a-9057-0e7dd35955be", + "desc": "Repository profile layout check" + }, + "test_18": { + "test_name": "Manifestation Profile", + "uri": "/profile/manifestation/9dcbb8f2-da22-411b-95d4-478da4267806", + "desc": "Manifestation profile layout check" + }, + "test_19": { + "test_name": "Image profile", + "uri": "/profile/image/4ac9b456-35c3-41d6-8828-6b2f9590d74c", + "desc": "Image profile layout check" + }, + "test_2": { + "test_name": "Search+", + "uri": "/advanced", + "desc": "Testing search+ page." + }, + "test_20": { + "test_name": "Broken URL", + "uri": "/broken-url", + "desc": "Broken layout page check" + }, + "test_3": { + "test_name": "Search+ specific section", + "uri": "/advanced#letters_section", + "desc": "Checking layout if we directly go on specific section." + }, + "test_4": { + "test_name": "About page", + "uri": "/about", + "desc": "Checking layout for about page" + }, + "test_5": { + "test_name": "Contribute", + "uri": "/contribute", + "desc": "Checking layout for contribute page." + }, + "test_6": { + "test_name": "Browse people with letter I", + "uri": "/browse/people?letter=i&filters=fe%2Cma%2Cun%2Cre%2Cwr%2Cme", + "desc": "Layout checking for peoples" + }, + "test_7": { + "test_name": "Browse locations", + "uri": "/browse/locations?letter=z&filters=fe,ma,un,re,wr,me", + "desc": "Layout checking for locations" + }, + "test_8": { + "test_name": "Browse organizations", + "uri": "/browse/organizations?filters=fe,ma,un,re,wr,me", + "desc": "Layout checking for organizations" + }, + "test_9": { + "test_name": "Browse repos with letter O", + "uri": "/browse/repositories?letter=o", + "desc": "Layout checking for repositories" + } +} \ No newline at end of file diff --git a/test-runner/test_run.py b/test-runner/test_run.py new file mode 100644 index 0000000..5516df3 --- /dev/null +++ b/test-runner/test_run.py @@ -0,0 +1,506 @@ +from __future__ import annotations + +import json +import logging +import queue +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Optional + +from flask import Blueprint, Response, jsonify, request, send_from_directory + +# ── Logging ─────────────────────────────────────────────────────────────────── +logger = logging.getLogger("test_run") +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] [%(levelname)s] test_run: %(message)s", + datefmt="%H:%M:%S", +) + +# ── Config ──────────────────────────────────────────────────────────────────── +# Everything lives flat inside the test-runner folder. +BASE_DIR = Path(__file__).resolve().parent +TEST_DIR = BASE_DIR # all files are in the same folder +TESTS_FILE = BASE_DIR / "test_cases.json" +CONFIG_FILE = BASE_DIR / "runner_config.json" + +RUNNER_SCRIPT = BASE_DIR / "runner.py" + +STATE_RUN_DIR = BASE_DIR / ".test_run" +RESULTS_DIR = STATE_RUN_DIR / "results" +STATE_FILE = STATE_RUN_DIR / "_run_state.json" + +logger.info("BASE_DIR = %s", BASE_DIR) +logger.info("TEST_DIR = %s", TEST_DIR) +logger.info("TESTS_FILE = %s exists=%s", TESTS_FILE, TESTS_FILE.exists()) +logger.info("CONFIG_FILE = %s exists=%s", CONFIG_FILE, CONFIG_FILE.exists()) +logger.info("RUNNER = %s exists=%s", RUNNER_SCRIPT, RUNNER_SCRIPT.exists()) +logger.info("STATE_FILE = %s", STATE_FILE) + +test_run_bp = Blueprint("test_run", __name__) + +_run_lock = threading.Lock() +_run_active = False +_event_queues = [] + +DEFAULT_CONFIG = { + "sites": { + "cl": {"name": "Cottagelabs", "base_url": ""}, + "ox": {"name": "Bodleian", "base_url": ""}, + }, + "auth": { + "cl": {"required": False, "username": "", "password": ""}, + "ox": {"required": False, "username": "", "password": ""}, + }, + "config": { + "capture_all_screenshots": False, + "fail_fast": False, + "clean_old_results": True, + "diff_threshold": 0.1, + "max_diff_pixels": 0, + "viewport_width": 1440, + "viewport_height": 900, + }, +} + + +# ── Helpers ─────────────────────────────────────────────────────────────────── +def _broadcast(event: str, data: dict): + msg = f"event: {event}\ndata: {json.dumps(data)}\n\n" + for q in list(_event_queues): + try: + q.put_nowait(msg) + except queue.Full: + logger.warning("SSE queue full — dropping '%s'", event) + + +def _write_state(state: dict): + try: + STATE_RUN_DIR.mkdir(parents=True, exist_ok=True) + tmp = STATE_FILE.with_suffix(".tmp") + tmp.write_text(json.dumps(state, indent=2)) + tmp.replace(STATE_FILE) + except Exception as exc: + logger.error("Failed to write state file: %s", exc) + + +def _read_state() -> Optional[dict]: + if not STATE_FILE.exists(): + return None + try: + data = json.loads(STATE_FILE.read_text()) + logger.info("State loaded: status=%s tests=%d", data.get("status"), len(data.get("tests", {}))) + return data + except Exception as exc: + logger.error("Failed to read state file: %s", exc) + return None + + +def _read_config() -> dict: + if not CONFIG_FILE.exists(): + logger.warning("runner_config.json not found — returning defaults") + return DEFAULT_CONFIG + try: + with open(CONFIG_FILE) as f: + return json.load(f) + except Exception as exc: + logger.error("Failed to read runner_config.json: %s", exc) + return DEFAULT_CONFIG + + +def _write_config(data: dict): + try: + CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True) + tmp = CONFIG_FILE.with_suffix(".tmp") + tmp.write_text(json.dumps(data, indent=2)) + tmp.replace(CONFIG_FILE) + logger.info("runner_config.json written to %s", CONFIG_FILE) + except Exception as exc: + logger.error("Failed to write runner_config.json: %s", exc) + raise + + +def _is_running() -> bool: + """Check if a run is currently active (thread flag OR state file).""" + if _run_active: + return True + state = _read_state() + return bool(state and state.get("status") == "running") + + +def _build_initial_state(tests: dict) -> dict: + return { + "status": "running", + "started_at": time.time(), + "finished_at": None, + "exit_code": None, + "report": None, + "logs": [], + "tests_snapshot": dict(tests), # locked copy — immune to file edits during run + "tests": { + tid: { + "test_name": t.get("test_name", tid), + "uri": t.get("uri", ""), + "status": "queued", + "matching_percentage": None, + "cl_load_time_ms": None, + "ox_load_time_ms": None, + "duration_ms": None, + "execution_error": None, + "cl_site_url": None, + "ox_site_url": None, + "artifacts": { + "cl_site_image": None, + "ox_site_image": None, + "diff_image": None, + }, + } + for tid, t in tests.items() + }, + } + + +# ── Startup ─────────────────────────────────────────────────────────────────── +def _reset_stale_run(): + """On startup, if state file says 'running' it's a stale state from a previous + crashed/restarted process. Mark it as error so _is_running() doesn't stay True forever.""" + state = _read_state() + if state and state.get("status") == "running": + logger.warning("Stale 'running' state found on startup — marking as error") + state["status"] = "error" + state["logs"] = state.get("logs", []) + ["[STARTUP] Run was interrupted by server restart"] + _write_state(state) + +_reset_stale_run() + + +# ── UI ──────────────────────────────────────────────────────────────────────── +@test_run_bp.route("/test-run") +def testRun(): + return send_from_directory(str(TEST_DIR), "index.html") + + +# ── Test cases CRUD ─────────────────────────────────────────────────────────── +@test_run_bp.route("/api/tests", methods=["GET"]) +def get_tests(): + if not TESTS_FILE.exists(): + return jsonify({}) + try: + with open(TESTS_FILE) as f: + return jsonify(json.load(f)) + except Exception as exc: + logger.error("Failed to read test_cases.json: %s", exc) + return jsonify({"error": f"Could not read test_cases.json: {exc}"}), 500 + + +@test_run_bp.route("/api/tests", methods=["POST"]) +def save_tests(): + if _is_running(): + return jsonify({"error": "Cannot edit tests while a run is in progress"}), 409 + data = request.get_json(force=True) + if data is None: + return jsonify({"error": "Invalid JSON body"}), 400 + if not isinstance(data, dict): + return jsonify({"error": "Expected a JSON object"}), 400 + try: + TESTS_FILE.parent.mkdir(parents=True, exist_ok=True) + tmp = TESTS_FILE.with_suffix(".tmp") + tmp.write_text(json.dumps(data, indent=2)) + tmp.replace(TESTS_FILE) + logger.info("Saved %d test cases to %s", len(data), TESTS_FILE) + except Exception as exc: + logger.error("Failed to write test_cases.json: %s", exc) + return jsonify({"error": f"Failed to save: {exc}"}), 500 + return jsonify({"ok": True}) + + +# ── Runner config ───────────────────────────────────────────────────────────── +@test_run_bp.route("/api/settings", methods=["GET"]) +def get_settings(): + return jsonify(_read_config()) + + +@test_run_bp.route("/api/settings", methods=["POST"]) +def save_settings(): + if _is_running(): + return jsonify({"error": "Cannot edit settings while a run is in progress"}), 409 + data = request.get_json(force=True) + if data is None: + return jsonify({"error": "Invalid JSON body"}), 400 + # Deep-merge into existing config so partial updates don't wipe unrelated keys + existing = _read_config() + for section in ("sites", "auth", "config"): + if section in data: + if isinstance(data[section], dict) and isinstance(existing.get(section), dict): + for k, v in data[section].items(): + if isinstance(v, dict) and isinstance(existing[section].get(k), dict): + existing[section][k].update(v) + else: + existing[section][k] = v + else: + existing[section] = data[section] + try: + _write_config(existing) + except Exception as exc: + return jsonify({"error": f"Failed to save config: {exc}"}), 500 + logger.info("runner_config.json saved") + return jsonify({"ok": True}) + + +# ── Run tests ───────────────────────────────────────────────────────────────── +@test_run_bp.route("/api/run", methods=["POST"]) +def run_tests(): + global _run_active + + with _run_lock: + existing = _read_state() + if existing and existing.get("status") == "running": + return jsonify({"error": "A run is already in progress"}), 409 + _run_active = True + + # Validate + for label, path in [("test_cases.json", TESTS_FILE), ("runner_config.json", CONFIG_FILE), ("runner.py", RUNNER_SCRIPT)]: + if not path.exists(): + _run_active = False + msg = f"{label} not found at {path}" + logger.error(msg) + return jsonify({"error": msg}), 500 + + try: + with open(TESTS_FILE) as f: + tests = json.load(f) + except Exception as exc: + _run_active = False + return jsonify({"error": f"Could not parse test_cases.json: {exc}"}), 500 + + if not tests: + _run_active = False + return jsonify({"error": "No test cases found"}), 400 + + # Write initial state before thread starts + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + run_state = _build_initial_state(tests) + _write_state(run_state) + logger.info("Initial state written — %d tests queued", len(tests)) + + def _stream_run(): + global _run_active + logger.info("Run thread started") + try: + _broadcast("run_start", {"ts": run_state["started_at"]}) + # Use the snapshot so the UI shows exactly what was locked in at run start + snapshot = run_state["tests_snapshot"] + _broadcast("test_list", {"ids": list(snapshot.keys()), "tests": snapshot}) + + proc = subprocess.Popen( + [sys.executable, str(RUNNER_SCRIPT)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + cwd=str(BASE_DIR), + ) + logger.info("Subprocess PID: %s", proc.pid) + + current_test = None + + for raw_line in proc.stdout: + line = raw_line.rstrip() + if not line: + continue + + run_state["logs"].append(line) + if len(run_state["logs"]) > 300: + run_state["logs"] = run_state["logs"][-300:] + + _broadcast("log", {"line": line}) + + # ── Structured result (rich, immediate) ─────────────────────── + if "##RESULT##" in line: + try: + json_part = line[line.index("##RESULT##") + len("##RESULT##"):].strip() + result = json.loads(json_part) + result_name = result.get("test_name") + normalized = result.get("id") + if result_name or normalized: + status = result.get("status", "unknown") + # Map back to test_cases.json key + state_key = next( + (k for k, v in run_state["tests"].items() + if v.get("test_name") == result_name or k == normalized), + normalized, + ) + logger.info(" ##RESULT## %s (key=%s) => %s match=%s", + result_name, state_key, status, + result.get("matching_percentage")) + run_state["tests"][state_key] = { + **run_state["tests"].get(state_key, {}), + "status": status, + "matching_percentage": result.get("matching_percentage"), + "cl_load_time_ms": result.get("cl_load_time_ms"), + "ox_load_time_ms": result.get("ox_load_time_ms"), + "duration_ms": result.get("duration_ms"), + "execution_error": result.get("execution_error"), + "cl_site_url": result.get("cl_site_url"), + "ox_site_url": result.get("ox_site_url"), + "artifacts": result.get("artifacts", { + "cl_site_image": None, + "ox_site_image": None, + "diff_image": None, + }), + } + _write_state(run_state) + _broadcast("test_result", { + "id": state_key, + "status": status, + "pct": result.get("matching_percentage"), + "error": result.get("execution_error"), + "result": {**result, "id": state_key}, + }) + current_test = None + except Exception as exc: + logger.error("Failed to parse ##RESULT##: %s | %s", line, exc) + + # ── Test start marker ───────────────────────────────────────── + elif "Executing test:" in line: + name = line.split("Executing test:", 1)[1].strip() + tid = next( + (k for k, v in tests.items() if v.get("test_name") == name), + name, + ) + current_test = tid + logger.info(" → Running: %s (%s)", name, tid) + if tid not in run_state["tests"]: + run_state["tests"][tid] = {"test_name": name, "status": "running"} + else: + run_state["tests"][tid]["status"] = "running" + _write_state(run_state) + _broadcast("test_start", {"id": tid, "name": name}) + + proc.wait() + exit_code = proc.returncode + logger.info("Subprocess exited — code=%s", exit_code) + + # If runner crashed before emitting any results (e.g. import error, + # missing module), all tests will still be queued — mark them failed + # so the UI doesn't hang forever. + for tid, t in run_state["tests"].items(): + if t.get("status") in ("queued", "running"): + logger.warning("Test %s never completed — marking as execution-error", tid) + run_state["tests"][tid]["status"] = "execution-error" + run_state["tests"][tid]["execution_error"] = ( + f"Runner exited with code {exit_code} before this test ran. " + "Check logs — likely a missing dependency or import error." + ) + _broadcast("test_result", { + "id": tid, + "status": "execution-error", + "error": run_state["tests"][tid]["execution_error"], + "result": run_state["tests"][tid], + }) + _write_state(run_state) + + # Load final report for summary + report_data = None + reports = sorted(RESULTS_DIR.glob("reports_*.json"), reverse=True) + if reports: + try: + with open(reports[0]) as f: + report_data = json.load(f) + logger.info("Loaded final report: %s", reports[0].name) + except Exception as exc: + logger.error("Failed to read report: %s", exc) + else: + logger.warning("No reports_*.json found in %s", RESULTS_DIR) + + run_state["status"] = "finished" + run_state["finished_at"] = time.time() + run_state["exit_code"] = exit_code + run_state["report"] = report_data + _write_state(run_state) + logger.info("Run finished — state saved") + + _broadcast("run_end", { + "exit_code": exit_code, + "report": report_data, + "run_state": run_state, + }) + + except Exception as exc: + logger.exception("Unexpected error in run thread: %s", exc) + try: + run_state["status"] = "error" + run_state["logs"].append(f"[INTERNAL ERROR] {exc}") + _write_state(run_state) + except Exception: + pass + _broadcast("run_error", {"error": str(exc)}) + + finally: + with _run_lock: + _run_active = False + logger.info("Run thread exiting") + + threading.Thread(target=_stream_run, daemon=True).start() + return jsonify({"ok": True}) + + +# ── Run status (for UI to check if locked) ─────────────────────────────────── +@test_run_bp.route("/api/run/status", methods=["GET"]) +def run_status(): + return jsonify({"running": _is_running()}) + + +# ── SSE ─────────────────────────────────────────────────────────────────────── +@test_run_bp.route("/api/events") +def events(): + q = queue.Queue(maxsize=200) + _event_queues.append(q) + logger.info("SSE client connected (total=%d)", len(_event_queues)) + + def generate(): + try: + yield "event: connected\ndata: {}\n\n" + while True: + try: + yield q.get(timeout=30) + except queue.Empty: + yield ": ping\n\n" + finally: + if q in _event_queues: + _event_queues.remove(q) + logger.info("SSE client disconnected (total=%d)", len(_event_queues)) + + return Response( + generate(), + mimetype="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +# ── Last report ─────────────────────────────────────────────────────────────── +@test_run_bp.route("/api/last-report", methods=["GET"]) +def last_report(): + state = _read_state() + if state: + return jsonify(state) + if not RESULTS_DIR.exists(): + return jsonify(None) + reports = sorted(RESULTS_DIR.glob("reports_*.json"), reverse=True) + if not reports: + return jsonify(None) + try: + with open(reports[0]) as f: + return jsonify(json.load(f)) + except Exception as exc: + logger.error("Failed to read fallback report: %s", exc) + return jsonify(None) + + +# ── Serve result images ─────────────────────────────────────────────────────── +@test_run_bp.route("/results/") +def result_file(filename): + return send_from_directory(str(RESULTS_DIR), filename) \ No newline at end of file diff --git a/views/test_runner.py b/views/test_runner.py new file mode 100644 index 0000000..6afcb6b --- /dev/null +++ b/views/test_runner.py @@ -0,0 +1,259 @@ +""" + +Proxy blueprint for the test-runner service. +""" +import os + +import requests as _http +from flask import Blueprint, Response, request + +# Docker sets this via docker-compose. Locally falls back to localhost. +TEST_RUNNER_URL = os.getenv("TEST_RUNNER_URL", "http://test-runner:8085") + +test_runner_bp = Blueprint("test_runner", __name__) + +ERROR_PAGE_HTML = """ + + + + +Service Unavailable — EMLO Tests + + + + + + + +
+
+ + + + + +
+ +

Test Runner Unavailable

+

The service is not running. Start the backend container to resume visual regression testing.

+ +
+
Terminal Action Required
+ docker compose up test-runner +
+ +
+ + Return to Dashboard +
+
+ + + +""" + +def _proxy(path: str): + url = f"{TEST_RUNNER_URL}/{path}" + print(f"starting test runner proxy on {url}") + try: + resp = _http.request( + method=request.method, + url=url, + headers={k: v for k, v in request.headers if k.lower() != "host"}, + data=request.get_data(), + params=request.args, + stream=True, # required for SSE /api/events + timeout=None, # SSE streams are long-lived + ) + return Response( + resp.iter_content(chunk_size=1), + status=resp.status_code, + content_type=resp.headers.get("Content-Type", "application/octet-stream"), + headers={ + "X-Accel-Buffering": "no", + "Cache-Control": "no-cache", + }, + ) + except _http.exceptions.ConnectionError: + return Response( + ERROR_PAGE_HTML, + status=503, + content_type="text/html", + ) + + +@test_runner_bp.route("/test-run", methods=["GET", "POST"]) +def proxy_page(): + return _proxy("test-run") + + +@test_runner_bp.route("/api/tests", methods=["GET", "POST"]) +@test_runner_bp.route("/api/settings", methods=["GET", "POST"]) +@test_runner_bp.route("/api/run", methods=["POST"]) +@test_runner_bp.route("/api/run/status", methods=["GET"]) +@test_runner_bp.route("/api/events", methods=["GET"]) +@test_runner_bp.route("/api/last-report", methods=["GET"]) +def proxy_api(): + return _proxy(request.path.lstrip("/")) + + +@test_runner_bp.route("/results/") +def proxy_results(filename): + return _proxy(f"results/{filename}") \ No newline at end of file