Record and save traces on test failures

Remove step name that may be bad syntax

Update step 'if' syntax

Fix traces path and ensure traces are enabled

Try shortening step name

Fail a step intentionally to get a trace

Add step name

Try removing the new step

Re-add step

.github/workflows/ci.yml

@@ -136,7 +136,7 @@ jobs:
       - name: Build Docker Image
         run: docker build --target web -t ons .
 
-  functional_tests:
+  functional-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -158,6 +158,8 @@ jobs:
       PG_DUMP: /home/linuxbrew/.linuxbrew/opt/postgresql@16/bin/pg_dump
       PG_RESTORE: /home/linuxbrew/.linuxbrew/opt/postgresql@16/bin/pg_restore
       BROWSER: ${{ matrix.browser }}
+      PLAYWRIGHT_TRACES_DIR: playwright_traces
+      PLAYWRIGHT_TRACE: 'true'
 
     services:
       postgres:
@@ -186,10 +188,11 @@ jobs:
       - name: Install dependencies
         run: make install-dev
 
-      - name: Install Playwright
+      - name: Playwright Install
         run: poetry run python -m playwright install --with-deps ${{ matrix.browser }}
 
       - name: Install/Update Postgres
+        # We need pg_dump and pg_restore for the functional tests
         # The Ubuntu runner does come with postgresql, but the wrong version, so we must acquire v16
         run: /home/linuxbrew/.linuxbrew/bin/brew install postgresql@16
 
@@ -206,3 +209,10 @@ jobs:
 
       - name: Run Functional Tests
         run: poetry run behave functional_tests
+
+      - name: Upload Failure Traces
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: playwright-traces
+          path: playwright_traces/

.gitignore

@@ -190,3 +190,6 @@ static/
 
 # Media files
 /media
+
+# Playwright Traces
+/tmp_traces

functional_tests/environment.py

@@ -1,7 +1,9 @@
 import os
+from pathlib import Path
 
 from behave import use_fixture
 from behave.model import Scenario
+from behave.model_core import Status
 from behave.runner import Context
 from django.db import close_old_connections
 from playwright.sync_api import BrowserContext, Page, Playwright, sync_playwright
@@ -24,8 +26,10 @@ def before_all(context: Context):
 
     context.playwright: Playwright = sync_playwright().start()
     browser_type = os.getenv("BROWSER", "chromium")
-    headless = os.getenv("HEADLESS", "True") == "True"
+    headless = str_to_bool(os.getenv("HEADLESS", "True"))
     slowmo = int(os.getenv("SLOWMO", "0"))
+    context.playwright_trace = str_to_bool(os.getenv("PLAYWRIGHT_TRACE", "True"))
+    context.playwright_traces_dir = Path(os.getenv("PLAYWRIGHT_TRACES_DIR", str(Path.cwd().joinpath("tmp_traces"))))
 
     browser_kwargs = {
         "headless": headless,
@@ -45,31 +49,64 @@ def before_all(context: Context):
     context.browser_context: BrowserContext = context.browser.new_context()
     context.browser_context.set_default_timeout(10_000)
 
+    if context.playwright_trace:
+        context.playwright_traces_dir.mkdir(exist_ok=True)
+        # Start the main trace for this browser context, we will record individual scenario traces in chunks
+        context.browser_context.tracing.start(screenshots=True, snapshots=True, sources=True)
+
 
 def after_all(context: Context):
     """Runs once after all tests.
     Cleans up playwright objects.
     """
+    if context.playwright_trace:
+        context.browser_context.tracing.stop()
+
     context.browser_context.close()
     context.browser.close()
     context.playwright.stop()
 
 
-def before_scenario(context: Context, _scenario: Scenario):
+def before_scenario(context: Context, scenario: Scenario):
     """Runs before each scenario.
     Create a new playwright page to be used by the scenario, through the context.
     """
     # Register our django test case fixture so every scenario is wrapped in a Django test case
     use_fixture(django_test_case, context=context)
 
-    context.page: Page = context.browser.new_page()
+    context.page: Page = context.browser_context.new_page()
+
+    if context.playwright_trace:
+        # Start a tracing chunk to capture each scenario separately
+        context.browser_context.tracing.start_chunk(name=scenario.name, title=scenario.name)
 
 
-def after_scenario(context: Context, _scenario: Scenario):
+def after_scenario(context: Context, scenario: Scenario):
     """Runs after each scenario.
     Close the playwright page and tidy up any DB connections so they don't block the database teardown.
     """
-    context.page.close()
-
     # Prevent any remaining connections from blocking teardown
     close_old_connections()
+
+    if context.playwright_trace and scenario.status == Status.failed:
+        # If the scenario failed, write the trace chunk out to a file, which will be prefixed with the scenario name
+        context.browser_context.tracing.stop_chunk(
+            path=context.playwright_traces_dir.joinpath(f"{scenario.name}_failure_trace.zip")
+        )
+
+    elif context.playwright_trace:
+        # Else end the trace chunk without saving
+        context.browser_context.tracing.stop_chunk()
+
+    context.page.close()
+
+
+def str_to_bool(bool_string: str) -> bool:
+    """Takes a string argument which indicates a boolean, and returns the corresponding boolean value.
+    raises ValueError if input string is not one of the recognized boolean like values.
+    """
+    if bool_string.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    if bool_string.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    raise ValueError(f"Invalid input: {bool_string}")

functional_tests/steps/example.py

@@ -55,3 +55,4 @@ def user_sees_admin_homepage(context: Context) -> None:
     expect(context.page).to_have_url(f"{context.base_url}/admin/")
     expect(context.page.get_by_role("heading", name="Office For National Statistics")).to_be_visible()
     expect(context.page.get_by_label("Dashboard")).to_be_visible()
+    assert False, "Fail a test intentionally"  # noqa: B011, S101