Initial e2e test setting up for the fab

e2e_tests/README.MD

@@ -0,0 +1,56 @@
+# Python E2E tests
+
+These are FAB E2E tests written using [Playwright](https://playwright.dev/python/).
+
+# Pre Reqs
+
+- Playwright and other dependencies installed:
+    - Run `uv run playwright install --with-deps chromium` (this is a one-time setup step and may take a little while)
+- If running against your local machine, you will need the fab services up and running - see
+  the [docker runner](https://github.com/communitiesuk/funding-service-design-docker-runner) for details on this.
+- If running from your local machine against `dev` or `test`, you will also need
+  `aws-vault` [installed](https://github.com/99designs/aws-vault/blob/master/README.md)
+  and [setup](https://mhclgdigital.atlassian.net/wiki/spaces/FS/pages/5241813/Using+AWS+Vault+SSO#Install-AWS-Vault)
+  with profiles for your target environment.
+
+# Running
+
+## Locally—Against your local machine
+
+- These tests can be run the same way you normally run `pytest` - either from the command line or using an IDE
+  extension.
+
+```
+    uv run pytest e2e_tests
+```
+
+- You can optionally add the `headed` switch to see the browser window while the tests run
+
+```
+    uv run pytest e2e_test --headed
+```
+
+## Locally - Against dev or test
+
+- To run against a non-local environment, pass the environment you want to target, and the aws-vault profile name to
+  use, and run with pytest as normal:
+
+```
+    uv run pytest e2e_test --e2e-env dev --e2e-aws-vault-profile fsd-dev
+```
+
+In this case, `fsd-dev` is the name of the profile in the aws-vault config file (~/.aws/config):
+
+```
+    [profile fsd-dev]
+    region=eu-west-2
+    source_profile=XXXXX
+    role_arn=arn:aws:iam::123123123123:role/developer
+    mfa_serial=arn:aws:iam::123123123123:mfa/user.name
+```
+
+## Parallel test runs
+
+By default the tests are set to spawn 3 workers and run in parallel. This is configured
+in [pyproject.toml](../pyproject.toml) under the pytest options. The project
+installs [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/index.html) to enable this.

e2e_tests/config.py

@@ -0,0 +1,67 @@
+import json
+import subprocess
+from typing import Literal, Protocol
+
+import boto3
+from playwright.sync_api import HttpCredentials
+
+
+class EndToEndTestSecrets(Protocol):
+    @property
+    def HTTP_BASIC_AUTH(self) -> HttpCredentials | None: ...
+
+
+class LocalEndToEndSecrets:
+    @property
+    def HTTP_BASIC_AUTH(self) -> None:
+        return None
+
+
+class AWSEndToEndSecrets:
+    def __init__(self, e2e_env: Literal["dev", "test"], e2e_aws_vault_profile: str | None):
+        self.e2e_env = e2e_env
+        self.e2e_aws_vault_profile = e2e_aws_vault_profile
+
+        if self.e2e_env == "prod":  # type: ignore[comparison-overlap]
+            # It shouldn't be possible to set e2e_env to `prod` based on current setup; this is a safeguard against it
+            # being added in the future without thinking about this fixture. When it comes to prod secrets, remember:
+            #   keep it secret; keep it safe.
+            raise ValueError("Refusing to init against prod environment because it would read production secrets")
+
+    def _read_aws_parameter_store_value(self, parameter):
+        # This flow is used to collect secrets when running tests *from* your local machine
+        if self.e2e_aws_vault_profile:
+            value = json.loads(
+                subprocess.check_output(
+                    [
+                        "aws-vault",
+                        "exec",
+                        self.e2e_aws_vault_profile,
+                        "--",
+                        "aws",
+                        "ssm",
+                        "get-parameter",
+                        "--name",
+                        parameter,
+                        "--with-decryption",
+                    ],
+                ).decode()
+            )["Parameter"]["Value"]
+
+        # This flow is used when running tests *in* CI/CD, where AWS credentials are available from OIDC auth
+        else:
+            ssm_client = boto3.client("ssm")
+            value = ssm_client.get_parameter(Name=parameter, WithDecryption=True)["Parameter"]["Value"]
+
+        return value
+
+    @property
+    def HTTP_BASIC_AUTH(self) -> HttpCredentials:
+        return {
+            "username": self._read_aws_parameter_store_value(
+                f"/copilot/pre-award/{self.e2e_env}/secrets/BASIC_AUTH_USERNAME"
+            ),
+            "password": self._read_aws_parameter_store_value(
+                f"/copilot/pre-award/{self.e2e_env}/secrets/BASIC_AUTH_PASSWORD"
+            ),
+        }

e2e_tests/conftest.py

@@ -0,0 +1,81 @@
+import os
+import secrets
+import sys
+
+import pytest
+from pytest_playwright import CreateContextCallback
+
+from e2e_tests.config import AWSEndToEndSecrets, EndToEndTestSecrets, LocalEndToEndSecrets
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--e2e-aws-vault-profile",
+        action="store",
+        help="the aws-vault profile matching the env set in --e2e-env (for `dev` or `test` only)",
+    )
+    parser.addoption(
+        "--e2e-env",
+        action="store",
+        default="local",
+        help="choose the environment that e2e tests will target",
+        choices=("local", "dev", "test"),
+    )
+
+
+@pytest.fixture
+def get_e2e_params(request):
+    e2e_env = request.config.getoption("e2e_env", "local")
+    vault_profile = request.config.getoption("e2e_aws_vault_profile", None)
+    session_token_from_env = os.getenv("AWS_SESSION_TOKEN", None)
+    if not session_token_from_env and e2e_env != "local" and not vault_profile:
+        sys.exit("Must supply e2e-aws-vault-profile with e2e-env")
+    yield {
+        "e2e_env": e2e_env,
+        "e2e_aws_vault_profile": vault_profile,
+    }
+
+
+@pytest.fixture
+def unique_token():
+    yield secrets.token_urlsafe(8)
+
+
+@pytest.fixture()
+def domains(request: pytest.FixtureRequest, get_e2e_params) -> str:
+    e2e_env = get_e2e_params["e2e_env"]
+    match e2e_env:
+        case "local":
+            return "https://fund-application-builder.levellingup.gov.localhost:3011"
+        case "dev":
+            return "https://fund-application-builder.dev.access-funding.test.levellingup.gov.uk"
+        case "test":
+            return "https://fund-application-builder.test.access-funding.test.levellingup.gov.uk"
+        case _:
+            raise ValueError(f"not configured for {e2e_env}")
+
+
+@pytest.fixture
+def context(
+        new_context: CreateContextCallback,
+        request: pytest.FixtureRequest,
+        e2e_test_secrets: EndToEndTestSecrets,
+        get_e2e_params,
+):
+    e2e_env = get_e2e_params["e2e_env"]
+    http_credentials = e2e_test_secrets.HTTP_BASIC_AUTH if e2e_env in {"dev", "test"} else None
+    return new_context(http_credentials=http_credentials)
+
+
+@pytest.fixture
+def e2e_test_secrets(request: pytest.FixtureRequest, get_e2e_params) -> EndToEndTestSecrets:
+    e2e_env = get_e2e_params["e2e_env"]
+    e2e_aws_vault_profile = get_e2e_params["e2e_aws_vault_profile"]
+
+    if e2e_env == "local":
+        return LocalEndToEndSecrets()
+
+    if e2e_env in {"dev", "test"}:
+        return AWSEndToEndSecrets(e2e_env=e2e_env, e2e_aws_vault_profile=e2e_aws_vault_profile)
+
+    raise ValueError(f"Unknown e2e_env: {e2e_env}.")

pyproject.toml

@@ -34,13 +34,13 @@ target-version = "py310"
 
 [tool.ruff.lint]
 select = [
-    "E",  # pycodestyle
-    "W",  # pycodestyle
-    "F",  # pyflakes
-    "I",  # isort
-    "B",  # flake8-bugbear
-    "C90",  # mccabe cyclomatic complexity
-    "G",  # flake8-logging-format
+    "E", # pycodestyle
+    "W", # pycodestyle
+    "F", # pyflakes
+    "I", # isort
+    "B", # flake8-bugbear
+    "C90", # mccabe cyclomatic complexity
+    "G", # flake8-logging-format
 ]
 ignore = []
 exclude = [
@@ -64,4 +64,5 @@ dev = [
     "pytest==8.3.4",
     "python-dotenv==1.0.1",
     "ruff==0.9.7",
+    "pytest-playwright==0.6.2",
 ]