Initial e2e test setting up for the fab

restructure tests

document update

pyproject.toml

@@ -34,13 +34,13 @@ target-version = "py310"
 
 [tool.ruff.lint]
 select = [
-    "E",  # pycodestyle
-    "W",  # pycodestyle
-    "F",  # pyflakes
-    "I",  # isort
-    "B",  # flake8-bugbear
-    "C90",  # mccabe cyclomatic complexity
-    "G",  # flake8-logging-format
+    "E", # pycodestyle
+    "W", # pycodestyle
+    "F", # pyflakes
+    "I", # isort
+    "B", # flake8-bugbear
+    "C90", # mccabe cyclomatic complexity
+    "G", # flake8-logging-format
 ]
 ignore = []
 exclude = [
@@ -64,4 +64,6 @@ dev = [
     "pytest==8.3.4",
     "python-dotenv==1.0.1",
     "ruff==0.9.7",
+    "pytest-playwright==0.6.2",
+    "pytest-xdist==3.6.1",
 ]

tests/conftest.py

@@ -1,157 +1,36 @@
-import json
-import os
-import shutil
-from unittest.mock import patch
-
 import pytest
-from flask import current_app
-from flask_migrate import upgrade
-from sqlalchemy import text
-
-from app.create_app import create_app
-from app.import_config.load_form_json import load_form_jsons
-from config import Config
-from tests.seed_test_data import init_unit_test_data, insert_test_data, fund_without_assessment
 
 pytest_plugins = ["fsd_test_utils.fixtures.db_fixtures"]
 
 
-@pytest.fixture(scope="session")
-def temp_output_dir():
-    temp_dir = Config.TEMP_FILE_PATH
-    yield temp_dir
-    if temp_dir.exists():
-        shutil.rmtree(temp_dir)
-
-
-@pytest.fixture(scope="function")
-def seed_dynamic_data(request, app, clear_test_data, _db, enable_preserve_test_data):
-    marker = request.node.get_closest_marker("seed_config")
-
-    if marker is None:
-        fab_seed_data = init_unit_test_data()
-    else:
-        fab_seed_data = marker.args[0]
-    insert_test_data(db=_db, test_data=fab_seed_data)
-    yield fab_seed_data
-    # cleanup data after test
-    # rollback incase of any errors during test session
-    _db.session.rollback()
-    # disable foreign key checks
-    _db.session.execute(text("SET session_replication_role = replica"))
-    # delete all data from tables
-    for table in reversed(_db.metadata.sorted_tables):
-        _db.session.execute(table.delete())
-    # reset foreign key checks
-    _db.session.execute(text("SET session_replication_role = DEFAULT"))
-    _db.session.commit()
-
-
-@pytest.fixture(scope="function")
-def seed_fund_without_assessment(request, app, clear_test_data, _db, enable_preserve_test_data):
-    marker = request.node.get_closest_marker("seed_config")
-
-    if marker is None:
-        fab_seed_data = fund_without_assessment()
+def pytest_addoption(parser):
+    parser.addoption(
+        "--e2e", action="store_true", default=False, help="Run end-to-end tests"
+    )
+    parser.addoption(
+        "--e2e-aws-vault-profile",
+        action="store",
+        help="the aws-vault profile matching the env set in --e2e-env (for `dev` or `test` only)",
+    )
+    parser.addoption(
+        "--e2e-env",
+        action="store",
+        default="local",
+        help="choose the environment that e2e tests will target",
+        choices=("local", "dev", "test"),
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    skip_e2e = pytest.mark.skip(reason="only running unit tests")
+    skip_non_e2e = pytest.mark.skip(reason="only running e2e tests")
+
+    e2e_run = config.getoption("--e2e")
+    if e2e_run:
+        for item in items:
+            if "e2e" not in item.keywords:
+                item.add_marker(skip_non_e2e)
     else:
-        fab_seed_data = marker.args[0]
-    insert_test_data(db=_db, test_data=fab_seed_data)
-    yield fab_seed_data
-    # cleanup data after test
-    # rollback incase of any errors during test session
-    _db.session.rollback()
-    # disable foreign key checks
-    _db.session.execute(text("SET session_replication_role = replica"))
-    # delete all data from tables
-    for table in reversed(_db.metadata.sorted_tables):
-        _db.session.execute(table.delete())
-    # reset foreign key checks
-    _db.session.execute(text("SET session_replication_role = DEFAULT"))
-    _db.session.commit()
-
-
-@pytest.fixture(scope="function")
-def db_with_templates(app, _db):
-    """Ensures a clean database but with templates already loaded"""
-    with app.app_context():
-        script_dir = os.path.dirname(__file__)
-        test_data_dir = os.path.join(script_dir, "test_data")
-
-        form_configs = []
-        file_path = os.path.join(test_data_dir, "asset-information.json")
-        if os.path.exists(file_path):
-            with open(file_path, "r") as json_file:
-                input_form = json.load(json_file)
-                input_form["filename"] = "asset-information"
-                form_configs.append(input_form)
-        load_form_jsons(form_configs)
-    yield _db
-
-
-@pytest.fixture(scope="function")
-def clean_db(app, _db):
-    """Ensures a clean database before each test runs"""
-    with app.app_context():
-        # Rollback any existing transactions
-        _db.session.rollback()
-        # Disable foreign key constraints
-        _db.session.execute(text("SET session_replication_role = replica"))
-        # Clear all tables
-        for table in reversed(_db.metadata.sorted_tables):
-            _db.session.execute(table.delete())
-        # Re-enable foreign key constraints
-        _db.session.execute(text("SET session_replication_role = DEFAULT"))
-        _db.session.commit()
-    yield _db
-
-
-@pytest.fixture(scope="session")
-def app():
-    app = create_app()
-    # this will enable the usage of url_for but use the app context in the test
-    app.config["TESTING"] = True
-    app.config["SERVER_NAME"] = "localhost"
-    with app.app_context():
-        yield app
-
-
-@pytest.fixture(scope="function")
-def flask_test_client(app):
-    with app.app_context():
-        upgrade()
-        with app.test_client() as test_client:
-            with test_client.session_transaction() as session:
-                session['visited_pages'] = []  # Initialize the session for the test
-            yield test_client
-
-
-@pytest.fixture
-def set_auth_cookie(flask_test_client):
-    # This fixture sets the authentication cookie on every test.
-    user_token_cookie_name = current_app.config.get("FSD_USER_TOKEN_COOKIE_NAME", "fsd_user_token")
-    flask_test_client.set_cookie(key=user_token_cookie_name, value="dummy_jwt_token")
-    yield
-
-
-@pytest.fixture
-def patch_validate_token_rs256_internal_user():
-    # This fixture patches validate_token_rs256 for all tests automatically.
-    with patch("fsd_utils.authentication.decorators.validate_token_rs256") as mock_validate_token_rs256:
-        mock_validate_token_rs256.return_value = {
-            "accountId": "test-account-id",
-            "roles": [],
-            "email": "[email protected]",
-        }
-        yield mock_validate_token_rs256
-
-
-@pytest.fixture
-def patch_validate_token_rs256_external_user():
-    # This fixture patches validate_token_rs256 for all tests automatically.
-    with patch("fsd_utils.authentication.decorators.validate_token_rs256") as mock_validate_token_rs256:
-        mock_validate_token_rs256.return_value = {
-            "accountId": "test-account-id",
-            "roles": [],
-            "email": "[email protected]",
-        }
-        yield mock_validate_token_rs256
+        for item in items:
+            if "e2e" in item.keywords:
+                item.add_marker(skip_e2e)

tests/e2e/README.MD

@@ -0,0 +1,56 @@
+# Python E2E tests
+
+These are FAB E2E tests written using [Playwright](https://playwright.dev/python/).
+
+# Pre Reqs
+
+- Playwright and other dependencies installed:
+    - Run `uv run playwright install --with-deps chromium` (this is a one-time setup step and may take a little while)
+- If running against your local machine, you will need the fab services up and running - see
+  the [docker runner](https://github.com/communitiesuk/funding-service-design-docker-runner) for details on this.
+- If running from your local machine against `dev` or `test`, you will also need
+  `aws-vault` [installed](https://github.com/99designs/aws-vault/blob/master/README.md)
+  and [setup](https://mhclgdigital.atlassian.net/wiki/spaces/FS/pages/5241813/Using+AWS+Vault+SSO#Install-AWS-Vault)
+  with profiles for your target environment.
+
+# Running
+
+## Locally—Against your local machine
+
+- These tests can be run the same way you normally run `pytest` - either from the command line or using an IDE
+  extension.
+
+```
+    uv run pytest --e2e
+```
+
+- You can optionally add the `headed` switch to see the browser window while the tests run
+
+```
+    uv run pytest --e2e --headed
+```
+
+## Locally - Against dev or test
+
+- To run against a non-local environment, pass the environment you want to target, and the aws-vault profile name to
+  use, and run with pytest as normal:
+
+```
+    uv run pytest --e2e --e2e-env dev --e2e-aws-vault-profile fsd-dev
+```
+
+In this case, `fsd-dev` is the name of the profile in the aws-vault config file (~/.aws/config):
+
+```
+    [profile fsd-dev]
+    region=eu-west-2
+    source_profile=XXXXX
+    role_arn=arn:aws:iam::123123123123:role/developer
+    mfa_serial=arn:aws:iam::123123123123:mfa/user.name
+```
+
+## Parallel test runs
+
+By default the tests are set to spawn 3 workers and run in parallel. This is configured
+in [pyproject.toml](../pyproject.toml) under the pytest options. The project
+installs [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/index.html) to enable this.

tests/e2e/config.py

@@ -0,0 +1,67 @@
+import json
+import subprocess
+from typing import Literal, Protocol
+
+import boto3
+from playwright.sync_api import HttpCredentials
+
+
+class EndToEndTestSecrets(Protocol):
+    @property
+    def HTTP_BASIC_AUTH(self) -> HttpCredentials | None: ...
+
+
+class LocalEndToEndSecrets:
+    @property
+    def HTTP_BASIC_AUTH(self) -> None:
+        return None
+
+
+class AWSEndToEndSecrets:
+    def __init__(self, e2e_env: Literal["dev", "test"], e2e_aws_vault_profile: str | None):
+        self.e2e_env = e2e_env
+        self.e2e_aws_vault_profile = e2e_aws_vault_profile
+
+        if self.e2e_env == "prod":  # type: ignore[comparison-overlap]
+            # It shouldn't be possible to set e2e_env to `prod` based on current setup; this is a safeguard against it
+            # being added in the future without thinking about this fixture. When it comes to prod secrets, remember:
+            #   keep it secret; keep it safe.
+            raise ValueError("Refusing to init against prod environment because it would read production secrets")
+
+    def _read_aws_parameter_store_value(self, parameter):
+        # This flow is used to collect secrets when running tests *from* your local machine
+        if self.e2e_aws_vault_profile:
+            value = json.loads(
+                subprocess.check_output(
+                    [
+                        "aws-vault",
+                        "exec",
+                        self.e2e_aws_vault_profile,
+                        "--",
+                        "aws",
+                        "ssm",
+                        "get-parameter",
+                        "--name",
+                        parameter,
+                        "--with-decryption",
+                    ],
+                ).decode()
+            )["Parameter"]["Value"]
+
+        # This flow is used when running tests *in* CI/CD, where AWS credentials are available from OIDC auth
+        else:
+            ssm_client = boto3.client("ssm")
+            value = ssm_client.get_parameter(Name=parameter, WithDecryption=True)["Parameter"]["Value"]
+
+        return value
+
+    @property
+    def HTTP_BASIC_AUTH(self) -> HttpCredentials:
+        return {
+            "username": self._read_aws_parameter_store_value(
+                f"/copilot/pre-award/{self.e2e_env}/secrets/BASIC_AUTH_USERNAME"
+            ),
+            "password": self._read_aws_parameter_store_value(
+                f"/copilot/pre-award/{self.e2e_env}/secrets/BASIC_AUTH_PASSWORD"
+            ),
+        }