diff --git a/poetry.lock b/poetry.lock index a9fc5d89c..6501b4b10 100644 --- a/poetry.lock +++ b/poetry.lock @@ -988,6 +988,21 @@ files = [ [package.dependencies] PyYAML = "==6.*" +[[package]] +name = "jubilant-backports" +version = "1.0.0a1" +description = "Extends Jubilant to include support for Juju 2.9" +optional = false +python-versions = ">=3.8" +groups = ["integration"] +files = [ + {file = "jubilant_backports-1.0.0a1-py3-none-any.whl", hash = "sha256:ff8d73e17afaae4418c588496978ac42ee9eb9d6d4e77ce103102772038796cc"}, + {file = "jubilant_backports-1.0.0a1.tar.gz", hash = "sha256:03f0788a2301e1a71ebab56bc59515361c37e5686e40a985caba5b2907514e3f"}, +] + +[package.dependencies] +jubilant = ">=1.2,<2.0" + [[package]] name = "juju" version = "3.6.0.0" @@ -2522,4 +2537,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "ab835ca1f001d45d3bf5dc48d4bb56403f2e5ef98da2aacdce981b45e9d65068" +content-hash = "8b0ec063b95e882f19e59ea53a211b1b432d249528d0c88bcd0aac9acbc87937" diff --git a/pyproject.toml b/pyproject.toml index ca759beec..e50cd37fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ pyyaml = "^6.0" urllib3 = "^2.0.0" allure-pytest = "^2.13.2" allure-pytest-default-results = "^0.1.2" -jubilant = "^1.0.1" +jubilant-backports = "^1.0.0a1" [tool.coverage.run] branch = true diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 051a9456c..5e973148b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,7 +3,7 @@ import os import uuid -import jubilant +import jubilant_backports import pytest from . import architecture @@ -57,11 +57,11 @@ def juju(request: pytest.FixtureRequest): keep_models = bool(request.config.getoption("--keep-models")) if model: - juju = jubilant.Juju(model=model) # type: ignore + juju = jubilant_backports.Juju(model=model) # type: ignore yield juju log = juju.debug_log(limit=1000) else: - with jubilant.temp_model(keep=keep_models) as juju: + with jubilant_backports.temp_model(keep=keep_models) as juju: yield juju log = juju.debug_log(limit=1000) diff --git a/tests/integration/high_availability/high_availability_helpers_new.py b/tests/integration/high_availability/high_availability_helpers_new.py index c41f34661..7329ca0bc 100644 --- a/tests/integration/high_availability/high_availability_helpers_new.py +++ b/tests/integration/high_availability/high_availability_helpers_new.py @@ -2,11 +2,13 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. +import json +import subprocess from collections.abc import Callable -import jubilant -from jubilant import Juju -from jubilant.statustypes import Status, UnitStatus +import jubilant_backports +from jubilant_backports import Juju +from jubilant_backports.statustypes import Status, UnitStatus from tenacity import Retrying, stop_after_delay, wait_fixed from constants import SERVER_CONFIG_USERNAME @@ -19,19 +21,22 @@ JujuAppsStatusFn = Callable[[Status, str], bool] -async def check_mysql_units_writes_increment(juju: Juju, app_name: str) -> None: +async def check_mysql_units_writes_increment( + juju: Juju, app_name: str, app_units: list[str] | None = None +) -> None: """Ensure that continuous writes is incrementing on all units. Also, ensure that all continuous writes up to the max written value is available on all units (ensure that no committed data is lost). """ - mysql_app_units = get_app_units(juju, app_name) - mysql_app_primary = get_mysql_primary_unit(juju, app_name) + if not app_units: + app_units = get_app_units(juju, app_name) - app_max_value = await get_mysql_max_written_value(juju, app_name, mysql_app_primary) + app_primary = get_mysql_primary_unit(juju, app_name) + app_max_value = await get_mysql_max_written_value(juju, app_name, app_primary) juju.model_config({"update-status-hook-interval": "15s"}) - for unit_name in mysql_app_units: + for unit_name in app_units: for attempt in Retrying( reraise=True, stop=stop_after_delay(5 * MINUTE_SECS), @@ -72,6 +77,17 @@ def get_app_units(juju: Juju, app_name: str) -> dict[str, UnitStatus]: return app_status.units +def get_unit_by_number(juju: Juju, app_name: str, unit_number: int) -> str: + """Get unit by number.""" + model_status = juju.status() + app_status = model_status.apps[app_name] + for name in app_status.units: + if name == f"{app_name}/{unit_number}": + return name + + raise Exception("No application unit found") + + def get_unit_ip(juju: Juju, app_name: str, unit_name: str) -> str: """Get the application unit IP.""" model_status = juju.status() @@ -83,13 +99,67 @@ def get_unit_ip(juju: Juju, app_name: str, unit_name: str) -> str: raise Exception("No application unit found") -def get_mysql_cluster_status(juju: Juju, unit: str, cluster_set: bool | None = False) -> dict: +def get_unit_info(juju: Juju, unit_name: str) -> dict: + """Return a dictionary with the show-unit data.""" + output = subprocess.check_output( + ["juju", "show-unit", f"--model={juju.model}", "--format=json", unit_name], + text=True, + ) + + return json.loads(output) + + +def get_unit_status_log(juju: Juju, unit_name: str, log_lines: int = 0) -> list[dict]: + """Get the status log for a unit. + + Args: + juju: The juju instance to use. + unit_name: The name of the unit to retrieve the status log for + log_lines: The number of status logs to retrieve (optional) + """ + # fmt: off + output = subprocess.check_output( + ["juju", "show-status-log", f"--model={juju.model}", "--format=json", unit_name, "-n", f"{log_lines}"], + text=True, + ) + + return json.loads(output) + + +def get_relation_data(juju: Juju, app_name: str, rel_name: str) -> list[dict]: + """Returns a list that contains the relation-data. + + Args: + juju: The juju instance to use. + app_name: The name of the application + rel_name: name of the relation to get connection data from + + Returns: + A list that contains the relation-data + """ + app_leader = get_app_leader(juju, app_name) + app_leader_info = get_unit_info(juju, app_leader) + if not app_leader_info: + raise ValueError(f"No unit info could be grabbed for unit {app_leader}") + + relation_data = [ + value + for value in app_leader_info[app_leader]["relation-info"] + if value["endpoint"] == rel_name + ] + if not relation_data: + raise ValueError(f"No relation data could be grabbed for relation {rel_name}") + + return relation_data + + +def get_mysql_cluster_status(juju: Juju, unit: str, cluster_set: bool = False) -> dict: """Get the cluster status by running the get-cluster-status action. Args: juju: The juju instance to use. unit: The unit on which to execute the action on - cluster_set: Whether to get the cluster-set instead + cluster_set: Whether to get the cluster-set instead (optional) Returns: A dictionary representing the cluster status @@ -97,7 +167,7 @@ def get_mysql_cluster_status(juju: Juju, unit: str, cluster_set: bool | None = F task = juju.run( unit=unit, action="get-cluster-status", - params={"cluster-set": bool(cluster_set)}, + params={"cluster-set": cluster_set}, wait=5 * MINUTE_SECS, ) task.raise_on_failure() @@ -185,6 +255,20 @@ def wait_for_apps_status(jubilant_status_func: JujuAppsStatusFn, *apps: str) -> Juju model status function. """ return lambda status: all(( - jubilant.all_agents_idle(status, *apps), + jubilant_backports.all_agents_idle(status, *apps), jubilant_status_func(status, *apps), )) + + +def wait_for_unit_status(app_name: str, unit_name: str, unit_status: str) -> JujuModelStatusFn: + """Returns whether a Juju unit to have a specific status.""" + return lambda status: ( + status.apps[app_name].units[unit_name].workload_status.current == unit_status + ) + + +def wait_for_unit_message(app_name: str, unit_name: str, unit_message: str) -> JujuModelStatusFn: + """Returns whether a Juju unit to have a specific message.""" + return lambda status: ( + status.apps[app_name].units[unit_name].workload_status.message == unit_message + ) diff --git a/tests/integration/high_availability/test_async_replication.py b/tests/integration/high_availability/test_async_replication.py index 3da522ab5..a020fd88d 100644 --- a/tests/integration/high_availability/test_async_replication.py +++ b/tests/integration/high_availability/test_async_replication.py @@ -6,9 +6,9 @@ import time from collections.abc import Generator -import jubilant +import jubilant_backports import pytest -from jubilant import Juju +from jubilant_backports import Juju from .. import architecture from ..markers import juju3 @@ -98,11 +98,11 @@ def test_build_and_deploy(first_model: str, second_model: str, charm: str) -> No logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_1), timeout=10 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_2), timeout=10 * MINUTE_SECS, ) @@ -127,11 +127,11 @@ def test_async_relate(first_model: str, second_model: str) -> None: logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.any_blocked, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.any_blocked, MYSQL_APP_1), timeout=5 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.any_waiting, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.any_waiting, MYSQL_APP_2), timeout=5 * MINUTE_SECS, ) @@ -170,7 +170,7 @@ def test_deploy_router_and_app(first_model: str) -> None: ) model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_TEST_APP_NAME), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_TEST_APP_NAME), timeout=10 * MINUTE_SECS, ) @@ -192,11 +192,11 @@ def test_create_replication(first_model: str, second_model: str) -> None: logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_1), timeout=5 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_2), timeout=5 * MINUTE_SECS, ) @@ -326,11 +326,11 @@ async def test_unrelate_and_relate(first_model: str, second_model: str, continuo logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_1), timeout=10 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.all_blocked, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.all_blocked, MYSQL_APP_2), timeout=10 * MINUTE_SECS, ) @@ -340,7 +340,7 @@ async def test_unrelate_and_relate(first_model: str, second_model: str, continuo f"{MYSQL_APP_2}:replication", ) model_1.wait( - ready=wait_for_apps_status(jubilant.any_blocked, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.any_blocked, MYSQL_APP_1), timeout=5 * MINUTE_SECS, ) @@ -354,11 +354,11 @@ async def test_unrelate_and_relate(first_model: str, second_model: str, continuo logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_1), timeout=10 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_2), timeout=10 * MINUTE_SECS, ) diff --git a/tests/integration/high_availability/test_async_replication_upgrade.py b/tests/integration/high_availability/test_async_replication_upgrade.py index 6dcd08db8..4cd457495 100644 --- a/tests/integration/high_availability/test_async_replication_upgrade.py +++ b/tests/integration/high_availability/test_async_replication_upgrade.py @@ -6,9 +6,9 @@ import time from collections.abc import Generator -import jubilant +import jubilant_backports import pytest -from jubilant import Juju +from jubilant_backports import Juju from .. import architecture from ..markers import juju3 @@ -99,11 +99,11 @@ def test_build_and_deploy(first_model: str, second_model: str, charm: str) -> No logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_1), timeout=10 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_2), timeout=10 * MINUTE_SECS, ) @@ -128,11 +128,11 @@ def test_async_relate(first_model: str, second_model: str) -> None: logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.any_blocked, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.any_blocked, MYSQL_APP_1), timeout=5 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.any_waiting, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.any_waiting, MYSQL_APP_2), timeout=5 * MINUTE_SECS, ) @@ -158,7 +158,7 @@ def test_deploy_test_app(first_model: str) -> None: ) model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_TEST_APP_NAME), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_TEST_APP_NAME), timeout=10 * MINUTE_SECS, ) @@ -180,11 +180,11 @@ def test_create_replication(first_model: str, second_model: str) -> None: logging.info("Waiting for the applications to settle") model_1.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_1), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_1), timeout=5 * MINUTE_SECS, ) model_2.wait( - ready=wait_for_apps_status(jubilant.all_active, MYSQL_APP_2), + ready=wait_for_apps_status(jubilant_backports.all_active, MYSQL_APP_2), timeout=5 * MINUTE_SECS, ) @@ -275,13 +275,13 @@ async def run_upgrade_from_edge(juju: Juju, app_name: str, charm: str) -> None: logging.info("Wait for upgrade to start") juju.wait( - ready=lambda status: jubilant.any_maintenance(status, app_name), + ready=lambda status: jubilant_backports.any_maintenance(status, app_name), timeout=10 * MINUTE_SECS, ) logging.info("Wait for upgrade to complete") juju.wait( - ready=lambda status: jubilant.all_active(status, app_name), + ready=lambda status: jubilant_backports.all_active(status, app_name), timeout=20 * MINUTE_SECS, ) diff --git a/tests/integration/high_availability/test_primary_switchover.py b/tests/integration/high_availability/test_primary_switchover.py index 0c73247b4..6aa226686 100644 --- a/tests/integration/high_availability/test_primary_switchover.py +++ b/tests/integration/high_availability/test_primary_switchover.py @@ -5,19 +5,19 @@ from subprocess import run import pytest -from jubilant import Juju, all_active +from jubilant_backports import Juju, all_active -from ..markers import juju3 from .high_availability_helpers_new import ( get_app_name, get_app_units, get_mysql_primary_unit, + wait_for_unit_message, + wait_for_unit_status, ) logging.getLogger("jubilant.wait").setLevel(logging.WARNING) -@juju3 @pytest.mark.abort_on_fail def test_cluster_switchover(juju: Juju, highly_available_cluster) -> None: """Test that the primary node can be switched over.""" @@ -43,7 +43,6 @@ def test_cluster_switchover(juju: Juju, highly_available_cluster) -> None: assert get_mysql_primary_unit(juju, app_name) == new_primary_unit, "Switchover failed" -@juju3 @pytest.mark.abort_on_fail def test_cluster_failover_after_majority_loss(juju: Juju, highly_available_cluster) -> None: """Test the promote-to-primary command after losing the majority of nodes, with force flag.""" @@ -74,10 +73,11 @@ def test_cluster_failover_after_majority_loss(juju: Juju, highly_available_clust juju.model_config({"update-status-hook-interval": "45s"}) logging.info("Waiting to settle in error state") juju.wait( - lambda status: status.apps[app_name].units[unit_to_promote].workload_status.current - == "active" - and status.apps[app_name].units[units_to_kill[0]].workload_status.message == "offline" - and status.apps[app_name].units[units_to_kill[1]].workload_status.message == "offline", + ready=lambda status: all(( + wait_for_unit_status(app_name, unit_to_promote, "active")(status), + wait_for_unit_message(app_name, units_to_kill[0], "offline")(status), + wait_for_unit_message(app_name, units_to_kill[1], "offline")(status), + )), timeout=60 * 15, delay=15, ) diff --git a/tests/integration/high_availability/test_upgrade.py b/tests/integration/high_availability/test_upgrade.py index 139638619..f45e04e7d 100644 --- a/tests/integration/high_availability/test_upgrade.py +++ b/tests/integration/high_availability/test_upgrade.py @@ -1,179 +1,181 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. -import asyncio import json import logging -import os -import pathlib +import shutil import zipfile from pathlib import Path -from shutil import copy +import jubilant_backports import pytest -from pytest_operator.plugin import OpsTest - -from .. import juju_ -from ..helpers import ( - get_leader_unit, - get_primary_unit_wrapper, +from jubilant_backports import Juju + +from .high_availability_helpers_new import ( + check_mysql_units_writes_increment, + get_app_leader, + get_app_units, + get_mysql_primary_unit, + get_mysql_variable_value, get_relation_data, - retrieve_database_variable_value, -) -from .high_availability_helpers import ( - ensure_all_units_continuous_writes_incrementing, - relate_mysql_and_application, + wait_for_apps_status, ) -logger = logging.getLogger(__name__) +MYSQL_APP_NAME = "mysql" +MYSQL_TEST_APP_NAME = "mysql-test-app" -TIMEOUT = 20 * 60 +MINUTE_SECS = 60 -MYSQL_APP_NAME = "mysql" -TEST_APP_NAME = "mysql-test-app" +logging.getLogger("jubilant.wait").setLevel(logging.WARNING) @pytest.mark.abort_on_fail -async def test_deploy_latest(ops_test: OpsTest) -> None: - """Simple test to ensure that the mysql and application charms get deployed.""" - await asyncio.gather( - ops_test.model.deploy( - MYSQL_APP_NAME, - application_name=MYSQL_APP_NAME, - num_units=3, - channel="8.0/edge", - config={"profile": "testing"}, - base="ubuntu@22.04", - ), - ops_test.model.deploy( - TEST_APP_NAME, - application_name=TEST_APP_NAME, - num_units=1, - channel="latest/edge", - base="ubuntu@22.04", - ), +def test_deploy_latest(juju: Juju) -> None: + """Simple test to ensure that the MySQL and application charms get deployed.""" + logging.info("Deploying MySQL cluster") + juju.deploy( + charm=MYSQL_APP_NAME, + app=MYSQL_APP_NAME, + base="ubuntu@22.04", + channel="8.0/edge", + config={"profile": "testing"}, + num_units=3, ) - await relate_mysql_and_application(ops_test, MYSQL_APP_NAME, TEST_APP_NAME) - logger.info("Wait for applications to become active") - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME, TEST_APP_NAME], - status="active", - timeout=TIMEOUT, + juju.deploy( + charm=MYSQL_TEST_APP_NAME, + app=MYSQL_TEST_APP_NAME, + base="ubuntu@22.04", + channel="latest/edge", + num_units=1, ) - assert len(ops_test.model.applications[MYSQL_APP_NAME].units) == 3 + juju.integrate( + f"{MYSQL_APP_NAME}:database", + f"{MYSQL_TEST_APP_NAME}:database", + ) -@pytest.mark.abort_on_fail -async def test_pre_upgrade_check(ops_test: OpsTest) -> None: - """Test that the pre-upgrade-check action runs successfully.""" - mysql_units = ops_test.model.applications[MYSQL_APP_NAME].units + logging.info("Wait for applications to become active") + juju.wait( + ready=wait_for_apps_status( + jubilant_backports.all_active, MYSQL_APP_NAME, MYSQL_TEST_APP_NAME + ), + error=jubilant_backports.any_blocked, + timeout=20 * MINUTE_SECS, + ) - logger.info("Get leader unit") - leader_unit = await get_leader_unit(ops_test, MYSQL_APP_NAME) - assert leader_unit is not None, "No leader unit found" - logger.info("Run pre-upgrade-check action") - await juju_.run_action(leader_unit, "pre-upgrade-check") +@pytest.mark.abort_on_fail +async def test_pre_upgrade_check(juju: Juju) -> None: + """Test that the pre-upgrade-check action runs successfully.""" + mysql_leader = get_app_leader(juju, MYSQL_APP_NAME) + mysql_units = get_app_units(juju, MYSQL_APP_NAME) - logger.info("Assert slow shutdown is enabled") - for unit in mysql_units: - value = await retrieve_database_variable_value(ops_test, unit, "innodb_fast_shutdown") - assert value == 0, f"innodb_fast_shutdown not 0 at {unit.name}" + logging.info("Run pre-upgrade-check action") + task = juju.run(unit=mysql_leader, action="pre-upgrade-check") + task.raise_on_failure() - primary_unit = await get_primary_unit_wrapper(ops_test, MYSQL_APP_NAME) + logging.info("Assert slow shutdown is enabled") + for unit_name in mysql_units: + value = await get_mysql_variable_value( + juju, MYSQL_APP_NAME, unit_name, "innodb_fast_shutdown" + ) + assert value == 0 - logger.info("Assert primary is set to leader") - assert await primary_unit.is_leader_from_status(), "Primary unit not set to leader" + logging.info("Assert primary is set to leader") + mysql_primary = get_mysql_primary_unit(juju, MYSQL_APP_NAME) + assert mysql_primary == mysql_leader, "Primary unit not set to leader" @pytest.mark.abort_on_fail -async def test_upgrade_from_edge( - ops_test: OpsTest, - charm, - continuous_writes, -) -> None: - logger.info("Ensure continuous_writes") - await ensure_all_units_continuous_writes_incrementing(ops_test) - - application = ops_test.model.applications[MYSQL_APP_NAME] - - logger.info("Refresh the charm") - await application.refresh(path=charm) - - logger.info("Wait for upgrade to start") - await ops_test.model.block_until( - lambda: "waiting" in {unit.workload_status for unit in application.units}, - timeout=TIMEOUT, +async def test_upgrade_from_edge(juju: Juju, charm: str, continuous_writes) -> None: + """Update the second cluster.""" + logging.info("Ensure continuous writes are incrementing") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME) + + logging.info("Refresh the charm") + juju.refresh(app=MYSQL_APP_NAME, path=charm) + + logging.info("Wait for upgrade to start") + juju.wait( + ready=lambda status: jubilant_backports.any_maintenance(status, MYSQL_APP_NAME), + timeout=10 * MINUTE_SECS, ) - logger.info("Wait for upgrade to complete") - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME], status="active", idle_period=30, timeout=TIMEOUT + logging.info("Wait for upgrade to complete") + juju.wait( + ready=lambda status: jubilant_backports.all_active(status, MYSQL_APP_NAME), + timeout=20 * MINUTE_SECS, ) - logger.info("Ensure continuous_writes") - await ensure_all_units_continuous_writes_incrementing(ops_test) + logging.info("Ensure continuous writes are incrementing") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME) @pytest.mark.abort_on_fail -async def test_fail_and_rollback(ops_test, charm, continuous_writes) -> None: - logger.info("Get leader unit") - leader_unit = await get_leader_unit(ops_test, MYSQL_APP_NAME) +async def test_fail_and_rollback(juju: Juju, charm: str, continuous_writes) -> None: + """Test an upgrade failure and its rollback.""" + mysql_app_leader = get_app_leader(juju, MYSQL_APP_NAME) + mysql_app_units = get_app_units(juju, MYSQL_APP_NAME) - assert leader_unit is not None, "No leader unit found" + logging.info("Run pre-upgrade-check action") + task = juju.run(unit=mysql_app_leader, action="pre-upgrade-check") + task.raise_on_failure() - logger.info("Run pre-upgrade-check action") - await juju_.run_action(leader_unit, "pre-upgrade-check") + tmp_folder = Path("tmp") + tmp_folder.mkdir(exist_ok=True) + tmp_folder_charm = Path(tmp_folder, charm).absolute() - fault_charm = f"/tmp/{pathlib.Path(charm).name}" - copy(charm, fault_charm) + shutil.copy(charm, tmp_folder_charm) - logger.info("Inject dependency fault") - await inject_dependency_fault(ops_test, MYSQL_APP_NAME, fault_charm) + logging.info("Inject dependency fault") + inject_dependency_fault(juju, MYSQL_APP_NAME, tmp_folder_charm) - application = ops_test.model.applications[MYSQL_APP_NAME] + logging.info("Refresh the charm") + juju.refresh(app=MYSQL_APP_NAME, path=tmp_folder_charm) - logger.info("Refresh the charm") - await application.refresh(path=fault_charm) - - logger.info("Wait for upgrade to fail on leader") - await ops_test.model.block_until( - lambda: leader_unit.workload_status == "blocked", - timeout=TIMEOUT, + logging.info("Wait for upgrade to fail on leader") + juju.wait( + ready=wait_for_apps_status(jubilant_backports.any_blocked, MYSQL_APP_NAME), + timeout=10 * MINUTE_SECS, ) - logger.info("Ensure continuous_writes while in failure state") - await ensure_all_units_continuous_writes_incrementing(ops_test) + logging.info("Ensure continuous writes on all units") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME, list(mysql_app_units)) + + logging.info("Re-run pre-upgrade-check action") + task = juju.run(unit=mysql_app_leader, action="pre-upgrade-check") + task.raise_on_failure() - logger.info("Re-run pre-upgrade-check action") - await juju_.run_action(leader_unit, "pre-upgrade-check") + logging.info("Re-refresh the charm") + juju.refresh(app=MYSQL_APP_NAME, path=charm) + + logging.info("Wait for upgrade to start") + juju.wait( + ready=lambda status: jubilant_backports.any_maintenance(status, MYSQL_APP_NAME), + timeout=10 * MINUTE_SECS, + ) - logger.info("Re-refresh the charm") - await application.refresh(path=charm) - logger.info("Wait for upgrade to start") - await ops_test.model.block_until( - lambda: "waiting" in {unit.workload_status for unit in application.units}, - timeout=TIMEOUT, + logging.info("Wait for upgrade to complete") + juju.wait( + ready=lambda status: jubilant_backports.all_active(status, MYSQL_APP_NAME), + timeout=20 * MINUTE_SECS, ) - await ops_test.model.wait_for_idle(apps=[MYSQL_APP_NAME], status="active", timeout=TIMEOUT) - logger.info("Ensure continuous_writes after rollback procedure") - await ensure_all_units_continuous_writes_incrementing(ops_test) + logging.info("Ensure continuous writes after rollback procedure") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME, list(mysql_app_units)) - # remove fault charm file - os.remove(fault_charm) + # Remove fault charm file + tmp_folder_charm.unlink() -async def inject_dependency_fault( - ops_test: OpsTest, application_name: str, charm_file: str | Path -) -> None: +def inject_dependency_fault(juju: Juju, app_name: str, charm_file: str | Path) -> None: """Inject a dependency fault into the mysql charm.""" # Open dependency.json and load current charm version with open("src/dependency.json") as dependency_file: current_charm_version = json.load(dependency_file)["charm"]["version"] - # query running dependency to overwrite with incompatible version - relation_data = await get_relation_data(ops_test, application_name, "upgrade") + # Query running dependency to overwrite with incompatible version + relation_data = get_relation_data(juju, app_name, "upgrade") loaded_dependency_dict = json.loads(relation_data[0]["application-data"]["dependencies"]) loaded_dependency_dict["charm"]["upgrade_supported"] = f">{current_charm_version}" diff --git a/tests/integration/high_availability/test_upgrade_from_stable.py b/tests/integration/high_availability/test_upgrade_from_stable.py index e6b8a1063..d272d0647 100644 --- a/tests/integration/high_availability/test_upgrade_from_stable.py +++ b/tests/integration/high_availability/test_upgrade_from_stable.py @@ -1,97 +1,106 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. -import asyncio import logging +import jubilant_backports import pytest -from pytest_operator.plugin import OpsTest - -from .. import juju_ -from ..helpers import get_leader_unit, get_primary_unit_wrapper, retrieve_database_variable_value -from .high_availability_helpers import ( - ensure_all_units_continuous_writes_incrementing, - relate_mysql_and_application, +from jubilant_backports import Juju + +from .high_availability_helpers_new import ( + check_mysql_units_writes_increment, + get_app_leader, + get_app_units, + get_mysql_primary_unit, + get_mysql_variable_value, + wait_for_apps_status, ) -logger = logging.getLogger(__name__) +MYSQL_APP_NAME = "mysql" +MYSQL_TEST_APP_NAME = "mysql-test-app" -TIMEOUT = 20 * 60 +MINUTE_SECS = 60 -MYSQL_APP_NAME = "mysql" -TEST_APP_NAME = "mysql-test-app" +logging.getLogger("jubilant.wait").setLevel(logging.WARNING) @pytest.mark.abort_on_fail -async def test_deploy_stable(ops_test: OpsTest) -> None: - """Simple test to ensure that the mysql and application charms get deployed.""" - await asyncio.gather( - ops_test.model.deploy( - MYSQL_APP_NAME, - application_name=MYSQL_APP_NAME, - num_units=3, - channel="8.0/stable", - base="ubuntu@22.04", - config={"profile": "testing"}, - ), - ops_test.model.deploy( - TEST_APP_NAME, - application_name=TEST_APP_NAME, - num_units=1, - channel="latest/edge", - base="ubuntu@22.04", - ), +def test_deploy_stable(juju: Juju) -> None: + """Simple test to ensure that the MySQL and application charms get deployed.""" + logging.info("Deploying MySQL cluster") + juju.deploy( + charm=MYSQL_APP_NAME, + app=MYSQL_APP_NAME, + base="ubuntu@22.04", + channel="8.0/stable", + config={"profile": "testing"}, + num_units=3, ) - await relate_mysql_and_application(ops_test, MYSQL_APP_NAME, TEST_APP_NAME) - logger.info("Wait for applications to become active") - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME, TEST_APP_NAME], - status="active", - timeout=TIMEOUT, + juju.deploy( + charm=MYSQL_TEST_APP_NAME, + app=MYSQL_TEST_APP_NAME, + base="ubuntu@22.04", + channel="latest/edge", + num_units=1, ) - assert len(ops_test.model.applications[MYSQL_APP_NAME].units) == 3 - -@pytest.mark.abort_on_fail -async def test_pre_upgrade_check(ops_test: OpsTest) -> None: - """Test that the pre-upgrade-check action runs successfully.""" - mysql_units = ops_test.model.applications[MYSQL_APP_NAME].units - - logger.info("Get leader unit") - leader_unit = await get_leader_unit(ops_test, MYSQL_APP_NAME) + juju.integrate( + f"{MYSQL_APP_NAME}:database", + f"{MYSQL_TEST_APP_NAME}:database", + ) - assert leader_unit is not None, "No leader unit found" - logger.info("Run pre-upgrade-check action") - await juju_.run_action(leader_unit, "pre-upgrade-check") + logging.info("Wait for applications to become active") + juju.wait( + ready=wait_for_apps_status( + jubilant_backports.all_active, MYSQL_APP_NAME, MYSQL_TEST_APP_NAME + ), + error=jubilant_backports.any_blocked, + timeout=20 * MINUTE_SECS, + ) - logger.info("Assert slow shutdown is enabled") - for unit in mysql_units: - value = await retrieve_database_variable_value(ops_test, unit, "innodb_fast_shutdown") - assert value == 0, f"innodb_fast_shutdown not 0 at {unit.name}" - primary_unit = await get_primary_unit_wrapper(ops_test, MYSQL_APP_NAME) +@pytest.mark.abort_on_fail +async def test_pre_upgrade_check(juju: Juju) -> None: + """Test that the pre-upgrade-check action runs successfully.""" + mysql_leader = get_app_leader(juju, MYSQL_APP_NAME) + mysql_units = get_app_units(juju, MYSQL_APP_NAME) - logger.info("Assert primary is set to leader") - assert await primary_unit.is_leader_from_status(), "Primary unit not set to leader" + logging.info("Run pre-upgrade-check action") + task = juju.run(unit=mysql_leader, action="pre-upgrade-check") + task.raise_on_failure() + logging.info("Assert slow shutdown is enabled") + for unit_name in mysql_units: + value = await get_mysql_variable_value( + juju, MYSQL_APP_NAME, unit_name, "innodb_fast_shutdown" + ) + assert value == 0 -async def test_upgrade_from_stable(ops_test: OpsTest, charm): - """Test updating from stable channel.""" - application = ops_test.model.applications[MYSQL_APP_NAME] + logging.info("Assert primary is set to leader") + mysql_primary = get_mysql_primary_unit(juju, MYSQL_APP_NAME) + assert mysql_primary == mysql_leader, "Primary unit not set to leader" - logger.info("Refresh the charm") - await application.refresh(path=charm) - logger.info("Wait for upgrade to start") - await ops_test.model.block_until( - lambda: "maintenance" in {unit.workload_status for unit in application.units}, - timeout=TIMEOUT, +@pytest.mark.abort_on_fail +async def test_upgrade_from_stable(juju: Juju, charm: str, continuous_writes) -> None: + """Update the second cluster.""" + logging.info("Ensure continuous writes are incrementing") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME) + + logging.info("Refresh the charm") + juju.refresh(app=MYSQL_APP_NAME, path=charm) + + logging.info("Wait for upgrade to start") + juju.wait( + ready=lambda status: jubilant_backports.any_maintenance(status, MYSQL_APP_NAME), + timeout=10 * MINUTE_SECS, ) - logger.info("Wait for upgrade to complete") - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME], status="active", idle_period=30, timeout=TIMEOUT + logging.info("Wait for upgrade to complete") + juju.wait( + ready=lambda status: jubilant_backports.all_active(status, MYSQL_APP_NAME), + timeout=20 * MINUTE_SECS, ) - logger.info("Ensure continuous_writes") - await ensure_all_units_continuous_writes_incrementing(ops_test) + logging.info("Ensure continuous writes are incrementing") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME) diff --git a/tests/integration/high_availability/test_upgrade_rollback_incompat.py b/tests/integration/high_availability/test_upgrade_rollback_incompat.py index 925faffc0..d0ccf86e1 100644 --- a/tests/integration/high_availability/test_upgrade_rollback_incompat.py +++ b/tests/integration/high_availability/test_upgrade_rollback_incompat.py @@ -1,208 +1,219 @@ # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. -import ast import json import logging -import pathlib import shutil -from time import sleep -from zipfile import ZipFile +import time +import zipfile +from ast import literal_eval +from collections.abc import Generator +from pathlib import Path +import jubilant_backports import pytest -from pytest_operator.plugin import OpsTest +from jubilant_backports import Juju -from .. import juju_, markers -from ..helpers import ( - get_leader_unit, +from ..markers import amd64_only +from .high_availability_helpers_new import ( + check_mysql_units_writes_increment, + get_app_leader, get_relation_data, - get_status_log, - get_unit_by_index, + get_unit_by_number, + get_unit_status_log, + wait_for_apps_status, + wait_for_unit_status, ) -from .high_availability_helpers import ( - ensure_all_units_continuous_writes_incrementing, - relate_mysql_and_application, -) - -logger = logging.getLogger(__name__) -TIMEOUT = 20 * 60 MYSQL_APP_NAME = "mysql" -TEST_APP = "mysql-test-app" +MYSQL_TEST_APP_NAME = "mysql-test-app" + +MINUTE_SECS = 60 + +logging.getLogger("jubilant.wait").setLevel(logging.WARNING) + + +@pytest.fixture() +def continuous_writes(juju: Juju) -> Generator: + """Starts continuous writes to the MySQL cluster for a test and clear the writes at the end.""" + test_app_leader = get_app_leader(juju, MYSQL_TEST_APP_NAME) + logging.info("Clearing continuous writes") + juju.run(test_app_leader, "clear-continuous-writes") + logging.info("Starting continuous writes") + juju.run(test_app_leader, "start-continuous-writes") -# TODO: remove after next incompatible MySQL server version released in our snap + yield + + logging.info("Clearing continuous writes") + juju.run(test_app_leader, "clear-continuous-writes") + + +# TODO: remove AMD64 marker after next incompatible MySQL server version is released in our snap # (details: https://github.com/canonical/mysql-operator/pull/472#discussion_r1659300069) -@markers.amd64_only +@amd64_only @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest, charm) -> None: - """Simple test to ensure that the mysql and application charms get deployed.""" - snap_revisions = pathlib.Path("snap_revisions.json") +async def test_build_and_deploy(juju: Juju, charm: str) -> None: + """Simple test to ensure that the MySQL and application charms get deployed.""" + snap_revisions = Path("snap_revisions.json") with snap_revisions.open("r") as file: - old_revisions: dict = json.load(file) - new_revisions = old_revisions.copy() + old_revisions = json.load(file) + # TODO: support arm64 & s390x + new_revisions = old_revisions.copy() new_revisions["x86_64"] = "69" + with snap_revisions.open("w") as file: json.dump(new_revisions, file) - local_charm = await charm_local_build(ops_test, charm) + + local_charm = get_locally_built_charm(charm) with snap_revisions.open("w") as file: json.dump(old_revisions, file) - async with ops_test.fast_forward("30s"): - await ops_test.model.deploy( - local_charm, - application_name=MYSQL_APP_NAME, - num_units=3, - base="ubuntu@22.04", - config={"profile": "testing", "plugin-audit-enabled": "false"}, - ) - - await ops_test.model.deploy( - TEST_APP, - application_name=TEST_APP, - channel="latest/edge", - num_units=1, - base="ubuntu@22.04", - config={"auto_start_writes": False, "sleep_interval": "500"}, - ) - - await relate_mysql_and_application(ops_test, MYSQL_APP_NAME, TEST_APP) - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME, TEST_APP], - status="active", - timeout=TIMEOUT, - ) - - -# TODO: remove after next incompatible MySQL server version released in our snap + juju.deploy( + charm=local_charm, + app=MYSQL_APP_NAME, + base="ubuntu@22.04", + config={"profile": "testing", "plugin-audit-enabled": False}, + num_units=3, + ) + juju.deploy( + charm=MYSQL_TEST_APP_NAME, + app=MYSQL_TEST_APP_NAME, + base="ubuntu@22.04", + channel="latest/edge", + config={"auto_start_writes": False, "sleep_interval": 500}, + num_units=1, + ) + + juju.integrate( + f"{MYSQL_APP_NAME}:database", + f"{MYSQL_TEST_APP_NAME}:database", + ) + + logging.info("Wait for applications to become active") + juju.wait( + ready=wait_for_apps_status( + jubilant_backports.all_active, MYSQL_APP_NAME, MYSQL_TEST_APP_NAME + ), + error=jubilant_backports.any_blocked, + timeout=20 * MINUTE_SECS, + ) + + +# TODO: remove AMD64 marker after next incompatible MySQL server version is released in our snap # (details: https://github.com/canonical/mysql-operator/pull/472#discussion_r1659300069) -@markers.amd64_only +@amd64_only @pytest.mark.abort_on_fail -async def test_pre_upgrade_check(ops_test: OpsTest) -> None: +async def test_pre_upgrade_check(juju: Juju) -> None: """Test that the pre-upgrade-check action runs successfully.""" - logger.info("Get leader unit") - leader_unit = await get_leader_unit(ops_test, MYSQL_APP_NAME) + mysql_leader = get_app_leader(juju, MYSQL_APP_NAME) - assert leader_unit is not None, "No leader unit found" - logger.info("Run pre-upgrade-check action") - await juju_.run_action(leader_unit, "pre-upgrade-check") + logging.info("Run pre-upgrade-check action") + task = juju.run(unit=mysql_leader, action="pre-upgrade-check") + task.raise_on_failure() -# TODO: remove after next incompatible MySQL server version released in our snap +# TODO: remove AMD64 marker after next incompatible MySQL server version is released in our snap # (details: https://github.com/canonical/mysql-operator/pull/472#discussion_r1659300069) -@markers.amd64_only +@amd64_only @pytest.mark.abort_on_fail -async def test_upgrade_to_failling( - ops_test: OpsTest, - charm, - continuous_writes, -) -> None: - logger.info("Ensure continuous_writes") - await ensure_all_units_continuous_writes_incrementing(ops_test) - - application = ops_test.model.applications[MYSQL_APP_NAME] +async def test_upgrade_to_failing(juju: Juju, charm: str, continuous_writes) -> None: + logging.info("Ensure continuous_writes") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME) with InjectFailure( path="src/upgrade.py", original_str="self.charm.recover_unit_after_restart()", replace_str="raise Exception", ): - logger.info("Build charm with failure injected") - new_charm = await charm_local_build(ops_test, charm, refresh=True) + logging.info("Build charm with failure injected") + new_charm = get_locally_built_charm(charm) - logger.info("Refresh the charm") - await application.refresh(path=new_charm) + logging.info("Refresh the charm") + juju.refresh(app=MYSQL_APP_NAME, path=new_charm) - logger.info("Wait for upgrade to start") - await ops_test.model.block_until( - lambda: "waiting" in {unit.workload_status for unit in application.units}, - timeout=TIMEOUT, - ) - logger.info("Get first upgrading unit") - relation_data = await get_relation_data(ops_test, MYSQL_APP_NAME, "upgrade") - upgrade_stack = relation_data[0]["application-data"]["upgrade-stack"] - upgrading_unit = get_unit_by_index( - MYSQL_APP_NAME, application.units, ast.literal_eval(upgrade_stack)[-1] + logging.info("Wait for upgrade to start") + juju.wait( + ready=lambda status: jubilant_backports.any_maintenance(status, MYSQL_APP_NAME), + timeout=10 * MINUTE_SECS, ) - assert upgrading_unit is not None, "No upgrading unit found" + logging.info("Get first upgrading unit") + relation_data = get_relation_data(juju, MYSQL_APP_NAME, "upgrade") + upgrade_stack = relation_data[0]["application-data"]["upgrade-stack"] + upgrade_unit = get_unit_by_number(juju, MYSQL_APP_NAME, literal_eval(upgrade_stack)[-1]) - logger.info("Wait for upgrade to fail on upgrading unit") - await ops_test.model.block_until( - lambda: upgrading_unit.workload_status == "blocked", - timeout=TIMEOUT, + logging.info("Wait for upgrade to fail on upgrading unit") + juju.wait( + ready=wait_for_unit_status(MYSQL_APP_NAME, upgrade_unit, "blocked"), + timeout=10 * MINUTE_SECS, ) -# TODO: remove after next incompatible MySQL server version released in our snap +# TODO: remove AMD64 marker after next incompatible MySQL server version is released in our snap # (details: https://github.com/canonical/mysql-operator/pull/472#discussion_r1659300069) -@markers.amd64_only +@amd64_only @pytest.mark.abort_on_fail -async def test_rollback(ops_test, charm, continuous_writes) -> None: - application = ops_test.model.applications[MYSQL_APP_NAME] - - relation_data = await get_relation_data(ops_test, MYSQL_APP_NAME, "upgrade") +async def test_rollback(juju: Juju, charm: str, continuous_writes) -> None: + """Test upgrade rollback to a healthy revision.""" + relation_data = get_relation_data(juju, MYSQL_APP_NAME, "upgrade") upgrade_stack = relation_data[0]["application-data"]["upgrade-stack"] - upgrading_unit = get_unit_by_index( - MYSQL_APP_NAME, application.units, ast.literal_eval(upgrade_stack)[-1] - ) - assert upgrading_unit is not None, "No upgrading unit found" - assert upgrading_unit.workload_status == "blocked", "Upgrading unit's status is not blocked" + upgrade_unit = get_unit_by_number(juju, MYSQL_APP_NAME, literal_eval(upgrade_stack)[-1]) - snap_revisions = pathlib.Path("snap_revisions.json") + snap_revisions = Path("snap_revisions.json") with snap_revisions.open("r") as file: - old_revisions: dict = json.load(file) - new_revisions = old_revisions.copy() + old_revisions = json.load(file) + # TODO: support arm64 & s390x + new_revisions = old_revisions.copy() new_revisions["x86_64"] = "69" + with snap_revisions.open("w") as file: json.dump(new_revisions, file) - local_charm = await charm_local_build(ops_test, charm, refresh=True) - - logger.info("Get leader unit") - leader_unit = await get_leader_unit(ops_test, MYSQL_APP_NAME) - assert leader_unit is not None, "No leader unit found" + mysql_leader = get_app_leader(juju, MYSQL_APP_NAME) + local_charm = get_locally_built_charm(charm) - sleep(10) - logger.info("Run pre-upgrade-check action") - await juju_.run_action(leader_unit, "pre-upgrade-check") + time.sleep(10) - sleep(20) - logger.info("Refresh with previous charm") - await application.refresh(path=local_charm) + logging.info("Run pre-upgrade-check action") + task = juju.run(unit=mysql_leader, action="pre-upgrade-check") + task.raise_on_failure() - logger.info("Wait for upgrade to start") - await ops_test.model.block_until( - lambda: "waiting" in {unit.workload_status for unit in application.units}, - timeout=TIMEOUT, - ) - await ops_test.model.wait_for_idle(apps=[MYSQL_APP_NAME], status="active", timeout=TIMEOUT) + time.sleep(20) - logger.info("Ensure rollback has taken place") + logging.info("Refresh with previous charm") + juju.refresh(app=MYSQL_APP_NAME, path=local_charm) - status_logs = await get_status_log(ops_test, upgrading_unit.name, 100) + logging.info("Wait for upgrade to start") + juju.wait( + ready=lambda status: jubilant_backports.any_maintenance(status, MYSQL_APP_NAME), + timeout=10 * MINUTE_SECS, + ) + juju.wait( + ready=lambda status: jubilant_backports.all_active(status, MYSQL_APP_NAME), + timeout=20 * MINUTE_SECS, + ) - upgrade_failed_index = -1 - for index, status_log in enumerate(status_logs): - if "upgrade failed. Check logs for rollback instruction" in status_log: - upgrade_failed_index = index - break - assert upgrade_failed_index > -1, "Upgrade failed status log not found" + logging.info("Ensure rollback has taken place") + unit_status_logs = get_unit_status_log(juju, upgrade_unit, 100) - post_upgrade_failed_status_logs = status_logs[upgrade_failed_index:] + upgrade_failed_index = get_unit_log_message( + status_logs=unit_status_logs[:], + unit_message="upgrade failed. Check logs for rollback instruction", + ) + assert upgrade_failed_index is not None - upgrade_complete_index = -1 - for index, status_log in enumerate(post_upgrade_failed_status_logs): - if "upgrade completed" in status_log: - upgrade_complete_index = index - break - assert upgrade_complete_index > -1, "Upgrade completed status log not found for rollback" + upgrade_complete_index = get_unit_log_message( + status_logs=unit_status_logs[upgrade_failed_index:], + unit_message="upgrade completed", + ) + assert upgrade_complete_index is not None - logger.info("Ensure continuous_writes after rollback procedure") - await ensure_all_units_continuous_writes_incrementing(ops_test) + logging.info("Ensure continuous writes after rollback procedure") + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME) class InjectFailure: @@ -214,7 +225,7 @@ def __init__(self, path: str, original_str: str, replace_str: str): self.original_content = file.read() def __enter__(self): - logger.info("Injecting failure") + logging.info("Injecting failure") assert self.original_str in self.original_content, "replace content not found" new_content = self.original_content.replace(self.original_str, self.replace_str) assert self.original_str not in new_content, "original string not replaced" @@ -222,33 +233,37 @@ def __enter__(self): file.write(new_content) def __exit__(self, exc_type, exc_value, traceback): - logger.info("Reverting failure") + logging.info("Reverting failure") with open(self.path, "w") as file: file.write(self.original_content) -async def charm_local_build(ops_test: OpsTest, charm, refresh: bool = False): +def get_unit_log_message(status_logs: list[dict], unit_message: str) -> int | None: + """Returns the index of a status log containing the desired message.""" + for index, status_log in enumerate(status_logs): + if status_log.get("message") == unit_message: + return index + + return None + + +def get_locally_built_charm(charm: str) -> str: """Wrapper for a local charm build zip file updating.""" - local_charms = pathlib.Path().glob("local-*.charm") - for lc in local_charms: - # clean up local charms from previous runs to avoid - # pytest_operator_cache globbing them - lc.unlink() + local_charm_paths = Path().glob("local-*.charm") - update_files = ["snap_revisions.json", "src/upgrade.py"] + # Clean up local charms from previous runs + # to avoid pytest_operator_cache globbing them + for charm_path in local_charm_paths: + charm_path.unlink() - # create a copy of the charm to avoid modifying the original - local_charm = pathlib.Path(shutil.copy(charm, f"local-{pathlib.Path(charm).stem}.charm")) + # Create a copy of the charm to avoid modifying the original + local_charm_path = shutil.copy(charm, f"local-{Path(charm).stem}.charm") + local_charm_path = Path(local_charm_path) - for path in update_files: + for path in ["snap_revisions.json", "src/upgrade.py"]: with open(path) as f: content = f.read() - - with ZipFile(local_charm, mode="a") as charm_zip: + with zipfile.ZipFile(local_charm_path, mode="a") as charm_zip: charm_zip.writestr(path, content) - if refresh: - # when refreshing, return posix path - return local_charm - # when deploying, return prefixed full path - return f"local:{local_charm.resolve()}" + return f"{local_charm_path.resolve()}" diff --git a/tests/integration/high_availability/test_upgrade_skip_pre_upgrade_check.py b/tests/integration/high_availability/test_upgrade_skip_pre_upgrade_check.py index 69ff73092..c6031d0b2 100644 --- a/tests/integration/high_availability/test_upgrade_skip_pre_upgrade_check.py +++ b/tests/integration/high_availability/test_upgrade_skip_pre_upgrade_check.py @@ -1,110 +1,97 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. -import asyncio import logging -import subprocess -from time import sleep +import jubilant_backports import pytest -from pytest_operator.plugin import OpsTest +from jubilant_backports import Juju -from .high_availability_helpers import ( - ensure_all_units_continuous_writes_incrementing, - relate_mysql_and_application, +from .high_availability_helpers_new import ( + check_mysql_units_writes_increment, + get_app_units, + wait_for_apps_status, + wait_for_unit_status, ) -logger = logging.getLogger(__name__) +MYSQL_APP_NAME = "mysql" +MYSQL_TEST_APP_NAME = "mysql-test-app" -TIMEOUT = 20 * 60 +MINUTE_SECS = 60 -MYSQL_APP_NAME = "mysql" -TEST_APP_NAME = "mysql-test-app" +logging.getLogger("jubilant.wait").setLevel(logging.WARNING) @pytest.mark.abort_on_fail -async def test_deploy_stable(ops_test: OpsTest) -> None: - """Simple test to ensure that the mysql and application charms get deployed.""" - await asyncio.gather( - ops_test.model.deploy( - MYSQL_APP_NAME, - application_name=MYSQL_APP_NAME, - num_units=3, - channel="8.0/stable", - base="ubuntu@22.04", - config={"profile": "testing"}, - ), - ops_test.model.deploy( - TEST_APP_NAME, - application_name=TEST_APP_NAME, - num_units=1, - channel="latest/edge", - base="ubuntu@22.04", - config={"sleep_interval": 50}, - ), +def test_deploy_stable(juju: Juju) -> None: + """Simple test to ensure that the MySQL and application charms get deployed.""" + logging.info("Deploying MySQL cluster") + juju.deploy( + charm=MYSQL_APP_NAME, + app=MYSQL_APP_NAME, + base="ubuntu@22.04", + channel="8.0/stable", + config={"profile": "testing"}, + num_units=3, + ) + juju.deploy( + charm=MYSQL_TEST_APP_NAME, + app=MYSQL_TEST_APP_NAME, + base="ubuntu@22.04", + channel="latest/edge", + config={"sleep_interval": 50}, + num_units=1, ) - await relate_mysql_and_application(ops_test, MYSQL_APP_NAME, TEST_APP_NAME) - logger.info("Wait for applications to become active") - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME, TEST_APP_NAME], - status="active", - timeout=TIMEOUT, + + juju.integrate( + f"{MYSQL_APP_NAME}:database", + f"{MYSQL_TEST_APP_NAME}:database", + ) + + logging.info("Wait for applications to become active") + juju.wait( + ready=wait_for_apps_status( + jubilant_backports.all_active, MYSQL_APP_NAME, MYSQL_TEST_APP_NAME + ), + error=jubilant_backports.any_blocked, + timeout=20 * MINUTE_SECS, ) - assert len(ops_test.model.applications[MYSQL_APP_NAME].units) == 3 -async def test_refresh_without_pre_upgrade_check(ops_test: OpsTest, charm): +@pytest.mark.abort_on_fail +async def test_refresh_without_pre_upgrade_check(juju: Juju, charm: str) -> None: """Test updating from stable channel.""" - application = ops_test.model.applications[MYSQL_APP_NAME] - - logger.info("Refresh the charm") - await application.refresh(path=charm) - - # Refresh without pre-upgrade-check can have two immediate effects: - # 1. None, if there's no configuration change - # 2. Rolling restart, if there's a configuration change - # for both, operations should continue to work - # and there's a mismatch between the charm and the snap - logger.info("Wait (120s) for rolling restart OR continue to writes") - count = 0 - while count < 2 * 60: - if "maintenance" in {unit.workload_status for unit in application.units}: - # Case when refresh triggers a rolling restart - logger.info("Waiting for rolling restart to complete") - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME], status="active", idle_period=30, timeout=TIMEOUT - ) - break - else: - count += 1 - sleep(1) - - await ensure_all_units_continuous_writes_incrementing(ops_test) - - -async def test_rollback_without_pre_upgrade_check(ops_test: OpsTest): - """Test refresh back to stable channel.""" - application = ops_test.model.applications[MYSQL_APP_NAME] + logging.info("Refresh the charm") + juju.refresh(app=MYSQL_APP_NAME, path=charm) + + logging.info("Wait for rolling restart") + app_units = get_app_units(juju, MYSQL_APP_NAME) + app_units_funcs = [wait_for_unit_status(MYSQL_APP_NAME, unit, "error") for unit in app_units] + + juju.wait( + ready=lambda status: any(status_func(status) for status_func in app_units_funcs), + timeout=10 * MINUTE_SECS, + successes=1, + ) - logger.info("Refresh the charm back to stable channel") - # pylibjuju refresh dont work for switch: - # https://github.com/juju/python-libjuju/issues/924 - subprocess.check_output( - f"juju refresh {MYSQL_APP_NAME} --switch ch:{MYSQL_APP_NAME} --channel 8.0/stable".split() + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME) + + +@pytest.mark.abort_on_fail +async def test_rollback_without_pre_upgrade_check(juju: Juju, charm: str) -> None: + """Test refresh back to stable channel.""" + # Early Jubilant 1.X.Y versions do not support the `switch` option + logging.info("Refresh the charm to stable channel") + juju.cli("refresh", "--channel=8.0/stable", f"--switch={MYSQL_APP_NAME}", MYSQL_APP_NAME) + + logging.info("Wait for rolling restart") + app_units = get_app_units(juju, MYSQL_APP_NAME) + app_units_funcs = [wait_for_unit_status(MYSQL_APP_NAME, unit, "error") for unit in app_units] + + juju.wait( + ready=lambda status: any(status_func(status) for status_func in app_units_funcs), + timeout=10 * MINUTE_SECS, + successes=1, ) - logger.info("Wait (120s) for rolling restart OR continue to writes") - count = 0 - while count < 2 * 60: - if "maintenance" in {unit.workload_status for unit in application.units}: - # Case when refresh triggers a rolling restart - logger.info("Waiting for rolling restart to complete") - await ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME], status="active", idle_period=30, timeout=TIMEOUT - ) - break - else: - count += 1 - sleep(1) - - await ensure_all_units_continuous_writes_incrementing(ops_test) + await check_mysql_units_writes_increment(juju, MYSQL_APP_NAME)