From c7753f9d99e8e80fa35a5038cc93355fbc5a2187 Mon Sep 17 00:00:00 2001 From: Damien Thenot Date: Mon, 31 Mar 2025 11:33:02 +0200 Subject: [PATCH] feat(coalesce): Add tests for coalesce Add a vdi-type parameter to parametrize the vdi_type fixture, it default to vhd at the moment, will be used to run test on other vdi type, e.g. qcow2. The tests create a VDI, connect it to Dom0 then use the tapdev to write random data somewhere in it. It then compare the data of the VDI to the original data we have to see if it has changed. The test create snapshot/clone before deleting them and waiting for the coalesce to have happened by observing sm-config:vhd-parent before checking the integrity of the data. Signed-off-by: Damien Thenot --- conftest.py | 13 ++++ lib/host.py | 14 ++++ lib/vdi.py | 10 +-- tests/storage/coalesce/conftest.py | 62 +++++++++++++++++ tests/storage/coalesce/test_coalesce.py | 93 +++++++++++++++++++++++++ 5 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 tests/storage/coalesce/conftest.py create mode 100644 tests/storage/coalesce/test_coalesce.py diff --git a/conftest.py b/conftest.py index 67955a20..4d56b6bf 100644 --- a/conftest.py +++ b/conftest.py @@ -75,6 +75,13 @@ def pytest_addoption(parser): "4KiB blocksize to be formatted and used in storage tests. " "Set it to 'auto' to let the fixtures auto-detect available disks." ) + parser.addoption( + "--image-format", + action="append", + default=[], + help="Format of VDI to execute tests on." + "Example: vhd,qcow2" + ) def pytest_configure(config): global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner') @@ -87,6 +94,12 @@ def pytest_generate_tests(metafunc): vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default metafunc.parametrize("vm_ref", vms, indirect=True, scope="module") + if "image_format" in metafunc.fixturenames: + image_format = metafunc.config.getoption("image_format") + if len(image_format) == 0: + image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd + metafunc.parametrize("image_format", image_format, scope="session") + def pytest_collection_modifyitems(items, config): # Automatically mark tests based on fixtures they require. # Check pytest.ini or pytest --markers for marker descriptions. diff --git a/lib/host.py b/lib/host.py index 29ffd9b0..8def9526 100644 --- a/lib/host.py +++ b/lib/host.py @@ -6,6 +6,7 @@ import subprocess import tempfile import uuid +from typing import Optional from packaging import version from typing import Dict, List, Literal, Optional, overload, TYPE_CHECKING, Union @@ -658,3 +659,16 @@ def enable_hsts_header(self): def disable_hsts_header(self): self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf']) self.restart_toolstack(verify=True) + + def get_dom0_uuid(self): + output = self.ssh(["grep", "-e", "\"CONTROL_DOMAIN_UUID=\"", "/etc/xensource-inventory"]) + return output.split("=")[1].replace("'", "") + + def get_sr_from_vdi_uuid(self, vdi_uuid) -> Optional[SR]: + sr_uuid = self.xe("vdi-param-get", + {"param-name": "sr-uuid", + "uuid": vdi_uuid, + }) + if sr_uuid is None: + return None + return SR(sr_uuid, self.pool) diff --git a/lib/vdi.py b/lib/vdi.py index d260ebe0..47c123e6 100644 --- a/lib/vdi.py +++ b/lib/vdi.py @@ -1,4 +1,5 @@ import logging +from typing import Optional from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set, strtobool from typing import Literal, Optional, overload, TYPE_CHECKING @@ -23,11 +24,7 @@ def __init__(self, uuid, *, host=None, sr=None): # TODO: use a different approach when migration is possible if sr is None: assert host - sr_uuid = host.pool.get_vdi_sr_uuid(uuid) - # avoid circular import - # FIXME should get it from Host instead - from lib.sr import SR - self.sr = SR(sr_uuid, host.pool) + self.sr = host.get_sr_from_vdi_uuid(self.uuid) else: self.sr = sr @@ -48,6 +45,9 @@ def readonly(self) -> bool: def __str__(self): return f"VDI {self.uuid} on SR {self.sr.uuid}" + def get_parent(self) -> Optional[str]: + return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True) + @overload def param_get(self, param_name: str, key: Optional[str] = ..., accept_unknown_key: Literal[False] = ...) -> str: diff --git a/tests/storage/coalesce/conftest.py b/tests/storage/coalesce/conftest.py new file mode 100644 index 00000000..a0e13577 --- /dev/null +++ b/tests/storage/coalesce/conftest.py @@ -0,0 +1,62 @@ +import pytest +import logging + +from lib.vdi import VDI + +MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB + +@pytest.fixture(scope="module") +def vdi_on_local_sr(host, local_sr_on_hostA1, image_format): + sr_uuid = local_sr_on_hostA1.uuid + vdi_uuid = host.xe("vdi-create", + {"sr-uuid": sr_uuid, + "name-label": "testVDI", + "virtual-size": str(MAX_LENGTH), + "sm-config:type": image_format, + }) + logging.info(">> Created VDI {} of type {}".format(vdi_uuid, image_format)) + + vdi = VDI(vdi_uuid, host=host) + + yield vdi + + logging.info("<< Destroying VDI {}".format(vdi_uuid)) + host.xe("vdi-destroy", {"uuid": vdi_uuid}) + +@pytest.fixture(scope="module") +def vdi_with_vbd_on_dom0(host, vdi_on_local_sr): + vdi_uuid = vdi_on_local_sr.uuid + dom0_uuid = host.get_dom0_uuid() + logging.info(f">> Plugging VDI {vdi_uuid} on Dom0") + vbd_uuid = host.xe("vbd-create", + {"vdi-uuid": vdi_uuid, + "vm-uuid": dom0_uuid, + "device": "autodetect", + }) + host.xe("vbd-plug", {"uuid": vbd_uuid}) + + yield vdi_on_local_sr + + logging.info(f"<< Unplugging VDI {vdi_uuid} from Dom0") + host.xe("vbd-unplug", {"uuid": vbd_uuid}) + host.xe("vbd-destroy", {"uuid": vbd_uuid}) + +@pytest.fixture(scope="class") +def data_file_on_host(host): + filename = "/root/data.bin" + logging.info(f">> Creating data file {filename} on host") + size = 1 * 1024 * 1024 # 1MiB + assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size" + + host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"]) + + yield filename + + logging.info("<< Deleting data file") + host.ssh(["rm", filename]) + +@pytest.fixture(scope="module") +def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0): + sr_uuid = local_sr_on_hostA1.uuid + vdi_uuid = vdi_with_vbd_on_dom0.uuid + yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}" diff --git a/tests/storage/coalesce/test_coalesce.py b/tests/storage/coalesce/test_coalesce.py new file mode 100644 index 00000000..a4a4844e --- /dev/null +++ b/tests/storage/coalesce/test_coalesce.py @@ -0,0 +1,93 @@ +import logging +import time + +from lib.host import Host + +def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int): + """ + if offset == 0: + off = "0" + else: + off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3 + """ + bs = 1 + off = int(offset / bs) + count = length / bs + count += length % bs + count = int(count) + cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"] + host.ssh(cmd) + +def get_data(host: Host, file: str, offset: int, length: int, checksum: bool = False) -> str: + cmd = ["xxd", "-p", "-seek", str(offset), "-len", str(length), file] + if checksum: + cmd = cmd + ["|", "sha256sum"] + return host.ssh(cmd) + +def get_hashed_data(host: Host, file: str, offset: int, length: int): + return get_data(host, file, offset, length, True).split()[0] + +def snapshot_vdi(host: Host, vdi_uuid: str): + vdi_snap = host.xe("vdi-snapshot", {"uuid": vdi_uuid}) + logging.info(f"Snapshot VDI {vdi_uuid}: {vdi_snap}") + return vdi_snap + +def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool: + logging.info("Getting data from VDI and file") + vdi_checksum = get_hashed_data(host, tapdev, offset, length) + file_checksum = get_hashed_data(host, data_file, 0, length) + logging.info(f"VDI: {vdi_checksum}") + logging.info(f"FILE: {file_checksum}") + + return vdi_checksum == file_checksum + +def test_write_data(host, tapdev, data_file_on_host): + length = 1 * 1024 * 1024 + offset = 0 + + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + + assert compare_data(host, tapdev, data_file_on_host, offset, length) + +def test_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): + vdi = vdi_with_vbd_on_dom0 + vdi_uuid = vdi.uuid + length = 1 * 1024 * 1024 + offset = 0 + + vdi_snap = snapshot_vdi(host, vdi_uuid) + + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + + logging.info("Removing VDI snapshot") + host.xe("vdi-destroy", {"uuid": vdi_snap}) + + logging.info("Waiting for coalesce") + while vdi.get_parent() is not None: + time.sleep(1) + logging.info("Coalesce done") + + assert compare_data(host, tapdev, data_file_on_host, offset, length) + +def test_clone_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host): + vdi = vdi_with_vbd_on_dom0 + vdi_uuid = vdi.uuid + length = 1 * 1024 * 1024 + offset = 0 + + clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid}) + logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}") + + logging.info("Copying data to tapdev") + copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length) + + host.xe("vdi-destroy", {"uuid": clone_uuid}) + + logging.info("Waiting for coalesce") + while vdi.get_parent() is not None: + time.sleep(1) + logging.info("Coalesce done") + + assert compare_data(host, tapdev, data_file_on_host, offset, length)