feat(coalesce): Add tests for coalesce

conftest.py

@@ -75,6 +75,13 @@ def pytest_addoption(parser):
              "4KiB blocksize to be formatted and used in storage tests. "
              "Set it to 'auto' to let the fixtures auto-detect available disks."
     )
+    parser.addoption(
+        "--image-format",
+        action="append",
+        default=[],
+        help="Format of VDI to execute tests on."
+        "Example: vhd,qcow2"
+    )
 
 def pytest_configure(config):
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
@@ -87,6 +94,12 @@ def pytest_generate_tests(metafunc):
             vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default
         metafunc.parametrize("vm_ref", vms, indirect=True, scope="module")
 
+    if "image_format" in metafunc.fixturenames:
+        image_format = metafunc.config.getoption("image_format")
+        if len(image_format) == 0:
+            image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
+        metafunc.parametrize("image_format", image_format, scope="session")
+
 def pytest_collection_modifyitems(items, config):
     # Automatically mark tests based on fixtures they require.
     # Check pytest.ini or pytest --markers for marker descriptions.

lib/host.py

@@ -6,6 +6,7 @@
 import subprocess
 import tempfile
 import uuid
+from typing import Optional
 
 from packaging import version
 from typing import Dict, List, Literal, Optional, overload, TYPE_CHECKING, Union
@@ -658,3 +659,16 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def get_dom0_uuid(self):
+        output = self.ssh(["grep", "-e", "\"CONTROL_DOMAIN_UUID=\"", "/etc/xensource-inventory"])
+        return output.split("=")[1].replace("'", "")
+
+    def get_sr_from_vdi_uuid(self, vdi_uuid) -> Optional[SR]:
+        sr_uuid = self.xe("vdi-param-get",
+                          {"param-name": "sr-uuid",
+                           "uuid": vdi_uuid,
+                           })
+        if sr_uuid is None:
+            return None
+        return SR(sr_uuid, self.pool)

lib/vdi.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set, strtobool
 from typing import Literal, Optional, overload, TYPE_CHECKING
@@ -23,11 +24,7 @@ def __init__(self, uuid, *, host=None, sr=None):
         # TODO: use a different approach when migration is possible
         if sr is None:
             assert host
-            sr_uuid = host.pool.get_vdi_sr_uuid(uuid)
-            # avoid circular import
-            # FIXME should get it from Host instead
-            from lib.sr import SR
-            self.sr = SR(sr_uuid, host.pool)
+            self.sr = host.get_sr_from_vdi_uuid(self.uuid)
         else:
             self.sr = sr
 
@@ -48,6 +45,9 @@ def readonly(self) -> bool:
     def __str__(self):
         return f"VDI {self.uuid} on SR {self.sr.uuid}"
 
+    def get_parent(self) -> Optional[str]:
+        return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True)
+
     @overload
     def param_get(self, param_name: str, key: Optional[str] = ...,
                   accept_unknown_key: Literal[False] = ...) -> str:

tests/storage/coalesce/conftest.py

@@ -0,0 +1,62 @@
+import pytest
+import logging
+
+from lib.vdi import VDI
+
+MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB
+
+@pytest.fixture(scope="module")
+def vdi_on_local_sr(host, local_sr_on_hostA1, image_format):
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = host.xe("vdi-create",
+                       {"sr-uuid": sr_uuid,
+                        "name-label": "testVDI",
+                        "virtual-size": str(MAX_LENGTH),
+                        "sm-config:type": image_format,
+                        })
+    logging.info(">> Created VDI {} of type {}".format(vdi_uuid, image_format))
+
+    vdi = VDI(vdi_uuid, host=host)
+
+    yield vdi
+
+    logging.info("<< Destroying VDI {}".format(vdi_uuid))
+    host.xe("vdi-destroy", {"uuid": vdi_uuid})
+
+@pytest.fixture(scope="module")
+def vdi_with_vbd_on_dom0(host, vdi_on_local_sr):
+    vdi_uuid = vdi_on_local_sr.uuid
+    dom0_uuid = host.get_dom0_uuid()
+    logging.info(f">> Plugging VDI {vdi_uuid} on Dom0")
+    vbd_uuid = host.xe("vbd-create",
+                       {"vdi-uuid": vdi_uuid,
+                        "vm-uuid": dom0_uuid,
+                        "device": "autodetect",
+                        })
+    host.xe("vbd-plug", {"uuid": vbd_uuid})
+
+    yield vdi_on_local_sr
+
+    logging.info(f"<< Unplugging VDI {vdi_uuid} from Dom0")
+    host.xe("vbd-unplug", {"uuid": vbd_uuid})
+    host.xe("vbd-destroy", {"uuid": vbd_uuid})
+
+@pytest.fixture(scope="class")
+def data_file_on_host(host):
+    filename = "/root/data.bin"
+    logging.info(f">> Creating data file {filename} on host")
+    size = 1 * 1024 * 1024 # 1MiB
+    assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size"
+
+    host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"])
+
+    yield filename
+
+    logging.info("<< Deleting data file")
+    host.ssh(["rm", filename])
+
+@pytest.fixture(scope="module")
+def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0):
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = vdi_with_vbd_on_dom0.uuid
+    yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}"

tests/storage/coalesce/test_coalesce.py

@@ -0,0 +1,93 @@
+import logging
+import time
+
+from lib.host import Host
+
+def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int):
+    """
+    if offset == 0:
+        off = "0"
+    else:
+        off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3
+    """
+    bs = 1
+    off = int(offset / bs)
+    count = length / bs
+    count += length % bs
+    count = int(count)
+    cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"]
+    host.ssh(cmd)
+
+def get_data(host: Host, file: str, offset: int, length: int, checksum: bool = False) -> str:
+    cmd = ["xxd", "-p", "-seek", str(offset), "-len", str(length), file]
+    if checksum:
+        cmd = cmd + ["|", "sha256sum"]
+    return host.ssh(cmd)
+
+def get_hashed_data(host: Host, file: str, offset: int, length: int):
+    return get_data(host, file, offset, length, True).split()[0]
+
+def snapshot_vdi(host: Host, vdi_uuid: str):
+    vdi_snap = host.xe("vdi-snapshot", {"uuid": vdi_uuid})
+    logging.info(f"Snapshot VDI {vdi_uuid}: {vdi_snap}")
+    return vdi_snap
+
+def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool:
+    logging.info("Getting data from VDI and file")
+    vdi_checksum = get_hashed_data(host, tapdev, offset, length)
+    file_checksum = get_hashed_data(host, data_file, 0, length)
+    logging.info(f"VDI: {vdi_checksum}")
+    logging.info(f"FILE: {file_checksum}")
+
+    return vdi_checksum == file_checksum
+
+def test_write_data(host, tapdev, data_file_on_host):
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+def test_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+    vdi = vdi_with_vbd_on_dom0
+    vdi_uuid = vdi.uuid
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    vdi_snap = snapshot_vdi(host, vdi_uuid)
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    logging.info("Removing VDI snapshot")
+    host.xe("vdi-destroy", {"uuid": vdi_snap})
+
+    logging.info("Waiting for coalesce")
+    while vdi.get_parent() is not None:
+        time.sleep(1)
+    logging.info("Coalesce done")
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+def test_clone_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+    vdi = vdi_with_vbd_on_dom0
+    vdi_uuid = vdi.uuid
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid})
+    logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}")
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    host.xe("vdi-destroy", {"uuid": clone_uuid})
+
+    logging.info("Waiting for coalesce")
+    while vdi.get_parent() is not None:
+        time.sleep(1)
+    logging.info("Coalesce done")
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)