feat(coalesce): Add tests for coalesce

conftest.py

@@ -86,6 +86,13 @@ def pytest_addoption(parser):
              "4KiB blocksize to be formatted and used in storage tests. "
              "Set it to 'auto' to let the fixtures auto-detect available disks."
     )
+    parser.addoption(
+        "--image-format",
+        action="append",
+        default=[],
+        help="Format of VDI to execute tests on."
+        "Example: vhd,qcow2"
+    )
 
 def pytest_configure(config):
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
@@ -98,6 +105,12 @@ def pytest_generate_tests(metafunc):
             vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default
         metafunc.parametrize("vm_ref", vms, indirect=True, scope="module")
 
+    if "image_format" in metafunc.fixturenames:
+        image_format = metafunc.config.getoption("image_format")
+        if len(image_format) == 0:
+            image_format = ["vhd"] # Not giving image-format will default to doing tests on vhd
+        metafunc.parametrize("image_format", image_format, scope="session")
+
 def pytest_collection_modifyitems(items, config):
     # Automatically mark tests based on fixtures they require.
     # Check pytest.ini or pytest --markers for marker descriptions.
@@ -296,14 +309,21 @@ def host_no_ipv6(host):
     if is_ipv6(host.hostname_or_ip):
         pytest.skip(f"This test requires an IPv4 XCP-ng")
 
+@pytest.fixture(scope="session")
+def shared_sr(host):
+    sr = host.pool.first_shared_sr()
+    assert sr, "No shared SR available on hosts"
+    logging.info(">> Shared SR on host present: {} of type {}".format(sr.uuid, sr.get_type()))
+    yield sr
+
 @pytest.fixture(scope='session')
 def local_sr_on_hostA1(hostA1):
     """ A local SR on the pool's master. """
     srs = hostA1.local_vm_srs()
     assert len(srs) > 0, "a local SR is required on the pool's master"
     # use the first local SR found
     sr = srs[0]
-    logging.info(">> local SR on hostA1 present : %s" % sr.uuid)
+    logging.info(">> local SR on hostA1 present: {} of type {}".format(sr.uuid, sr.get_type()))
     yield sr
 
 @pytest.fixture(scope='session')
@@ -313,7 +333,7 @@ def local_sr_on_hostA2(hostA2):
     assert len(srs) > 0, "a local SR is required on the pool's second host"
     # use the first local SR found
     sr = srs[0]
-    logging.info(">> local SR on hostA2 present : %s" % sr.uuid)
+    logging.info(">> local SR on hostA2 present: {} of type {}".format(sr.uuid, sr.get_type()))
     yield sr
 
 @pytest.fixture(scope='session')
@@ -323,7 +343,7 @@ def local_sr_on_hostB1(hostB1):
     assert len(srs) > 0, "a local SR is required on the second pool's master"
     # use the first local SR found
     sr = srs[0]
-    logging.info(">> local SR on hostB1 present : %s" % sr.uuid)
+    logging.info(">> local SR on hostB1 present: {} of type {}".format(sr.uuid, sr.get_type()))
     yield sr
 
 @pytest.fixture(scope='session')

lib/basevm.py

@@ -1,13 +1,14 @@
 import logging
 
-from typing import Any, Literal, Optional, overload, TYPE_CHECKING
+from typing import Any, Literal, Optional, overload, TYPE_CHECKING, List
 
 import lib.commands as commands
 if TYPE_CHECKING:
     import lib.host
 
 from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set
 from lib.sr import SR
+from lib.vdi import VDI
 
 class BaseVM:
     """ Base class for VM and Snapshot. """
@@ -19,6 +20,15 @@ def __init__(self, uuid: str, host: 'lib.host.Host'):
         logging.info("New %s: %s", type(self).__name__, uuid)
         self.uuid = uuid
         self.host = host
+        try:
+            self.vdis = [VDI(vdi_uuid, host=host) for vdi_uuid in self.vdi_uuids()]
+        except commands.SSHCommandFailed as e:
+            # Doesn't work with Dom0 since `vm-disk-list` doesn't work on it so we create empty list
+            if e.stdout == "Error: No matching VMs found":
+                logging.info("Couldn't get disks list. We are Dom0. Continuing...")
+                self.vdis = []
+            else:
+                raise
 
     @overload
     def param_get(self, param_name: str, key: Optional[str] = ...,
@@ -56,11 +66,31 @@ def name(self) -> str:
         assert isinstance(n, str)
         return n
 
+    def connect_vdi(self, vdi: VDI, device: str = "autodetect") -> str:
+        logging.info(f">> Plugging VDI {vdi.uuid} on VM {self.uuid}")
+        vbd_uuid = self.host.xe("vbd-create",
+                                {
+                                    "vdi-uuid": vdi.uuid,
+                                    "vm-uuid": self.uuid,
+                                    "device": device,
+                                })
+        self.host.xe("vbd-plug", {"uuid": vbd_uuid})
+
+        self.vdis.append(vdi)
+
+        return vbd_uuid
+
+    def disconnect_vdi(self, vdi: VDI):
+        logging.info(f"<< Unplugging VDI {vdi.uuid} from VM {self.uuid}")
+        vbd_uuid = self.host.xe("vbd-list", {"vdi-uuid": vdi.uuid, "vm-uuid": self.uuid}, minimal=True)
+        self.host.xe("vbd-unplug", {"uuid": vbd_uuid})
+        self.host.xe("vbd-destroy", {"uuid": vbd_uuid})
+
     # @abstractmethod
     def _disk_list(self):
         raise NotImplementedError()
 
-    def vdi_uuids(self, sr_uuid=None):
+    def vdi_uuids(self, sr_uuid=None) -> List[str]:
         output = self._disk_list()
         if output == '':
             return []
@@ -78,6 +108,9 @@ def vdi_uuids(self, sr_uuid=None):
 
     def destroy_vdi(self, vdi_uuid: str) -> None:
         self.host.xe('vdi-destroy', {'uuid': vdi_uuid})
+        for vdi in self.vdis:
+            if vdi.uuid == vdi_uuid:
+                self.vdis.remove(vdi)
 
     def all_vdis_on_host(self, host):
         for vdi_uuid in self.vdi_uuids():

lib/host.py

@@ -6,6 +6,7 @@
 import subprocess
 import tempfile
 import uuid
+from typing import Optional
 
 from packaging import version
 from typing import Dict, List, Literal, Optional, overload, TYPE_CHECKING, Union
@@ -55,6 +56,7 @@ def __init__(self, pool: 'lib.pool.Pool', hostname_or_ip):
         self.uuid = self.inventory['INSTALLATION_UUID']
         self.xcp_version = version.parse(self.inventory['PRODUCT_VERSION'])
         self.xcp_version_short = f"{self.xcp_version.major}.{self.xcp_version.minor}"
+        self._dom0: Optional[VM] = None
 
     def __str__(self):
         return self.hostname_or_ip
@@ -699,3 +701,20 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def get_dom0_uuid(self):
+        return self.inventory["CONTROL_DOMAIN_UUID"]
+
+    def get_dom0_VM(self) -> VM:
+        if not self._dom0:
+            self._dom0 = VM(self.get_dom0_uuid(), self)
+        return self._dom0
+
+    def get_sr_from_vdi_uuid(self, vdi_uuid) -> Optional[SR]:
+        sr_uuid = self.xe("vdi-param-get",
+                          {"param-name": "sr-uuid",
+                           "uuid": vdi_uuid,
+                           })
+        if sr_uuid is None:
+            return None
+        return SR(sr_uuid, self.pool)

lib/pool.py

@@ -1,7 +1,7 @@
 import logging
 import os
 import traceback
-from typing import Any, Dict, Optional, cast
+from typing import Any, Dict, Optional
 
 from packaging import version
 

lib/sr.py

@@ -1,5 +1,6 @@
 import logging
 import time
+from typing import Optional
 
 import lib.commands as commands
 
@@ -12,6 +13,7 @@ def __init__(self, uuid, pool):
         self.pool = pool
         self._is_shared = None # cached value for is_shared()
         self._main_host = None # cached value for main_host()
+        self._type = None # cache value for get_type()
 
     def pbd_uuids(self):
         return safe_split(self.pool.master.xe('pbd-list', {'sr-uuid': self.uuid}, minimal=True))
@@ -153,13 +155,21 @@ def is_shared(self):
                                                             {'uuid': self.uuid, 'param-name': 'shared'}))
         return self._is_shared
 
-    def create_vdi(self, name_label, virtual_size=64):
+    def get_type(self) -> str:
+        if self._type is None:
+            self._type = self.pool.master.xe("sr-param-get", {"uuid": self.uuid, "param-name": "type"})
+        return self._type
+
+    def create_vdi(self, name_label: str, virtual_size: int = 64, image_format: Optional[str] = None) -> VDI:
         logging.info("Create VDI %r on SR %s", name_label, self.uuid)
-        vdi_uuid = self.pool.master.xe('vdi-create', {
+        args = {
             'name-label': prefix_object_name(name_label),
             'virtual-size': str(virtual_size),
-            'sr-uuid': self.uuid
-        })
+            'sr-uuid': self.uuid,
+        }
+        if image_format:
+            args["sm-config:image-format"] = image_format
+        vdi_uuid = self.pool.master.xe('vdi-create', args)
         return VDI(vdi_uuid, sr=self)
 
     def run_quicktest(self):

lib/vdi.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set, strtobool
 from typing import Literal, Optional, overload, TYPE_CHECKING
@@ -23,11 +24,7 @@ def __init__(self, uuid, *, host=None, sr=None):
         # TODO: use a different approach when migration is possible
         if sr is None:
             assert host
-            sr_uuid = host.pool.get_vdi_sr_uuid(uuid)
-            # avoid circular import
-            # FIXME should get it from Host instead
-            from lib.sr import SR
-            self.sr = SR(sr_uuid, host.pool)
+            self.sr = host.get_sr_from_vdi_uuid(self.uuid)
         else:
             self.sr = sr
 
@@ -48,6 +45,12 @@ def readonly(self) -> bool:
     def __str__(self):
         return f"VDI {self.uuid} on SR {self.sr.uuid}"
 
+    def get_parent(self) -> Optional[str]:
+        return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True)
+
+    def get_image_format(self) -> Optional[str]:
+        return self.param_get("sm-config", key="image-format", accept_unknown_key=True)
+
     @overload
     def param_get(self, param_name: str, key: Optional[str] = ...,
                   accept_unknown_key: Literal[False] = ...) -> str:

tests/storage/coalesce/conftest.py

@@ -0,0 +1,44 @@
+import pytest
+import logging
+
+MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB
+
+@pytest.fixture(scope="module")
+def vdi_on_local_sr(host, local_sr_on_hostA1, image_format):
+    sr = local_sr_on_hostA1
+    vdi = sr.create_vdi("testVDI", MAX_LENGTH, image_format=image_format)
+    logging.info(">> Created VDI {} of type {}".format(vdi.uuid, image_format))
+
+    yield vdi
+
+    logging.info("<< Destroying VDI {}".format(vdi.uuid))
+    vdi.destroy()
+
+@pytest.fixture(scope="module")
+def vdi_with_vbd_on_dom0(host, vdi_on_local_sr):
+    dom0 = host.get_dom0_VM()
+    vbd_uuid = dom0.connect_vdi(vdi_on_local_sr)
+
+    yield vdi_on_local_sr
+
+    dom0.disconnect_vdi(vdi_on_local_sr)
+
+@pytest.fixture(scope="class")
+def data_file_on_host(host):
+    filename = "/root/data.bin"
+    logging.info(f">> Creating data file {filename} on host")
+    size = 1 * 1024 * 1024 # 1MiB
+    assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size"
+
+    host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"])
+
+    yield filename
+
+    logging.info("<< Deleting data file")
+    host.ssh(["rm", filename])
+
+@pytest.fixture(scope="module")
+def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0):
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = vdi_with_vbd_on_dom0.uuid
+    yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}"

tests/storage/coalesce/test_coalesce.py

@@ -0,0 +1,50 @@
+import logging
+
+from .utils import wait_for_vdi_coalesce, copy_data_to_tapdev, snapshot_vdi, compare_data
+
+class Test:
+    def test_write_data(self, host, tapdev, data_file_on_host):
+        length = 1 * 1024 * 1024
+        offset = 0
+
+        logging.info("Copying data to tapdev")
+        copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+        assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+    def test_coalesce(self, host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+        vdi = vdi_with_vbd_on_dom0
+        vdi_uuid = vdi.uuid
+        length = 1 * 1024 * 1024
+        offset = 0
+
+        vdi_snap = snapshot_vdi(host, vdi_uuid)
+
+        logging.info("Copying data to tapdev")
+        copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+        logging.info("Removing VDI snapshot")
+        host.xe("vdi-destroy", {"uuid": vdi_snap})
+
+        wait_for_vdi_coalesce(vdi)
+
+        assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+    def test_clone_coalesce(self, host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+        vdi = vdi_with_vbd_on_dom0
+        vdi_uuid = vdi.uuid
+        length = 1 * 1024 * 1024
+        offset = 0
+
+        clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid})
+        logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}")
+
+        logging.info("Copying data to tapdev")
+        copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+        logging.info("Removing VDI clone")
+        host.xe("vdi-destroy", {"uuid": clone_uuid})
+
+        wait_for_vdi_coalesce(vdi)
+
+        assert compare_data(host, tapdev, data_file_on_host, offset, length)