diff --git a/docs/source/adapters/site.rst b/docs/source/adapters/site.rst index ce9094a5..e9050241 100644 --- a/docs/source/adapters/site.rst +++ b/docs/source/adapters/site.rst @@ -579,15 +579,15 @@ Available machine type configuration options +----------------+-------------------------------------------------------------------------------+-----------------+ | args | Arguments for the containers that run in your pods. | **Required** | +----------------+-------------------------------------------------------------------------------+-----------------+ - | hpa | Set True\False to enable\disable kubernetes horizontal pod autoscaler feature.| **Required** | + | hpa | Set True\False to enable\disable kubernetes horizontal pod autoscaler feature.| **Optional** | +----------------+-------------------------------------------------------------------------------+-----------------+ - | min_replicas | Minimum number of pods to scale to. (Only required when hpa is set to True) | **Required** | + | min_replicas | Minimum number of pods to scale to. (Required when hpa is set to True) | **Optional** | +----------------+-------------------------------------------------------------------------------+-----------------+ - | max_replicas | Maximum number of pods to scale to. (Only required when hpa is set to True) | **Required** | + | max_replicas | Maximum number of pods to scale to. (Required when hpa is set to True) | **Optional** | +----------------+-------------------------------------------------------------------------------+-----------------+ - | cpu_utilization| Average Cpu utilization to maintain across pods of a deployment. | **Required** | + | cpu_utilization| Average Cpu utilization to maintain across pods of a deployment. | **Optional** | + + + + - | | (Only required when hpa is set to True) | | + | | (Required when hpa is set to True) | | +----------------+-------------------------------------------------------------------------------+-----------------+ .. content-tabs:: right-col diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 13718623..d873b8c9 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,4 +1,4 @@ -.. Created by changelog.py at 2022-09-16, command +.. Created by changelog.py at 2022-10-06, command '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.10/bin/changelog docs/source/changes compile --output=docs/source/changelog.rst' based on the format of 'https://keepachangelog.com/' @@ -6,7 +6,7 @@ CHANGELOG ######### -[Unreleased] - 2022-09-16 +[Unreleased] - 2022-10-06 ========================= Added diff --git a/setup.py b/setup.py index f4205726..537e6055 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def get_cryptography_version(): get_cryptography_version(), "CloudStackAIO>=0.0.8", "PyYAML", - "AsyncOpenStackClient", + "AsyncOpenStackClient>=0.9.0", "cobald>=0.12.3", "asyncssh", "aiotelegraf", diff --git a/tardis/adapters/sites/cloudstack.py b/tardis/adapters/sites/cloudstack.py index efa618f4..e2c2561b 100644 --- a/tardis/adapters/sites/cloudstack.py +++ b/tardis/adapters/sites/cloudstack.py @@ -2,14 +2,18 @@ from tardis.exceptions.tardisexceptions import TardisError from tardis.exceptions.tardisexceptions import TardisQuotaExceeded from tardis.exceptions.tardisexceptions import TardisResourceStatusUpdateFailed -from tardis.interfaces.siteadapter import ResourceStatus -from tardis.interfaces.siteadapter import SiteAdapter +from tardis.interfaces.siteadapter import ( + ResourceStatus, + SiteAdapter, + SiteAdapterBaseModel, +) from tardis.utilities.attributedict import AttributeDict from tardis.utilities.staticmapping import StaticMapping from aiohttp import ClientConnectionError from CloudStackAIO.CloudStack import CloudStack from CloudStackAIO.CloudStack import CloudStackClientException +from pydantic import AnyUrl from contextlib import contextmanager from datetime import datetime @@ -21,7 +25,19 @@ logger = logging.getLogger("cobald.runtime.tardis.adapters.sites.cloudstack") +class CloudStackAdapterConfigurationModel(SiteAdapterBaseModel): + """ + pydantic model for the input validation of the CloudStack site adapter configuration + """ + + end_point: AnyUrl + api_key: str + api_secret: str + + class CloudStackAdapter(SiteAdapter): + _configuration_validation_model = CloudStackAdapterConfigurationModel + def __init__(self, machine_type: str, site_name: str): self._machine_type = machine_type self._site_name = site_name diff --git a/tardis/adapters/sites/fakesite.py b/tardis/adapters/sites/fakesite.py index 656e3844..0b501f12 100644 --- a/tardis/adapters/sites/fakesite.py +++ b/tardis/adapters/sites/fakesite.py @@ -1,6 +1,6 @@ from ...exceptions.tardisexceptions import TardisError -from ...interfaces.siteadapter import ResourceStatus -from ...interfaces.siteadapter import SiteAdapter +from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel +from ...interfaces.simulator import Simulator from ...utilities.attributedict import AttributeDict from ...utilities.staticmapping import StaticMapping @@ -13,7 +13,18 @@ import asyncio +class FakeSiteAdapterConfigurationModel(SiteAdapterBaseModel): + """ + pydantic model for the input validation of the fake site adapter configuration + """ + + api_response_delay: Simulator + resource_boot_time: Simulator + + class FakeSiteAdapter(SiteAdapter): + _configuration_validation_model = FakeSiteAdapterConfigurationModel + def __init__(self, machine_type: str, site_name: str) -> None: self._machine_type = machine_type self._site_name = site_name diff --git a/tardis/adapters/sites/htcondor.py b/tardis/adapters/sites/htcondor.py index b0d610fb..0f02ce54 100644 --- a/tardis/adapters/sites/htcondor.py +++ b/tardis/adapters/sites/htcondor.py @@ -1,9 +1,7 @@ -from typing import Iterable, Tuple, Awaitable from ...exceptions.executorexceptions import CommandExecutionFailure from ...exceptions.tardisexceptions import TardisError from ...exceptions.tardisexceptions import TardisResourceStatusUpdateFailed -from ...interfaces.siteadapter import SiteAdapter -from ...interfaces.siteadapter import ResourceStatus +from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel from ...interfaces.executor import Executor from ...utilities.asynccachemap import AsyncCacheMap from ...utilities.attributedict import AttributeDict @@ -12,10 +10,13 @@ from ...utilities.asyncbulkcall import AsyncBulkCall from ...utilities.utils import csv_parser, machine_meta_data_translation +from pydantic import PositiveInt, PositiveFloat + from contextlib import contextmanager from datetime import datetime from functools import partial from string import Template +from typing import Iterable, Tuple, Awaitable, Optional import warnings import logging @@ -74,7 +75,7 @@ def _submit_description(resource_jdls: Tuple[JDL, ...]) -> str: FutureWarning, ) else: - commands.append("queue 1") + commands.append("queue 1\n") return "\n".join(commands) @@ -181,10 +182,22 @@ async def _condor_tool( } +class HTCondorAdapterConfigurationModel(SiteAdapterBaseModel): + """ + pydantic model for the input validation of the HTCondor site adapter configuration + """ + + executor: Optional[Executor] = ShellExecutor() + bulk_size: Optional[PositiveInt] = 100 + bulk_delay: Optional[PositiveFloat] = 1.0 + max_age: PositiveInt + + class HTCondorAdapter(SiteAdapter): htcondor_machine_meta_data_translation_mapping = AttributeDict( Cores=1, Memory=1024, Disk=1024 * 1024 ) + _configuration_validation_model = HTCondorAdapterConfigurationModel def __init__( self, @@ -193,9 +206,9 @@ def __init__( ): self._machine_type = machine_type self._site_name = site_name - self._executor = getattr(self.configuration, "executor", ShellExecutor()) - bulk_size = getattr(self.configuration, "bulk_size", 100) - bulk_delay = getattr(self.configuration, "bulk_delay", 1.0) + self._executor = self.configuration.executor + bulk_size = self.configuration.bulk_size + bulk_delay = self.configuration.bulk_delay self._condor_submit, self._condor_suspend, self._condor_rm = ( AsyncBulkCall( partial(tool, executor=self._executor), diff --git a/tardis/adapters/sites/kubernetes.py b/tardis/adapters/sites/kubernetes.py index 7ef09ce7..f92212de 100644 --- a/tardis/adapters/sites/kubernetes.py +++ b/tardis/adapters/sites/kubernetes.py @@ -1,22 +1,86 @@ from kubernetes_asyncio import client as k8s_client from kubernetes_asyncio.client.rest import ApiException as K8SApiException from ...exceptions.tardisexceptions import TardisError -from ...interfaces.siteadapter import SiteAdapter -from ...interfaces.siteadapter import ResourceStatus +from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel from ...utilities.attributedict import AttributeDict from ...utilities.staticmapping import StaticMapping from ...utilities.utils import convert_to +from pydantic import AnyUrl, BaseModel, conint, constr, PositiveInt, validator + from functools import partial from datetime import datetime from contextlib import contextmanager +from typing import Any, List, Optional import logging logger = logging.getLogger("cobald.runtime.tardis.adapters.sites.kubernetes") +class KubernetesMachineTypeConfigurationModel(BaseModel): + args: List[str] + cpu_utilization: Optional[conint(ge=0, le=100)] + hpa: bool = False + image: constr(regex=r"^\S+:\S+$") # noqa F722 + max_replicas: Optional[PositiveInt] + min_replicas: Optional[PositiveInt] + namespace: str = "default" + + +class KubernetesAdapterConfigurationModel(SiteAdapterBaseModel): + """ + pydantic model for the input validation of the Kubernetes site adapter configuration + """ + + host: AnyUrl + token: str + + @validator("MachineTypeConfiguration") + def validate_machine_type_configuration( + cls, # noqa B902 + machine_type_configurations: AttributeDict[ + str, Optional[AttributeDict[str, AttributeDict[str, Any]]] + ], + ): + validated_configurations = AttributeDict() + + for ( + machine_type, + machine_type_configuration, + ) in machine_type_configurations.items(): + validated_configuration = KubernetesMachineTypeConfigurationModel( + **machine_type_configuration + ) + hpa_required_attrs = ("cpu_utilization", "max_replicas", "min_replicas") + if validated_configuration.hpa: + if not all( # check all required attributes are present + getattr(validated_configuration, attr) + for attr in hpa_required_attrs + ): + raise ValueError( + f"You need to supply {hpa_required_attrs} in case you activate " + "the Kubernetes Horizontal Autopod Scaler (HPA)!" + ) + if ( # check consistency of min_replicas and max_replicas + validated_configuration.min_replicas + > validated_configuration.max_replicas + ): + raise ValueError( + "max_replicas have to be larger/equal as/to min_replicas!" + ) + setattr( + validated_configurations, + machine_type, + AttributeDict(validated_configuration), + ) + + return validated_configurations + + class KubernetesAdapter(SiteAdapter): + _configuration_validation_model = KubernetesAdapterConfigurationModel + def __init__(self, machine_type: str, site_name: str): self._machine_type = machine_type self._site_name = site_name diff --git a/tardis/adapters/sites/moab.py b/tardis/adapters/sites/moab.py index 5b757251..c6a4a291 100644 --- a/tardis/adapters/sites/moab.py +++ b/tardis/adapters/sites/moab.py @@ -2,8 +2,8 @@ from ...exceptions.tardisexceptions import TardisError from ...exceptions.tardisexceptions import TardisTimeout from ...exceptions.tardisexceptions import TardisResourceStatusUpdateFailed -from ...interfaces.siteadapter import ResourceStatus -from ...interfaces.siteadapter import SiteAdapter +from ...interfaces.executor import Executor +from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel from ...utilities.staticmapping import StaticMapping from ...utilities.attributedict import AttributeDict from ...utilities.attributedict import convert_to_attribute_dict @@ -11,10 +11,13 @@ from ...utilities.asynccachemap import AsyncCacheMap from ...utilities.utils import submit_cmd_option_formatter +from pydantic import PositiveInt + from asyncio import TimeoutError from contextlib import contextmanager from functools import partial from datetime import datetime +from typing import Optional import asyncssh import logging @@ -47,7 +50,19 @@ async def moab_status_updater(executor): return moab_resource_status +class MoabAdapterConfigurationModel(SiteAdapterBaseModel): + """ + pydantic model for the input validation of the Moab site adapter configuration + """ + + executor: Optional[Executor] = ShellExecutor() + StatusUpdate: PositiveInt + StartupCommand: Optional[str] + + class MoabAdapter(SiteAdapter): + _configuration_validation_model = MoabAdapterConfigurationModel + def __init__(self, machine_type: str, site_name: str): self._machine_type = machine_type self._site_name = site_name @@ -55,7 +70,7 @@ def __init__(self, machine_type: str, site_name: str): try: self._startup_command = self.machine_type_configuration.StartupCommand except AttributeError: - if not hasattr(self.configuration, "StartupCommand"): + if self.configuration.StartupCommand is None: raise warnings.warn( "StartupCommand has been moved to the machine_type_configuration!", @@ -63,7 +78,7 @@ def __init__(self, machine_type: str, site_name: str): ) self._startup_command = self.configuration.StartupCommand - self._executor = getattr(self.configuration, "executor", ShellExecutor()) + self._executor = self.configuration.executor self._moab_status = AsyncCacheMap( update_coroutine=partial(moab_status_updater, self._executor), diff --git a/tardis/adapters/sites/openstack.py b/tardis/adapters/sites/openstack.py index 389e0859..13382a01 100644 --- a/tardis/adapters/sites/openstack.py +++ b/tardis/adapters/sites/openstack.py @@ -4,14 +4,18 @@ from simple_rest_client.exceptions import ClientError from aiohttp import ClientConnectionError from aiohttp import ContentTypeError +from pydantic import AnyHttpUrl, root_validator from tardis.exceptions.tardisexceptions import TardisAuthError from tardis.exceptions.tardisexceptions import TardisDroneCrashed from tardis.exceptions.tardisexceptions import TardisError from tardis.exceptions.tardisexceptions import TardisTimeout from tardis.exceptions.tardisexceptions import TardisResourceStatusUpdateFailed -from tardis.interfaces.siteadapter import ResourceStatus -from tardis.interfaces.siteadapter import SiteAdapter +from tardis.interfaces.siteadapter import ( + ResourceStatus, + SiteAdapter, + SiteAdapterBaseModel, +) from tardis.utilities.attributedict import AttributeDict from tardis.utilities.staticmapping import StaticMapping @@ -19,13 +23,73 @@ from contextlib import contextmanager from datetime import datetime from functools import partial +from typing import Any, Dict, Optional import logging logger = logging.getLogger("cobald.runtime.tardis.adapters.sites.openstack") +class OpenStackAdapterConfigurationModel(SiteAdapterBaseModel): + """ + pydantic model for the input validation of the OpenStack site adapter configuration + """ + + auth_url: AnyHttpUrl + username: Optional[str] = None + password: Optional[str] = None + project_name: Optional[str] = None + user_domain_name: Optional[str] = None + project_domain_name: Optional[str] = None + application_credential_id: Optional[str] = None + application_credential_secret: Optional[str] = None + + class Config: + extra = "forbid" + + @root_validator + def validate_openstack_config( + cls, values: Dict[str, Any] # noqa B902 + ) -> Dict[str, Any]: + username = values.get("username") + password = values.get("password") + project_name = values.get("project_name") + user_domain_name = values.get("user_domain_name") + project_domain_name = values.get("project_domain_name") + application_credential_id = values.get("application_credential_id") + application_credential_secret = values.get("application_credential_secret") + + def check_option_group_exclusivity(option_group1, option_group2): + return ( + all(value is not None for value in option_group1) + and all(value is None for value in option_group2) + ) or ( + all(value is not None for value in option_group2) + and all(value is None for value in option_group1) + ) + + if not check_option_group_exclusivity( + option_group1=(application_credential_id, application_credential_secret), + option_group2=( + username, + password, + project_name, + user_domain_name, + project_domain_name, + ), + ): + raise ValueError( + "OpenStackAdapter exclusively requires either" + "(application_credential_id, application_credential_secret) or " + "(username, password, project_name, user_domain_name," + "project_domain_name) to be set." + ) + return values + + class OpenStackAdapter(SiteAdapter): + _configuration_validation_model = OpenStackAdapterConfigurationModel + def __init__(self, machine_type: str, site_name: str): self._machine_type = machine_type self._site_name = site_name @@ -33,10 +97,11 @@ def __init__(self, machine_type: str, site_name: str): auth = AuthPassword( auth_url=self.configuration.auth_url, username=self.configuration.username, - password=self.configuration.password, project_name=self.configuration.project_name, user_domain_name=self.configuration.user_domain_name, project_domain_name=self.configuration.project_domain_name, + application_credential_id=self.configuration.application_credential_id, + application_credential_secret=self.configuration.application_credential_secret, # noqa B950 ) self.nova = NovaClient(session=auth) diff --git a/tardis/adapters/sites/slurm.py b/tardis/adapters/sites/slurm.py index f5fd0b81..9ca9971e 100644 --- a/tardis/adapters/sites/slurm.py +++ b/tardis/adapters/sites/slurm.py @@ -2,8 +2,8 @@ from ...exceptions.tardisexceptions import TardisError from ...exceptions.tardisexceptions import TardisTimeout from ...exceptions.tardisexceptions import TardisResourceStatusUpdateFailed -from ...interfaces.siteadapter import ResourceStatus -from ...interfaces.siteadapter import SiteAdapter +from ...interfaces.executor import Executor +from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel from ...utilities.staticmapping import StaticMapping from ...utilities.attributedict import AttributeDict from ...utilities.attributedict import convert_to_attribute_dict @@ -11,10 +11,13 @@ from ...utilities.asynccachemap import AsyncCacheMap from ...utilities.utils import convert_to, csv_parser, submit_cmd_option_formatter +from pydantic import PositiveInt + from asyncio import TimeoutError from contextlib import contextmanager from functools import partial from datetime import datetime +from typing import Optional import logging import re @@ -45,7 +48,19 @@ async def slurm_status_updater(executor): return slurm_resource_status +class SlurmAdapterConfigurationModel(SiteAdapterBaseModel): + """ + pydantic model for the input validation of the Slurm site adapter configuration + """ + + executor: Optional[Executor] = ShellExecutor() + StatusUpdate: PositiveInt + StartupCommand: Optional[str] + + class SlurmAdapter(SiteAdapter): + _configuration_validation_model = SlurmAdapterConfigurationModel + def __init__(self, machine_type: str, site_name: str): self._machine_type = machine_type self._site_name = site_name @@ -53,7 +68,7 @@ def __init__(self, machine_type: str, site_name: str): try: self._startup_command = self.machine_type_configuration.StartupCommand except AttributeError: - if not hasattr(self.configuration, "StartupCommand"): + if self.configuration.StartupCommand is None: raise warnings.warn( "StartupCommand has been moved to the machine_type_configuration!", @@ -61,7 +76,7 @@ def __init__(self, machine_type: str, site_name: str): ) self._startup_command = self.configuration.StartupCommand - self._executor = getattr(self.configuration, "executor", ShellExecutor()) + self._executor = self.configuration.executor self._slurm_status = AsyncCacheMap( update_coroutine=partial(slurm_status_updater, self._executor), diff --git a/tardis/interfaces/siteadapter.py b/tardis/interfaces/siteadapter.py index 27fb0eba..11ec18a9 100644 --- a/tardis/interfaces/siteadapter.py +++ b/tardis/interfaces/siteadapter.py @@ -6,14 +6,76 @@ from cobald.utility.primitives import infinity as inf from enum import Enum from functools import lru_cache -from pydantic import BaseModel, conint, validator -from typing import Optional +from pydantic import BaseModel, conint, root_validator, validator +from typing import Any, Callable, Dict, List, Optional import logging logger = logging.getLogger("cobald.runtime.tardis.interfaces.site") +class SiteAdapterBaseModel(BaseModel): + """ + pydantic BaseModel for the input validation of site adapters + """ + + MachineTypes: List[str] + MachineTypeConfiguration: "AttributeDict[str, Optional[AttributeDict[str, Any]]]" + MachineMetaData: "AttributeDict[str, AttributeDict[str, Any]]" + # Use Any to avoid automated conversion to int or float here, validate later + + class Config: + arbitrary_types_allowed = True + + @root_validator(skip_on_failure=True, pre=True) # skip if previous validator failed + def validate(cls, values: Dict[str, Any]) -> Dict[str, Any]: # noqa B902 + """ + Validate that MachineTypeConfiguration and MachineMetaData is available + for each MachineType defined. + """ + if "MachineTypes" not in values: + raise ValueError( + "You have to add MachineTypes to the site configuration" + ) from None + + for machine_type in values["MachineTypes"]: + for config_block in ("MachineTypeConfiguration", "MachineMetaData"): + if machine_type not in values.get(config_block, {}): + raise ValueError( + f"You have to specify {config_block} for MachineType " + f"{machine_type}." + ) + return values + + @validator("MachineMetaData") + def validate_machine_meta_data( + cls, # noqa B902 + machine_meta_data: "AttributeDict[str, AttributeDict[str, Any]]", + ): + for machine_type, machine_meta_data_item in machine_meta_data.items(): + for entry, allowed_types in ( + ("Cores", (int,)), + ("Memory", (int, float)), + ("Disk", (int, float)), + ): + if entry not in machine_meta_data_item.keys(): + raise ValueError( + f"You have to supply the {entry} entry in the " + f"MachineMetaData for MachineType {machine_type}!" + ) from None + + # validate types here + if not isinstance(machine_meta_data_item[entry], allowed_types): + raise ValueError( + f"You supplied a wrong type " + f"{type(machine_meta_data_item[entry])} in the " + f"MachineMetaData for machine_type {machine_type} entry " + f"'{entry}: {machine_meta_data_item[entry]}'!\n" + f"The allowed types are {allowed_types}" + ) from None + return machine_meta_data + + class SiteConfigurationModel(BaseModel): """ pydantic BaseModel for the input validation of the generic site configuration @@ -54,13 +116,34 @@ class SiteAdapter(metaclass=ABCMeta): """ @property + @lru_cache(maxsize=16) def configuration(self) -> AttributeDict: """ - Property to provide access to SiteAdapter specific configuration. + Property to provide access to SiteAdapter specific configuration and + perform input validation. :return: returns the Site Adapter specific configuration :rtype: AttributeDict """ - return getattr(Configuration(), self.site_name) + configuration = getattr(Configuration(), self.site_name) + validated_configuration = self.configuration_validation_model(**configuration) + return AttributeDict(validated_configuration) + + @property + def configuration_validation_model(self) -> Callable: + """ + Property to access the configuration_validation model of a site adapter + implementation and ensuring that all sub-classes of the SiteAdapter have + a _configuration_validation_model class variable. + :return: The configuration validation model of a site adapter implementation. + :rtype: str + """ + try: + # noinspection PyUnresolvedReferences + return self._configuration_validation_model + except AttributeError as ae: + raise AttributeError( + f"Class {self.__class__.__name__} must have an '_configuration_validation_model' class variable" # noqa + ) from ae @abstractmethod async def deploy_resource( @@ -217,6 +300,20 @@ def machine_type_configuration(self) -> AttributeDict: """ return self.configuration.MachineTypeConfiguration[self.machine_type] + @classmethod + def refresh_configuration(cls) -> None: + """ + To increase performance, the configuration and site_configuration + properties are cached using a lru_cache. This helper class method clears + those caches in order to reload a potentially changed configuration. + :return: Returns None + :rtype None + """ + # noinspection PyUnresolvedReferences + cls.configuration.fget.cache_clear() + # noinspection PyUnresolvedReferences + cls.site_configuration.fget.cache_clear() + @abstractmethod async def resource_status( self, resource_attributes: AttributeDict diff --git a/tardis/utilities/attributedict.py b/tardis/utilities/attributedict.py index 755e7eac..408d5b18 100644 --- a/tardis/utilities/attributedict.py +++ b/tardis/utilities/attributedict.py @@ -1,3 +1,9 @@ +from typing import Dict, TypeVar + +K = TypeVar("K") +V = TypeVar("V") + + def convert_to_attribute_dict(obj): if isinstance(obj, dict): for key, value in obj.items(): @@ -9,7 +15,7 @@ def convert_to_attribute_dict(obj): return obj -class AttributeDict(dict): +class AttributeDict(Dict[K, V]): def __getattr__(self, item): try: return self[item] diff --git a/tests/adapters_t/sites_t/test_cloudstack.py b/tests/adapters_t/sites_t/test_cloudstack.py index 4ce2ef55..3e6d9984 100644 --- a/tests/adapters_t/sites_t/test_cloudstack.py +++ b/tests/adapters_t/sites_t/test_cloudstack.py @@ -11,6 +11,7 @@ from tests.utilities.utilities import run_async from aiohttp import ClientConnectionError +from pydantic.error_wrappers import ValidationError from unittest import TestCase from unittest.mock import patch @@ -34,23 +35,24 @@ def tearDownClass(cls): cls.mock_cloudstack_api_patcher.stop() def setUp(self): - config = self.mock_config.return_value - test_site_config = config.TestSite - test_site_config.end_point = "https://test.cloudstack.local/compute" - test_site_config.api_key = "1234567890abcdef" - test_site_config.api_secret = "fedcba0987654321" - test_site_config.MachineTypeConfiguration = AttributeDict( - test2large=AttributeDict( - templateid="1b0b9253-929e-4865-874b-7d2c3491987b", - serviceofferingid="74bfaf4e-7d67-4adf-9322-12b9a36e84f7", - zoneid="35eb7739-d19e-45f7-a581-4687c54d6d02", - keypair="MG", - rootdisksize=500, - ) - ) - - test_site_config.MachineMetaData = AttributeDict( - test2large=AttributeDict(Cores=128) + self.config = self.mock_config.return_value + self.config.TestSite = AttributeDict( + end_point="https://test.cloudstack.local/compute", + api_key="1234567890abcdef", + api_secret="fedcba0987654321", + MachineTypes=["test2large"], + MachineTypeConfiguration=AttributeDict( + test2large=AttributeDict( + templateid="1b0b9253-929e-4865-874b-7d2c3491987b", + serviceofferingid="74bfaf4e-7d67-4adf-9322-12b9a36e84f7", + zoneid="35eb7739-d19e-45f7-a581-4687c54d6d02", + keypair="MG", + rootdisksize=500, + ) + ), + MachineMetaData=AttributeDict( + test2large=AttributeDict(Cores=128, Memory=256, Disk=1000) + ), ) cloudstack_api = self.mock_cloudstack_api.return_value @@ -84,6 +86,13 @@ def setUp(self): def tearDown(self): self.mock_cloudstack_api.reset_mock() + def test_configuration_validation(self): + self.config.TestSite.end_point = "NotAValidURl" + + with self.assertRaises(ValidationError): + # noinspection PyStatementEffect + CloudStackAdapter(machine_type="test2large", site_name="TestSite") + def test_deploy_resource(self): self.assertEqual( run_async( @@ -108,7 +117,8 @@ def test_deploy_resource(self): def test_machine_meta_data(self): self.assertEqual( - self.cloudstack_adapter.machine_meta_data, AttributeDict(Cores=128) + self.cloudstack_adapter.machine_meta_data, + AttributeDict(Cores=128, Memory=256, Disk=1000), ) def test_machine_type(self): diff --git a/tests/adapters_t/sites_t/test_fakesite.py b/tests/adapters_t/sites_t/test_fakesite.py index a77f5ff3..ec67fb85 100644 --- a/tests/adapters_t/sites_t/test_fakesite.py +++ b/tests/adapters_t/sites_t/test_fakesite.py @@ -3,7 +3,9 @@ from tardis.interfaces.siteadapter import ResourceStatus from tardis.utilities.attributedict import AttributeDict -from ...utilities.utilities import run_async +from ...utilities.utilities import MockedSimulator, run_async + +from pydantic.error_wrappers import ValidationError from datetime import datetime from datetime import timedelta @@ -24,23 +26,34 @@ def tearDownClass(cls): cls.mock_config_patcher.stop() def setUp(self): - config = self.mock_config.return_value - test_site_config = config.TestSite - test_site_config.MachineMetaData = self.machine_meta_data - test_site_config.MachineTypeConfiguration = self.machine_type_configuration - test_site_config.api_response_delay = AttributeDict(get_value=lambda: 0) - test_site_config.resource_boot_time = AttributeDict(get_value=lambda: 100) + self.config = self.mock_config.return_value + self.config.TestSite = AttributeDict( + MachineMetaData=self.machine_meta_data, + MachineTypes=["test2large"], + MachineTypeConfiguration=self.machine_type_configuration, + api_response_delay=MockedSimulator(0), + resource_boot_time=MockedSimulator(100), + ) self.adapter = FakeSiteAdapter(machine_type="test2large", site_name="TestSite") @property def machine_meta_data(self): - return AttributeDict(test2large=AttributeDict(Cores=8, Memory=32)) + return AttributeDict(test2large=AttributeDict(Cores=8, Memory=32, Disk=1000)) @property def machine_type_configuration(self): return AttributeDict(test2large=AttributeDict(jdl="submit.jdl")) + def test_configuration_validation(self): + for variable in ("api_response_delay", "api_response_delay"): + old_value = getattr(self.config.TestSite, variable) + setattr(self.config.TestSite, variable, "DoesNotWork") + with self.assertRaises(ValidationError): + # noinspection PyStatementEffect + FakeSiteAdapter(machine_type="test2large", site_name="TestSite") + setattr(self.config.TestSite, variable, old_value) + def test_deploy_resource(self): response = run_async(self.adapter.deploy_resource, AttributeDict()) self.assertEqual(response.resource_status, ResourceStatus.Booting) diff --git a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py index 2c80c242..2d72fbd8 100644 --- a/tests/adapters_t/sites_t/test_htcondorsiteadapter.py +++ b/tests/adapters_t/sites_t/test_htcondorsiteadapter.py @@ -4,8 +4,9 @@ from tardis.exceptions.tardisexceptions import TardisResourceStatusUpdateFailed from tardis.interfaces.siteadapter import ResourceStatus from tardis.utilities.attributedict import AttributeDict -from ...utilities.utilities import mock_executor_run_command -from ...utilities.utilities import run_async +from ...utilities.utilities import mock_executor_run_command, run_async + +from pydantic.error_wrappers import ValidationError from datetime import datetime from datetime import timedelta @@ -69,7 +70,8 @@ request_memory=32768 request_disk=167772160 -queue 1""" # noqa: B950 +queue 1 +""" # noqa: B950 CONDOR_SUBMIT_PER_ARGUMENTS_JDL_CONDOR_OBS = """executable = start_pilot.sh arguments=--cores=8 --memory=32768 --disk=167772160 --uuid=test-123 @@ -84,7 +86,8 @@ request_memory=32768 request_disk=167772160 -queue 1""" # noqa: B950 +queue 1 +""" # noqa: B950 CONDOR_SUBMIT_JDL_SPARK_OBS = """executable = start_pilot.sh transfer_input_files = setup_pilot.sh @@ -100,7 +103,8 @@ request_memory=32768 request_disk=167772160 -queue 1""" # noqa: B950 +queue 1 +""" # noqa: B950 class TestHTCondorSiteAdapter(TestCase): @@ -112,7 +116,7 @@ def setUpClass(cls): cls.mock_config_patcher = patch("tardis.interfaces.siteadapter.Configuration") cls.mock_config = cls.mock_config_patcher.start() cls.mock_executor_patcher = patch( - "tardis.adapters.sites.htcondor.ShellExecutor" + "tardis.adapters.sites.htcondor.ShellExecutor", autospec=True ) cls.mock_executor = cls.mock_executor_patcher.start() @@ -122,14 +126,16 @@ def tearDownClass(cls): cls.mock_executor_patcher.stop() def setUp(self): - config = self.mock_config.return_value - test_site_config = config.TestSite - test_site_config.MachineMetaData = self.machine_meta_data - test_site_config.MachineTypeConfiguration = self.machine_type_configuration - test_site_config.executor = self.mock_executor.return_value - test_site_config.bulk_size = 100 - test_site_config.bulk_delay = 0.1 - test_site_config.max_age = 10 + self.config = self.mock_config.return_value + self.config.TestSite = AttributeDict( + MachineTypes=["test2large"], + MachineMetaData=self.machine_meta_data, + MachineTypeConfiguration=self.machine_type_configuration, + executor=self.mock_executor.return_value, + bulk_size=100, + bulk_delay=0.1, + max_age=10, + ) self.adapter = HTCondorAdapter(machine_type="test2large", site_name="TestSite") @@ -140,6 +146,7 @@ def machine_meta_data(self): test2large_args=AttributeDict(Cores=8, Memory=32, Disk=160), test2large_deprecated=AttributeDict(Cores=8, Memory=32, Disk=160), testunkownresource=AttributeDict(Cores=8, Memory=32, Disk=160, Foo=3), + testdeprecated=AttributeDict(Cores=8, Memory=32, Disk=160), ) @property @@ -149,8 +156,23 @@ def machine_type_configuration(self): test2large_args=AttributeDict(jdl="tests/data/submit_per_arguments.jdl"), test2large_deprecated=AttributeDict(jdl="tests/data/submit_deprecated.jdl"), testunkownresource=AttributeDict(jdl="tests/data/submit.jdl"), + testdeprecated=AttributeDict(jdl="tests/data/submit_deprecated.jdl"), ) + def test_configuration_validation(self): + for variable, new_value in ( + ("executor", "DoesNotWork"), + ("bulk_size", -1), + ("bulk_delay", -1), + ("max_age", -1), + ): + old_value = getattr(self.config.TestSite, variable) + setattr(self.config.TestSite, variable, new_value) + with self.assertRaises(ValidationError): + # noinspection PyStatementEffect + HTCondorAdapter(machine_type="test2large", site_name="TestSite") + setattr(self.config.TestSite, variable, old_value) + @mock_executor_run_command(stdout=CONDOR_SUBMIT_OUTPUT) def test_deploy_resource_htcondor_obs(self): response = run_async( @@ -164,6 +186,7 @@ def test_deploy_resource_htcondor_obs(self): ), ), ) + self.assertEqual(response.remote_resource_uuid, "1351043.0") self.assertFalse(response.created - datetime.now() > timedelta(seconds=1)) self.assertFalse(response.updated - datetime.now() > timedelta(seconds=1)) @@ -173,7 +196,7 @@ def test_deploy_resource_htcondor_obs(self): kwargs["stdin_input"], CONDOR_SUBMIT_JDL_CONDOR_OBS, ) - self.mock_executor.reset() + self.mock_executor.reset_mock() run_async( self.adapter.deploy_resource, @@ -192,7 +215,9 @@ def test_deploy_resource_htcondor_obs(self): kwargs["stdin_input"], CONDOR_SUBMIT_JDL_SPARK_OBS, ) - self.mock_executor.reset() + self.mock_executor.reset_mock() + + adapter = HTCondorAdapter(machine_type="testdeprecated", site_name="TestSite") self.adapter = HTCondorAdapter( machine_type="test2large_deprecated", site_name="TestSite" @@ -201,15 +226,22 @@ def test_deploy_resource_htcondor_obs(self): # "queue 1" deprecation with self.assertWarns(FutureWarning): run_async( - self.adapter.deploy_resource, + adapter.deploy_resource, AttributeDict( drone_uuid="test-123", obs_machine_meta_data_translation_mapping=AttributeDict( - Cores=1, Memory=1, Disk=1 + Cores=1, + Memory=1024, + Disk=1024 * 1024, ), ), ) - self.mock_executor.reset() + _, kwargs = self.mock_executor.return_value.run_command.call_args + self.assertEqual( + kwargs["stdin_input"], + CONDOR_SUBMIT_JDL_CONDOR_OBS, + ) + self.mock_executor.reset_mock() self.adapter = HTCondorAdapter( machine_type="test2large_args", site_name="TestSite" @@ -232,7 +264,7 @@ def test_deploy_resource_htcondor_obs(self): kwargs["stdin_input"], CONDOR_SUBMIT_PER_ARGUMENTS_JDL_CONDOR_OBS, ) - self.mock_executor.reset() + self.mock_executor.reset_mock() def test_translate_resources_raises_logs(self): self.adapter = HTCondorAdapter( diff --git a/tests/adapters_t/sites_t/test_kubernetes.py b/tests/adapters_t/sites_t/test_kubernetes.py index 223a7d6c..f3175cbd 100644 --- a/tests/adapters_t/sites_t/test_kubernetes.py +++ b/tests/adapters_t/sites_t/test_kubernetes.py @@ -11,6 +11,7 @@ import logging from kubernetes_asyncio import client +from pydantic import ValidationError class TestKubernetesStackAdapter(TestCase): @@ -38,25 +39,27 @@ def tearDownClass(cls): cls.mock_kubernetes_hpa_patcher.stop() def setUp(self): - config = self.mock_config.return_value - test_site_config = config.TestSite - # Endpoint of Kube cluster - test_site_config.host = "https://127.0.0.1:443" - # Barer token we are going to use to authenticate - test_site_config.token = "31ada4fd-adec-460c-809a-9e56ceb75269" - test_site_config.MachineTypeConfiguration = AttributeDict( - test2large=AttributeDict( - namespace="default", - image="busybox:1.26.1", - args=["sleep", "3600"], - hpa="True", - min_replicas="1", - max_replicas="2", - cpu_utilization="50", - ) - ) - test_site_config.MachineMetaData = AttributeDict( - test2large=AttributeDict(Cores=2, Memory=4) + self.config = self.mock_config.return_value + self.config.TestSite = AttributeDict( + # Endpoint of Kube cluster + host="https://127.0.0.1:443", + # Barer token we are going to use to authenticate + token="31ada4fd-adec-460c-809a-9e56ceb75269", + MachineTypes=["test2large"], + MachineTypeConfiguration=AttributeDict( + test2large=AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization=50, + ) + ), + MachineMetaData=AttributeDict( + test2large=AttributeDict(Cores=2, Memory=4, Disk=1000) + ), ) kubernetes_api = self.mock_kubernetes_api.return_value kubernetes_hpa = self.mock_kubernetes_hpa.return_value @@ -71,13 +74,15 @@ def setUp(self): name="testsite-089123", resources=client.V1ResourceRequirements( requests={ - "cpu": test_site_config.MachineMetaData.test2large.Cores, - "memory": test_site_config.MachineMetaData.test2large.Memory * 1e9, + "cpu": self.config.TestSite.MachineMetaData.test2large.Cores, + "memory": self.config.TestSite.MachineMetaData.test2large.Memory + * 1e9, } ), env=[ client.V1EnvVar(name="TardisDroneCores", value="2"), client.V1EnvVar(name="TardisDroneMemory", value="4096"), + client.V1EnvVar(name="TardisDroneDisk", value="1048576000"), client.V1EnvVar(name="TardisDroneUuid", value="testsite-089123"), ], ) @@ -152,6 +157,255 @@ def update_delete_hpa_side_effect(self, exception): def tearDown(self): self.mock_kubernetes_api.reset_mock() + def test_adapter_configuration_validation(self): + old_value = self.config.TestSite.host + self.config.TestSite.host = "NotAnUrl" + + with self.assertRaises(ValidationError): + k8s_adapter = KubernetesAdapter( + machine_type="test2large", site_name="TestSite" + ) + # noinspection PyStatementEffect + k8s_adapter.configuration # config needs to be accessed to run validation + + self.config.TestSite.host = old_value + + self.config.TestSite.token = ValueError + + with self.assertRaises(ValidationError): + k8s_adapter = KubernetesAdapter( + machine_type="test2large", site_name="TestSite" + ) + # noinspection PyStatementEffect + k8s_adapter.configuration # config needs to be accessed to run validation + + def test_machine_type_configuration_validation(self): + def run_validation_test(configuration, expected_outcome): + self.config.TestSite.MachineTypeConfiguration.test2large = configuration + k8s_adapter = KubernetesAdapter( + machine_type="test2large", site_name="TestSite" + ) + try: + issubclass(expected_outcome, Exception) + except TypeError: + self.assertEqual( + expected_outcome, k8s_adapter.machine_type_configuration + ) + else: + with self.assertRaises(expected_outcome): + # config needs to be accessed to run validation + # noinspection PyStatementEffect + k8s_adapter.machine_type_configuration + + run_validation_test( + self.config.TestSite.MachineTypeConfiguration.test2large, + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization=50, + ), + ) + + run_validation_test( + AttributeDict( + namespace="tardis", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=False, + ), + AttributeDict( + namespace="tardis", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=False, + min_replicas=None, + max_replicas=None, + cpu_utilization=None, + ), + ) + + run_validation_test( + AttributeDict( + namespace="default", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="InvalidImage", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args="Invalid", + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + max_replicas=2, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas="invalid", + max_replicas=2, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=-1, + max_replicas=2, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas="invalid", + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=-1, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=2, + max_replicas=1, + cpu_utilization=50, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization="invalid", + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization=-1, + ), + ValidationError, + ) + + run_validation_test( + AttributeDict( + namespace="default", + image="busybox:1.26.1", + args=["sleep", "3600"], + hpa=True, + min_replicas=1, + max_replicas=2, + cpu_utilization=999, + ), + ValidationError, + ) + @patch("kubernetes_asyncio.client.rest.aiohttp") def test_deploy_resource(self, mocked_aiohttp): self.assertEqual( @@ -178,7 +432,7 @@ def test_deploy_resource(self, mocked_aiohttp): def test_machine_meta_data(self): self.assertEqual( self.kubernetes_adapter.machine_meta_data, - AttributeDict(Cores=2, Memory=4), + AttributeDict(Cores=2, Memory=4, Disk=1000), ) def test_machine_type(self): diff --git a/tests/adapters_t/sites_t/test_moab.py b/tests/adapters_t/sites_t/test_moab.py index 417b6d25..d7726bbd 100644 --- a/tests/adapters_t/sites_t/test_moab.py +++ b/tests/adapters_t/sites_t/test_moab.py @@ -9,7 +9,7 @@ from tests.utilities.utilities import run_async from unittest import TestCase -from unittest.mock import MagicMock, patch +from unittest.mock import patch from datetime import datetime, timedelta from warnings import filterwarnings @@ -18,7 +18,6 @@ import asyncssh import logging -__all__ = ["TestMoabAdapter"] TEST_RESOURCE_STATUS_RESPONSE = """ @@ -144,7 +143,9 @@ class TestMoabAdapter(TestCase): def setUpClass(cls): cls.mock_config_patcher = patch("tardis.interfaces.siteadapter.Configuration") cls.mock_config = cls.mock_config_patcher.start() - cls.mock_executor_patcher = patch("tardis.adapters.sites.moab.ShellExecutor") + cls.mock_executor_patcher = patch( + "tardis.adapters.sites.moab.ShellExecutor", autospec=True + ) cls.mock_executor = cls.mock_executor_patcher.start() @classmethod @@ -153,20 +154,14 @@ def tearDownClass(cls): cls.mock_executor_patcher.stop() def setUp(self): - config = self.mock_config.return_value - config.TestSite = MagicMock( - spec=[ - "MachineMetaData", - "StatusUpdate", - "MachineTypeConfiguration", - "executor", - ] + self.config = self.mock_config.return_value + self.config.TestSite = AttributeDict( + MachineTypes=["test2large"], + MachineMetaData=self.machine_meta_data, + StatusUpdate=10, + MachineTypeConfiguration=self.machine_type_configuration, + executor=self.mock_executor.return_value, ) - self.test_site_config = config.TestSite - self.test_site_config.MachineMetaData = self.machine_meta_data - self.test_site_config.StatusUpdate = 10 - self.test_site_config.MachineTypeConfiguration = self.machine_type_configuration - self.test_site_config.executor = self.mock_executor.return_value self.moab_adapter = MoabAdapter(machine_type="test2large", site_name="TestSite") @@ -175,7 +170,7 @@ def tearDown(self): @property def machine_meta_data(self): - return AttributeDict(test2large=AttributeDict(Cores=128, Memory="120")) + return AttributeDict(test2large=AttributeDict(Cores=128, Memory=120, Disk=1000)) @property def machine_type_configuration(self): @@ -204,16 +199,20 @@ def resource_attributes(self): def test_start_up_command_deprecation_warning(self): # Necessary to avoid annoying message in PyCharm filterwarnings(action="ignore", message="unclosed", category=ResourceWarning) - del self.test_site_config.MachineTypeConfiguration.test2large.StartupCommand + del self.config.TestSite.MachineTypeConfiguration.test2large.StartupCommand with self.assertRaises(AttributeError): + from tardis.adapters.sites.moab import MoabAdapter + self.moab_adapter = MoabAdapter( machine_type="test2large", site_name="TestSite" ) - self.test_site_config.StartupCommand = "startVM.py" + self.config.TestSite.StartupCommand = "startVM.py" with self.assertWarns(DeprecationWarning): + from tardis.adapters.sites.moab import MoabAdapter + self.moab_adapter = MoabAdapter( machine_type="test2large", site_name="TestSite" ) @@ -250,7 +249,7 @@ def test_deploy_resource(self): @mock_executor_run_command(TEST_DEPLOY_RESOURCE_RESPONSE) def test_deploy_resource_w_submit_options(self): - self.test_site_config.MachineTypeConfiguration.test2large.SubmitOptions = ( + self.config.TestSite.MachineTypeConfiguration.test2large.SubmitOptions = ( AttributeDict( short=AttributeDict(M="someone@somewhere.com"), long=AttributeDict(timeout=60), diff --git a/tests/adapters_t/sites_t/test_openstack.py b/tests/adapters_t/sites_t/test_openstack.py index adf8b4ec..9c207f00 100644 --- a/tests/adapters_t/sites_t/test_openstack.py +++ b/tests/adapters_t/sites_t/test_openstack.py @@ -11,6 +11,7 @@ from aiohttp import ClientConnectionError from aiohttp import ContentTypeError +from pydantic.error_wrappers import ValidationError from simple_rest_client.exceptions import AuthError from simple_rest_client.exceptions import ClientError @@ -40,19 +41,23 @@ def tearDownClass(cls): cls.mock_openstack_api_patcher.stop() def setUp(self): - config = self.mock_config.return_value - test_site_config = config.TestSite - test_site_config.auth_url = "https://test.nova.client.local" - test_site_config.username = "TestUser" - test_site_config.password = "test123" - test_site_config.project_name = "TestProject" - test_site_config.user_domain_name = "TestDomain" - test_site_config.project_domain_name = "TestProjectDomain" - test_site_config.MachineTypeConfiguration = AttributeDict( - test2large=AttributeDict(imageRef="bc613271-6a54-48ca-9222-47e009dc0c29") - ) - test_site_config.MachineMetaData = AttributeDict( - test2large=AttributeDict(Cores=128) + self.config = self.mock_config.return_value + self.config.TestSite = AttributeDict( + auth_url="https://test.nova.client.local", + username="TestUser", + password="test123", + project_name="TestProject", + user_domain_name="TestDomain", + project_domain_name="TestProjectDomain", + MachineTypes=["test2large"], + MachineTypeConfiguration=AttributeDict( + test2large=AttributeDict( + imageRef="bc613271-6a54-48ca-9222-47e009dc0c29" + ) + ), + MachineMetaData=AttributeDict( + test2large=AttributeDict(Cores=128, Memory=256, Disk=1000) + ), ) openstack_api = self.mock_openstack_api.return_value @@ -90,6 +95,40 @@ def setUp(self): def tearDown(self): self.mock_openstack_api.reset_mock() + def test_configuration_validation(self): + self.config.TestSite.auth_url = "NotAnUrl" + + with self.assertRaises(ValidationError): + # noinspection PyStatementEffect + OpenStackAdapter(machine_type="test2large", site_name="TestSite") + + self.config.TestSite.auth_url = "https://test.nova.client.local" + + self.config.TestSite.application_credential_id = "test123" + self.config.TestSite.application_credential_secret = "secret123" + with self.assertRaises(ValidationError) as ve: + # noinspection PyStatementEffect + OpenStackAdapter(machine_type="test2large", site_name="TestSite") + self.assertEqual( + "OpenStackAdapter exclusively requires either" + "(application_credential_id, application_credential_secret) or " + "(username, password, project_name, user_domain_name," + "project_domain_name) to be set.", + ve.exception.errors()[0]["msg"], + ) + + for variable in ( + "username", + "password", + "project_name", + "user_domain_name", + "project_domain_name", + ): + delattr(self.config.TestSite, variable) + + # noinspection PyStatementEffect + OpenStackAdapter(machine_type="test2large", site_name="TestSite") + def test_deploy_resource(self): self.assertEqual( run_async( @@ -110,7 +149,8 @@ def test_deploy_resource(self): def test_machine_meta_data(self): self.assertEqual( - self.openstack_adapter.machine_meta_data, AttributeDict(Cores=128) + self.openstack_adapter.machine_meta_data, + AttributeDict(Cores=128, Memory=256, Disk=1000), ) def test_machine_type(self): diff --git a/tests/adapters_t/sites_t/test_slurm.py b/tests/adapters_t/sites_t/test_slurm.py index a372abb0..c140ad25 100644 --- a/tests/adapters_t/sites_t/test_slurm.py +++ b/tests/adapters_t/sites_t/test_slurm.py @@ -5,11 +5,15 @@ from tardis.exceptions.executorexceptions import CommandExecutionFailure from tardis.interfaces.siteadapter import ResourceStatus from tardis.utilities.attributedict import AttributeDict -from ...utilities.utilities import mock_executor_run_command -from ...utilities.utilities import run_async +from ...utilities.utilities import ( + assert_awaited_once, + assert_awaited_with, + mock_executor_run_command, + run_async, +) from unittest import TestCase -from unittest.mock import MagicMock, patch +from unittest.mock import patch from datetime import datetime, timedelta from warnings import filterwarnings @@ -17,7 +21,6 @@ import asyncio import logging -__all__ = ["TestSlurmAdapter"] TEST_RESOURCE_STATUS_RESPONSE = """ 1390065||PENDING @@ -81,7 +84,9 @@ def check_attribute_dicts( def setUpClass(cls): cls.mock_config_patcher = patch("tardis.interfaces.siteadapter.Configuration") cls.mock_config = cls.mock_config_patcher.start() - cls.mock_executor_patcher = patch("tardis.adapters.sites.slurm.ShellExecutor") + cls.mock_executor_patcher = patch( + "tardis.adapters.sites.slurm.ShellExecutor", autospec=True + ) cls.mock_executor = cls.mock_executor_patcher.start() @classmethod @@ -90,25 +95,21 @@ def tearDownClass(cls): cls.mock_executor_patcher.stop() def setUp(self): - config = self.mock_config.return_value - config.TestSite = MagicMock( - spec=[ - "MachineMetaData", - "StatusUpdate", - "MachineTypeConfiguration", - "executor", - ] - ) - self.test_site_config = config.TestSite - self.test_site_config.MachineMetaData = self.machine_meta_data - self.test_site_config.StatusUpdate = 10 - self.test_site_config.MachineTypeConfiguration = self.machine_type_configuration - self.test_site_config.executor = self.mock_executor.return_value + self.config = self.mock_config.return_value + self.config.TestSite = AttributeDict( + MachineTypes=["test2large"], + MachineMetaData=self.machine_meta_data, + StatusUpdate=10, + MachineTypeConfiguration=self.machine_type_configuration, + executor=self.mock_executor.return_value, + ) self.slurm_adapter = SlurmAdapter( machine_type="test2large", site_name="TestSite" ) + self.mock_executor.reset_mock() + def tearDown(self): pass @@ -137,14 +138,14 @@ def resource_attributes(self): def test_start_up_command_deprecation_warning(self): # Necessary to avoid annoying message in PyCharm filterwarnings(action="ignore", message="unclosed", category=ResourceWarning) - del self.test_site_config.MachineTypeConfiguration.test2large.StartupCommand + del self.config.TestSite.MachineTypeConfiguration.test2large.StartupCommand with self.assertRaises(AttributeError): self.slurm_adapter = SlurmAdapter( machine_type="test2large", site_name="TestSite" ) - self.test_site_config.StartupCommand = "pilot.sh" + self.config.TestSite.StartupCommand = "pilot.sh" with self.assertWarns(DeprecationWarning): self.slurm_adapter = SlurmAdapter( @@ -189,8 +190,9 @@ def test_deploy_resource(self): ) self.mock_executor.reset_mock() + self.slurm_adapter.refresh_configuration() - self.test_site_config.MachineMetaData.test2large.Memory = 2.5 + self.config.TestSite.MachineMetaData.test2large.Memory = 2.5 run_async(self.slurm_adapter.deploy_resource, resource_attributes) @@ -199,8 +201,9 @@ def test_deploy_resource(self): ) self.mock_executor.reset_mock() + self.slurm_adapter.refresh_configuration() - self.test_site_config.MachineMetaData.test2large.Memory = 2.546372129 + self.config.TestSite.MachineMetaData.test2large.Memory = 2.546372129 run_async(self.slurm_adapter.deploy_resource, resource_attributes) @@ -210,7 +213,7 @@ def test_deploy_resource(self): @mock_executor_run_command(TEST_DEPLOY_RESOURCE_RESPONSE) def test_deploy_resource_w_submit_options(self): - self.test_site_config.MachineTypeConfiguration.test2large.SubmitOptions = ( + self.config.TestSite.MachineTypeConfiguration.test2large.SubmitOptions = ( AttributeDict(long=AttributeDict(gres="tmp:1G")) ) @@ -334,10 +337,10 @@ def test_resource_state_translation(self): ) self.assertEqual(returned_resource_attributes.resource_status, value) - self.mock_executor.return_value.run_command.called_once() - - self.mock_executor.return_value.run_command.assert_called_with( - 'squeue -o "%A|%N|%T" -h -t all' + assert_awaited_once(self.mock_executor.return_value.run_command) + assert_awaited_with( + self.mock_executor.return_value.run_command, + 'squeue -o "%A|%N|%T" -h -t all', ) def test_resource_status_raise_update_failed(self): diff --git a/tests/interfaces_t/test_siteadapter.py b/tests/interfaces_t/test_siteadapter.py index 614b3ac2..2c7e5c70 100644 --- a/tests/interfaces_t/test_siteadapter.py +++ b/tests/interfaces_t/test_siteadapter.py @@ -1,4 +1,4 @@ -from tardis.interfaces.siteadapter import SiteAdapter +from tardis.interfaces.siteadapter import SiteAdapter, SiteAdapterBaseModel from tardis.utilities.attributedict import AttributeDict from ..utilities.utilities import run_async @@ -41,17 +41,28 @@ def setUp(self) -> None: self.site_adapter = SiteAdapter() self.site_adapter._site_name = "TestSite" self.site_adapter._machine_type = "TestMachineType" + self.site_adapter._configuration_validation_model = SiteAdapterBaseModel def test_configuration(self): self.assertEqual(self.site_adapter.configuration, self.config.TestSite) + def test_configuration_validation_model(self): + self.assertEqual( + SiteAdapterBaseModel, self.site_adapter.configuration_validation_model + ) + + # noinspection PyUnresolvedReferences + del self.site_adapter._configuration_validation_model + + with self.assertRaises(AttributeError): + # noinspection PyStatementEffect + self.site_adapter.configuration_validation_model + def test_deploy_resource(self): with self.assertRaises(NotImplementedError): run_async(self.site_adapter.deploy_resource, dict()) def test_drone_environment(self): - self.site_adapter._machine_type = "TestMachineType" - self.assertEqual( AttributeDict(Cores=128, Memory=524288, Disk=104857600, Uuid="test-123"), self.site_adapter.drone_environment( @@ -79,14 +90,12 @@ def test_drone_heartbeat_interval(self): self.assertEqual(self.site_adapter.drone_heartbeat_interval, 60) # lru_cache needs to be cleared before manipulating site configuration - # noinspection PyUnresolvedReferences - SiteAdapter.site_configuration.fget.cache_clear() + self.site_adapter.refresh_configuration() self.config.Sites[0]["drone_heartbeat_interval"] = 10 self.assertEqual(self.site_adapter.drone_heartbeat_interval, 10) - # noinspection PyUnresolvedReferences - SiteAdapter.site_configuration.fget.cache_clear() + self.site_adapter.refresh_configuration() self.config.Sites[0]["drone_heartbeat_interval"] = -1 with self.assertRaises(ValidationError): @@ -97,14 +106,13 @@ def test_drone_minimum_lifetime(self): self.assertEqual(self.site_adapter.drone_minimum_lifetime, None) # lru_cache needs to be cleared before manipulating site configuration - # noinspection PyUnresolvedReferences - SiteAdapter.site_configuration.fget.cache_clear() + self.site_adapter.refresh_configuration() self.config.Sites[0]["drone_minimum_lifetime"] = 10 self.assertEqual(self.site_adapter.drone_minimum_lifetime, 10) # noinspection PyUnresolvedReferences - SiteAdapter.site_configuration.fget.cache_clear() + self.site_adapter.refresh_configuration() self.config.Sites[0]["drone_minimum_lifetime"] = -1 with self.assertRaises(ValidationError): @@ -177,6 +185,52 @@ def test_machine_meta_data(self): # noinspection PyStatementEffect self.site_adapter.machine_meta_data + def test_machine_meta_data_validation(self): + def assert_raised_validation_error_existence(meta_data_entry): + self.site_adapter.refresh_configuration() + with self.assertRaises(ValidationError) as ve: + # noinspection PyStatementEffect + self.site_adapter.configuration + self.assertTrue( + f"You have to supply the {meta_data_entry} entry in the " + f"MachineMetaData for MachineType TestMachineType!" + in ve.exception.errors()[0]["msg"] + ) + + def assert_raised_validation_error_wrong_type(meta_data_entry): + self.site_adapter.refresh_configuration() + with self.assertRaises(ValidationError) as ve: + # noinspection PyStatementEffect + self.site_adapter.configuration + self.assertTrue( + "You supplied a wrong type in the " + "MachineMetaData for machine_type TestMachineType entry " + f"'{meta_data_entry}: 123'!\n" + f"The allowed types are " in ve.exception.errors()[0]["msg"] + ) + + for meta_data_entry in ("Cores", "Disk", "Memory"): + current_meta_data_entry = getattr( + self.config.TestSite.MachineMetaData.TestMachineType, + meta_data_entry, + ) + setattr( + self.config.TestSite.MachineMetaData.TestMachineType, + meta_data_entry, + "123", + ) + assert_raised_validation_error_wrong_type(meta_data_entry) + + delattr( + self.config.TestSite.MachineMetaData.TestMachineType, meta_data_entry + ) + assert_raised_validation_error_existence(meta_data_entry) + setattr( + self.config.TestSite.MachineMetaData.TestMachineType, + meta_data_entry, + current_meta_data_entry, + ) + def test_machine_type(self): self.assertEqual(self.site_adapter.machine_type, "TestMachineType") @@ -187,6 +241,41 @@ def test_machine_type(self): # noinspection PyStatementEffect self.site_adapter.machine_type + def test_machine_type_configuration_and_meta_data_existence(self): + def assert_raised_validation_error(config_block): + self.site_adapter.refresh_configuration() + with self.assertRaises(ValidationError) as ve: + # noinspection PyStatementEffect + self.site_adapter.configuration + self.assertTrue( + f"You have to specify {config_block} for MachineType TestMachineType" # noqa B950 + in ve.exception.errors()[0]["msg"] + ) + + for config_block in ("MachineTypeConfiguration", "MachineMetaData"): + current_config_block = AttributeDict( + getattr(self.config.TestSite, config_block) + ) + del getattr(self.config.TestSite, config_block).TestMachineType + assert_raised_validation_error(config_block) + + setattr(self.config.TestSite, config_block, current_config_block) + delattr(self.config.TestSite, config_block) + assert_raised_validation_error(config_block) + + setattr(self.config.TestSite, config_block, current_config_block) + + def test_machine_type_validation_not_exists(self): + del self.config.TestSite.MachineTypes + self.site_adapter.refresh_configuration() + with self.assertRaises(ValidationError) as ve: + # noinspection PyStatementEffect + self.site_adapter.configuration + self.assertTrue( + "You have to add MachineTypes to the site configuration" + in ve.exception.errors()[0]["msg"] + ) + def test_machine_type_configuration(self): self.assertEqual( self.site_adapter.machine_type_configuration, @@ -200,6 +289,21 @@ def test_machine_type_configuration(self): # noinspection PyStatementEffect self.site_adapter.machine_type_configuration + def test_refresh_configuration(self): + current_config = self.site_adapter.site_configuration + self.config.Sites[0]["quota"] = 123 + self.assertEqual(current_config, self.site_adapter.site_configuration) + self.site_adapter.refresh_configuration() + current_config["quota"] = 123 + self.assertEqual(current_config, self.site_adapter.site_configuration) + + current_config = self.site_adapter.configuration + self.config.TestSite.MachineTypeConfiguration.TestMachineType.test_id = "xy789" + self.assertEqual(current_config, self.site_adapter.configuration) + self.site_adapter.refresh_configuration() + current_config.MachineTypeConfiguration.TestMachineType.test_id = "xy789" + self.assertEqual(current_config, self.site_adapter.configuration) + def test_resource_status(self): with self.assertRaises(NotImplementedError): run_async(self.site_adapter.resource_status, dict()) @@ -216,8 +320,7 @@ def test_site_configuration(self): ), ) - # noinspection PyUnresolvedReferences - SiteAdapter.site_configuration.fget.cache_clear() + self.site_adapter.refresh_configuration() del self.config.Sites[0]["quota"] @@ -232,8 +335,7 @@ def test_site_configuration(self): ), ) - # noinspection PyUnresolvedReferences - SiteAdapter.site_configuration.fget.cache_clear() + self.site_adapter.refresh_configuration() self.config.Sites[0]["extra"] = "Should fail!" @@ -249,8 +351,7 @@ def test_site_configuration(self): ), ) - # noinspection PyUnresolvedReferences - SiteAdapter.site_configuration.fget.cache_clear() + self.site_adapter.refresh_configuration() self.config.Sites[0]["quota"] = 0 diff --git a/tests/utilities/utilities.py b/tests/utilities/utilities.py index 66691ab4..f9aa8e39 100644 --- a/tests/utilities/utilities.py +++ b/tests/utilities/utilities.py @@ -1,9 +1,25 @@ +from tardis.interfaces.simulator import Simulator from tardis.utilities.attributedict import AttributeDict +from unittest.mock import Mock, MagicMock import asyncio import socket +def assert_awaited_once(mock: Mock): + if not isinstance(mock, MagicMock): + return mock.assert_awaited_once() + else: + return mock.assert_called_once() + + +def assert_awaited_with(mock: Mock, *args, **kwargs): + if not isinstance(mock, MagicMock): + return mock.assert_awaited_with(*args, **kwargs) + else: + return mock.assert_called_with(*args, **kwargs) + + def async_return(*args, return_value=None, **kwargs): loop = asyncio.get_event_loop_policy().get_event_loop() f = loop.create_future() @@ -19,15 +35,29 @@ def get_free_port(): # from https://gist.github.com/dbrgn/3979133 return port +class MockedSimulator(Simulator): + def __init__(self, return_value): + self._return_value = return_value + + def get_value(self) -> float: + return self._return_value + + def mock_executor_run_command(stdout, stderr="", exit_code=0, raise_exception=None): def decorator(func): def wrapper(self): executor = self.mock_executor.return_value - executor.run_command.return_value = async_return( - return_value=AttributeDict( - stdout=stdout, stderr=stderr, exit_code=exit_code - ) + return_value = AttributeDict( + stdout=stdout, stderr=stderr, exit_code=exit_code ) + if not isinstance( + executor.run_command, MagicMock + ): # since python 3.8, AsyncMock is returned, no need for async_return + executor.run_command.return_value = return_value + else: # before python 3,8 MagicMock is returned, requires async_return + executor.run_command.return_value = async_return( + return_value=return_value + ) executor.run_command.side_effect = raise_exception func(self) executor.run_command.side_effect = None @@ -38,6 +68,7 @@ def wrapper(self): def run_async(coroutine, *args, **kwargs): + # return asyncio.run(coroutine(*args, **kwargs)) loop = asyncio.get_event_loop_policy().get_event_loop() return loop.run_until_complete(coroutine(*args, **kwargs))