Initial implementation of pydantic validation for site adapters

docs/source/adapters/site.rst

@@ -579,15 +579,15 @@ Available machine type configuration options
     +----------------+-------------------------------------------------------------------------------+-----------------+
     | args           | Arguments for the containers that run in your pods.                           |  **Required**   |
     +----------------+-------------------------------------------------------------------------------+-----------------+
-    | hpa            | Set True\False to enable\disable kubernetes horizontal pod autoscaler feature.|  **Required**   |
+    | hpa            | Set True\False to enable\disable kubernetes horizontal pod autoscaler feature.|  **Optional**   |
     +----------------+-------------------------------------------------------------------------------+-----------------+
-    | min_replicas   | Minimum number of pods to scale to. (Only required when hpa is set to True)   |  **Required**   |
+    | min_replicas   | Minimum number of pods to scale to. (Required when hpa is set to True)        |  **Optional**   |
     +----------------+-------------------------------------------------------------------------------+-----------------+
-    | max_replicas   | Maximum number of pods to scale to. (Only required when hpa is set to True)   |  **Required**   |
+    | max_replicas   | Maximum number of pods to scale to. (Required when hpa is set to True)        |  **Optional**   |
     +----------------+-------------------------------------------------------------------------------+-----------------+
-    | cpu_utilization| Average Cpu utilization to maintain across pods of a deployment.              |  **Required**   |
+    | cpu_utilization| Average Cpu utilization to maintain across pods of a deployment.              |  **Optional**   |
     +                +                                                                               +                 +
-    |                | (Only required when hpa is set to True)                                       |                 |
+    |                | (Required when hpa is set to True)                                            |                 |
     +----------------+-------------------------------------------------------------------------------+-----------------+
 
 .. content-tabs:: right-col

docs/source/changelog.rst

@@ -1,12 +1,12 @@
-.. Created by changelog.py at 2022-09-16, command
+.. Created by changelog.py at 2022-10-06, command
    '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.10/bin/changelog docs/source/changes compile --output=docs/source/changelog.rst'
    based on the format of 'https://keepachangelog.com/'
 
 #########
 CHANGELOG
 #########
 
-[Unreleased] - 2022-09-16
+[Unreleased] - 2022-10-06
 =========================
 
 Added

setup.py

@@ -84,7 +84,7 @@ def get_cryptography_version():
         get_cryptography_version(),
         "CloudStackAIO>=0.0.8",
         "PyYAML",
-        "AsyncOpenStackClient",
+        "AsyncOpenStackClient>=0.9.0",
         "cobald>=0.12.3",
         "asyncssh",
         "aiotelegraf",

tardis/adapters/sites/cloudstack.py

@@ -2,14 +2,18 @@
 from tardis.exceptions.tardisexceptions import TardisError
 from tardis.exceptions.tardisexceptions import TardisQuotaExceeded
 from tardis.exceptions.tardisexceptions import TardisResourceStatusUpdateFailed
-from tardis.interfaces.siteadapter import ResourceStatus
-from tardis.interfaces.siteadapter import SiteAdapter
+from tardis.interfaces.siteadapter import (
+    ResourceStatus,
+    SiteAdapter,
+    SiteAdapterBaseModel,
+)
 from tardis.utilities.attributedict import AttributeDict
 from tardis.utilities.staticmapping import StaticMapping
 
 from aiohttp import ClientConnectionError
 from CloudStackAIO.CloudStack import CloudStack
 from CloudStackAIO.CloudStack import CloudStackClientException
+from pydantic import AnyUrl
 
 from contextlib import contextmanager
 from datetime import datetime
@@ -21,7 +25,19 @@
 logger = logging.getLogger("cobald.runtime.tardis.adapters.sites.cloudstack")
 
 
+class CloudStackAdapterConfigurationModel(SiteAdapterBaseModel):
+    """
+    pydantic model for the input validation of the CloudStack site adapter configuration
+    """
+
+    end_point: AnyUrl
+    api_key: str
+    api_secret: str
+
+
 class CloudStackAdapter(SiteAdapter):
+    _configuration_validation_model = CloudStackAdapterConfigurationModel
+
     def __init__(self, machine_type: str, site_name: str):
         self._machine_type = machine_type
         self._site_name = site_name

tardis/adapters/sites/fakesite.py

@@ -1,6 +1,6 @@
 from ...exceptions.tardisexceptions import TardisError
-from ...interfaces.siteadapter import ResourceStatus
-from ...interfaces.siteadapter import SiteAdapter
+from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel
+from ...interfaces.simulator import Simulator
 from ...utilities.attributedict import AttributeDict
 from ...utilities.staticmapping import StaticMapping
 
@@ -13,7 +13,18 @@
 import asyncio
 
 
+class FakeSiteAdapterConfigurationModel(SiteAdapterBaseModel):
+    """
+    pydantic model for the input validation of the fake site adapter configuration
+    """
+
+    api_response_delay: Simulator
+    resource_boot_time: Simulator
+
+
 class FakeSiteAdapter(SiteAdapter):
+    _configuration_validation_model = FakeSiteAdapterConfigurationModel
+
     def __init__(self, machine_type: str, site_name: str) -> None:
         self._machine_type = machine_type
         self._site_name = site_name

tardis/adapters/sites/htcondor.py

@@ -1,9 +1,7 @@
-from typing import Iterable, Tuple, Awaitable
 from ...exceptions.executorexceptions import CommandExecutionFailure
 from ...exceptions.tardisexceptions import TardisError
 from ...exceptions.tardisexceptions import TardisResourceStatusUpdateFailed
-from ...interfaces.siteadapter import SiteAdapter
-from ...interfaces.siteadapter import ResourceStatus
+from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel
 from ...interfaces.executor import Executor
 from ...utilities.asynccachemap import AsyncCacheMap
 from ...utilities.attributedict import AttributeDict
@@ -12,10 +10,13 @@
 from ...utilities.asyncbulkcall import AsyncBulkCall
 from ...utilities.utils import csv_parser, machine_meta_data_translation
 
+from pydantic import PositiveInt, PositiveFloat
+
 from contextlib import contextmanager
 from datetime import datetime
 from functools import partial
 from string import Template
+from typing import Iterable, Tuple, Awaitable, Optional
 
 import warnings
 import logging
@@ -74,7 +75,7 @@ def _submit_description(resource_jdls: Tuple[JDL, ...]) -> str:
                 FutureWarning,
             )
         else:
-            commands.append("queue 1")
+            commands.append("queue 1\n")
     return "\n".join(commands)
 
 
@@ -181,10 +182,22 @@ async def _condor_tool(
 }
 
 
+class HTCondorAdapterConfigurationModel(SiteAdapterBaseModel):
+    """
+    pydantic model for the input validation of the HTCondor site adapter configuration
+    """
+
+    executor: Optional[Executor] = ShellExecutor()
+    bulk_size: Optional[PositiveInt] = 100
+    bulk_delay: Optional[PositiveFloat] = 1.0
+    max_age: PositiveInt
+
+
 class HTCondorAdapter(SiteAdapter):
     htcondor_machine_meta_data_translation_mapping = AttributeDict(
         Cores=1, Memory=1024, Disk=1024 * 1024
     )
+    _configuration_validation_model = HTCondorAdapterConfigurationModel
 
     def __init__(
         self,
@@ -193,9 +206,9 @@ def __init__(
     ):
         self._machine_type = machine_type
         self._site_name = site_name
-        self._executor = getattr(self.configuration, "executor", ShellExecutor())
-        bulk_size = getattr(self.configuration, "bulk_size", 100)
-        bulk_delay = getattr(self.configuration, "bulk_delay", 1.0)
+        self._executor = self.configuration.executor
+        bulk_size = self.configuration.bulk_size
+        bulk_delay = self.configuration.bulk_delay
         self._condor_submit, self._condor_suspend, self._condor_rm = (
             AsyncBulkCall(
                 partial(tool, executor=self._executor),

tardis/adapters/sites/kubernetes.py

@@ -1,22 +1,86 @@
 from kubernetes_asyncio import client as k8s_client
 from kubernetes_asyncio.client.rest import ApiException as K8SApiException
 from ...exceptions.tardisexceptions import TardisError
-from ...interfaces.siteadapter import SiteAdapter
-from ...interfaces.siteadapter import ResourceStatus
+from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel
 from ...utilities.attributedict import AttributeDict
 from ...utilities.staticmapping import StaticMapping
 from ...utilities.utils import convert_to
 
+from pydantic import AnyUrl, BaseModel, conint, constr, PositiveInt, validator
+
 from functools import partial
 from datetime import datetime
 from contextlib import contextmanager
+from typing import Any, List, Optional
 
 import logging
 
 logger = logging.getLogger("cobald.runtime.tardis.adapters.sites.kubernetes")
 
 
+class KubernetesMachineTypeConfigurationModel(BaseModel):
+    args: List[str]
+    cpu_utilization: Optional[conint(ge=0, le=100)]
+    hpa: bool = False
+    image: constr(regex=r"^\S+:\S+$")  # noqa F722
+    max_replicas: Optional[PositiveInt]
+    min_replicas: Optional[PositiveInt]
+    namespace: str = "default"
+
+
+class KubernetesAdapterConfigurationModel(SiteAdapterBaseModel):
+    """
+    pydantic model for the input validation of the Kubernetes site adapter configuration
+    """
+
+    host: AnyUrl
+    token: str
+
+    @validator("MachineTypeConfiguration")
+    def validate_machine_type_configuration(
+        cls,  # noqa B902
+        machine_type_configurations: AttributeDict[
+            str, Optional[AttributeDict[str, AttributeDict[str, Any]]]
+        ],
+    ):
+        validated_configurations = AttributeDict()
+
+        for (
+            machine_type,
+            machine_type_configuration,
+        ) in machine_type_configurations.items():
+            validated_configuration = KubernetesMachineTypeConfigurationModel(
+                **machine_type_configuration
+            )
+            hpa_required_attrs = ("cpu_utilization", "max_replicas", "min_replicas")
+            if validated_configuration.hpa:
+                if not all(  # check all required attributes are present
+                    getattr(validated_configuration, attr)
+                    for attr in hpa_required_attrs
+                ):
+                    raise ValueError(
+                        f"You need to supply {hpa_required_attrs} in case you activate "
+                        "the Kubernetes Horizontal Autopod Scaler (HPA)!"
+                    )
+                if (  # check consistency of min_replicas and max_replicas
+                    validated_configuration.min_replicas
+                    > validated_configuration.max_replicas
+                ):
+                    raise ValueError(
+                        "max_replicas have to be larger/equal as/to min_replicas!"
+                    )
+            setattr(
+                validated_configurations,
+                machine_type,
+                AttributeDict(validated_configuration),
+            )
+
+        return validated_configurations
+
+
 class KubernetesAdapter(SiteAdapter):
+    _configuration_validation_model = KubernetesAdapterConfigurationModel
+
     def __init__(self, machine_type: str, site_name: str):
         self._machine_type = machine_type
         self._site_name = site_name

tardis/adapters/sites/moab.py

@@ -2,19 +2,22 @@
 from ...exceptions.tardisexceptions import TardisError
 from ...exceptions.tardisexceptions import TardisTimeout
 from ...exceptions.tardisexceptions import TardisResourceStatusUpdateFailed
-from ...interfaces.siteadapter import ResourceStatus
-from ...interfaces.siteadapter import SiteAdapter
+from ...interfaces.executor import Executor
+from ...interfaces.siteadapter import ResourceStatus, SiteAdapter, SiteAdapterBaseModel
 from ...utilities.staticmapping import StaticMapping
 from ...utilities.attributedict import AttributeDict
 from ...utilities.attributedict import convert_to_attribute_dict
 from ...utilities.executors.shellexecutor import ShellExecutor
 from ...utilities.asynccachemap import AsyncCacheMap
 from ...utilities.utils import submit_cmd_option_formatter
 
+from pydantic import PositiveInt
+
 from asyncio import TimeoutError
 from contextlib import contextmanager
 from functools import partial
 from datetime import datetime
+from typing import Optional
 
 import asyncssh
 import logging
@@ -47,23 +50,35 @@ async def moab_status_updater(executor):
     return moab_resource_status
 
 
+class MoabAdapterConfigurationModel(SiteAdapterBaseModel):
+    """
+    pydantic model for the input validation of the Moab site adapter configuration
+    """
+
+    executor: Optional[Executor] = ShellExecutor()
+    StatusUpdate: PositiveInt
+    StartupCommand: Optional[str]
+
+
 class MoabAdapter(SiteAdapter):
+    _configuration_validation_model = MoabAdapterConfigurationModel
+
     def __init__(self, machine_type: str, site_name: str):
         self._machine_type = machine_type
         self._site_name = site_name
 
         try:
             self._startup_command = self.machine_type_configuration.StartupCommand
         except AttributeError:
-            if not hasattr(self.configuration, "StartupCommand"):
+            if self.configuration.StartupCommand is None:
                 raise
             warnings.warn(
                 "StartupCommand has been moved to the machine_type_configuration!",
                 DeprecationWarning,
             )
             self._startup_command = self.configuration.StartupCommand
 
-        self._executor = getattr(self.configuration, "executor", ShellExecutor())
+        self._executor = self.configuration.executor
 
         self._moab_status = AsyncCacheMap(
             update_coroutine=partial(moab_status_updater, self._executor),