From 5d653aadb9530da9e7e5a977a75c9e998042a84b Mon Sep 17 00:00:00 2001 From: liquidiert Date: Mon, 30 Sep 2024 11:44:12 +0200 Subject: [PATCH 1/2] feat: custom cluster configs :sparkles: - add cluster config option in ClusterOptions - add cluster config parsing - add cluster config pytest arg - add unit tests --- .github/workflows/pr.yaml | 4 +- README.md | 18 ++++++ pytest_kubernetes/options.py | 1 + pytest_kubernetes/plugin.py | 18 ++++-- pytest_kubernetes/providers/base.py | 16 ++++-- pytest_kubernetes/providers/k3d.py | 42 ++++++++++++-- pytest_kubernetes/providers/kind.py | 21 +++++-- pytest_kubernetes/providers/minikube.py | 73 +++++++++++++++++++++---- tests/fixtures/k3d_config.yaml | 3 + tests/fixtures/kind_config.yaml | 3 + tests/fixtures/mk_config.yaml | 10 ++++ tests/test_providers.py | 40 ++++++++++++++ 12 files changed, 218 insertions(+), 31 deletions(-) create mode 100644 tests/fixtures/k3d_config.yaml create mode 100644 tests/fixtures/kind_config.yaml create mode 100644 tests/fixtures/mk_config.yaml diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e581531..f82c13d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,5 +34,5 @@ jobs: run: poetry install --no-interaction --no-root - name: Create k8s Kind Cluster uses: helm/kind-action@v1 - - name: Run Tests - run: poetry run pytest -k Testkind -v + - name: Run Tests with kind + run: poetry run pytest -k test_custom_cluster_config -v diff --git a/README.md b/README.md index ff2f187..57061e2 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,23 @@ In this example, the cluster remains active for the entire session and is only d > Note that `yield` notation that is prefered by pytest to express clean up tasks for this fixture. +#### Cluster configs +You can pass a cluster config file in the create method of a cluster: +```python + cluster = select_provider_manager("k3d")("my-cluster") + # bind ports of this k3d cluster + cluster.create( + cluster_options=ClusterOptions( + cluster_config=Path("my_cluster_config.yaml") + ) + ) +``` +For the different providers you have to submit different kinds of configuration files. +- kind: https://kind.sigs.k8s.io/docs/user/configuration/#getting-started +- k3d: https://k3d.io/v5.1.0/usage/configfile/ +- minikube: Has to be a custom yaml file that corresponds to the `minikube config` command. An example can be found in the [fixtures directory](https://github.com/Blueshoe/pytest-kubernetes/tree/main/tests/fixtures/mk_config.yaml) of this repository. + + #### Special cluster options You can pass more options using `kwargs['options']: List[str]` to the `create(options=...)` function when creating the cluster like so: ```python @@ -136,5 +153,6 @@ You can pass more options using `kwargs['options']: List[str]` to the `create(op cluster.create(options=["--agents", "1", "-p", "8080:80@agent:0", "-p", "31820:31820/UDP@agent:0"]) ``` + ## Examples Please find more examples in *tests/vendor.py* in this repository. These test cases are written as users of pytest-kubernetes would write test cases in their projects. \ No newline at end of file diff --git a/pytest_kubernetes/options.py b/pytest_kubernetes/options.py index 1f2d01a..c78632c 100644 --- a/pytest_kubernetes/options.py +++ b/pytest_kubernetes/options.py @@ -7,4 +7,5 @@ class ClusterOptions: api_version: str = field(default="1.25.3") # nodes: int = None kubeconfig_path: Path | None = None + cluster_config: Path | None = None # Path to a Provider cluster config file cluster_timeout: int = field(default=240) diff --git a/pytest_kubernetes/plugin.py b/pytest_kubernetes/plugin.py index 597d81e..a7df96c 100644 --- a/pytest_kubernetes/plugin.py +++ b/pytest_kubernetes/plugin.py @@ -1,5 +1,6 @@ from typing import Dict, Type import pytest +from pytest import FixtureRequest from pytest_kubernetes.providers import select_provider_manager from pytest_kubernetes.providers.base import AClusterManager @@ -8,7 +9,7 @@ @pytest.fixture -def k8s(request): +def k8s(request: FixtureRequest): """Provide a Kubernetes cluster as test fixture.""" provider = None @@ -19,6 +20,7 @@ def k8s(request): provider = req.get("provider") cluster_name = req.get("cluster_name") or cluster_name keep = req.get("keep") + cluster_config = req.get("cluster_config") if not provider: provider = provider = request.config.getoption("k8s_provider") if not cluster_name: @@ -31,7 +33,7 @@ def k8s(request): manager = cluster_cache[cache_key] del cluster_cache[cache_key] else: - manager: AClusterManager = manager_klass(cluster_name) + manager: AClusterManager = manager_klass(cluster_name, cluster_config) def delete_cluster(): manager.delete() @@ -52,17 +54,21 @@ def remaining_clusters_teardown(): def pytest_addoption(parser): - group = parser.getgroup("k8s") - group.addoption( + k8s_group = parser.getgroup("k8s") + k8s_group.addoption( "--k8s-cluster-name", default="pytest", help="Name of the Kubernetes cluster (default 'pytest').", ) - group.addoption( + k8s_group.addoption( "--k8s-provider", help="The default cluster provider; selects k3d, kind, minikube depending on what is available", ) - group.addoption( + k8s_group.addoption( "--k8s-version", help="The default cluster provider; selects k3d, kind, minikube depending on what is available", ) + k8s_group.addoption( + "--k8s-cluster-config", + help="Path to a Provider cluster config file", + ) diff --git a/pytest_kubernetes/providers/base.py b/pytest_kubernetes/providers/base.py index 732abd4..29e10bc 100644 --- a/pytest_kubernetes/providers/base.py +++ b/pytest_kubernetes/providers/base.py @@ -56,8 +56,10 @@ class AClusterManager(ABC): cluster_name = "" context = None - def __init__(self, cluster_name: str) -> None: + def __init__(self, cluster_name: str, cluster_config: str | None = None) -> None: self.cluster_name = f"pytest-{cluster_name}" + if cluster_config: + self._cluster_options.cluster_config = Path(cluster_config) self._ensure_executable() @classmethod @@ -212,12 +214,18 @@ def create( self._on_create(self._cluster_options, **kwargs) _i = 0 # check if this cluster is ready: readyz check passed and default service account is available + ready = "Nope" + sa_available = "Nope" while _i < timeout: sleep(1) try: - ready = self.kubectl(["get", "--raw='/readyz?verbose'"], as_dict=False) - sa_available = self.kubectl( - ["get", "sa", "default", "-n", "default"], as_dict=False + ready = str( + self.kubectl(["get", "--raw='/readyz?verbose'"], as_dict=False) + ) + sa_available = str( + self.kubectl( + ["get", "sa", "default", "-n", "default"], as_dict=False + ) ) except RuntimeError: _i += 1 diff --git a/pytest_kubernetes/providers/k3d.py b/pytest_kubernetes/providers/k3d.py index 52ee5d1..74c7c38 100644 --- a/pytest_kubernetes/providers/k3d.py +++ b/pytest_kubernetes/providers/k3d.py @@ -1,5 +1,8 @@ from pytest_kubernetes.providers.base import AClusterManager from pytest_kubernetes.options import ClusterOptions +import subprocess +import yaml +import re class K3dManager(AClusterManager): @@ -7,15 +10,38 @@ class K3dManager(AClusterManager): def get_binary_name(self) -> str: return "k3d" + @classmethod + def get_k3d_version(self) -> str: + version_proc = subprocess.run( + "k3d --version", + shell=True, + capture_output=True, + check=True, + timeout=10, + ) + version_match = re.match( + r"k3d version v(\d+\.\d+\.\d+)", version_proc.stdout.decode() + ) + if not version_match: + return "0.0.0" + return version_match.group(1) + def _translate_version(self, version: str) -> str: return f"rancher/k3s:v{version}-k3s1" def _on_create(self, cluster_options: ClusterOptions, **kwargs) -> None: opts = kwargs.get("options", []) - self._exec( - [ - "cluster", - "create", + + # see https://k3d.io/v5.1.0/usage/configfile/ + if cluster_options.cluster_config and K3dManager.get_k3d_version() >= "4.0.0": + opts += [ + "--config", + str(cluster_options.cluster_config), + ] + config_yaml = yaml.safe_load(cluster_options.cluster_config.read_text()) + else: + opts += [ + "--name", self.cluster_name, "--kubeconfig-update-default=0", "--image", @@ -23,13 +49,19 @@ def _on_create(self, cluster_options: ClusterOptions, **kwargs) -> None: "--wait", f"--timeout={cluster_options.cluster_timeout}s", ] + + self._exec( + [ + "cluster", + "create", + ] + opts ) self._exec( [ "kubeconfig", "get", - self.cluster_name, + self.cluster_name if not config_yaml else config_yaml["name"], ">", str(cluster_options.kubeconfig_path), ] diff --git a/pytest_kubernetes/providers/kind.py b/pytest_kubernetes/providers/kind.py index c50de81..3a80f5b 100644 --- a/pytest_kubernetes/providers/kind.py +++ b/pytest_kubernetes/providers/kind.py @@ -9,10 +9,17 @@ def get_binary_name(self) -> str: def _on_create(self, cluster_options: ClusterOptions, **kwargs) -> None: opts = kwargs.get("options", []) - _ = self._exec( - [ - "create", - "cluster", + + # see https://kind.sigs.k8s.io/docs/user/configuration/#getting-started + if cluster_options.cluster_config: + opts += [ + "--config", + str(cluster_options.cluster_config), + "--kubeconfig", + str(cluster_options.kubeconfig_path), + ] + else: + opts += [ "--name", self.cluster_name, "--kubeconfig", @@ -20,6 +27,12 @@ def _on_create(self, cluster_options: ClusterOptions, **kwargs) -> None: "--image", f"kindest/node:v{cluster_options.api_version}", ] + + _ = self._exec( + [ + "create", + "cluster", + ] + opts ) diff --git a/pytest_kubernetes/providers/minikube.py b/pytest_kubernetes/providers/minikube.py index 06601ec..74f737f 100644 --- a/pytest_kubernetes/providers/minikube.py +++ b/pytest_kubernetes/providers/minikube.py @@ -1,5 +1,6 @@ from pytest_kubernetes.providers.base import AClusterManager from pytest_kubernetes.options import ClusterOptions +import yaml class MinikubeManager(AClusterManager): @@ -17,17 +18,42 @@ def load_image(self, image: str) -> None: class MinikubeKVM2Manager(MinikubeManager): def _on_create(self, cluster_options: ClusterOptions, **kwargs) -> None: opts = kwargs.get("options", []) - self._exec( - [ - "start", - "-p", - self.cluster_name, + + if cluster_options.cluster_config: + config_yaml = yaml.safe_load(cluster_options.cluster_config.read_text()) + try: + for config in config_yaml["cluster"]["configs"]: + self._exec( + [ + "config", + config["name"], + f"{config['value']}", + "-p", + f"{config_yaml['cluster']['profile']}", + ], + additional_env={ + "KUBECONFIG": str(cluster_options.kubeconfig_path) + }, + ) + except KeyError as ex: + raise ValueError( + f"Missing key: {ex}; cluster_config for minikube setup invalid. Please refer to the docs!" + ) + else: + opts += [ "--driver", "kvm2", "--embed-certs", "--kubernetes-version", f"v{cluster_options.api_version}", ] + + self._exec( + [ + "start", + "-p", + self.cluster_name, + ] + opts, additional_env={"KUBECONFIG": str(cluster_options.kubeconfig_path)}, ) @@ -36,17 +62,44 @@ def _on_create(self, cluster_options: ClusterOptions, **kwargs) -> None: class MinikubeDockerManager(MinikubeManager): def _on_create(self, cluster_options: ClusterOptions, **kwargs) -> None: opts = kwargs.get("options", []) - self._exec( - [ - "start", - "-p", - self.cluster_name, + + if cluster_options.cluster_config: + config_yaml = yaml.safe_load(cluster_options.cluster_config.read_text()) + try: + for config in config_yaml["cluster"]["configs"]: + self._exec( + [ + "config", + config["name"], + f"{config['value']}", + "-p", + f"{config_yaml['cluster']['profile']}", + ], + additional_env={ + "KUBECONFIG": str(cluster_options.kubeconfig_path) + }, + ) + except KeyError as ex: + raise ValueError( + f"Missing key: {ex}; cluster_config for minikube setup invalid. Please refer to the docs!" + ) + else: + opts += [ "--driver", "docker", "--embed-certs", "--kubernetes-version", f"v{cluster_options.api_version}", ] + + self._exec( + [ + "start", + "-p", + self.cluster_name + if not config_yaml + else config_yaml["cluster"]["profile"], + ] + opts, additional_env={"KUBECONFIG": str(cluster_options.kubeconfig_path)}, ) diff --git a/tests/fixtures/k3d_config.yaml b/tests/fixtures/k3d_config.yaml new file mode 100644 index 0000000..832b377 --- /dev/null +++ b/tests/fixtures/k3d_config.yaml @@ -0,0 +1,3 @@ +apiVersion: k3d.io/v1alpha3 +kind: Simple +name: pytest-k3d-cluster \ No newline at end of file diff --git a/tests/fixtures/kind_config.yaml b/tests/fixtures/kind_config.yaml new file mode 100644 index 0000000..8b24313 --- /dev/null +++ b/tests/fixtures/kind_config.yaml @@ -0,0 +1,3 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: pytest-kind-cluster \ No newline at end of file diff --git a/tests/fixtures/mk_config.yaml b/tests/fixtures/mk_config.yaml new file mode 100644 index 0000000..8181ef2 --- /dev/null +++ b/tests/fixtures/mk_config.yaml @@ -0,0 +1,10 @@ +cluster: + profile: pytest-mk-cluster + # config options have to be named the same as in the minikube config docs + configs: + - name: driver + value: kvm2 + - name: EmbedCerts + value: true + - name: kubernetes-version + value: "v1.25.3" \ No newline at end of file diff --git a/tests/test_providers.py b/tests/test_providers.py index b7f8aff..73cdfc9 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -4,6 +4,7 @@ import pytest +from pytest_kubernetes.options import ClusterOptions from pytest_kubernetes.providers import ( AClusterManager, K3dManager, @@ -164,14 +165,53 @@ def setup_method(self, method): class Testk3d(KubernetesManagerTest): manager = K3dManager + def test_custom_cluster_config(self): + self.cluster.create( + cluster_options=ClusterOptions( + cluster_config=Path(__file__).parent + / Path("./fixtures/k3d_config.yaml"), + ) + ) + cluster_name = self.cluster.kubectl( + ["config", "view", "--minify", "-o", "jsonpath='{.clusters[].name}'"], + as_dict=False, + ) + assert cluster_name == "k3d-pytest-k3d-cluster" + class Testkind(KubernetesManagerTest): manager = KindManager + def test_custom_cluster_config(self): + self.cluster.create( + cluster_options=ClusterOptions( + cluster_config=Path(__file__).parent + / Path("./fixtures/kind_config.yaml"), + ) + ) + cluster_name = self.cluster.kubectl( + ["config", "view", "--minify", "-o", "jsonpath='{.clusters[].name}'"], + as_dict=False, + ) + assert cluster_name == "kind-pytest-kind-cluster" + class TestDockerminikube(KubernetesManagerTest): manager = MinikubeDockerManager + def test_custom_cluster_config(self): + self.cluster.create( + cluster_options=ClusterOptions( + cluster_config=Path(__file__).parent + / Path("./fixtures/mk_config.yaml"), + ) + ) + cluster_name = self.cluster.kubectl( + ["config", "view", "--minify", "-o", "jsonpath='{.clusters[].name}'"], + as_dict=False, + ) + assert cluster_name == "pytest-mk-cluster" + class TestKVM2minikube(KubernetesManagerTest): manager = MinikubeKVM2Manager From 1ff853ea0cd069923a29739a3dbc4345d0217114 Mon Sep 17 00:00:00 2001 From: liquidiert Date: Mon, 30 Sep 2024 11:45:31 +0200 Subject: [PATCH 2/2] test: extend PR tests again --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f82c13d..41c2310 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -35,4 +35,4 @@ jobs: - name: Create k8s Kind Cluster uses: helm/kind-action@v1 - name: Run Tests with kind - run: poetry run pytest -k test_custom_cluster_config -v + run: poetry run pytest -k Testkind -v