diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26d2af055..3d2927304 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,13 +17,15 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install packages - run: sudo apt install podman golang-github-containernetworking-plugin-dnsname + run: sudo apt install podman golang-github-containernetworking-plugin-dnsname sqlite3 jq - name: Create virtualenv run: python3 -m venv venv - name: Install run: ./venv/bin/pip3 install -e . - name: Set owner for /dev/loop-control run: sudo chown $(whoami) /dev/loop-control + - name: Configure + run: ./venv/bin/ceph-devstack config set containers.postgres.count 0 - name: Doctor run: ./venv/bin/ceph-devstack -v doctor --fix - name: Build @@ -37,6 +39,31 @@ jobs: - name: Dump logs if: success() || failure() run: podman logs -f teuthology + - name: Create archive + if: success() || failure() + run: | + mkdir -p /tmp/artifacts + - name: Dump job data + if: success() || failure() + run: | + podman cp paddles:/paddles/dev.db /tmp/ + sqlite3 /tmp/dev.db ".output stdout" ".mode json" "select * from jobs" | jq | tee /tmp/artifacts/jobs.json + - name: Upload jobs.json + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: jobs + path: /tmp/artifacts/jobs.json + - name: Create tarball of log archive + if: success() || failure() + run: | + tar -czf /tmp/artifacts/archive.tar ~/.local/share/ceph-devstack/archive/ + - name: Upload log archive + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: archive + path: /tmp/artifacts/archive.tar - name: Stop run: ./venv/bin/ceph-devstack -v stop - name: Remove diff --git a/Jenkinsfile b/Jenkinsfile index 456b31e92..fbcf4def2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { else python3 -c "import yaml; print(yaml.safe_dump({'containers': {'teuthology': {'repo': '${env.WORKSPACE}/teuthology'}}, 'data_dir': '${env.WORKSPACE}/data'}))" > ${env.CDS_CONF} fi - ceph-devstack --config-file ${env.CDS_CONF} show-conf + ceph-devstack --config-file ${env.CDS_CONF} config dump ceph-devstack --config-file ${env.CDS_CONF} doctor """ } diff --git a/README.md b/README.md index 474f8686b..d27c11ee4 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ python3 -m pip install git+https://github.com/zmc/ceph-devstack.git ## Configuration `ceph-devstack` 's default configuration is [here](./ceph_devstack/config.yml). It can be extended by placing a file at `~/.config/ceph-devstack/config.yml` or by using the `--config-file` flag. -`ceph-devstack show-conf` will output the current configuration. +`ceph-devstack config dump` will output the current configuration. As an example, the following configuration will use a local image for paddles with the tag `TEST`; it will also create ten testnode containers; and will build its teuthology container from the git repo at `~/src/teuthology`: ``` diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index c2c88d660..fc0cc8d94 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -1,9 +1,11 @@ import argparse import logging.config -import yaml +import tomlkit +import tomlkit.items +import tomlkit.exceptions -from pathlib import Path, PosixPath -from typing import List, Optional +from pathlib import Path +from typing import List, Optional, Union VERBOSE = 15 @@ -12,17 +14,7 @@ logger = logging.getLogger("ceph-devstack") PROJECT_ROOT = Path(__file__).parent -DEFAULT_CONFIG_PATH = Path("~/.config/ceph-devstack/config.yml") - - -def represent_path(dumper: yaml.dumper.SafeDumper, data: PosixPath) -> yaml.Node: - return dumper.represent_scalar("tag:yaml.org,2002:str", str(data)) - - -yaml.SafeDumper.add_representer( - PosixPath, - represent_path, -) +DEFAULT_CONFIG_PATH = Path("~/.config/ceph-devstack/config.toml") def parse_args(args: List[str]) -> argparse.Namespace: @@ -50,6 +42,14 @@ def parse_args(args: List[str]) -> argparse.Namespace: help="Path to the ceph-devstack config file", ) subparsers = parser.add_subparsers(dest="command") + parser_config = subparsers.add_parser("config", help="Get or set config items") + subparsers_config = parser_config.add_subparsers(dest="config_op") + subparsers_config.add_parser("dump", help="show the configuration") + parser_config_get = subparsers_config.add_parser("get") + parser_config_get.add_argument("name") + parser_config_set = subparsers_config.add_parser("set") + parser_config_set.add_argument("name") + parser_config_set.add_argument("value") parser_doc = subparsers.add_parser( "doctor", help="Check that the system meets requirements" ) @@ -104,7 +104,6 @@ def parse_args(args: List[str]) -> argparse.Namespace: "container", help="The container to wait for", ) - subparsers.add_parser("show-conf", help="show the configuration") return parser.parse_args(args) @@ -119,21 +118,61 @@ def deep_merge(*maps): class Config(dict): + __slots__ = ["user_obj", "user_path"] + def load(self, config_path: Optional[Path] = None): - self.update(yaml.safe_load((Path(__file__).parent / "config.yml").read_text())) + parsed = tomlkit.parse((Path(__file__).parent / "config.toml").read_text()) + self.update(parsed) if config_path: - user_path = config_path.expanduser() - if user_path.exists(): - user_obj = yaml.safe_load(user_path.read_text()) or {} - self.update(deep_merge(config, user_obj)) - elif user_path != DEFAULT_CONFIG_PATH.expanduser(): - raise OSError(f"Config file at {user_path} not found!") - - -yaml.SafeDumper.add_representer( - Config, - yaml.representer.SafeRepresenter.represent_dict, -) + self.user_path = config_path.expanduser() + if self.user_path.exists(): + self.user_obj: dict = tomlkit.parse(self.user_path.read_text()) or {} + self.update(deep_merge(config, self.user_obj)) + elif self.user_path != DEFAULT_CONFIG_PATH.expanduser(): + raise OSError(f"Config file at {self.user_path} not found!") + else: + self.user_obj = {} + + def dump(self): + return tomlkit.dumps(self) + + def get_value(self, name: str) -> str: + path = name.split(".") + obj = config + i = 0 + while i < len(path): + sub_path = path[i] + try: + obj = obj[sub_path] + except KeyError: + logger.error(f"{name} not found in config") + raise + i += 1 + if isinstance(obj, (str, int, bool)): + return str(obj) + return tomlkit.dumps(obj).strip() + + def set_value(self, name: str, value: str): + path = name.split(".") + obj = self.user_obj + i = 0 + last_index = len(path) - 1 + item: Union[tomlkit.items.Item, str] = value + try: + item = tomlkit.value(item) + except tomlkit.exceptions.UnexpectedCharError: + pass + except tomlkit.exceptions.InternalParserError: + pass + while i <= last_index: + if i < last_index: + obj = obj.setdefault(path[i], {}) + elif i == last_index: + obj[path[i]] = item + self.update(self.user_obj) + self.user_path.parent.mkdir(exist_ok=True) + self.user_path.write_text(tomlkit.dumps(self.user_obj).strip()) + i += 1 config = Config() diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index b3116f94d..ccb02ef83 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -1,7 +1,6 @@ import asyncio import logging import sys -import yaml from pathlib import Path @@ -17,8 +16,13 @@ def main(): for handler in logging.getLogger("root").handlers: if not isinstance(handler, logging.FileHandler): handler.setLevel(VERBOSE) - if args.command == "show-conf": - print(yaml.safe_dump(config)) + if args.command == "config": + if args.config_op == "dump": + print(config.dump()) + if args.config_op == "get": + print(config.get_value(args.name)) + elif args.config_op == "set": + config.set_value(args.name, args.value) return config["args"] = vars(args) data_path = Path(config["data_dir"]).expanduser() diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml new file mode 100644 index 000000000..af07ab673 --- /dev/null +++ b/ceph_devstack/config.toml @@ -0,0 +1,23 @@ +data_dir = "~/.local/share/ceph-devstack" + +[containers.archive] +image = "python:alpine" + +[containers.beanstalk] +image = "quay.io/ceph-infra/teuthology-beanstalkd:latest" + +[containers.paddles] +image = "quay.io/ceph-infra/paddles:latest" + +[containers.postgres] +image = "quay.io/ceph-infra/teuthology-postgresql:latest" + +[containers.pulpito] +image = "quay.io/ceph-infra/pulpito:latest" + +[containers.testnode] +count = 3 +image = "quay.io/ceph-infra/teuthology-testnode:latest" + +[containers.teuthology] +image = "quay.io/ceph-infra/teuthology-dev:latest" diff --git a/ceph_devstack/config.yml b/ceph_devstack/config.yml deleted file mode 100644 index 208c11536..000000000 --- a/ceph_devstack/config.yml +++ /dev/null @@ -1,17 +0,0 @@ -containers: - postgres: - image: "quay.io/ceph-infra/teuthology-postgresql:latest" - paddles: - image: "quay.io/ceph-infra/paddles:latest" - beanstalk: - image: "quay.io/ceph-infra/teuthology-beanstalkd:latest" - archive: - image: "python:alpine" - pulpito: - image: "quay.io/ceph-infra/pulpito:latest" - testnode: - count: 3 - image: "quay.io/ceph-infra/teuthology-testnode:latest" - teuthology: - image: "quay.io/ceph-infra/teuthology-dev:latest" -data_dir: ~/.local/share/ceph-devstack diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 95a8754e2..344cc16a6 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -3,7 +3,6 @@ import os import tempfile -from collections import OrderedDict from subprocess import CalledProcessError from ceph_devstack import config, logger @@ -87,26 +86,38 @@ class CephDevStack: networks = [CephDevStackNetwork] secrets = [SSHKeyPair] - async def get_containers(self): - return OrderedDict( - [ - (Postgres, 1), - (Paddles, 1), - (Beanstalk, 1), - (Pulpito, 1), - (Teuthology, 1), - (TestNode, await self.get_testnode_count()), - (Archive, 1), - ] - ) - - async def get_testnode_count(self) -> int: - teuth = Teuthology() - try: - data = await teuth.inspect() - return int(data[0]["config"]["Labels"]["testnode_count"]) - except (KeyError, IndexError, CalledProcessError): - return config["containers"]["testnode"]["count"] + def __init__(self): + services = [ + Postgres, + Paddles, + Beanstalk, + Pulpito, + Teuthology, + TestNode, + Archive, + ] + self.service_specs = {} + for service in services: + name = service.__name__.lower() + count = config["containers"][name].get("count", 1) + if count == 0: + continue + self.service_specs[name] = { + "obj": service, + "count": count, + } + if count == 1: + self.service_specs[name]["objects"] = [service()] + elif count > 1: + self.service_specs[name]["objects"] = [ + service(name=f"{name}_{i}") for i in range(count) + ] + if postgres_spec := self.service_specs.get("postgres"): + postgres_obj = postgres_spec["objects"][0] + paddles_obj = self.service_specs["paddles"]["objects"][0] + paddles_obj.env_vars["PADDLES_SQLALCHEMY_URL"] = ( + postgres_obj.paddles_sqla_url + ) async def check_requirements(self): result = True @@ -130,45 +141,32 @@ async def check_requirements(self): async def apply(self, action): return await getattr(self, action)() - async def get_container_names(self, kind): - count = (await self.get_containers())[kind] - name = kind.__name__.lower() - if count > 1: - return [f"{name}_{i}" for i in range(count)] - return [""] - async def pull(self): logger.info("Pulling images...") - images = config["args"]["image"] - for kind in (await self.get_containers()).keys(): - if images and str(kind.__name__).lower() not in images: - continue - await kind().pull() + for spec in self.service_specs.values(): + await spec["objects"][0].pull() async def build(self): logger.info("Building images...") - images = config["args"]["image"] - for kind in (await self.get_containers()).keys(): - if images and str(kind.__name__).lower() not in images: - continue - await kind().build() + for spec in self.service_specs.values(): + await spec["objects"][0].build() async def create(self): logger.info("Creating containers...") await CephDevStackNetwork().create() await SSHKeyPair().create() containers = [] - for kind in (await self.get_containers()).keys(): - for name in await self.get_container_names(kind): - containers.append(kind(name=name).create()) + for spec in self.service_specs.values(): + for object in spec["objects"]: + containers.append(object.create()) await asyncio.gather(*containers) async def start(self): await self.create() logger.info("Starting containers...") - for kind in (await self.get_containers()).keys(): - for name in await self.get_container_names(kind): - await kind(name=name).start() + for spec in self.service_specs.values(): + for object in spec["objects"]: + await object.start() logger.info( "All containers are running. To monitor teuthology, try running: podman " "logs -f teuthology" @@ -179,17 +177,17 @@ async def start(self): async def stop(self): logger.info("Stopping containers...") containers = [] - for kind in (await self.get_containers()).keys(): - for name in await self.get_container_names(kind): - containers.append(kind(name=name).stop()) + for spec in self.service_specs.values(): + for object in spec["objects"]: + containers.append(object.stop()) await asyncio.gather(*containers) async def remove(self): logger.info("Removing containers...") containers = [] - for kind in (await self.get_containers()).keys(): - for name in await self.get_container_names(kind): - containers.append(kind(name=name).remove()) + for spec in self.service_specs.values(): + for object in spec["objects"]: + containers.append(object.remove()) await asyncio.gather(*containers) await CephDevStackNetwork().remove() await SSHKeyPair().remove() @@ -197,11 +195,11 @@ async def remove(self): async def watch(self): logger.info("Watching containers; will replace any that are stopped") containers = [] - for kind, count in (await self.get_containers()).items(): - if not count > 0: + for spec in self.service_specs.values(): + if not spec["count"] > 0: continue - for name in await self.get_container_names(kind): - containers.append(kind(name=name)) + for object in spec["objects"]: + containers.append(object) logger.info(f"Watching {containers}") while True: try: @@ -222,10 +220,9 @@ async def watch(self): break async def wait(self, container_name: str): - for kind in (await self.get_containers()).keys(): - for name in await self.get_container_names(kind): - container = kind(name=name) - if container.name == container_name: - return await container.wait() + for spec in self.service_specs.values(): + for object in spec["objects"]: + if object.name == container_name: + return await object.wait() logger.error(f"Could not find container {container_name}") return 1 diff --git a/ceph_devstack/resources/ceph/containers.py b/ceph_devstack/resources/ceph/containers.py index 66a08f834..c36d5b781 100644 --- a/ceph_devstack/resources/ceph/containers.py +++ b/ceph_devstack/resources/ceph/containers.py @@ -42,6 +42,15 @@ class Postgres(Container): "APP_DB_NAME": "paddles", } + def __init__(self, name: str = ""): + super().__init__(name) + username = self.env_vars["APP_DB_USER"] + password = self.env_vars["APP_DB_PASS"] + db_name = self.env_vars["APP_DB_NAME"] + self.paddles_sqla_url = ( + f"postgresql+psycopg2://{username}:{password}@postgres:5432/{db_name}" + ) + class Beanstalk(Container): _name = "beanstalk" @@ -84,7 +93,6 @@ class Paddles(Container): ] env_vars = { "PADDLES_SERVER_HOST": "0.0.0.0", - "PADDLES_SQLALCHEMY_URL": "postgresql+psycopg2://admin:password@postgres:5432/paddles", "PADDLES_JOB_LOG_HREF_TEMPL": f"http://{host.hostname()}:8000" "/{run_name}/{job_id}/teuthology.log", } diff --git a/pytest.ini b/pytest.ini index 2f4c80e30..c8c9c7579 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] asyncio_mode = auto +asyncio_default_fixture_loop_scope = function diff --git a/setup.cfg b/setup.cfg index 0a219eae0..71dded2e0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ install_requires = packaging pre-commit PyYAML + tomlkit python_requires = >=3.8 include_package_data = True