Breeze: make breeze down discover and stop every compose project

dev/breeze/doc/03_developer_tasks.rst

@@ -694,6 +694,19 @@ You can always stop it via:
 
    breeze down
 
+``breeze down`` discovers every running docker compose project that breeze knows
+about — ``breeze shell``, ``breeze testing``, ``breeze build-docs``, ``breeze db``,
+release-management, registry, ``breeze run``, and prek-hook compose projects — by
+reading the ``com.docker.compose.project`` label that compose sets on every container
+it creates. Each matching project is brought down with ``--remove-orphans`` and
+``--volumes`` (unless ``--preserve-volumes`` is passed). One ``breeze down`` is
+enough to leave the host clean.
+
+If you have an unrelated docker compose project running on the host that does not
+match any breeze prefix, it is left alone by default. Pass ``--all-projects`` to
+also bring those down. To restrict the cleanup to a single named project (useful
+in CI steps), pass ``--project-name <name>``.
+
 These are all available flags of ``down`` command:
 
 .. image:: ./images/output_down.svg

dev/breeze/doc/images/output_down.txt

@@ -1 +1 @@
-5c5712f40f3397d077afce24bc6b5d4d
+cde032283bb58d1cf97ee92f145f1ea8

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -113,6 +113,7 @@
 from airflow_breeze.utils.confirm import Answer, user_confirm
 from airflow_breeze.utils.console import console_print
 from airflow_breeze.utils.docker_command_utils import (
+    bring_all_compose_projects_down,
     bring_compose_project_down,
     check_docker_resources,
     enter_shell,
@@ -904,7 +905,17 @@ def build_docs(
     sys.exit(result.returncode)
 
 
-@main.command(name="down", help="Stop running breeze environment.")
+@main.command(
+    name="down",
+    help=(
+        "Stop every docker compose project breeze knows about. Discovers running "
+        "projects via the `com.docker.compose.project` label and brings each one "
+        "down with `--remove-orphans` (and `--volumes` unless `--preserve-volumes` "
+        "is passed). Covers `breeze shell`, `breeze testing`, `breeze build-docs`, "
+        "`breeze db`, release-management, registry, and prek-hook compose projects "
+        "in a single command."
+    ),
+)
 @click.option(
     "-p",
     "--preserve-volumes",
@@ -923,12 +934,46 @@ def build_docs(
     help="Additionally cleanup Build (pip/uv) cache.",
     is_flag=True,
 )
+@click.option(
+    "--all-projects",
+    help=(
+        "Also bring down docker compose projects whose names do not match any known "
+        "breeze prefix. Off by default to avoid touching unrelated projects on the host."
+    ),
+    is_flag=True,
+)
+@click.option(
+    "--project-name",
+    help=(
+        "Restrict the cleanup to a single docker compose project name and skip "
+        "discovery. Useful in CI steps that want to bring exactly one project down."
+    ),
+    default=None,
+)
 @option_verbose
 @option_dry_run
-def down(preserve_volumes: bool, cleanup_mypy_cache: bool, cleanup_build_cache: bool):
+def down(
+    preserve_volumes: bool,
+    cleanup_mypy_cache: bool,
+    cleanup_build_cache: bool,
+    all_projects: bool,
+    project_name: str | None,
+):
     perform_environment_checks()
-    shell_params = ShellParams(backend="all", include_mypy_volume=cleanup_mypy_cache)
-    bring_compose_project_down(preserve_volumes=preserve_volumes, shell_params=shell_params)
+    brought_down, skipped = bring_all_compose_projects_down(
+        preserve_volumes=preserve_volumes,
+        include_unknown=all_projects,
+        only_project=project_name,
+    )
+    if not brought_down and not project_name:
+        console_print("[info]No running breeze-managed docker compose projects found.[/]")
+    elif brought_down:
+        console_print(f"[success]Brought down {len(brought_down)} compose project(s): {brought_down}[/]")
+    if skipped:
+        console_print(
+            f"[warning]Left {len(skipped)} unrelated compose project(s) running: {skipped}\n"
+            f"Use `breeze down --all-projects` to also bring those down.[/]"
+        )
     if cleanup_mypy_cache:
         command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"]
         run_command(command_to_execute)
@@ -1071,8 +1116,9 @@ def doctor(ctx):
     if not get_dry_run() and given_answer == Answer.YES:
         cleanup_python_generated_files()
 
-    shell_params = ShellParams(backend="all", include_mypy_volume=True)
-    bring_compose_project_down(preserve_volumes=False, shell_params=shell_params)
+    # Doctor is the heal-everything command, so it sweeps EVERY compose project
+    # on the host (not only the known-prefix ones that `breeze down` defaults to).
+    bring_all_compose_projects_down(preserve_volumes=False, include_unknown=True)
 
     given_answer = user_confirm("Are you sure with the removal of mypy cache and build cache dir?")
     if given_answer == Answer.YES:
@@ -1178,7 +1224,6 @@ def run(
     from airflow_breeze.params.shell_params import ShellParams
     from airflow_breeze.utils.ci_group import ci_group
     from airflow_breeze.utils.docker_command_utils import (
-        bring_compose_project_down,
         execute_command_in_shell,
         fix_ownership_using_docker,
         remove_docker_networks,

dev/breeze/src/airflow_breeze/commands/developer_commands_config.py

@@ -347,6 +347,13 @@
                 "--cleanup-build-cache",
             ],
         },
+        {
+            "name": "Project selection",
+            "options": [
+                "--all-projects",
+                "--project-name",
+            ],
+        },
     ],
     "breeze build-docs": [
         {

dev/breeze/src/airflow_breeze/commands/registry_commands.py

@@ -31,7 +31,12 @@
 from airflow_breeze.params.shell_params import ShellParams
 from airflow_breeze.utils.ci_group import ci_group
 from airflow_breeze.utils.click_utils import BreezeGroup
-from airflow_breeze.utils.docker_command_utils import execute_command_in_shell, fix_ownership_using_docker
+from airflow_breeze.utils.docker_command_utils import (
+    bring_compose_project_down,
+    execute_command_in_shell,
+    fix_ownership_using_docker,
+    remove_docker_networks,
+)
 from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH
 from airflow_breeze.utils.run_utils import run_command
 
@@ -111,15 +116,18 @@ def extract_data(python: str, provider: str | None, allow_unreleased: bool):
         f"python dev/registry/extract_connections.py{provider_flag}"
     )
 
-    with ci_group("Extracting registry data"):
-        result = execute_command_in_shell(
-            shell_params=shell_params,
-            project_name=unique_project_name,
-            command=command,
-            preserve_backend=True,
-        )
-
-    fix_ownership_using_docker()
+    try:
+        with ci_group("Extracting registry data"):
+            result = execute_command_in_shell(
+                shell_params=shell_params,
+                project_name=unique_project_name,
+                command=command,
+                preserve_backend=True,
+            )
+    finally:
+        bring_compose_project_down(preserve_volumes=False, shell_params=shell_params)
+        remove_docker_networks([f"{unique_project_name}_default"])
+        fix_ownership_using_docker()
     sys.exit(result.returncode)
 
 
@@ -364,6 +372,8 @@ def _backfill_docker(
                 failed.append(f"{version}/docker-extraction")
     finally:
         shutil.rmtree(backfill_tmp_dir, ignore_errors=True)
+        bring_compose_project_down(preserve_volumes=False, shell_params=shell_params)
+        remove_docker_networks([f"{unique_project_name}_default"])
         fix_ownership_using_docker()
 
     return failed

dev/breeze/src/airflow_breeze/global_constants.py

@@ -146,6 +146,28 @@
 ALLOWED_TTY = ["auto", "enabled", "disabled"]
 ALLOWED_TERMINAL_MULTIPLEXERS = ["mprocs", "tmux"]
 ALLOWED_DOCKER_COMPOSE_PROJECTS = ["breeze", "prek", "docker-compose"]
+
+# Every docker compose project name that any breeze command, prek hook, or
+# CI workflow uses. `breeze down` discovers running compose projects via the
+# `com.docker.compose.project` label and only touches the ones that match
+# either an exact entry in `KNOWN_DOCKER_COMPOSE_PROJECT_NAMES` or one of the
+# prefixes in `KNOWN_DOCKER_COMPOSE_PROJECT_PREFIXES`. When you add a new
+# project_name pattern anywhere (new breeze command, new prek hook, new CI
+# step), update this list so `breeze down` stays a one-shot cleanup.
+KNOWN_DOCKER_COMPOSE_PROJECT_NAMES = [
+    "breeze",  # default `breeze shell` / `breeze start-airflow`
+    "prek",  # prek hooks (see scripts/ci/prek/common_prek_utils.py)
+    "docker-compose",  # legacy name kept for migration_tests CI
+    "docs",  # `breeze build-docs`
+    "db",  # `breeze db ...`
+    "providers",  # release-management providers builds
+]
+KNOWN_DOCKER_COMPOSE_PROJECT_PREFIXES = [
+    "breeze-",  # breeze-registry-*, breeze-backfill-*, *-run-*
+    "airflow-test",  # airflow-test, airflow-test-<test-type>
+    "constraints-",  # constraints-<python-version>
+    "providers-",  # providers-<index> (parallel provider builds)
+]
 ALLOWED_LOG_LEVELS = ["INFO", "DEBUG", "WARNING", "ERROR", "CRITICAL"]
 DEFAULT_LOG_LEVEL = ALLOWED_LOG_LEVELS[0]
 

dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

@@ -54,6 +54,8 @@
     ALLOWED_DEBIAN_VERSIONS,
     DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
     DOCKER_DEFAULT_PLATFORM,
+    KNOWN_DOCKER_COMPOSE_PROJECT_NAMES,
+    KNOWN_DOCKER_COMPOSE_PROJECT_PREFIXES,
     MIN_DOCKER_COMPOSE_VERSION,
     MIN_DOCKER_VERSION,
 )
@@ -834,6 +836,83 @@ def bring_compose_project_down(preserve_volumes: bool, shell_params: ShellParams
     )
 
 
+def discover_running_compose_projects() -> set[str]:
+    """
+    Return the set of compose project names of every container on the host.
+
+    Reads the ``com.docker.compose.project`` label that ``docker compose``
+    sets on every container/network/volume it creates. Returns an empty set
+    if docker is unreachable or no compose-managed containers exist.
+    """
+    result = run_command(
+        [
+            "docker",
+            "ps",
+            "--all",
+            "--filter",
+            "label=com.docker.compose.project",
+            "--format",
+            '{{ index .Labels "com.docker.compose.project" }}',
+        ],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0 or not result.stdout:
+        return set()
+    return {line.strip() for line in result.stdout.splitlines() if line.strip()}
+
+
+def is_known_breeze_compose_project(name: str) -> bool:
+    """Return True if ``name`` matches a project breeze knows it owns."""
+    if name in KNOWN_DOCKER_COMPOSE_PROJECT_NAMES:
+        return True
+    return any(name.startswith(prefix) for prefix in KNOWN_DOCKER_COMPOSE_PROJECT_PREFIXES)
+
+
+def bring_all_compose_projects_down(
+    *,
+    preserve_volumes: bool = False,
+    include_unknown: bool = False,
+    only_project: str | None = None,
+) -> tuple[list[str], list[str]]:
+    """
+    Discover and bring down every docker compose project breeze manages.
+
+    :param preserve_volumes: if True, pass ``--volumes`` is omitted so DB
+        volumes survive (matches the existing ``--preserve-volumes`` flag
+        on ``breeze down``).
+    :param include_unknown: if True, also bring down projects whose names
+        do not match any known breeze prefix. Useful as an emergency
+        cleanup; can wipe out unrelated docker compose projects on the
+        host, so off by default.
+    :param only_project: if set, restrict to exactly this project name and
+        skip discovery entirely (for the ``--project-name`` flag).
+    :returns: ``(brought_down, skipped)`` lists of project names, both
+        sorted, suitable for printing in a session summary.
+    """
+    if only_project:
+        targets = {only_project}
+        skipped: set[str] = set()
+    else:
+        running = discover_running_compose_projects()
+        if include_unknown:
+            targets = running
+            skipped = set()
+        else:
+            targets = {name for name in running if is_known_breeze_compose_project(name)}
+            skipped = running - targets
+    brought_down: list[str] = []
+    for name in sorted(targets):
+        console_print(f"[info]Bringing down docker compose project: {name}[/]")
+        cmd = ["docker", "compose", "--project-name", name, "down", "--remove-orphans"]
+        if not preserve_volumes:
+            cmd.append("--volumes")
+        run_command(cmd, text=True, check=False)
+        brought_down.append(name)
+    return brought_down, sorted(skipped)
+
+
 def execute_command_in_shell(
     shell_params: ShellParams,
     project_name: str,