diff --git a/.github/workflows/testpr.yml b/.github/workflows/testpr.yml index e6e2742ba..0e53aa32f 100644 --- a/.github/workflows/testpr.yml +++ b/.github/workflows/testpr.yml @@ -134,6 +134,12 @@ jobs: run: | pixi run vinca-gha --platform ${{ matrix.platform }} --trigger-branch dummy_build_branch_as_it_is_unused -d ./recipes + - name: Verify platform outputs are complete before publish + if: ${{ success() && env.IGNORE_CACHE_AND_DO_FULL_REBUILD == 'true' }} + shell: bash -l {0} + run: | + pixi run python publish_conda_outputs.py --dry-run --platform ${{ matrix.platform }} --output-dir "${{ matrix.folder_cache }}/.." + - name: Upload build cache as artifact if: ${{ always() && env.SAVE_CACHE_AS_ARTIFACT == 'true' }} uses: actions/upload-artifact@v6 diff --git a/.scripts/build_unix.sh b/.scripts/build_unix.sh index c4ac97060..24428edc7 100755 --- a/.scripts/build_unix.sh +++ b/.scripts/build_unix.sh @@ -35,6 +35,8 @@ else cross_compile="" fi +skip_upload="${VINCA_SKIP_UPLOAD:-0}" + for recipe in ${CURRENT_RECIPES[@]}; do pixi run -v rattler-build build \ @@ -48,6 +50,11 @@ for recipe in ${CURRENT_RECIPES[@]}; do done +if [[ "$skip_upload" == "1" ]]; then + echo "VINCA_SKIP_UPLOAD=1, skipping immediate upload for $target" + exit 0 +fi + # Check if it build something, this is a hotfix for the skips inside additional_recipes if compgen -G "${CONDA_BLD_PATH}/${target}*/*.conda" > /dev/null; then pixi run upload "${CONDA_BLD_PATH}/${target}"*/*.conda --force diff --git a/.scripts/build_win.bat b/.scripts/build_win.bat index aa49c1936..0b8c07ae3 100644 --- a/.scripts/build_win.bat +++ b/.scripts/build_win.bat @@ -8,6 +8,8 @@ rmdir /Q/S "C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\" set "FEEDSTOCK_ROOT=%cd%" +if "%VINCA_SKIP_UPLOAD%"=="" set "VINCA_SKIP_UPLOAD=0" + mkdir %CONDA_BLD_PATH% :: Enable long path names on Windows @@ -25,6 +27,11 @@ for %%X in (%CURRENT_RECIPES%) do ( rem -m %FEEDSTOCK_ROOT%\.ci_support\conda_forge_pinnings.yaml ) +if "%VINCA_SKIP_UPLOAD%"=="1" ( + echo VINCA_SKIP_UPLOAD=1, skipping immediate upload for win-64 + exit /b 0 +) + :: Check if .conda files exist in the win-64 directory if exist "%CONDA_BLD_PATH%\win-64\*.conda" ( echo Found .conda files, starting upload... diff --git a/AGENTS.md b/AGENTS.md index eb88f5a4a..50765631b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -206,8 +206,11 @@ pixi run build ``` ## Full rebuilds + For full rebuilds also remember: - refresh snapshot: `pixi run create_snapshot` +- validate that the published channel is not in a mixed mutex state: `pixi run check-channel-mutex-consistency` +- validate that local outputs are complete for the platform you are about to publish: `pixi run check-release-readiness --platform ` - update `conda_build_config.yaml` for active migrations. You can use https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/main/recipe/conda_build_config.yaml as a base, and then also apply migrations that are mostly done; you can check the status at https://conda-forge.org/status/. - bump `build_number` - bump mutex minor and update hardcoded mutex refs where needed diff --git a/build_gap_report.py b/build_gap_report.py index 47c671996..713976ed1 100644 --- a/build_gap_report.py +++ b/build_gap_report.py @@ -41,6 +41,16 @@ def parse_args() -> argparse.Namespace: "If omitted, all detected platform folders are inspected." ), ) + parser.add_argument( + "--fail-on-extra", + action="store_true", + help="Exit with code 1 if built artifacts exist without a matching recipe directory.", + ) + parser.add_argument( + "--fail-on-missing", + action="store_true", + help="Exit with code 1 if any generated recipe is missing a built artifact.", + ) return parser.parse_args() @@ -106,6 +116,16 @@ def recipe_directories(recipes_dir: Path) -> Set[str]: return {entry.name for entry in recipes_dir.iterdir() if entry.is_dir()} +def gap_report_for_platform( + output_root: Path, recipes_dir: Path, platform: str +) -> tuple[Set[str], Set[str], Set[str]]: + built = built_packages_for_platform(output_root, platform) + recipes = recipe_directories(recipes_dir) + extra = built - recipes + missing = recipes - built + return built, extra, missing + + def print_list(title: str, values: Iterable[str]) -> None: values = sorted(values) print(f"{title}: {len(values)}") @@ -132,16 +152,17 @@ def main() -> int: ) return 1 + exit_code = 0 + for idx, platform in enumerate(selected_platforms): - built = built_packages_for_platform(output_root, platform) + built, extra, missing = gap_report_for_platform(output_root, recipes_dir, platform) print(f"Platform: {platform}") print_list( "Built package artifacts without matching recipe directory", - built - recipes, + extra, ) print() - missing = recipes - built print( f"Recipe directories without built artifact on this platform: " f"{len(missing)} out of {len(recipes)}" @@ -150,10 +171,15 @@ def main() -> int: for recipe in sorted(missing): print(f" - {recipe}") + if args.fail_on_extra and extra: + exit_code = 1 + if args.fail_on_missing and missing: + exit_code = 1 + if idx != len(selected_platforms) - 1: print("\n" + "-" * 72 + "\n") - return 0 + return exit_code if __name__ == "__main__": diff --git a/check_channel_mutex_consistency.py b/check_channel_mutex_consistency.py new file mode 100644 index 000000000..1b14f3170 --- /dev/null +++ b/check_channel_mutex_consistency.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Validate that the latest published ROS packages use a consistent mutex generation. + +This checks the current published channel state, not the local recipes. It is meant to +catch partial migrations where some latest packages still depend on an older +``ros2-distro-mutex`` generation while the repository has already moved ``vinca.yaml`` +to a newer one. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import urllib.error +import urllib.request +from collections import Counter +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_CHANNEL_BASE = "https://prefix.dev/robostack-kilted" +DEFAULT_PLATFORMS = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64", "win-64"] +USER_AGENT = "Mozilla/5.0 (compatible; RoboStack mutex consistency checker)" +PACKAGE_PREFIX = "ros-kilted-" +MUTEX_NAME = "ros2-distro-mutex" + + +@dataclass(frozen=True) +class LatestPackage: + name: str + filename: str + version: str + build_number: int + depends: tuple[str, ...] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Check that the latest published ROS packages in a channel all depend on " + "the mutex generation declared in vinca.yaml." + ) + ) + parser.add_argument( + "--channel-base", + default=DEFAULT_CHANNEL_BASE, + help=( + "Channel base URL, without platform suffix " + f"(default: {DEFAULT_CHANNEL_BASE})" + ), + ) + parser.add_argument( + "--platform", + action="append", + default=[], + help=( + "Platform subdir to inspect (repeatable). " + "Defaults to the standard RoboStack platforms." + ), + ) + parser.add_argument( + "--vinca-file", + default="vinca.yaml", + help="Path to the vinca.yaml file that declares mutex_package.version.", + ) + parser.add_argument( + "--expected-mutex-version", + default=None, + help="Override the expected mutex version instead of reading it from vinca.yaml.", + ) + parser.add_argument( + "--package-prefix", + default=PACKAGE_PREFIX, + help=f"Only inspect packages with this prefix (default: {PACKAGE_PREFIX}).", + ) + parser.add_argument( + "--ignore-package", + action="append", + default=[], + help="Package name to ignore (repeatable).", + ) + parser.add_argument( + "--max-report", + type=int, + default=20, + help="Maximum number of offending packages to print per category (default: 20).", + ) + return parser.parse_args() + + +def read_expected_mutex_version(vinca_file: Path) -> str: + in_mutex_block = False + for raw_line in vinca_file.read_text(encoding="utf-8").splitlines(): + line = raw_line.rstrip() + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if not line.startswith(" ") and stripped == "mutex_package:": + in_mutex_block = True + continue + if in_mutex_block and not line.startswith(" "): + break + if in_mutex_block: + match = re.match(r'\s*version:\s*["\']?([^"\']+)["\']?\s*$', line) + if match: + return match.group(1) + raise ValueError(f"Could not find mutex_package.version in {vinca_file}") + + +def normalize_mutex_generation(version_or_spec: str) -> str | None: + match = re.search(r"(\d+)\.(\d+)", version_or_spec) + if not match: + return None + return f"{match.group(1)}.{match.group(2)}" + + +def version_key(version: str) -> tuple[tuple[int, int | str], ...]: + key: list[tuple[int, int | str]] = [] + for part in re.split(r"[^0-9A-Za-z]+", version): + if not part: + continue + if part.isdigit(): + key.append((0, int(part))) + else: + key.append((1, part)) + return tuple(key) + + +def fetch_repodata(channel_base: str, platform: str) -> dict: + url = f"{channel_base.rstrip('/')}/{platform}/repodata.json" + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=90) as response: + return json.load(response) + + +def latest_packages(repodata: dict, package_prefix: str, ignored_packages: set[str]) -> dict[str, LatestPackage]: + latest: dict[str, tuple[tuple[tuple[int, int | str], ...], int, LatestPackage]] = {} + for package_set in ("packages.conda", "packages"): + for filename, metadata in repodata.get(package_set, {}).items(): + name = metadata.get("name") + if not name or not name.startswith(package_prefix) or name in ignored_packages: + continue + candidate = LatestPackage( + name=name, + filename=filename, + version=str(metadata.get("version", "")), + build_number=int(metadata.get("build_number", 0)), + depends=tuple(metadata.get("depends", [])), + ) + ordering = (version_key(candidate.version), candidate.build_number, candidate) + current = latest.get(name) + if current is None or ordering[:2] > current[:2]: + latest[name] = ordering + return {name: item[2] for name, item in latest.items()} + + +def mutex_generations(depends: tuple[str, ...]) -> set[str]: + generations: set[str] = set() + for dependency in depends: + if not dependency.startswith(MUTEX_NAME): + continue + generation = normalize_mutex_generation(dependency) + if generation: + generations.add(generation) + return generations + + +def print_package_list(title: str, packages: list[LatestPackage], max_report: int) -> None: + print(f"{title}: {len(packages)}") + for package in packages[:max_report]: + mutex_deps = [dep for dep in package.depends if dep.startswith(MUTEX_NAME)] + print( + " - " + f"{package.name} {package.version} build {package.build_number} " + f"({package.filename}) -> {mutex_deps or ['']}" + ) + remaining = len(packages) - max_report + if remaining > 0: + print(f" ... {remaining} more") + + +def main() -> int: + args = parse_args() + expected_version = args.expected_mutex_version + if expected_version is None: + expected_version = read_expected_mutex_version(Path(args.vinca_file)) + expected_generation = normalize_mutex_generation(expected_version) + if expected_generation is None: + raise ValueError(f"Could not parse expected mutex generation from {expected_version!r}") + + platforms = args.platform or DEFAULT_PLATFORMS + ignored_packages = set(args.ignore_package) + exit_code = 0 + + for index, platform in enumerate(platforms): + try: + repodata = fetch_repodata(args.channel_base, platform) + except urllib.error.URLError as exc: + print(f"Platform: {platform}") + print(f"Failed to fetch repodata: {exc}") + exit_code = 1 + if index != len(platforms) - 1: + print("\n" + "-" * 72 + "\n") + continue + + latest = latest_packages(repodata, args.package_prefix, ignored_packages) + generation_counts: Counter[str] = Counter() + missing_mutex: list[LatestPackage] = [] + unexpected_mutex: list[LatestPackage] = [] + + for package in latest.values(): + generations = mutex_generations(package.depends) + if not generations: + missing_mutex.append(package) + continue + generation_counts.update(generations) + if generations != {expected_generation}: + unexpected_mutex.append(package) + + print(f"Platform: {platform}") + print(f"Expected mutex generation: {expected_generation}") + print(f"Latest packages checked: {len(latest)}") + if generation_counts: + counts = ", ".join( + f"{generation}={count}" for generation, count in sorted(generation_counts.items()) + ) + print(f"Observed mutex generations: {counts}") + else: + print("Observed mutex generations: none") + + print_package_list("Packages on an unexpected mutex generation", sorted(unexpected_mutex, key=lambda item: item.name), args.max_report) + print_package_list("Packages missing a mutex dependency", sorted(missing_mutex, key=lambda item: item.name), args.max_report) + + if unexpected_mutex or missing_mutex: + exit_code = 1 + + if index != len(platforms) - 1: + print("\n" + "-" * 72 + "\n") + + if exit_code == 0: + print("Channel mutex consistency check passed.") + else: + print( + "Channel mutex consistency check failed. " + "This usually means the published channel is in a partial migration state." + ) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/pixi.toml b/pixi.toml index 166e37eb4..b8aa5bd5b 100644 --- a/pixi.toml +++ b/pixi.toml @@ -35,8 +35,12 @@ vinca = { git ="https://github.com/RoboStack/vinca.git", rev = "0e75b9673808de39 generate-recipes = { cmd = "vinca -m", depends-on = ["remove-recipes"] } generate-gha-workflows = { cmd = "vinca-gha --trigger-branch dummy_build_branch_as_it_is_unused -d ./recipes", depends-on = ["generate-recipes"] } check-patches = { cmd = "python check_patches_clean_apply.py", depends-on = ["generate-recipes"] } +check-channel-mutex-consistency = { cmd = "python check_channel_mutex_consistency.py" } create_snapshot = { cmd = "vinca-snapshot -d kilted -o rosdistro_snapshot.yaml" } -upload = "rattler-build upload anaconda -o robostack-kilted -a $ANACONDA_API_TOKEN" +check-release-readiness = { cmd = "python publish_conda_outputs.py --dry-run" } +upload-built-artifacts = { cmd = "python publish_conda_outputs.py" } +upload = { cmd = "python publish_conda_outputs.py" } +upload-raw = "rattler-build upload anaconda -o robostack-kilted -a $ANACONDA_API_TOKEN" build_continue_on_failure = { cmd = "rattler-build build --recipe-dir ./recipes -m ./conda_build_config.yaml -c robostack-kilted -c https://repo.prefix.dev/conda-forge --continue-on-failure --skip-existing", depends-on = ["generate-recipes"] } [tasks.build] diff --git a/publish_conda_outputs.py b/publish_conda_outputs.py new file mode 100644 index 000000000..73cdd2a44 --- /dev/null +++ b/publish_conda_outputs.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Upload built artifacts only after validating the selected platforms are complete. + +The script regenerates the expected recipes per platform in an isolated temporary copy of +the repository, compares them with the local output artifacts, and refuses to upload when +any generated recipe is still missing a package artifact. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Iterable + +from build_gap_report import CONDA_SUFFIX, TARBZ2_SUFFIX, gap_report_for_platform, recipe_directories + +DEFAULT_OWNER = "robostack-kilted" +DEFAULT_BATCH_SIZE = 200 + + +def default_api_key() -> str | None: + return os.environ.get("ANACONDA_API_KEY") or os.environ.get("ANACONDA_API_TOKEN") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Validate that the selected platforms have complete local build outputs " + "before uploading any artifacts to Anaconda.org." + ) + ) + parser.add_argument( + "--repo-root", + default=".", + help="Repository root used to regenerate recipes (default: current directory).", + ) + parser.add_argument( + "--output-dir", + default="output", + help="Directory containing built artifacts grouped by platform (default: output).", + ) + parser.add_argument( + "--platform", + action="append", + default=[], + help=( + "Platform to validate and upload (repeatable). " + "Defaults to the platform folders that currently contain package artifacts." + ), + ) + parser.add_argument( + "--owner", + default=DEFAULT_OWNER, + help=f"Anaconda owner to upload to (default: {DEFAULT_OWNER}).", + ) + parser.add_argument( + "--api-key", + default=default_api_key(), + help="Anaconda API key. Defaults to ANACONDA_API_KEY or ANACONDA_API_TOKEN if set.", + ) + parser.add_argument( + "--channel", + default=None, + help="Optional Anaconda label/channel to upload to, for example rc.", + ) + parser.add_argument( + "--url", + default=None, + help="Optional alternate Anaconda server URL.", + ) + parser.add_argument( + "--batch-size", + type=int, + default=DEFAULT_BATCH_SIZE, + help=f"How many package files to upload per command invocation (default: {DEFAULT_BATCH_SIZE}).", + ) + parser.add_argument( + "--force", + action="store_true", + help="Pass --force to rattler-build upload.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate readiness and print the upload plan without uploading.", + ) + parser.add_argument( + "--skip-validate", + action="store_true", + help="Skip completeness validation and upload whatever is present in output/. Use sparingly.", + ) + return parser.parse_args() + + +def chunked(values: list[Path], chunk_size: int) -> Iterable[list[Path]]: + for index in range(0, len(values), chunk_size): + yield values[index : index + chunk_size] + + +def artifact_files_for_platform(output_dir: Path, platform: str) -> list[Path]: + platform_dir = output_dir / platform + if not platform_dir.exists(): + return [] + return [ + path + for path in sorted(platform_dir.iterdir()) + if path.is_file() + and (path.name.endswith(CONDA_SUFFIX) or path.name.endswith(TARBZ2_SUFFIX)) + ] + + +def discover_platforms(output_dir: Path) -> list[str]: + if not output_dir.exists(): + return [] + platforms: list[str] = [] + for entry in sorted(output_dir.iterdir()): + if not entry.is_dir(): + continue + artifact_files = artifact_files_for_platform(output_dir, entry.name) + if artifact_files: + platforms.append(entry.name) + return platforms + + +def create_repo_copy(repo_root: Path, temp_root: Path, platform: str) -> Path: + copy_root = temp_root / platform + shutil.copytree( + repo_root, + copy_root, + ignore=shutil.ignore_patterns( + ".git", + ".pixi", + "output", + "recipes", + "recipes_only_patch", + "__pycache__", + "*.pyc", + ), + ) + return copy_root + + +def generate_recipes(repo_copy: Path, platform: str) -> Path: + recipes_dir = repo_copy / "recipes" + subprocess.run( + ["vinca", "--platform", platform, "-m", "-n"], + cwd=repo_copy, + check=True, + ) + if not recipes_dir.exists(): + raise RuntimeError(f"vinca did not generate recipes for {platform}") + return recipes_dir + + +def validate_platforms(repo_root: Path, output_dir: Path, platforms: list[str]) -> int: + exit_code = 0 + with tempfile.TemporaryDirectory(prefix="ros-kilted-release-") as temp_dir_str: + temp_dir = Path(temp_dir_str) + for index, platform in enumerate(platforms): + repo_copy = create_repo_copy(repo_root, temp_dir, platform) + recipes_dir = generate_recipes(repo_copy, platform) + recipes = recipe_directories(recipes_dir) + _, extra, missing = gap_report_for_platform(output_dir, recipes_dir, platform) + + print(f"Platform: {platform}") + print(f"Generated recipes: {len(recipes)}") + print(f"Built artifacts without matching recipe: {len(extra)}") + if extra: + for package in sorted(extra)[:20]: + print(f" - {package}") + if len(extra) > 20: + print(f" ... {len(extra) - 20} more") + print(f"Missing built artifacts: {len(missing)}") + if missing: + for package in sorted(missing)[:20]: + print(f" - {package}") + if len(missing) > 20: + print(f" ... {len(missing) - 20} more") + exit_code = 1 + + if index != len(platforms) - 1: + print("\n" + "-" * 72 + "\n") + + return exit_code + + +def upload_files(args: argparse.Namespace, files: list[Path]) -> int: + if not args.api_key: + print("No Anaconda API key configured. Set ANACONDA_API_KEY or ANACONDA_API_TOKEN.") + return 1 + + for batch_index, batch in enumerate(chunked(files, args.batch_size), start=1): + cmd = [ + "rattler-build", + "upload", + "anaconda", + "--owner", + args.owner, + "--api-key", + args.api_key, + ] + if args.channel: + cmd.extend(["--channel", args.channel]) + if args.url: + cmd.extend(["--url", args.url]) + if args.force: + cmd.append("--force") + cmd.extend(str(path) for path in batch) + print( + f"Uploading batch {batch_index} with {len(batch)} artifacts to owner {args.owner}" + + (f" channel {args.channel}" if args.channel else "") + ) + subprocess.run(cmd, check=True) + return 0 + + +def main() -> int: + args = parse_args() + repo_root = Path(args.repo_root).resolve() + output_dir = (repo_root / args.output_dir).resolve() + + if not output_dir.exists(): + print(f"Output directory does not exist: {output_dir}") + return 1 + + platforms = args.platform or discover_platforms(output_dir) + if not platforms: + print( + "No platform output folders with built package artifacts were found under " + f"{output_dir}." + ) + return 1 + + if not args.skip_validate: + validation_exit_code = validate_platforms(repo_root, output_dir, platforms) + if validation_exit_code != 0: + print("Release readiness check failed. Refusing to upload partial outputs.") + return validation_exit_code + + files_to_upload: list[Path] = [] + for platform in platforms: + files = artifact_files_for_platform(output_dir, platform) + print(f"Platform {platform} artifacts queued for upload: {len(files)}") + files_to_upload.extend(files) + + if not files_to_upload: + print("No package artifacts found to upload.") + return 1 + + if args.dry_run: + print("Dry run only. Validation passed; no upload was performed.") + return 0 + + return upload_files(args, files_to_upload) + + +if __name__ == "__main__": + sys.exit(main())