|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Guard already-published documentation URLs against silent removal. |
| 3 | +
|
| 4 | +The 10.0 restructure renamed and moved pages, which turned every external |
| 5 | +link written against the old layout into a 404. ``docs/conf.py`` now carries |
| 6 | +a ``redirects`` table so those URLs resolve again. This script stops the same |
| 7 | +thing happening a second time. |
| 8 | +
|
| 9 | +The invariant: **every URL the published site serves must keep resolving in a |
| 10 | +fresh build.** A page may be renamed, merged, or deleted — but only if a |
| 11 | +redirect takes its place. |
| 12 | +
|
| 13 | +That invariant is checked against ``docs/published-urls.txt``, a committed |
| 14 | +inventory of the URLs the site serves. The check is offline and deterministic: |
| 15 | +the same commit always produces the same verdict, with no network call and no |
| 16 | +model in the loop. ``--update`` is the only mode that reads the live site. |
| 17 | +
|
| 18 | +Removing a page without a redirect fails the check. Adding the redirect fixes |
| 19 | +it for free: sphinx-reredirects writes a real stub file at the old path, so |
| 20 | +the same file-existence test covers "page still here" and "page moved, with a |
| 21 | +forwarding stub" without needing to tell them apart. |
| 22 | +
|
| 23 | +Usage:: |
| 24 | +
|
| 25 | + # CI: verify the build against the committed inventory. |
| 26 | + uv run python scripts/check_docs_urls.py |
| 27 | +
|
| 28 | + # Maintainer: fold newly added pages (and anything live that the |
| 29 | + # inventory is missing) into docs/published-urls.txt, then commit it. |
| 30 | + uv run python scripts/check_docs_urls.py --update |
| 31 | +
|
| 32 | +``--update`` only ever *adds*. Running it after deleting a page will not make |
| 33 | +the check pass, because the deleted URL stays in the inventory — which is the |
| 34 | +point. Retiring a URL for real means deleting its line by hand, in a diff a |
| 35 | +reviewer can see, and that is a decision to make only for a URL that was |
| 36 | +never publicly reachable. |
| 37 | +""" |
| 38 | + |
| 39 | +from __future__ import annotations |
| 40 | + |
| 41 | +import argparse |
| 42 | +import sys |
| 43 | +import xml.etree.ElementTree as ET |
| 44 | +from pathlib import Path |
| 45 | +from urllib.parse import urljoin |
| 46 | + |
| 47 | +REPO_ROOT = Path(__file__).resolve().parents[1] |
| 48 | +DEFAULT_BUILD_DIR = REPO_ROOT / "docs" / "_build" / "html" |
| 49 | +DEFAULT_BASELINE = REPO_ROOT / "docs" / "published-urls.txt" |
| 50 | +DEFAULT_SITEMAP_URL = "https://sdk.pinecone.io/python/sitemap.xml" |
| 51 | + |
| 52 | +SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} |
| 53 | + |
| 54 | +# URLs deliberately outside the stability promise. The _modules/ tree is |
| 55 | +# viewcode's source listings, named after Python module paths — holding those |
| 56 | +# URLs stable would mean never renaming an internal module. The other three are |
| 57 | +# Sphinx infrastructure that every build regenerates unconditionally. |
| 58 | +EXCLUDED_PREFIXES = ("_modules/",) |
| 59 | +EXCLUDED_NAMES = frozenset({"genindex.html", "py-modindex.html", "search.html"}) |
| 60 | + |
| 61 | +BASELINE_HEADER = """\ |
| 62 | +# Documentation URLs that sdk.pinecone.io/python has published. |
| 63 | +# |
| 64 | +# Checked by scripts/check_docs_urls.py against the pages `make -C docs html` |
| 65 | +# produces, so a page cannot be removed without a redirect taking its place. |
| 66 | +# Paths are relative to https://sdk.pinecone.io/python/. |
| 67 | +# |
| 68 | +# To add newly published pages: uv run python scripts/check_docs_urls.py --update |
| 69 | +# Removing a line retires a URL for good — only do that for one that was never |
| 70 | +# publicly reachable. |
| 71 | +""" |
| 72 | + |
| 73 | + |
| 74 | +def _is_tracked(relative_path: str) -> bool: |
| 75 | + if relative_path.startswith(EXCLUDED_PREFIXES): |
| 76 | + return False |
| 77 | + return relative_path.rsplit("/", 1)[-1] not in EXCLUDED_NAMES |
| 78 | + |
| 79 | + |
| 80 | +def built_urls(build_dir: Path) -> set[str]: |
| 81 | + """URL paths the Sphinx build emits, including sphinx-reredirects stubs.""" |
| 82 | + if not build_dir.is_dir(): |
| 83 | + raise SystemExit( |
| 84 | + f"error: no docs build at {build_dir}\n" |
| 85 | + " run `make -C docs html` first, or pass --build-dir." |
| 86 | + ) |
| 87 | + urls = {path.relative_to(build_dir).as_posix() for path in build_dir.rglob("*.html")} |
| 88 | + tracked = {url for url in urls if _is_tracked(url)} |
| 89 | + if not tracked: |
| 90 | + raise SystemExit(f"error: {build_dir} contains no documentation pages.") |
| 91 | + return tracked |
| 92 | + |
| 93 | + |
| 94 | +def baseline_urls(baseline: Path) -> set[str]: |
| 95 | + if not baseline.is_file(): |
| 96 | + raise SystemExit( |
| 97 | + f"error: no URL inventory at {baseline}\n" |
| 98 | + " create it with `--update` after building the docs." |
| 99 | + ) |
| 100 | + lines = baseline.read_text(encoding="utf-8").splitlines() |
| 101 | + return {line.strip() for line in lines if line.strip() and not line.startswith("#")} |
| 102 | + |
| 103 | + |
| 104 | +def published_urls(sitemap_url: str) -> set[str]: |
| 105 | + """URL paths the live sitemap advertises. |
| 106 | +
|
| 107 | + Redirect stubs are not Sphinx documents, so they never appear here. The |
| 108 | + sitemap is a lower bound on what the site serves, which is why --update |
| 109 | + unions it with the build rather than trusting it alone. |
| 110 | + """ |
| 111 | + # Imported here, not at module scope, so the offline check CI runs needs |
| 112 | + # nothing but the standard library. |
| 113 | + import httpx |
| 114 | + |
| 115 | + site_root = urljoin(sitemap_url, ".") |
| 116 | + try: |
| 117 | + response = httpx.get(sitemap_url, timeout=30, follow_redirects=True) |
| 118 | + response.raise_for_status() |
| 119 | + except httpx.HTTPError as exc: |
| 120 | + raise SystemExit( |
| 121 | + f"error: could not fetch {sitemap_url}: {exc}\n" |
| 122 | + " pass --no-fetch to update from the local build alone." |
| 123 | + ) from exc |
| 124 | + |
| 125 | + document = response.text |
| 126 | + # A sitemap has no legitimate use for a doctype or entity declaration, and |
| 127 | + # refusing them here is what keeps ElementTree's expat parser away from an |
| 128 | + # entity-expansion bomb -- the reason for the S314 suppression below. |
| 129 | + if "<!DOCTYPE" in document or "<!ENTITY" in document: |
| 130 | + raise SystemExit(f"error: {sitemap_url} declares entities; refusing to parse it.") |
| 131 | + |
| 132 | + root = ET.fromstring(document) # noqa: S314 - entity declarations rejected above |
| 133 | + urls = set() |
| 134 | + for location in root.findall(".//sm:url/sm:loc", SITEMAP_NS): |
| 135 | + url = (location.text or "").strip() |
| 136 | + if not url.startswith(site_root): |
| 137 | + raise SystemExit(f"error: sitemap entry outside {site_root}: {url}") |
| 138 | + urls.add(url[len(site_root) :]) |
| 139 | + if not urls: |
| 140 | + raise SystemExit(f"error: {sitemap_url} advertised no URLs.") |
| 141 | + return {url for url in urls if _is_tracked(url)} |
| 142 | + |
| 143 | + |
| 144 | +def check(build_dir: Path, baseline: Path) -> int: |
| 145 | + built = built_urls(build_dir) |
| 146 | + expected = baseline_urls(baseline) |
| 147 | + |
| 148 | + removed = sorted(expected - built) |
| 149 | + added = sorted(built - expected) |
| 150 | + |
| 151 | + if removed: |
| 152 | + print( |
| 153 | + f"error: {len(removed)} published URL(s) are no longer produced by this build:", |
| 154 | + file=sys.stderr, |
| 155 | + ) |
| 156 | + for url in removed: |
| 157 | + print(f" {url}", file=sys.stderr) |
| 158 | + print( |
| 159 | + "\nEvery published URL has to keep resolving. Either restore the page, or\n" |
| 160 | + "add an entry to `redirects` in docs/conf.py pointing at its replacement —\n" |
| 161 | + "sphinx-reredirects writes a forwarding stub at the old path, which\n" |
| 162 | + "satisfies this check.", |
| 163 | + file=sys.stderr, |
| 164 | + ) |
| 165 | + if added: |
| 166 | + print( |
| 167 | + f"error: {len(added)} page(s) are built but absent from {baseline.name}:", |
| 168 | + file=sys.stderr, |
| 169 | + ) |
| 170 | + for url in added: |
| 171 | + print(f" {url}", file=sys.stderr) |
| 172 | + print( |
| 173 | + "\nRun `uv run python scripts/check_docs_urls.py --update` and commit\n" |
| 174 | + f"docs/{baseline.name} so these URLs are protected from now on.", |
| 175 | + file=sys.stderr, |
| 176 | + ) |
| 177 | + if removed or added: |
| 178 | + return 1 |
| 179 | + |
| 180 | + print(f"ok: all {len(expected)} published URLs still resolve in this build.") |
| 181 | + return 0 |
| 182 | + |
| 183 | + |
| 184 | +def update(build_dir: Path, baseline: Path, sitemap_url: str, fetch: bool) -> int: |
| 185 | + known = baseline_urls(baseline) if baseline.is_file() else set() |
| 186 | + discovered = built_urls(build_dir) |
| 187 | + if fetch: |
| 188 | + discovered |= published_urls(sitemap_url) |
| 189 | + |
| 190 | + merged = known | discovered |
| 191 | + added = sorted(merged - known) |
| 192 | + |
| 193 | + baseline.write_text(BASELINE_HEADER + "\n".join(sorted(merged)) + "\n", encoding="utf-8") |
| 194 | + |
| 195 | + if added: |
| 196 | + print(f"added {len(added)} URL(s) to {baseline}:") |
| 197 | + for url in added: |
| 198 | + print(f" {url}") |
| 199 | + else: |
| 200 | + print(f"no new URLs; {baseline} already lists all {len(merged)}.") |
| 201 | + |
| 202 | + stale = sorted(known - discovered) |
| 203 | + if stale: |
| 204 | + print( |
| 205 | + f"\nnote: {len(stale)} inventoried URL(s) are not in this build and were kept.\n" |
| 206 | + " The check will fail until each is restored or redirected:", |
| 207 | + ) |
| 208 | + for url in stale: |
| 209 | + print(f" {url}") |
| 210 | + return 0 |
| 211 | + |
| 212 | + |
| 213 | +def main(argv: list[str] | None = None) -> int: |
| 214 | + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) |
| 215 | + parser.add_argument( |
| 216 | + "--update", |
| 217 | + action="store_true", |
| 218 | + help="add newly built and newly published URLs to the inventory", |
| 219 | + ) |
| 220 | + parser.add_argument( |
| 221 | + "--no-fetch", |
| 222 | + action="store_true", |
| 223 | + help="with --update, skip the live sitemap and use the local build alone", |
| 224 | + ) |
| 225 | + parser.add_argument("--build-dir", type=Path, default=DEFAULT_BUILD_DIR) |
| 226 | + parser.add_argument("--baseline", type=Path, default=DEFAULT_BASELINE) |
| 227 | + parser.add_argument("--sitemap-url", default=DEFAULT_SITEMAP_URL) |
| 228 | + args = parser.parse_args(argv) |
| 229 | + |
| 230 | + if args.no_fetch and not args.update: |
| 231 | + parser.error("--no-fetch only applies to --update") |
| 232 | + |
| 233 | + if args.update: |
| 234 | + return update(args.build_dir, args.baseline, args.sitemap_url, fetch=not args.no_fetch) |
| 235 | + return check(args.build_dir, args.baseline) |
| 236 | + |
| 237 | + |
| 238 | +if __name__ == "__main__": |
| 239 | + sys.exit(main()) |
0 commit comments