Skip to content

Commit fbd2f76

Browse files
shirgoldbirdclaude
andcommitted
fix(pipeline): skip hub pages as thin, refuse empty reads, lazy SDK import
Carries over the parts of #450 that #479 didn't cover: - detect_gaps: pages with child pages are exempt from the thin-page check. breaking-changes-change-notices is short because its content sits on its children; padding it out is the wrong fix. - generate: load_existing_page() raises when the page a gap edits is missing or empty, so a failed read can't become "write this page from scratch". - generate: import the Anthropic SDK only when a client is built, so --dry-run and the tests run without it. CI installs pyyaml for the new tests. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent ac0a31b commit fbd2f76

4 files changed

Lines changed: 124 additions & 6 deletions

File tree

‎.github/workflows/test-scripts.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,7 @@ jobs:
3030
- uses: actions/setup-python@v5
3131
with:
3232
python-version: "3.12"
33-
# Stdlib only — these tests stub out gh and never call the network.
33+
# pyyaml only: the tests stub out gh, never call the network, and the
34+
# Anthropic SDK is imported lazily so it isn't needed here.
35+
- run: pip install pyyaml
3436
- run: python -m unittest discover -s pipeline -p "test_*.py" -v

‎pipeline/detect_gaps.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,12 @@ def page_frontmatter(page_path):
177177
return None, None # page listed in nav but file missing
178178

179179

180+
def has_child_pages(all_pages, page_path):
181+
"""Whether other nav entries sit under this one, making it a hub page."""
182+
prefix = page_path + "/"
183+
return any(p.startswith(prefix) for p in all_pages)
184+
185+
180186
def slug(page_path):
181187
return page_path.rsplit("/", 1)[-1].lower()
182188

@@ -331,14 +337,17 @@ def detect_gaps(docs_json, endpoints, families, section=None, force=False):
331337

332338
# --- Doc-quality checks on non-reference pages ------------------------ #
333339
if not section:
334-
for p in all_doc_pages(docs_json, apiref_tab):
340+
doc_pages = all_doc_pages(docs_json, apiref_tab)
341+
for p in doc_pages:
335342
fm, content = page_frontmatter(p)
336343
if fm is None:
337344
continue
338345
if not fm.get("description"):
339346
gaps.append(gap("missing_description", "low", None, path=page_file(p) or p,
340347
desc=f"{p} has no frontmatter description"))
341-
if len(strip_frontmatter(content).split()) < 100:
348+
# A page with children is a hub: it's short because the content sits
349+
# on the child pages. Padding it out is the wrong fix, so don't ask.
350+
if len(strip_frontmatter(content).split()) < 100 and not has_child_pages(doc_pages, p):
342351
gaps.append(gap("thin_page", "medium", None, path=page_file(p) or p,
343352
desc=f"{p} has under 100 words"))
344353

‎pipeline/generate.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,21 @@ def load_file(path):
6767
return ""
6868

6969

70+
def load_existing_page(gap):
71+
"""Content of the page a gap edits. Raises rather than returning nothing.
72+
73+
An empty read would turn "expand this page" into "write one from scratch"
74+
and silently discard the real content.
75+
"""
76+
page_path = REPO_ROOT / gap.get("path", "")
77+
if not page_path.is_file():
78+
raise FileNotFoundError(f"no file at '{gap.get('path', '')}' for {gap['type']} gap")
79+
content = load_file(page_path)
80+
if not content.strip():
81+
raise ValueError(f"{gap['path']} is empty, refusing to rewrite it from nothing")
82+
return content
83+
84+
7085
def load_openapi_for_tags(tags):
7186
"""Extract the portion of the OpenAPI spec whose endpoints carry any of `tags`."""
7287
with open(OPENAPI_PATH) as f:
@@ -215,7 +230,7 @@ def build_generation_prompt(gap, family_name, openapi_context, existing_docs_con
215230

216231
elif gap_type == "thin_page":
217232
page_path = gap.get("path", "")
218-
page_content = load_file(REPO_ROOT / page_path)
233+
page_content = load_existing_page(gap)
219234
return f"""Expand this thin page. It currently has only {gap.get('word_count', 0)} words.
220235
221236
Current content of {page_path}:
@@ -232,7 +247,7 @@ def build_generation_prompt(gap, family_name, openapi_context, existing_docs_con
232247

233248
elif gap_type == "missing_code_examples":
234249
page_path = gap.get("path", "")
235-
page_content = load_file(REPO_ROOT / page_path)
250+
page_content = load_existing_page(gap)
236251
return f"""Add code examples to this page. It's a guide but has no runnable code.
237252
238253
Current content of {page_path}:
@@ -249,7 +264,7 @@ def build_generation_prompt(gap, family_name, openapi_context, existing_docs_con
249264

250265
elif gap_type == "missing_description":
251266
page_path = gap.get("path", "")
252-
page_content = load_file(REPO_ROOT / page_path)[:500]
267+
page_content = load_existing_page(gap)[:500]
253268
return f"""Generate a frontmatter description for this page.
254269
255270
Current content of {page_path} (first 500 chars):

‎pipeline/test_hub_and_reads.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/usr/bin/env python3
2+
"""Tests for the thin-page hub exemption and for reading the page a gap edits.
3+
4+
python pipeline/test_hub_and_reads.py
5+
6+
Runs against a temp docs tree, so no repo file is read and none is written.
7+
"""
8+
9+
import sys
10+
import tempfile
11+
import unittest
12+
from pathlib import Path
13+
14+
sys.path.insert(0, str(Path(__file__).resolve().parent))
15+
16+
import detect_gaps # noqa: E402
17+
import generate # noqa: E402
18+
19+
20+
class TempDocsTree(unittest.TestCase):
21+
def setUp(self):
22+
self._tmp = tempfile.TemporaryDirectory()
23+
self.root = Path(self._tmp.name)
24+
self._saved = (detect_gaps.REPO_ROOT, generate.REPO_ROOT)
25+
detect_gaps.REPO_ROOT = self.root
26+
generate.REPO_ROOT = self.root
27+
self.addCleanup(self._restore)
28+
29+
def _restore(self):
30+
detect_gaps.REPO_ROOT, generate.REPO_ROOT = self._saved
31+
self._tmp.cleanup()
32+
33+
def write(self, rel, text="---\ntitle: T\n---\n\nSome body copy.\n"):
34+
path = self.root / rel
35+
path.parent.mkdir(parents=True, exist_ok=True)
36+
path.write_text(text, encoding="utf-8")
37+
38+
39+
class HasChildPagesTest(unittest.TestCase):
40+
def test_a_page_with_children_is_a_hub(self):
41+
pages = ["docs/notices", "docs/notices/july-2024"]
42+
self.assertTrue(detect_gaps.has_child_pages(pages, "docs/notices"))
43+
44+
def test_a_leaf_is_not(self):
45+
self.assertFalse(detect_gaps.has_child_pages(["docs/a", "docs/b"], "docs/a"))
46+
47+
def test_a_name_prefix_is_not_a_child(self):
48+
"""'docs/x-ray' must not count as a child of 'docs/x'."""
49+
self.assertFalse(detect_gaps.has_child_pages(["docs/x", "docs/x-ray"], "docs/x"))
50+
51+
52+
class ThinPageDetectionTest(TempDocsTree):
53+
def test_hub_is_not_flagged_but_a_real_thin_page_is(self):
54+
short = "---\ntitle: T\ndescription: d\n---\n\nToo short.\n"
55+
for page in ("docs/hub", "docs/hub/child", "docs/lonely"):
56+
self.write(f"{page}.mdx", short)
57+
docs_json = {"navigation": {"tabs": [
58+
{"tab": "Home", "pages": ["docs/hub", "docs/hub/child", "docs/lonely"]},
59+
]}}
60+
61+
thin = [g["description"] for g in detect_gaps.detect_gaps(docs_json, [], {})
62+
if g["type"] == "thin_page"]
63+
self.assertEqual(sorted(thin), [
64+
"docs/hub/child has under 100 words", # a child can still be thin
65+
"docs/lonely has under 100 words",
66+
])
67+
68+
69+
class LoadExistingPageTest(TempDocsTree):
70+
def test_reads_the_page(self):
71+
self.write("docs/a.mdx", "---\ntitle: T\n---\n\nReal content.\n")
72+
page = generate.load_existing_page({"type": "thin_page", "path": "docs/a.mdx"})
73+
self.assertIn("Real content.", page)
74+
75+
def test_a_missing_file_raises_instead_of_returning_nothing(self):
76+
"""An empty read used to become "write this page from scratch"."""
77+
with self.assertRaises(FileNotFoundError):
78+
generate.load_existing_page({"type": "thin_page", "path": "docs/ghost.mdx"})
79+
80+
def test_a_directory_raises_file_not_found_not_is_a_directory(self):
81+
self.write("docs/a/child.mdx")
82+
with self.assertRaises(FileNotFoundError):
83+
generate.load_existing_page({"type": "thin_page", "path": "docs/a"})
84+
85+
def test_an_empty_page_raises(self):
86+
self.write("docs/blank.mdx", " \n")
87+
with self.assertRaises(ValueError):
88+
generate.load_existing_page({"type": "thin_page", "path": "docs/blank.mdx"})
89+
90+
91+
if __name__ == "__main__":
92+
unittest.main(verbosity=2)

0 commit comments

Comments
 (0)