diff --git a/agents/repo-finder.md b/agents/repo-finder.md index 9ddfec4..4f50d75 100644 --- a/agents/repo-finder.md +++ b/agents/repo-finder.md @@ -29,35 +29,81 @@ Search in this priority order. Spend more effort on higher-priority categories: ### Step 1: Build Search Queries -Run GitHub searches across each category. Use `gh` CLI for all API access. - -The repo slug field is `fullName`. `gh search repos` rejects `nameWithOwner` with -`Unknown JSON field` — that field belongs to `gh repo` and the GraphQL schema, not -to the search command. Passing it fails every query in this step, so the scan ends -with zero candidates and writes an empty shortlist that reads as a clean run. +Use `gh api search/repositories`. Do not use `gh search repos`: it cannot return +`topics[]`, which the shortlist requires. Every other required field ships inline +in the same response. + +#### Query shape rules + +Repeated qualifiers do not behave alike: + +| Qualifier | Repeated behavior | +|---|---| +| `language:` | OR — `language:go language:rust` returns Go ∪ Rust | +| `topic:` | AND — `topic:cli topic:terminal` returns only repos tagged both | + +- Union languages into a single query. Multi-language costs nothing. +- **Emit at most one `topic:` per query.** A second `topic:` intersects rather + than unions, returning a smaller or empty set with no error. +- Query count is `|topics|`, not `|langs| × |topics|`. +- Every `key:` token must be one of `{language, topic, stars, archived}`. An + unrecognized qualifier (`langauge:go`) is parsed as free text and returns + `total_count: 0` with exit 0. +- Every query must carry a `language:` or a `topic:`. A query with neither is a + catch-all that admits repos matching no criterion. +- Sort by `stars`, not `updated`. `updated` reorders hourly, making the per-query + cutoff non-deterministic. +- Pass `archived:false` as a qualifier so archived repos never consume a result + slot. ```bash -# Category 1: AI/ML -gh search repos --topic machine-learning --stars ">20000" --sort updated --limit 20 --json fullName,stargazersCount,updatedAt,description -gh search repos --topic llm --stars ">5000" --sort updated --limit 20 --json fullName,stargazersCount,updatedAt,description -gh search repos --topic artificial-intelligence --stars ">20000" --sort updated --limit 20 --json fullName,stargazersCount,updatedAt,description - -# Category 2: Language-specific -gh search repos --language java --stars ">20000" --sort updated --limit 20 --json fullName,stargazersCount,updatedAt,description -gh search repos --language python --stars ">20000" --sort updated --limit 20 --json fullName,stargazersCount,updatedAt,description - -# Category 3: Framework-specific -gh search repos "spring boot" --stars ">10000" --sort updated --limit 10 --json fullName,stargazersCount,updatedAt,description -gh search repos "fastapi" --stars ">5000" --sort updated --limit 10 --json fullName,stargazersCount,updatedAt,description - -# Category 4: Famous tools -gh search repos --topic developer-tools --stars ">20000" --sort updated --limit 20 --json fullName,stargazersCount,updatedAt,description +DEFAULT_QUERIES=" +topic:machine-learning stars:>20000 archived:false +topic:llm stars:>5000 archived:false +topic:artificial-intelligence stars:>20000 archived:false +topic:developer-tools stars:>20000 archived:false +language:java language:python stars:>20000 archived:false +" + +# Feed the loop by heredoc, not `echo ... | while`. A piped while runs in a +# subshell, where `exit` terminates only the subshell and the scan continues past +# a fatal error. +CANDIDATES="" +while IFS= read -r q; do + [ -z "$q" ] && continue + + # An error is not a verdict. A failed search must abort, never contribute zero + # rows to a scan that then reads as clean. + RESP=$(gh api -X GET search/repositories \ + -f q="$q" -f sort=stars -f order=desc -f per_page=50) \ + || { echo "FATAL: search failed for query: $q" >&2; exit 1; } + + # Field-name drift against the API yields null, not an error. Abort; do not skip + # the row. A partial candidate set is indistinguishable from a clean scan. + if printf '%s' "$RESP" | jq -e '.items | any(.full_name == null)' >/dev/null; then + echo "FATAL: null full_name in search response — projection drifted: $q" >&2 + exit 1 + fi -# Category 5: Other active repos (catch-all) -gh search repos --stars ">20000" --sort updated --limit 20 --json fullName,stargazersCount,updatedAt,description + # Report matched-of-total from the same response. Never widen a query to fill + # the list. + echo "$(printf '%s' "$RESP" | jq -r '.items | length') of \ +$(printf '%s' "$RESP" | jq -r '.total_count') matched: $q" >&2 + + CANDIDATES="$CANDIDATES$(printf '%s' "$RESP" | jq -c '.items[] | + {full_name, language, topics, stargazers_count, pushed_at, + archived, open_issues_count, default_branch, description}') +" +done <=YYYY-MM-DD`). Do NOT fetch `--state # merged --limit N` and filter on mergedAt: that list is ordered by CREATION @@ -545,11 +601,30 @@ If no issue scores 8+ after applying the hard filter, mark the repo as "no clear Save results to `~/.superhuman/global/repo-shortlist.json` (create the directory if missing; atomic temp-rename write): +Validate against `schemas/repo_shortlist.schema.json` before the rename. A +malformed shortlist must never reach the orchestrator. + ```bash +source "${CLAUDE_PLUGIN_ROOT}/scripts/lib/state.sh" + GLOBAL_DIR="$HOME/.superhuman/global" mkdir -p "$GLOBAL_DIR" TMP="$GLOBAL_DIR/repo-shortlist.json.tmp.$$" -printf '%s' "$SHORTLIST" | jq . > "$TMP" && mv "$TMP" "$GLOBAL_DIR/repo-shortlist.json" +printf '%s' "$SHORTLIST" | jq . > "$TMP" \ + || { echo "FATAL: shortlist is not valid JSON" >&2; rm -f "$TMP"; exit 1; } + +validate_json "${CLAUDE_PLUGIN_ROOT}/schemas/repo_shortlist.schema.json" "$TMP" \ + || { echo "FATAL: shortlist failed schema validation" >&2; rm -f "$TMP"; exit 1; } + +# Row invariant, asserted separately: validate_json degrades to a top-level +# required-key check when python jsonschema is absent (stock macOS runner), and +# `repo`/`scores.final` are nested inside repos[]. A null `repo` (dropped +# full_name -> repo mapping) or null score (written to `.score`) survives that +# fallback and propagates downstream as a valid-looking value. +jq -e '.repos | length > 0 and all(.repo != null and .scores.final != null)' "$TMP" >/dev/null \ + || { echo "FATAL: shortlist row missing repo or scores.final" >&2; rm -f "$TMP"; exit 1; } + +mv "$TMP" "$GLOBAL_DIR/repo-shortlist.json" ``` The orchestrator binds the top result via: @@ -558,7 +633,9 @@ The orchestrator binds the top result via: REPO=$(jq -r '.repos[0].repo' "$GLOBAL_DIR/repo-shortlist.json") ``` -Shortlist payload shape: +Shortlist payload shape. The row key is `repo`, not `full_name`: map Step 1's +`full_name` across on write, or the orchestrator's `.repos[0].repo` binds null. +`topics[]` comes from the Step 1 projection; cite it in the `notes` rationale. ```json { @@ -575,6 +652,7 @@ Shortlist payload shape: "stars": 45000, "category": "ai-ml", "language": "Python", + "topics": ["machine-learning", "llm", "python"], "description": "repo description", "scores": { "responsiveness": 9, diff --git a/commands/repo-finder.md b/commands/repo-finder.md index 14a4bdd..35c517e 100644 --- a/commands/repo-finder.md +++ b/commands/repo-finder.md @@ -58,7 +58,7 @@ if [ ! -f "$SHORTLIST" ]; then fi echo "Top $N candidates:" -jq -r '.repos[] | "\(.score)\t\(.repo)\t\(.notes // "")"' "$SHORTLIST" \ +jq -r '.repos[] | "\(.scores.final)\t\(.repo)\t\(.notes // "")"' "$SHORTLIST" \ | head -"$N" \ | awk -F'\t' 'BEGIN{printf "%-6s %-32s %s\n","score","repo","notes"} {printf "%-6s %-32s %s\n",$1,$2,$3}' diff --git a/schemas/repo_shortlist.schema.json b/schemas/repo_shortlist.schema.json new file mode 100644 index 0000000..900cc34 --- /dev/null +++ b/schemas/repo_shortlist.schema.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/gaurav0107/superhuman/schemas/repo_shortlist.schema.json", + "title": "repo_shortlist", + "description": "Ranked repos worth contributing to, with the single best issue candidate per repo. Consumed by opensource-contributor and the contribute/contribution-fleet commands, which bind a target via .repos[0].repo. Owner (sole writer): repo-finder.", + "type": "object", + "additionalProperties": true, + "required": ["generated_at", "repos"], + "properties": { + "generated_at": {"type": "string", "format": "date-time"}, + "criteria": { + "type": "object", + "additionalProperties": true, + "description": "the filter that produced this list, e.g. {\"min_stars\":20000,\"min_score\":60}" + }, + "repos": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true, + "required": ["repo", "scores"], + "properties": { + "rank": {"type": "integer", "minimum": 1}, + "repo": {"type": "string", "pattern": "^[^/]+/[^/]+$", "description": "owner/repo slug. Search returns this field as full_name; map it across on write or .repos[0].repo binds null downstream."}, + "stars": {"type": "integer", "minimum": 0}, + "category": {"type": "string"}, + "language": {"type": ["string", "null"], "description": "null for repos GitHub cannot classify"}, + "topics": {"type": "array", "items": {"type": "string"}, "description": "from the Step 1 search projection; cited by the notes rationale"}, + "description": {"type": ["string", "null"]}, + "scores": { + "type": "object", + "additionalProperties": true, + "required": ["final"], + "description": "commands/repo-finder.md renders scores.final. A score written to a bare .score key renders as null on every row.", + "properties": { + "responsiveness": {"type": "number"}, + "opportunity_quality": {"type": "number"}, + "outside_contributor_track": {"type": "number"}, + "ai_friendliness": {"type": "number"}, + "category_bonus": {"type": "number"}, + "final": {"type": "number"} + } + }, + "responsiveness_detail": {"type": "object", "additionalProperties": true}, + "best_issue": { + "type": "object", + "additionalProperties": true, + "required": ["number", "url"], + "properties": { + "number": {"type": "integer"}, + "title": {"type": "string"}, + "type": {"type": "string"}, + "labels": {"type": "array", "items": {"type": "string"}}, + "has_maintainer_approval": {"type": "boolean"}, + "issue_score": {"type": "number"}, + "url": {"type": "string"} + } + }, + "notes": {"type": "string"} + } + } + } + } +} diff --git a/tests/scripts/test_repo_shortlist_schema.sh b/tests/scripts/test_repo_shortlist_schema.sh new file mode 100755 index 0000000..46e0fdc --- /dev/null +++ b/tests/scripts/test_repo_shortlist_schema.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +set -euo pipefail +CLAUDE_PLUGIN_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +export CLAUDE_PLUGIN_ROOT +source "$CLAUDE_PLUGIN_ROOT/scripts/lib/state.sh" +SCHEMA="$CLAUDE_PLUGIN_ROOT/schemas/repo_shortlist.schema.json" +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT + +cat > "$tmpdir/ok.json" <<'EOF' +{ + "generated_at": "2026-07-13T00:00:00Z", + "criteria": {"min_stars": 20000, "min_score": 60}, + "repos": [ + { + "rank": 1, + "repo": "owner/repo", + "stars": 45000, + "category": "ai-ml", + "language": "Python", + "topics": ["machine-learning", "llm"], + "description": "a repo", + "scores": {"responsiveness": 9, "final": 87}, + "best_issue": {"number": 1234, "url": "https://github.com/owner/repo/issues/1234"}, + "notes": "tagged llm, responsive maintainers" + } + ] +} +EOF +validate_json "$SCHEMA" "$tmpdir/ok.json" || { echo "FAIL valid repo_shortlist rejected"; exit 1; } + +echo '{"generated_at":"2026-07-13T00:00:00Z"}' > "$tmpdir/no_repos.json" +validate_json "$SCHEMA" "$tmpdir/no_repos.json" 2>/dev/null && { + echo "FAIL repo_shortlist missing repos accepted"; exit 1; } + +# validate_json degrades to a top-level required-key check when python jsonschema +# is absent, and repo/scores.final are nested inside repos[]. agents/repo-finder.md +# Step 5 asserts them with jq at write time. Keep this guard in lockstep with it. +row_guard() { # row_guard -> 0 if every row has repo + scores.final + jq -e '.repos | length > 0 and all(.repo != null and .scores.final != null)' "$1" >/dev/null +} + +row_guard "$tmpdir/ok.json" || { echo "FAIL row guard rejected a valid shortlist"; exit 1; } + +# full_name -> repo mapping dropped on write. +cat > "$tmpdir/null_repo.json" <<'EOF' +{"generated_at":"2026-07-13T00:00:00Z","repos":[{"repo":null,"scores":{"final":87}}]} +EOF +row_guard "$tmpdir/null_repo.json" 2>/dev/null && { echo "FAIL null repo accepted"; exit 1; } + +# Score written to a bare .score key instead of .scores.final. +cat > "$tmpdir/bare_score.json" <<'EOF' +{"generated_at":"2026-07-13T00:00:00Z","repos":[{"repo":"owner/repo","score":87,"scores":{}}]} +EOF +row_guard "$tmpdir/bare_score.json" 2>/dev/null && { echo "FAIL bare .score accepted"; exit 1; } + +# commands/repo-finder.md must render scores.final, not score. +RENDERED=$(jq -r '.repos[] | "\(.scores.final)\t\(.repo)"' "$tmpdir/ok.json") +case "$RENDERED" in + "87 owner/repo") ;; + *) echo "FAIL render projection: got '$RENDERED'"; exit 1 ;; +esac +grep -q 'scores\.final' "$CLAUDE_PLUGIN_ROOT/commands/repo-finder.md" || { + echo "FAIL commands/repo-finder.md does not render .scores.final"; exit 1; } + +AGENT="$CLAUDE_PLUGIN_ROOT/agents/repo-finder.md" + +# Step 1's guards are executed, not grepped: a literal-text assertion is satisfied +# by prose or a comment and proves nothing about behavior. Extract the Step 1 +# block and run it against a stubbed gh. No network. +STEP1=$(awk ' + /^```bash$/ {inb=1; buf=""; next} + /^```$/ {if (inb && buf ~ /DEFAULT_QUERIES=/) {printf "%s", buf; exit} inb=0; buf=""; next} + inb {buf = buf $0 "\n"} +' "$AGENT") +[ -n "$STEP1" ] || { echo "FAIL could not extract the Step 1 query block"; exit 1; } +# Trailing newline is load-bearing: the block ends in a heredoc terminator, and +# command substitution strips it. Without it, anything appended below fuses onto +# the EOF line and the heredoc never closes. +printf '%s\n' "$STEP1" > "$tmpdir/step1.sh" +bash -n "$tmpdir/step1.sh" || { echo "FAIL Step 1 block is not valid bash"; exit 1; } + +mkdir -p "$tmpdir/bin" +cat > "$tmpdir/bin/gh" <<'EOF' +#!/usr/bin/env bash +case "${GH_STUB_MODE:-ok}" in + fail) echo "gh: HTTP 403 rate limit exceeded" >&2; exit 1 ;; + null) echo '{"total_count":2,"items":[{"full_name":null,"language":"Go"}]}' ;; + *) echo '{"total_count":63,"items":[{"full_name":"owner/repo","language":"Go","topics":["cli"],"stargazers_count":1,"pushed_at":"2026-07-01T00:00:00Z","archived":false,"open_issues_count":3,"default_branch":"main","description":"d"}]}' ;; +esac +EOF +chmod +x "$tmpdir/bin/gh" + +# Step 1 accumulates rows into $CANDIDATES without printing them, so append a dump +# to observe what the loop actually collected. An abort exits before reaching it. +{ cat "$tmpdir/step1.sh"; echo 'printf "%s" "$CANDIDATES"'; } > "$tmpdir/run.sh" + +run_step1() { # run_step1 -> Step 1's exit code; stdout/stderr in $tmpdir/out + GH_STUB_MODE="$1" PATH="$tmpdir/bin:$PATH" \ + bash "$tmpdir/run.sh" > "$tmpdir/out" 2>&1 +} + +# A failed search must abort the scan. Contributing zero rows instead leaves a +# partial candidate set that reads as a clean run. +run_step1 fail && { echo "FAIL Step 1 did not abort on a failed search"; exit 1; } +grep -q 'FATAL' "$tmpdir/out" || { echo "FAIL failed search did not report FATAL"; exit 1; } + +# Field-name drift against the API yields null, not an error. +run_step1 null && { echo "FAIL Step 1 did not abort on null full_name"; exit 1; } +grep -q 'FATAL' "$tmpdir/out" || { echo "FAIL null full_name did not report FATAL"; exit 1; } + +# Happy path: the projected row must actually be collected. Asserting only on the +# matched-of-total line is satisfied by "0 of 63 matched", so a regression that +# drops every candidate would pass. +run_step1 ok || { echo "FAIL Step 1 aborted on a valid response"; exit 1; } +grep -q '"full_name":"owner/repo"' "$tmpdir/out" || { + echo "FAIL Step 1 collected no candidate row"; cat "$tmpdir/out"; exit 1; } +grep -q '"default_branch":"main"' "$tmpdir/out" || { + echo "FAIL Step 1 projection dropped default_branch (raw() breaks without it)"; exit 1; } +grep -q '1 of 63 matched' "$tmpdir/out" || { + echo "FAIL Step 1 does not report matched-of-total"; cat "$tmpdir/out"; exit 1; } + +# Both aborts above only terminate the scan if the loop runs in the current shell. +# A piped `while` runs in a subshell, where `exit` kills the subshell and the scan +# continues — the guards would pass review and do nothing. Checked on code with +# comments stripped, so a comment cannot satisfy or trip it. +STEP1_CODE=$(printf '%s' "$STEP1" | sed 's/#.*//') +printf '%s' "$STEP1_CODE" | grep -qE '\|[[:space:]]*while' && { + echo "FAIL Step 1 query loop is piped; its aborts cannot exit the scan"; exit 1; } +printf '%s' "$STEP1_CODE" | grep -q 'done <<' || { + echo "FAIL Step 1 query loop is not heredoc-fed"; exit 1; } + +# The per-candidate repo call must stay deleted. Match the ROOT repos/OWNER/REPO +# endpoint, not the field names: `gh api "repos/$name"` returns all four projected +# fields whether or not a field name appears on that line, and a line-continued +# command hides the field list from a line-oriented grep entirely. +# +# Subresources are legitimate and must keep working: /git/trees, /contributors, +# /issues/comments, /pulls carry data no search result holds. Banning `gh api +# repos/` wholesale, as review proposed, would break all four. +# +# Normalize first: strip comments, join line continuations, collapse whitespace. +NORM=$(sed 's/#.*//' "$AGENT" \ + | sed -e :a -e '/\\$/{N; s/\\\n//; ta' -e '}' \ + | tr -s '[:space:]' ' ') +# +# Allowlist the subresources rather than pattern-matching the root form: the owner +# and repo may be a variable (`gh api "repos/$full_name"`), which carries no +# literal owner/repo slash to match on. Anything without a subresource is a root +# fetch, however it is spelled. +REFETCH=$(printf '%s\n' "$NORM" \ + | grep -oE "gh api [^|;]*repos/[^\"' )]*" \ + | grep -oE "repos/[^\"' )]*" \ + | grep -vE "/(git/trees|contributors|issues|pulls|commits|compare|releases|contents|labels)" \ + || true) +[ -z "$REFETCH" ] || { + echo "FAIL Step 2 re-fetches the root repos endpoint ($REFETCH); Step 1 already carries those fields" + exit 1; } + +echo "OK test_repo_shortlist_schema.sh"