|
| 1 | +"""Add git repositories to the catalog as entries in catalog/. |
| 2 | +
|
| 3 | +A repo that already carries metadata (dataherb.json, dataherb.yml or the |
| 4 | +legacy .dataherb/metadata.yml) gets a short pointer entry; the builder reads |
| 5 | +the metadata on every build. A repo without metadata is cloned and scanned |
| 6 | +like `dataherb create` does, and gets an inline entry with the inferred |
| 7 | +resources, so it shows up in the catalog without touching the data repo. |
| 8 | +""" |
| 9 | + |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +import json |
| 13 | +import subprocess |
| 14 | +import tempfile |
| 15 | +from dataclasses import dataclass |
| 16 | +from pathlib import Path |
| 17 | +from urllib.parse import quote |
| 18 | + |
| 19 | +import yaml |
| 20 | + |
| 21 | +from .config import Config |
| 22 | +from .infer import scaffold |
| 23 | +from .resolve import METADATA_CANDIDATES, load_entries |
| 24 | +from .stores import GitStore, StoreError, git_key, make_stores |
| 25 | +from .util import FetchError, env_token, http_get, slugify |
| 26 | + |
| 27 | + |
| 28 | +@dataclass |
| 29 | +class Added: |
| 30 | + repo: str |
| 31 | + id: str |
| 32 | + path: Path | None |
| 33 | + kind: str # pointer | inline | skipped |
| 34 | + note: str = "" |
| 35 | + |
| 36 | + |
| 37 | +def list_org_repos( |
| 38 | + org: str, |
| 39 | + match: str = "", |
| 40 | + api_url: str = "https://api.github.com", |
| 41 | + token: str | None = None, |
| 42 | + include_archived: bool = False, |
| 43 | +) -> list[str]: |
| 44 | + """owner/name of every repo of a GitHub org (or user) whose name starts with match.""" |
| 45 | + headers = {"Accept": "application/vnd.github+json"} |
| 46 | + if token: |
| 47 | + headers["Authorization"] = f"token {token}" |
| 48 | + api = api_url.rstrip("/") |
| 49 | + repos: list[dict] = [] |
| 50 | + for kind in ("orgs", "users"): |
| 51 | + repos, page = [], 1 |
| 52 | + try: |
| 53 | + while True: |
| 54 | + raw = http_get( |
| 55 | + f"{api}/{kind}/{quote(org)}/repos?per_page=100&type=all&page={page}", |
| 56 | + headers=headers, |
| 57 | + ) |
| 58 | + batch = json.loads(raw) |
| 59 | + repos.extend(batch) |
| 60 | + if len(batch) < 100: |
| 61 | + break |
| 62 | + page += 1 |
| 63 | + break |
| 64 | + except FetchError as e: |
| 65 | + if e.status == 404 and kind == "orgs": |
| 66 | + continue |
| 67 | + raise |
| 68 | + return sorted( |
| 69 | + r["full_name"] |
| 70 | + for r in repos |
| 71 | + if r["name"].lower().startswith(match.lower()) |
| 72 | + and (include_archived or not r.get("archived")) |
| 73 | + ) |
| 74 | + |
| 75 | + |
| 76 | +def default_id(repo: str, strip_prefix: str = "") -> str: |
| 77 | + name = repo.split("/")[-1] |
| 78 | + if strip_prefix and name.lower().startswith(strip_prefix.lower()): |
| 79 | + name = name[len(strip_prefix) :].lstrip("-_.") or name |
| 80 | + return slugify(name) |
| 81 | + |
| 82 | + |
| 83 | +def _remote_metadata(store: GitStore, repo: str, ref: str | None) -> str | None: |
| 84 | + for c in METADATA_CANDIDATES: |
| 85 | + try: |
| 86 | + store.read(git_key(repo, c, ref)) |
| 87 | + return c |
| 88 | + except FetchError: |
| 89 | + continue |
| 90 | + return None |
| 91 | + |
| 92 | + |
| 93 | +def _clone(url: str, ref: str | None, dest: Path) -> None: |
| 94 | + cmd = ["git", "clone", "--quiet", "--depth", "1"] |
| 95 | + if ref and ref != "HEAD": |
| 96 | + cmd += ["--branch", ref] |
| 97 | + subprocess.run(cmd + [url, str(dest)], check=True, capture_output=True, text=True) |
| 98 | + |
| 99 | + |
| 100 | +def add_repos( |
| 101 | + cfg: Config, |
| 102 | + repos: list[str], |
| 103 | + store_name: str | None = None, |
| 104 | + ref: str | None = None, |
| 105 | + strip_prefix: str = "", |
| 106 | + tags: tuple[str, ...] = (), |
| 107 | + clone_url_template: str | None = None, |
| 108 | + force: bool = False, |
| 109 | + dry_run: bool = False, |
| 110 | +) -> list[Added]: |
| 111 | + stores = make_stores(cfg.stores, cfg.root) |
| 112 | + git_stores = {n: s for n, s in stores.items() if isinstance(s, GitStore)} |
| 113 | + if store_name: |
| 114 | + if store_name not in git_stores: |
| 115 | + raise StoreError(f"'{store_name}' is not a git store in the config") |
| 116 | + store = git_stores[store_name] |
| 117 | + elif git_stores: |
| 118 | + store_name, store = next(iter(git_stores.items())) |
| 119 | + else: |
| 120 | + raise StoreError("the config has no store of type git") |
| 121 | + |
| 122 | + dirs = cfg.catalog.get("dirs") or ["catalog"] |
| 123 | + out_dir = cfg.path(dirs[0]) |
| 124 | + entries, _ = load_entries(cfg) |
| 125 | + known_repos = { |
| 126 | + str(e.get("repo") or e.get("repository")).lower(): e |
| 127 | + for e in entries |
| 128 | + if e.get("repo") or e.get("repository") |
| 129 | + } |
| 130 | + known_ids = {str(e["id"]) for e in entries} |
| 131 | + clone_tmpl = clone_url_template or ( |
| 132 | + store.conf.get("web_url_template", "https://github.com/{repo}") + ".git" |
| 133 | + ) |
| 134 | + # Only one git store: entries can leave `store` out and still resolve to it. |
| 135 | + explicit_store = len(git_stores) > 1 |
| 136 | + |
| 137 | + results = [] |
| 138 | + for repo in repos: |
| 139 | + repo = repo.strip().strip("/") |
| 140 | + if repo.count("/") != 1: |
| 141 | + results.append(Added(repo, "", None, "skipped", "expected owner/name")) |
| 142 | + continue |
| 143 | + existing = known_repos.get(repo.lower()) |
| 144 | + if existing and not force: |
| 145 | + results.append( |
| 146 | + Added( |
| 147 | + repo, |
| 148 | + str(existing["id"]), |
| 149 | + cfg.path(existing["_file"]), |
| 150 | + "skipped", |
| 151 | + f"already in {existing['_file']}", |
| 152 | + ) |
| 153 | + ) |
| 154 | + continue |
| 155 | + did = str(existing["id"]) if existing else default_id(repo, strip_prefix) |
| 156 | + target = ( |
| 157 | + cfg.path(existing["_file"]) if existing else out_dir / f"{did}.yml" |
| 158 | + ) |
| 159 | + if not existing and (did in known_ids or target.exists()) and not force: |
| 160 | + results.append( |
| 161 | + Added(repo, did, target, "skipped", f"id '{did}' is already taken") |
| 162 | + ) |
| 163 | + continue |
| 164 | + |
| 165 | + entry: dict = {"id": did} |
| 166 | + if explicit_store: |
| 167 | + entry["store"] = store_name |
| 168 | + entry["repo"] = repo |
| 169 | + if ref: |
| 170 | + entry["ref"] = ref |
| 171 | + |
| 172 | + found = _remote_metadata(store, repo, ref) |
| 173 | + inferred = None |
| 174 | + if found is None: |
| 175 | + # Not readable over raw URLs (no metadata, or a private repo without a |
| 176 | + # token): look at a shallow clone, which uses the local git credentials. |
| 177 | + with tempfile.TemporaryDirectory() as tmp: |
| 178 | + dest = Path(tmp) / "repo" |
| 179 | + try: |
| 180 | + _clone(clone_tmpl.format(repo=repo), ref, dest) |
| 181 | + except subprocess.CalledProcessError as e: |
| 182 | + results.append( |
| 183 | + Added( |
| 184 | + repo, |
| 185 | + did, |
| 186 | + None, |
| 187 | + "skipped", |
| 188 | + f"cannot clone: {(e.stderr or '').strip()}", |
| 189 | + ) |
| 190 | + ) |
| 191 | + continue |
| 192 | + found = next((c for c in METADATA_CANDIDATES if (dest / c).exists()), None) |
| 193 | + if found is None: |
| 194 | + inferred = scaffold(dest, dataset_id=did) |
| 195 | + |
| 196 | + if inferred is not None: |
| 197 | + entry["inline"] = True |
| 198 | + for k in ("name", "description", "datapackage"): |
| 199 | + entry[k] = inferred[k] |
| 200 | + kind, note = "inline", f"no metadata in the repo; inferred {len(inferred['datapackage']['resources'])} resource(s)" |
| 201 | + else: |
| 202 | + kind, note = "pointer", f"reads {found}" |
| 203 | + if tags: |
| 204 | + entry["tags"] = list(tags) |
| 205 | + |
| 206 | + if not dry_run: |
| 207 | + target.parent.mkdir(parents=True, exist_ok=True) |
| 208 | + target.write_text( |
| 209 | + yaml.safe_dump(entry, sort_keys=False, allow_unicode=True), |
| 210 | + encoding="utf-8", |
| 211 | + ) |
| 212 | + known_ids.add(did) |
| 213 | + known_repos[repo.lower()] = {"id": did, "_file": str(target)} |
| 214 | + results.append(Added(repo, did, target, kind, note)) |
| 215 | + return results |
| 216 | + |
| 217 | + |
| 218 | +def org_token(cfg: Config, store_name: str | None) -> str | None: |
| 219 | + for name, conf in (cfg.stores or {}).items(): |
| 220 | + if conf.get("type") == "git" and (store_name in (None, name)): |
| 221 | + return env_token(conf.get("token_env")) |
| 222 | + return None |
0 commit comments