Read Markdown catalog entries; catalog add writes them

catalog/<id>.md holds the entry fields in YAML front matter and free
text in the body, which becomes the dataset's documentation (before the
dataset's own). .md files without front matter, like a README, are
ignored. `dataherb catalog add` writes .md by default (--format yml for
the old style) and keeps the body when overwriting with --force.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ggg4AutCcRdK8c57NMxSqp

Author: claude

dataherb/catalog/add.py

@@ -1,5 +1,8 @@
 """Add git repositories to the catalog as entries in catalog/.
 
+Entries are Markdown files: the fields go in the YAML front matter and the
+body is free text shown on the dataset page.
+
 A repo that already carries metadata (dataherb.json, dataherb.yml or the
 legacy .dataherb/metadata.yml) gets a short pointer entry; the builder reads
 the metadata on every build. A repo without metadata is cloned and scanned
@@ -22,7 +25,14 @@
 from .infer import scaffold
 from .resolve import METADATA_CANDIDATES, load_entries
 from .stores import GitStore, StoreError, git_key, make_stores
-from .util import FetchError, env_token, http_get, slugify
+from .util import (
+    FetchError,
+    env_token,
+    front_matter_document,
+    http_get,
+    slugify,
+    split_front_matter,
+)
 
 
 @dataclass
@@ -107,6 +117,7 @@ def add_repos(
     clone_url_template: str | None = None,
     force: bool = False,
     dry_run: bool = False,
+    fmt: str = "md",
 ) -> list[Added]:
     stores = make_stores(cfg.stores, cfg.root)
     git_stores = {n: s for n, s in stores.items() if isinstance(s, GitStore)}
@@ -154,7 +165,7 @@ def add_repos(
             continue
         did = str(existing["id"]) if existing else default_id(repo, strip_prefix)
         target = (
-            cfg.path(existing["_file"]) if existing else out_dir / f"{did}.yml"
+            cfg.path(existing["_file"]) if existing else out_dir / f"{did}.{fmt}"
         )
         if not existing and (did in known_ids or target.exists()) and not force:
             results.append(
@@ -205,16 +216,24 @@ def add_repos(
 
         if not dry_run:
             target.parent.mkdir(parents=True, exist_ok=True)
-            target.write_text(
-                yaml.safe_dump(entry, sort_keys=False, allow_unicode=True),
-                encoding="utf-8",
-            )
+            if target.suffix == ".md":
+                text = front_matter_document(entry, _keep_body(target) if force else "")
+            else:
+                text = yaml.safe_dump(entry, sort_keys=False, allow_unicode=True)
+            target.write_text(text, encoding="utf-8")
         known_ids.add(did)
         known_repos[repo.lower()] = {"id": did, "_file": str(target)}
         results.append(Added(repo, did, target, kind, note))
     return results
 
 
+def _keep_body(path: Path) -> str:
+    """The Markdown body of an entry that is being overwritten, so --force keeps hand-written notes."""
+    if not path.exists():
+        return ""
+    return split_front_matter(path.read_text(encoding="utf-8"))[1]
+
+
 def org_token(cfg: Config, store_name: str | None) -> str | None:
     for name, conf in (cfg.stores or {}).items():
         if conf.get("type") == "git" and (store_name in (None, name)):

dataherb/catalog/build.py

@@ -33,7 +33,7 @@ def validate_inputs(cfg: Config) -> list[BuildIssue]:
     entries, load_issues = load_entries(cfg)
     issues.extend(load_issues)
     for e in entries:
-        doc = {k: v for k, v in e.items() if k != "_file"}
+        doc = {k: v for k, v in e.items() if not k.startswith("_")}
         for msg in errors("catalog-entry", doc):
             issues.append(BuildIssue("error", e.get("id"), f"{e['_file']} {msg}"))
         if e.get("inline"):

dataherb/catalog/resolve.py

@@ -20,7 +20,15 @@
     join,
     make_stores,
 )
-from .util import FetchError, iso, load_file, load_structured, log, utcnow
+from .util import (
+    FetchError,
+    iso,
+    load_file,
+    load_structured,
+    log,
+    split_front_matter,
+    utcnow,
+)
 
 METADATA_CANDIDATES = (
     "dataherb.json",
@@ -29,6 +37,10 @@
     ".dataherb/metadata.yml",
 )
 
+# Catalog entry files. Markdown entries keep the fields in YAML front matter
+# and free text (shown on the dataset page) in the body.
+ENTRY_SUFFIXES = (".md", ".yml", ".yaml", ".json")
+
 # Keys in a catalog entry that say where the dataset is, rather than describe it.
 LOCATOR_KEYS = {
     "store",
@@ -40,6 +52,7 @@
     "inline",
     "hidden",
     "_file",
+    "_body",
 }
 
 FORMAT_BY_EXT = {
@@ -109,10 +122,17 @@ def load_entries(cfg: Config) -> tuple[list[dict], list[BuildIssue]]:
             )
             continue
         for p in sorted(folder.rglob("*")):
-            if p.suffix not in (".yml", ".yaml", ".json") or p.name.startswith("_"):
+            if p.suffix not in ENTRY_SUFFIXES or p.name.startswith("_"):
                 continue
+            body = ""
             try:
-                data = load_file(p)
+                if p.suffix == ".md":
+                    data, body = split_front_matter(p.read_text(encoding="utf-8"))
+                    if data is None:  # a README or notes, not an entry
+                        continue
+                    data = data or {}
+                else:
+                    data = load_file(p)
             except Exception as e:
                 issues.append(
                     BuildIssue(
@@ -134,6 +154,8 @@ def load_entries(cfg: Config) -> tuple[list[dict], list[BuildIssue]]:
                     continue
                 item = dict(item)
                 item["_file"] = str(p.relative_to(cfg.root)).replace("\\", "/")
+                if body.strip():
+                    item["_body"] = body.strip()
                 if "id" not in item:
                     item["id"] = p.stem
                 entries.append(item)
@@ -320,7 +342,9 @@ def normalize(
         "id": str(merged.get("id")),
         "name": merged.get("name") or merged.get("title") or str(merged.get("id")),
         "description": merged.get("description") or "",
-        "documentation": merged.get("documentation") or "",
+        "documentation": "\n\n".join(
+            x for x in (entry.get("_body"), merged.get("documentation")) if x
+        ),
         "tags": sorted({str(t) for t in _as_list(merged.get("tags")) if t}),
         "domain": merged.get("domain"),
         "owner": _owner(merged.get("owner")),

dataherb/catalog/util.py

@@ -28,6 +28,24 @@ def load_structured(text: str | bytes, name: str = "") -> Any:
     return yaml.safe_load(text)
 
 
+_FRONT_MATTER = re.compile(r"\A---[ \t]*\r?\n(.*?)^(?:---|\.\.\.)[ \t]*(?:\r?\n|\Z)", re.S | re.M)
+
+
+def split_front_matter(text: str) -> tuple[Any, str]:
+    """Split Markdown into (YAML front matter, body). Front matter is None when absent."""
+    text = text.lstrip("\ufeff")
+    m = _FRONT_MATTER.match(text)
+    if not m:
+        return None, text
+    return yaml.safe_load(m.group(1)), text[m.end() :]
+
+
+def front_matter_document(data: dict, body: str = "") -> str:
+    """Markdown with data as YAML front matter."""
+    head = yaml.safe_dump(data, sort_keys=False, allow_unicode=True)
+    return f"---\n{head}---\n" + (f"\n{body.strip()}\n" if body.strip() else "")
+
+
 def load_file(path: Path) -> Any:
     return load_structured(path.read_text(encoding="utf-8"), path.name)
 

dataherb/cmd/catalog.py

@@ -143,7 +143,19 @@ def lint(config_path, min_score):
     help="GitHub API, for --org (GitHub Enterprise: https://HOST/api/v3).",
 )
 @click.option("--include-archived", is_flag=True, help="With --org: keep archived repos.")
-@click.option("--force", is_flag=True, help="Overwrite entries that already exist.")
+@click.option(
+    "--format",
+    "fmt",
+    type=click.Choice(["md", "yml"]),
+    default="md",
+    show_default=True,
+    help="Markdown with YAML front matter, or plain YAML.",
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Overwrite entries that already exist (a Markdown body is kept).",
+)
 @click.option("--dry-run", is_flag=True, help="Show what would be written.")
 def add_to_catalog(
     config_path,
@@ -156,10 +168,11 @@ def add_to_catalog(
     tags,
     api_url,
     include_archived,
+    fmt,
     force,
     dry_run,
 ):
-    """Add git repos (owner/name) to the catalog, one file per repo in catalog/.
+    """Add git repos (owner/name) to the catalog, one Markdown file per repo in catalog/.
 
     Repos with a dataherb.json/.yml get a pointer entry. Repos without one
     are cloned and scanned, and get an inline entry with the inferred files
@@ -192,6 +205,7 @@ def add_to_catalog(
         tags=tags,
         force=force,
         dry_run=dry_run,
+        fmt=fmt,
     )
     colors = {"pointer": "green", "inline": "green", "skipped": "yellow"}
     for r in results:

docs/changelog.md

@@ -5,6 +5,8 @@
 Added:
 
 - `dataherb catalog build | validate | lint | serve`: build a static DataHerb Explorer catalog site from `dataherb.config.yml` (git, S3, HTTP and local sources, S3 discovery, metadata quality scores).
+- `dataherb catalog add`: list git repos in the catalog (`--org ORG --match PREFIX` for a whole org). Pointer entries for repos with metadata, inline entries inferred from a clone otherwise.
+- Catalog entries can be Markdown files (`catalog/<id>.md`): fields in YAML front matter, the body is shown on the dataset page above the dataset's own documentation. `.yml`/`.json` entries still work.
 - `dataherb status emit | check`: write and check job status files (`dataherb.status/v1`).
 - DataHerb v2 metadata (owner, tags, license, classification, update frequency, status job, related datasets) with JSON Schemas in `dataherb/catalog/schemas`.
 - Optional extras: `dataherb[s3]` (boto3), `dataherb[infer]` (duckdb).

tests/catalog/test_add.py

@@ -7,6 +7,7 @@
 from dataherb.catalog.add import add_repos, default_id, list_org_repos
 from dataherb.catalog.config import load_config
 from dataherb.catalog.resolve import build_catalog
+from dataherb.catalog.util import split_front_matter
 from dataherb.command import dataherb
 
 
@@ -50,13 +51,14 @@ def test_add_pointer_inline_and_skip(project, tmp_path):
         "acme/raw": "inline",
         "acme/gone": "skipped",
     }
-    meta = yaml.safe_load((project / "catalog" / "meta.yml").read_text())
+    meta, body = split_front_matter((project / "catalog" / "meta.md").read_text())
     assert meta == {"id": "meta", "repo": "acme/meta", "tags": ["new"]}
-    raw = yaml.safe_load((project / "catalog" / "raw.yml").read_text())
+    assert body == ""
+    raw, _ = split_front_matter((project / "catalog" / "raw.md").read_text())
     assert raw["inline"] is True and raw["repo"] == "acme/raw"
     r0 = raw["datapackage"]["resources"][0]
     assert r0["path"] == "data/x.csv" and r0["rows"] == 2
-    assert not (project / "catalog" / "gone.yml").exists()
+    assert not (project / "catalog" / "gone.md").exists()
 
     ds = {d["id"]: d for d in build_catalog(load_config(project / "dataherb.config.yml")).datasets}
     assert ds["meta"]["name"] == "Meta"
@@ -94,4 +96,25 @@ def test_cli_dry_run(project):
     )
     assert res.exit_code == 0, res.output
     assert "pointer" in res.output and "would add 1 of 1" in res.output
-    assert not (project / "catalog" / "meta.yml").exists()
+    assert not (project / "catalog" / "meta.md").exists()
+
+
+def test_force_keeps_markdown_body(project):
+    meta_dir = project / "git" / "acme" / "meta" / "HEAD"
+    meta_dir.mkdir(parents=True)
+    (meta_dir / "dataherb.json").write_text("{}")
+    entry = project / "catalog" / "meta.md"
+    entry.write_text("---\nid: meta\nrepo: acme/meta\n---\n\nHand-written notes.\n")
+    cfg = load_config(project / "dataherb.config.yml")
+    [r] = add_repos(cfg, ["acme/meta"], tags=("x",), force=True)
+    assert r.kind == "pointer"
+    data, body = split_front_matter(entry.read_text())
+    assert data["tags"] == ["x"] and body.strip() == "Hand-written notes."
+
+
+def test_yml_format(project):
+    meta_dir = project / "git" / "acme" / "meta" / "HEAD"
+    meta_dir.mkdir(parents=True)
+    (meta_dir / "dataherb.json").write_text("{}")
+    add_repos(load_config(project / "dataherb.config.yml"), ["acme/meta"], fmt="yml")
+    assert yaml.safe_load((project / "catalog" / "meta.yml").read_text())["repo"] == "acme/meta"

tests/catalog/test_catalog.py

@@ -42,3 +42,31 @@ def test_discovered_and_catalog_entries_merge(project):
     res = build_catalog(load_config(project / "dataherb.config.yml"))
     # regions is both discovered (datasets/regions/dataherb.yml) and listed: one record.
     assert [d["id"] for d in res.datasets].count("regions") == 1
+
+
+def test_markdown_entries(project):
+    cat = project / "catalog"
+    (cat / "orders.yml").unlink()
+    (cat / "orders.md").write_text(
+        "---\nid: orders\nrepo: acme/orders\nref: main\ntags: [sales]\n---\n\n"
+        "## Caveats\n\nRefunds arrive a day late.\n"
+    )
+    (cat / "README.md").write_text("# About this folder\n")  # no front matter: not an entry
+    (cat / "notes.md").write_text("---\nid: notes-ds\ninline: true\nname: Notes\n---\n")
+    res = build_catalog(load_config(project / "dataherb.config.yml"))
+    ds = by_id(res)
+    assert "README" not in ds
+    assert ds["orders"]["documentation"].startswith("## Caveats")
+    assert ds["orders"]["tags"] == ["sales"]
+    assert ds["orders"]["catalog_file"] == "catalog/orders.md"
+    assert ds["orders"]["resources"][0]["url"].endswith("/data/orders.csv")
+    assert ds["notes-ds"]["documentation"] == ""
+
+
+def test_split_front_matter():
+    from dataherb.catalog.util import split_front_matter
+
+    assert split_front_matter("---\na: 1\n---\nbody\n") == ({"a": 1}, "body\n")
+    assert split_front_matter("---\n---\nx\n") == (None, "x\n")  # empty front matter
+    assert split_front_matter("# title\n---\n") == (None, "# title\n---\n")
+    assert split_front_matter("---\na: 1\n...\n") == ({"a": 1}, "")