Add catalog and job status commands for DataHerb Explorer

Adds `dataherb catalog build|validate|lint|serve` and
`dataherb status emit|check`, backed by a new dataherb.catalog package
(config, stores for git/s3/http/local, metadata resolution, JSON Schemas,
the dataherb.status/v1 job status spec, metadata quality lint).

`dataherb create` now scaffolds a v2 dataherb.json from the data files
(with --no-input for scripts), and `dataherb validate` adds a schema check
and a metadata quality score. `dataherb upload` uses the current folder
instead of the import-time cwd.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Zz2NMq7okdD9v71q6vtDq

Author: claude

MANIFEST.in

@@ -1,2 +1,3 @@
 include README.md
 graft dataherb/serve/mkdocs_template
+graft dataherb/catalog/schemas

README.md

@@ -56,10 +56,12 @@ dataherb download covid19_eu_data
 
 We provide a template for dataset creation.
 
-Within a dataset folder where the data files are located, use the following command line tool to create the metadata template.
+Within a dataset folder where the data files are located, use the following command line tool to create the metadata. Columns, types and row counts are inferred from csv, tsv, parquet and json files.
 
 ```bash
-dataherb create
+dataherb create            # interactive
+dataherb create --no-input # infer only
+dataherb validate          # check it
 ```
 
 ### Upload dataset to remote
@@ -70,6 +72,23 @@ Within the dataset folder, run
 dataherb upload
 ```
 
+### Catalog website and job status
+
+Build a static catalog and explorer website ([DataHerb Explorer](https://github.com/DataHerb/dataherb-explorer)) from a `dataherb.config.yml`:
+
+```bash
+dataherb catalog validate   # check config and catalog entries
+dataherb catalog lint       # metadata quality per dataset
+dataherb catalog build      # write the site to dist/
+```
+
+Report job runs so the catalog can show freshness and failures:
+
+```bash
+dataherb status emit --target s3://bucket/_dataherb/status/ --job-id my-crawler --status success --expected-interval P1D
+dataherb status check       # exit 1 if any job is failing, stuck or stale
+```
+
 ### UI for all the datasets in a flora
 
 

dataherb/catalog/__init__.py

@@ -0,0 +1,5 @@
+"""Static data catalog tooling: build a DataHerb Explorer site from a YAML
+config, validate and lint catalog entries, and read or write job status files.
+
+The site template lives in https://github.com/DataHerb/dataherb-explorer.
+"""

dataherb/catalog/build.py

@@ -0,0 +1,151 @@
+"""`dataherb catalog build`: config + catalog entries + job status -> a static site in dist/."""
+
+from __future__ import annotations
+
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+from dataherb.version import __version__
+from .resolve import BuildIssue, build_catalog, load_entries
+from .config import Config
+from .lint import lint_dataset
+from .status import HEALTH_ORDER, collect
+from .stores import LocalStore, make_stores
+from .util import dump_json, iso, log, utcnow
+from .validate import errors
+
+
+@dataclass
+class BuildSummary:
+    datasets: int
+    jobs: int
+    errors: int
+    warnings: int
+    out: Path
+
+
+def validate_inputs(cfg: Config) -> list[BuildIssue]:
+    issues = [
+        BuildIssue("error", None, f"dataherb.config.yml {e}")
+        for e in errors("config", cfg.data)
+    ]
+    entries, load_issues = load_entries(cfg)
+    issues.extend(load_issues)
+    for e in entries:
+        doc = {k: v for k, v in e.items() if k != "_file"}
+        for msg in errors("catalog-entry", doc):
+            issues.append(BuildIssue("error", e.get("id"), f"{e['_file']} {msg}"))
+        if e.get("inline"):
+            for msg in errors("dataset", doc):
+                issues.append(BuildIssue("warning", e.get("id"), f"{e['_file']} {msg}"))
+    return issues
+
+
+def link_jobs(datasets: list[dict], jobs: list[dict]) -> None:
+    """Attach job health to datasets, via status_job in metadata or datasets[] in status files."""
+    by_dataset: dict[str, set[str]] = {}
+    for j in jobs:
+        for d in (j.get("state") or {}).get("datasets") or []:
+            if d.get("id"):
+                by_dataset.setdefault(str(d["id"]), set()).add(j["id"])
+    job_by_id = {j["id"]: j for j in jobs}
+    for d in datasets:
+        ids = list(
+            dict.fromkeys(
+                [*d.get("status_jobs", []), *sorted(by_dataset.get(d["id"], ()))]
+            )
+        )
+        d["status_jobs"] = ids
+        healths = [job_by_id[i]["health"] for i in ids if i in job_by_id]
+        d["health"] = min(healths, key=HEALTH_ORDER.index) if healths else None
+        updated = []
+        for i in ids:
+            jb = job_by_id.get(i)
+            if not jb:
+                continue
+            updated.append(jb.get("last_success_at"))
+            for ds in (jb.get("state") or {}).get("datasets") or []:
+                if str(ds.get("id")) == d["id"] and ds.get("data_updated_at"):
+                    updated.append(ds["data_updated_at"])
+        d["updated_at"] = max((u for u in updated if u), default=None)
+
+
+def build(
+    cfg: Config, out: Path, site_dir: Path | None = None, strict: bool = False
+) -> BuildSummary:
+    started = utcnow()
+    site_dir = site_dir or cfg.root / "site"
+    if not (site_dir / "index.html").is_file():
+        raise FileNotFoundError(
+            f"no site template at {site_dir}; build from a fork of "
+            "https://github.com/DataHerb/dataherb-explorer or pass --site"
+        )
+    issues = validate_inputs(cfg)
+
+    log.info("resolving catalog")
+    cat = build_catalog(cfg)
+    issues.extend(cat.issues)
+
+    log.info("collecting job status")
+    jobs, status_issues = collect(cfg, now=started)
+    issues.extend(BuildIssue(**i) for i in status_issues)
+
+    link_jobs(cat.datasets, jobs)
+    known = {j["id"] for j in jobs}
+    for d in cat.datasets:
+        d["quality"] = lint_dataset(d, known_jobs=known)
+
+    # Assemble the site.
+    if out.exists():
+        shutil.rmtree(out)
+    shutil.copytree(site_dir, out, ignore=shutil.ignore_patterns("data", ".DS_Store"))
+    for store in make_stores(cfg.stores, cfg.root).values():
+        if isinstance(store, LocalStore):
+            store.publish(out)
+    if (
+        cfg.explorer.get("duckdb", {}).get("mode") == "vendored"
+        and not (out / "vendor" / "duckdb").exists()
+    ):
+        issues.append(
+            BuildIssue(
+                "error",
+                None,
+                "explorer.duckdb.mode is 'vendored' but site/vendor/duckdb is missing; run `npm ci && npm run vendor`",
+            )
+        )
+
+    visible = [d for d in cat.datasets if not d.get("hidden")]
+    n_err = sum(1 for i in issues if i.level == "error")
+    n_warn = sum(1 for i in issues if i.level == "warning")
+    generated = iso(utcnow())
+    dump_json(
+        {**cfg.public(), "generated_at": generated, "version": __version__},
+        out / "data" / "config.json",
+    )
+    dump_json(
+        {"generated_at": generated, "datasets": visible}, out / "data" / "catalog.json"
+    )
+    dump_json({"generated_at": generated, "jobs": jobs}, out / "data" / "status.json")
+    dump_json(
+        {
+            "generated_at": generated,
+            "duration_seconds": round((utcnow() - started).total_seconds(), 2),
+            "version": __version__,
+            "datasets": len(visible),
+            "jobs": len(jobs),
+            "errors": n_err,
+            "warnings": n_warn,
+            "issues": [i.as_dict() for i in issues],
+        },
+        out / "data" / "build.json",
+        pretty=True,
+    )
+    (out / ".nojekyll").write_text("")
+    if strict and n_err:
+        raise SystemExit(
+            f"build finished with {n_err} error(s); see {out / 'data' / 'build.json'}"
+        )
+    return BuildSummary(
+        datasets=len(visible), jobs=len(jobs), errors=n_err, warnings=n_warn, out=out
+    )

dataherb/catalog/config.py

@@ -0,0 +1,119 @@
+"""Load dataherb.config.yml and fill defaults."""
+
+from __future__ import annotations
+
+import copy
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .util import load_file
+
+DEFAULTS: dict[str, Any] = {
+    "site": {
+        "title": "DataHerb Explorer",
+        "description": "",
+        "logo": None,
+        "accent": "#2f7d4f",
+        "links": [],
+        "repository": None,
+    },
+    "stores": {
+        "github": {
+            "type": "git",
+            "raw_url_template": "https://raw.githubusercontent.com/{repo}/{ref}/{path}",
+            "web_url_template": "https://github.com/{repo}",
+            "default_ref": "HEAD",
+            "token_env": "GITHUB_TOKEN",
+        },
+        "local": {"type": "local", "path": "demo"},
+    },
+    "catalog": {"dirs": ["catalog"], "discover": []},
+    "status": {"sources": [], "live": True, "stale_grace": 0.5, "history": 30},
+    "explorer": {
+        "enabled": True,
+        "duckdb": {
+            "mode": "cdn",
+            "version": "1.29.0",
+            "cdn_base": "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@{version}",
+        },
+        "preview_rows": 200,
+        "max_browser_bytes": 500_000_000,
+    },
+    "snippets": [],
+}
+
+
+def _merge(base: Any, override: Any) -> Any:
+    if isinstance(base, dict) and isinstance(override, dict):
+        out = dict(base)
+        for k, v in override.items():
+            out[k] = _merge(base.get(k), v) if k in base else v
+        return out
+    return copy.deepcopy(override) if override is not None else copy.deepcopy(base)
+
+
+@dataclass
+class Config:
+    data: dict
+    root: Path
+
+    @property
+    def site(self) -> dict:
+        return self.data["site"]
+
+    @property
+    def stores(self) -> dict:
+        return self.data["stores"]
+
+    @property
+    def catalog(self) -> dict:
+        return self.data["catalog"]
+
+    @property
+    def status(self) -> dict:
+        return self.data["status"]
+
+    @property
+    def explorer(self) -> dict:
+        return self.data["explorer"]
+
+    def path(self, p: str) -> Path:
+        q = Path(p)
+        return q if q.is_absolute() else self.root / q
+
+    def public(self) -> dict:
+        """The subset of the config the browser gets. Never includes store credentials."""
+        stores = {}
+        for name, s in self.stores.items():
+            stores[name] = {
+                k: v
+                for k, v in s.items()
+                if k in ("type", "web_url_template", "public_base_url")
+            }
+        return {
+            "site": self.site,
+            "stores": stores,
+            "status": {k: self.status.get(k) for k in ("live", "stale_grace")},
+            "explorer": self.explorer,
+            "snippets": self.data.get("snippets", []),
+        }
+
+
+def load_config(path: str | Path = "dataherb.config.yml") -> Config:
+    path = Path(path).resolve()
+    raw = load_file(path) or {}
+    if not isinstance(raw, dict):
+        raise ValueError(f"{path}: expected a mapping at the top level")
+    # Stores are replaced wholesale when given, so a fork can drop the demo stores.
+    data = _merge(DEFAULTS, {k: v for k, v in raw.items() if k != "stores"})
+    data["stores"] = raw.get("stores") or copy.deepcopy(DEFAULTS["stores"])
+    for name, store in data["stores"].items():
+        if not isinstance(store, dict) or "type" not in store:
+            raise ValueError(f"store '{name}' needs a type (git, s3, http, local)")
+    return Config(data=data, root=path.parent)
+
+
+def load_config_defaults(root: str | Path = ".") -> Config:
+    """A Config with only the defaults, for checking a single dataset outside a catalog."""
+    return Config(data=copy.deepcopy(DEFAULTS), root=Path(root).resolve())