refactor(DRAFT): Migrate to datapackage.json only

Major switch from multiple github/npm endpoints -> a single file.
Was Only possible following vega/vega-datasets#665

Still need to rewrite/fill out the `Metadata` doc, then moving onto features

Author: dangotbanned

altair/datasets/_cache.py

@@ -43,9 +43,7 @@
 _T = TypeVar("_T")
 
 _URL: Final[Path] = Path(__file__).parent / "_metadata" / "url.csv.gz"
-_SCHEMA: Final[Path] = (
-    Path(__file__).parent / "_metadata" / "datapackage_schemas.json.gz"
-)
+_SCHEMA: Final[Path] = Path(__file__).parent / "_metadata" / "schemas.json.gz"
 
 _FIELD_TO_DTYPE: Mapping[FlFieldStr, type[DType]] = {
     "integer": nw.Int64,
@@ -118,7 +116,7 @@ def __init__(
         fp: Path,
         /,
         *,
-        columns: tuple[str, str] = ("dataset_name", "url_npm"),
+        columns: tuple[str, str],
         tp: type[MutableMapping[_KT, _VT]] = dict["_KT", "_VT"],
     ) -> None:
         self.fp: Path = fp
@@ -253,7 +251,7 @@ def download_all(self) -> None:
         stems = tuple(fp.stem for fp in self)
         predicates = (~(nw.col("sha").is_in(stems)),) if stems else ()
         frame = (
-            self._rd._scan_metadata(predicates, is_image=False)  # type: ignore
+            self._rd._scan_metadata(predicates, is_image=False)
             .select("sha", "suffix", "url")
             .unique("sha")
             .collect()
@@ -338,5 +336,7 @@ def _ensure_active(self) -> None:
             raise ValueError(msg)
 
 
-url_cache: UrlCache[Dataset | LiteralString, str] = UrlCache(_URL)
+url_cache: UrlCache[Dataset | LiteralString, str] = UrlCache(
+    _URL, columns=("dataset_name", "url")
+)
 schema_cache = SchemaCache(_SCHEMA)

altair/datasets/_metadata/datapackage_features.parquet

@@ -86,9 +86,6 @@
 __all__ = ["backend"]
 
 _METADATA: Final[Path] = Path(__file__).parent / "_metadata" / "metadata.parquet"
-_DATAPACKAGE: Final[Path] = (
-    Path(__file__).parent / "_metadata" / "datapackage_features.parquet"
-)
 
 
 class AltairDatasetsError(Exception): ...
@@ -209,7 +206,7 @@ def query(
     def _scan_metadata(
         self, *predicates: OneOrSeq[IntoExpr], **constraints: Unpack[Metadata]
     ) -> nw.LazyFrame:
-        frame = nw.from_native(self.scan_fn(_DATAPACKAGE)(_DATAPACKAGE)).lazy()
+        frame = nw.from_native(self.scan_fn(_METADATA)(_METADATA)).lazy()
         if predicates or constraints:
             return frame.filter(*predicates, **constraints)
         return frame

altair/datasets/_metadata/metadata.parquet

@@ -22,17 +22,10 @@
     from typing_extensions import TypeAlias
 
 
-__all__ = [
-    "EXTENSION_SUFFIXES",
-    "VERSION_LATEST",
-    "Dataset",
-    "Extension",
-    "Metadata",
-    "Version",
-    "is_ext_read",
-]
+__all__ = ["EXTENSION_SUFFIXES", "Dataset", "Extension", "Metadata", "is_ext_read"]
 
 Dataset: TypeAlias = Literal[
+    "7zip",
     "airports",
     "annual-precip",
     "anscombe",
@@ -42,13 +35,13 @@
     "budgets",
     "burtin",
     "cars",
-    "climate",
     "co2-concentration",
     "countries",
     "crimea",
     "disasters",
     "driving",
     "earthquakes",
+    "ffox",
     "flare",
     "flare-dependencies",
     "flights-10k",
@@ -61,12 +54,11 @@
     "football",
     "gapminder",
     "gapminder-health-income",
+    "gimp",
     "github",
     "global-temp",
-    "graticule",
     "income",
     "iowa-electricity",
-    "iris",
     "jobs",
     "la-riots",
     "londonBoroughs",
@@ -86,10 +78,8 @@
     "political-contributions",
     "population",
     "population_engineers_hurricanes",
-    "seattle-temps",
     "seattle-weather",
     "seattle-weather-hourly-normals",
-    "sf-temps",
     "sp500",
     "sp500-2000",
     "stocks",
@@ -102,71 +92,24 @@
     "us-state-capitals",
     "volcano",
     "weather",
-    "weball26",
+    "weekly-weather",
     "wheat",
     "windvectors",
     "world-110m",
     "zipcodes",
 ]
-Version: TypeAlias = Literal[
-    "v2.11.0",
-    "v2.10.0",
-    "v2.9.0",
-    "v2.8.1",
-    "v2.8.0",
-    "v2.7.0",
-    "v2.5.4",
-    "v2.5.3",
-    "v2.5.3-next.0",
-    "v2.5.2",
-    "v2.5.2-next.0",
-    "v2.5.1",
-    "v2.5.1-next.0",
-    "v2.5.0",
-    "v2.5.0-next.0",
-    "v2.4.0",
-    "v2.3.1",
-    "v2.3.0",
-    "v2.1.0",
-    "v2.0.0",
-    "v1.31.1",
-    "v1.31.0",
-    "v1.30.4",
-    "v1.30.3",
-    "v1.30.2",
-    "v1.30.1",
-    "v1.29.0",
-    "v1.24.0",
-    "v1.22.0",
-    "v1.21.1",
-    "v1.21.0",
-    "v1.20.0",
-    "v1.19.0",
-    "v1.18.0",
-    "v1.17.0",
-    "v1.16.0",
-    "v1.15.0",
-    "v1.14.0",
-    "v1.12.0",
-    "v1.11.0",
-    "v1.10.0",
-    "v1.8.0",
-    "v1.7.0",
-    "v1.5.0",
-]
-Extension: TypeAlias = Literal[".csv", ".json", ".tsv", ".arrow", ".parquet"]
-VERSION_LATEST: Literal["v2.11.0"] = "v2.11.0"
+Extension: TypeAlias = Literal[".arrow", ".csv", ".json", ".parquet", ".tsv"]
 EXTENSION_SUFFIXES: tuple[
+    Literal[".arrow"],
     Literal[".csv"],
     Literal[".json"],
-    Literal[".tsv"],
-    Literal[".arrow"],
     Literal[".parquet"],
-] = (".csv", ".json", ".tsv", ".arrow", ".parquet")
+    Literal[".tsv"],
+] = (".arrow", ".csv", ".json", ".parquet", ".tsv")
 
 
 def is_ext_read(suffix: Any) -> TypeIs[Extension]:
-    return suffix in {".csv", ".json", ".tsv", ".arrow", ".parquet"}
+    return suffix in {".arrow", ".csv", ".json", ".parquet", ".tsv"}
 
 
 class Metadata(TypedDict, total=False):
@@ -177,29 +120,34 @@ class Metadata(TypedDict, total=False):
     ----------
     dataset_name
         Name of the dataset/`Path.stem`_.
-    ext_supported
-        Dataset can be read as tabular data.
+    suffix
+        File extension/`Path.suffix`_.
     file_name
         Equivalent to `Path.name`_.
-    name_collision
-        Dataset is available via multiple formats.
-
-        .. note::
-            Requires specifying a preference in calls to ``data(name, suffix=...)``
+    bytes
+        File size in *bytes*.
+    is_image
+        _description_
+    is_tabular
+        Can be read as tabular data.
+    is_geo
+        _description_
+    is_topo
+        _description_
+    is_spatial
+        _description_
+    is_json
+        _description_
+    has_schema
+        Data types available for improved ``pandas`` parsing.
     sha
         Unique hash for the dataset.
 
         .. note::
-            If the dataset did *not* change between ``v1.0.0``-``v2.0.0``;
+            E.g. if the dataset did *not* change between ``v1.0.0``-``v2.0.0``;
 
-            then all ``tag``(s) in this range would **share** this value.
-    size
-        File size (*bytes*).
-    suffix
-        File extension/`Path.suffix`_.
-    tag
-        Version identifier for a `vega-datasets release`_.
-    url_npm
+            then this value would remain stable.
+    url
         Remote url used to access dataset.
 
     .. _Path.stem:
@@ -208,13 +156,14 @@ class Metadata(TypedDict, total=False):
         https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.name
     .. _Path.suffix:
         https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.suffix
-    .. _vega-datasets release:
-        https://github.com/vega/vega-datasets/releases
+
 
     Examples
     --------
     ``Metadata`` keywords form constraints to filter a table like the below sample:
 
+    ### FIXME: NEEDS UPDATING TO DATAPACKAGE VERSION
+
     ```
     shape: (2_879, 9)
     ┌───────────┬──────────┬──────────┬──────────┬───┬────────┬─────────┬──────────┐
@@ -249,14 +198,18 @@ class Metadata(TypedDict, total=False):
     """
 
     dataset_name: str
-    ext_supported: bool
+    suffix: str
     file_name: str
-    name_collision: bool
+    bytes: int
+    is_image: bool
+    is_tabular: bool
+    is_geo: bool
+    is_topo: bool
+    is_spatial: bool
+    is_json: bool
+    has_schema: bool
     sha: str
-    size: int
-    suffix: str
-    tag: str
-    url_npm: str
+    url: str
 
 
 FlFieldStr: TypeAlias = Literal[

altair/datasets/_metadata/datapackage_schemas.json.gz altair/datasets/_metadata/schemas.json.gz

@@ -16,7 +16,7 @@
 from narwhals.stable.v1 import dependencies as nw_dep
 
 from altair.datasets import Loader, url
-from altair.datasets._readers import _METADATA, AltairDatasetsError
+from altair.datasets._readers import AltairDatasetsError
 from altair.datasets._typing import Dataset, Extension, Metadata, is_ext_read
 from tests import skip_requires_pyarrow, slow
 
@@ -296,9 +296,6 @@ def test_url_no_backend(monkeypatch: pytest.MonkeyPatch) -> None:
     assert match_url("flights-10k", url("flights-10k"))
     assert match_url("flights-200k", url("flights-200k"))
 
-    with pytest.raises(TypeError, match="cannot be loaded via url"):
-        url("climate")
-
     with pytest.raises(TypeError, match="cannot be loaded via url"):
         url("flights-3m")
 
@@ -690,9 +687,7 @@ def test_no_remote_connection(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -
 def test_metadata_columns(backend: _Backend, metadata_columns: frozenset[str]) -> None:
     """Ensure all backends will query the same column names."""
     data = Loader.from_backend(backend)
-    fn = data._reader.scan_fn(_METADATA)
-    native = fn(_METADATA)
-    schema_columns = nw.from_native(native).lazy().collect().columns
+    schema_columns = data._reader._scan_metadata().collect().columns
     assert set(schema_columns) == metadata_columns