From 713d996aed7a8f7c30213a4f617ac8e5211f5723 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Fri, 3 Jul 2026 15:29:04 +0200 Subject: [PATCH] fix lance auth --- src/datasets/download/download_config.py | 6 +++++- src/datasets/packaged_modules/lance/lance.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/datasets/download/download_config.py b/src/datasets/download/download_config.py index 6efae72b671..de197da50e3 100644 --- a/src/datasets/download/download_config.py +++ b/src/datasets/download/download_config.py @@ -76,6 +76,10 @@ def __setattr__(self, name, value): if name == "token" and getattr(self, "storage_options", None) is not None: if "hf" not in self.storage_options: self.storage_options["hf"] = {"endpoint": config.HF_ENDPOINT, "token": value} - elif getattr(self.storage_options["hf"], "token", None) is None: + else: self.storage_options["hf"]["token"] = value super().__setattr__(name, value) + + def __post_init__(self): + # update storage_options + self.token = self.token diff --git a/src/datasets/packaged_modules/lance/lance.py b/src/datasets/packaged_modules/lance/lance.py index 017458ec94e..e7310e00270 100644 --- a/src/datasets/packaged_modules/lance/lance.py +++ b/src/datasets/packaged_modules/lance/lance.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional import pyarrow as pa -from huggingface_hub import HfApi +from huggingface_hub import HfApi, get_token import datasets from datasets import Audio, Image, Video @@ -119,7 +119,11 @@ def _split_generators(self, dl_manager): splits: list[datasets.SplitGenerator] = [] for split_name, files in data_files.items(): - storage_options = dl_manager.download_config.storage_options.get(files[0].split("://", 1)[0]) + protocol = files[0].split("://", 1)[0] + storage_options = dict(dl_manager.download_config.storage_options.get(protocol, {})) + # lance doesn't allow "token": None for hf and expects a string + if protocol == "hf" and storage_options.get("token") is None: + storage_options["token"] = get_token() lance_dataset_uris = resolve_dataset_uris(files) if lance_dataset_uris: