Fix lance auth

src/datasets/download/download_config.py

@@ -76,6 +76,10 @@ def __setattr__(self, name, value):
         if name == "token" and getattr(self, "storage_options", None) is not None:
             if "hf" not in self.storage_options:
                 self.storage_options["hf"] = {"endpoint": config.HF_ENDPOINT, "token": value}
-            elif getattr(self.storage_options["hf"], "token", None) is None:
+            else:
                 self.storage_options["hf"]["token"] = value
         super().__setattr__(name, value)
+
+    def __post_init__(self):
+        # update storage_options
+        self.token = self.token

src/datasets/packaged_modules/lance/lance.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING, Dict, List, Optional
 
 import pyarrow as pa
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, get_token
 
 import datasets
 from datasets import Audio, Image, Video
@@ -119,7 +119,11 @@ def _split_generators(self, dl_manager):
 
         splits: list[datasets.SplitGenerator] = []
         for split_name, files in data_files.items():
-            storage_options = dl_manager.download_config.storage_options.get(files[0].split("://", 1)[0])
+            protocol = files[0].split("://", 1)[0]
+            storage_options = dict(dl_manager.download_config.storage_options.get(protocol, {}))
+            # lance doesn't allow "token": None for hf and expects a string
+            if protocol == "hf" and storage_options.get("token") is None:
+                storage_options["token"] = get_token()
 
             lance_dataset_uris = resolve_dataset_uris(files)
             if lance_dataset_uris: