diff --git a/Data.toml b/Data.toml new file mode 100644 index 0000000..3f23f4c --- /dev/null +++ b/Data.toml @@ -0,0 +1,11 @@ +data_config_version=1 + +[[datasets]] +name="Pi" +uuid="76e82d27-3914-4472-abb6-38d476023211" + + [datasets.storage] + driver="DataDeps" + sha256="1f08958df70c30bbf3f7205ad5f9b2b9430e3378b4e4e6300063bd9fbd83d6e3" + localdir="~/.julia/datadeps/Pi" + remote_path=["https://www.angio.net/pi/digits/10000.txt"] diff --git a/Project.toml b/Project.toml index 854771d..d4ca209 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "0.7.7" [deps] BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +DataSets = "c9661210-8a83-48f0-b833-72e62abce419" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" diff --git a/src/DataDeps.jl b/src/DataDeps.jl index 94e82b9..fc17823 100644 --- a/src/DataDeps.jl +++ b/src/DataDeps.jl @@ -42,4 +42,12 @@ include("preupload.jl") include("fetch_helpers.jl") include("post_fetch_helpers.jl") +# Extension +using DataSets +include("datasets_storage_driver.jl") + +function __init__() + DataSets.add_storage_driver("DataDeps"=>DataDep_storage_driver) +end + end # module diff --git a/src/datasets_storage_driver.jl b/src/datasets_storage_driver.jl new file mode 100644 index 0000000..c80cb6f --- /dev/null +++ b/src/datasets_storage_driver.jl @@ -0,0 +1,21 @@ +# storage driver for DataSets.jl + +function DataDep_storage_driver(user_func, storage_config, dataset) + conf = dataset.conf + dp = DataDep( + conf["name"], + "", + storage_config["remote_path"], + get(storage_config, "sha256", "") + ) + localdir = expanduser(storage_config["localdir"]) + if !isdir(localdir) + try + DataDeps.download(dp, localdir; i_accept_the_terms_of_use=true) + catch + @error "failed to download dataset" + rm(localdir; recursive=true) + end + end + user_func(BlobTree(DataSets.FileSystemRoot(localdir), path".")) +end