From 73c2a68335743323db1842bd0e896697f13507f2 Mon Sep 17 00:00:00 2001 From: Sam Kaplan Date: Sun, 23 Jan 2022 13:08:11 -0600 Subject: [PATCH] add Base.cp methods (#31) * add Base.cp methods Co-authored-by: Luke Decker --- docs/src/example.md | 27 +++++++++++-- docs/src/reference.md | 2 + src/AzStorage.jl | 66 +++++++++++++++++++++++++++++++ test/runtests.jl | 91 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 3 deletions(-) diff --git a/docs/src/example.md b/docs/src/example.md index 3e0b7fc..e4e35a7 100644 --- a/docs/src/example.md +++ b/docs/src/example.md @@ -35,6 +35,15 @@ serialize(container, "myblob.bin", (a=rand(10), b=rand(10))) # read and deserialze data from the container x = deserialize(container, "myblob.bin") +# copy a blob to a local file +cp(container, "myblob.bin", "mylocalfile.bin") + +# copy a local file to a blob +cp("mylocalfile.bin", container, "myblob.bin") + +# copy from a blob to another blob +cp(container, "myblob.bin", container, "mycopyblob.bin") + # remove the container, and its contents rm(container) ``` @@ -42,9 +51,9 @@ rm(container) In addition, we can represent blob's, providing an API that is similar to handling POSIX files. ```julia -# create a handle to a blob in a container -io = open(AzContainer("foo"; storageaccount="mystorageaccount"), "myblob.bin") -io = joinpath(AzContainer("foo"; storageaccount="mystorageaccount"), "myblob.bin") # this is equivalent to the previous line. +# create a handle, io, to a blob, "myblob.bin", in a container, "foo", in storage account "mystorageaccount" +io = open(AzContainer("foo"; storageaccount="mystorageaccount", session), "myblob.bin") +io = joinpath(AzContainer("foo"; storageaccount="mystorageaccount", session), "myblob.bin") # this is equivalent to the previous line. # write to the blob write(io, rand(10)) @@ -60,6 +69,18 @@ isfile(io) serialize(io, (a=rand(10),b=rand(10))) x = deserialize(io) +write(io, rand(10)) + +# copy a blob, io, to a local file, mylocalfile.bin +cp(io, "mylocalfile.bin") + +# copy a local file, mylocalfile.bin, to a blob, io +cp("mylocalfile.bin", io) + +# copy from a blob to another blob +io2 = open(AzContainer("foo"; storageaccount="mystorageaccount", session), "mycopyblob.bin") +cp(io, io2) + # remove the blob rm(io) ``` diff --git a/docs/src/reference.md b/docs/src/reference.md index 5a8e913..74ea6e8 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -18,6 +18,8 @@ rm(::AzContainer) ## Blob methods ```@docs +cp(::AbstractString, ::AzContainer, ::AbstractString) +cp(::AbstractString, ::AzStorage.AzObject) deserialize filesize isfile diff --git a/src/AzStorage.jl b/src/AzStorage.jl index 75965de..eb7bca8 100644 --- a/src/AzStorage.jl +++ b/src/AzStorage.jl @@ -502,6 +502,72 @@ read and deserialize a blob `object::AzObject`. See `deserialize(container, "bl """ Serialization.deserialize(o::AzObject) = deserialize(o.container, o.name) +""" + cp(from..., to...) + +copy a blob to a local file, a local file to a blob, or a blob to a blob. + +# Examples + +## local file to blob +``` +cp("localfile.txt", AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob.txt") +``` + +## blob to local file +``` +cp(AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob.txt", "localfile.txt") +``` + +## blob to blob +``` +cp(AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob_in.txt", AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob_out.txt") +``` +""" +function Base.cp(in::AbstractString, outc::AzContainer, outb::AbstractString) + bytes = read!(in, Vector{UInt8}(undef,filesize(in))) + write(outc, outb, bytes) +end + +function Base.cp(inc::AzContainer, inb::AbstractString, out::AbstractString) + bytes = read!(inc, inb, Vector{UInt8}(undef, filesize(inc, inb))) + write(out, bytes) +end + +function Base.cp(inc::AzContainer, inb::AbstractString, outc::AzContainer, outb::AbstractString) + bytes = read!(inc, inb, Vector{UInt8}(undef, filesize(inc, inb))) + write(outc, outb, bytes) +end + +""" + cp(from, to) + +copy a blob to a local file, a local file to a blob, or a blob to a blob. + +# Examples + +## local file to blob +``` +cp("localfile.txt", open(AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob.txt")) +``` + +## blob to local file +``` +cp(open(AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob.txt"), "localfile.txt") +``` + +## blob to blob +``` +cp(open(AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob_in.txt"), open(AzContainer("mycontainer";storageaccount="mystorageaccount"), "remoteblob_out.txt")) +``` +""" +Base.cp(in::AbstractString, out::AzObject) = cp(in, out.container, out.name) +Base.cp(in::AzObject, out::AbstractString) = cp(in.container, in.name, out) +Base.cp(in::AzObject, out::AzObject) = cp(in.container, in.name, out.container, out.name) + +Base.cp(inc::AzContainer, inb::AbstractString, out::AzObject) = cp(inc, inb, out.container, out.name) +Base.cp(in::AzObject, outc::AzContainer, outb::AbstractString) = cp(in.container, in.name, outc, outb) + """ readdir(container) diff --git a/test/runtests.jl b/test/runtests.jl index a10dd25..0f0ca4c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -396,3 +396,94 @@ end @test _c["prefix"] == "" @test length(_c) == 3 end + +@testset "Container, copy blob to local file" begin + sleep(1) + r = lowercase(randstring(MersenneTwister(millisecond(now())+26))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + write(c, "foo.txt", "Hello world") + cp(c, "foo.txt", "foolocal.txt") + @test read("foolocal.txt", String) == "Hello world" + rm(c) + rm("foolocal.txt") +end + +@testset "Container, copy local file to blob" begin + sleep(1) + write("foolocal.txt", "Hello world") + r = lowercase(randstring(MersenneTwister(millisecond(now())+27))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + cp("foolocal.txt", c, "foo.txt") + @test read(c, "foo.txt", String) == "Hello world" + rm(c) + rm("foolocal.txt") +end + +@testset "Container, copy blob to blob" begin + sleep(1) + r = lowercase(randstring(MersenneTwister(millisecond(now())+28))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + write(c, "foo.txt", "Hello world") + cp(c, "foo.txt", c, "bar.txt") + @test read(c, "bar.txt", String) == "Hello world" + rm(c) +end + +@testset "Object, copy blob to local file" begin + sleep(1) + r = lowercase(randstring(MersenneTwister(millisecond(now())+29))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + write(open(c, "foo.txt"), "Hello world") + cp(open(c, "foo.txt"), "foolocal.txt") + @test read("foolocal.txt", String) == "Hello world" + rm(c) + rm("foolocal.txt") +end + +@testset "Object, copy local file to blob" begin + sleep(1) + write("foolocal.txt", "Hello world") + r = lowercase(randstring(MersenneTwister(millisecond(now())+30))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + cp("foolocal.txt", open(c, "foo.txt")) + @test read(open(c, "foo.txt"), String) == "Hello world" + rm(c) + rm("foolocal.txt") +end + +@testset "Object, copy blob to blob" begin + sleep(1) + r = lowercase(randstring(MersenneTwister(millisecond(now())+31))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + write(open(c, "foo.txt"), "Hello world") + cp(open(c, "foo.txt"), open(c, "bar.txt")) + @test read(open(c, "bar.txt"), String) == "Hello world" + rm(c) +end + +@testset "Object,Container, copy blob to blob" begin + sleep(1) + r = lowercase(randstring(MersenneTwister(millisecond(now())+32))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + write(open(c, "foo.txt"), "Hello world") + cp(open(c, "foo.txt"), c, "bar.txt") + @test read(open(c, "bar.txt"), String) == "Hello world" + rm(c) +end + +@testset "Container,Object, copy blob to blob" begin sleep(1) + r = lowercase(randstring(MersenneTwister(millisecond(now())+33))) + c = AzContainer("foo-$r-o", storageaccount=storageaccount, session=session, nthreads=2, nretry=10) + mkpath(c) + write(open(c, "foo.txt"), "Hello world") + cp(c, "foo.txt", open(c, "bar.txt")) + @test read(open(c, "bar.txt"), String) == "Hello world" + rm(c) +end