diff --git a/NOTICE.txt b/NOTICE.txt index ef12f9a58bdc..7c2d23002c31 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -500,6 +500,40 @@ See the License for the specific language governing permissions and limitations ------------------------------------------------------------------------------------------------- +License notice for Apache Arrow +------------------------------------------------------------------------------- + +apache-arrow-java (https://github.com/apache/arrow-java) +Copyright 2015-2024 The Apache Software Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License +is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. + +------------------------------------------------------------------------------------------------- + +License notice for FlatBuffers +------------------------------------------------------------------------------- + +flatbuffers (https://github.com/google/flatbuffers) +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License +is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. + +------------------------------------------------------------------------------------------------- + License notice for GraalVM ------------------------------------------------------------------------------ org.graalvm.sdk:graal-sdk - https://github.com/graalvm/native-build-tools/blob/master/common/junit-platform-native/LICENSE @@ -609,4 +643,4 @@ compliance with the License. You may obtain a copy of the License at: Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific -language governing permissions and limitations under the License. \ No newline at end of file +language governing permissions and limitations under the License. diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index b3e6c054a6ba..3ebefc965c96 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -25,6 +25,7 @@ com.github.spotbugs:spotbugs;4.8.3 com.github.spotbugs:spotbugs-annotations;4.8.3 com.github.spotbugs:spotbugs-maven-plugin;4.8.3.1 com.google.code.gson:gson;2.14.0 +com.google.flatbuffers:flatbuffers-java;25.2.10 com.google.guava:guava;33.6.0-jre com.h2database:h2;2.2.220 com.h3xstream.findsecbugs:findsecbugs-plugin;1.9.0 @@ -63,6 +64,7 @@ io.vertx:vertx-codegen;4.5.27 io.vertx:vertx-core;4.5.27 javax.websocket:javax.websocket-api;1.1 org.apache.ant:ant;1.10.15 +org.apache.arrow:arrow-format;19.0.0 org.apache.avro:avro;1.11.4 org.apache.avro:avro-maven-plugin;1.11.4 org.apache.commons:commons-lang3;3.18.0 diff --git a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml index 3a8920a67b35..4587e20a1bd6 100644 --- a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml +++ b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml @@ -108,6 +108,150 @@ matrix: durationMin: 60 imageBuildDir: "../../.." + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation + crc64downloadstreamsm: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and async client + crc64downloadstreamasyncsm: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and large payload + crc64downloadstreamlg: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation, async client, and large payload + crc64downloadstreamasynclg: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation + crc64downloadcontentsm: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and async client + crc64downloadcontentasyncsm: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and large payload + crc64downloadcontentlg: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation, async client, and large payload + crc64downloadcontentasynclg: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation + crc64downloadfilesm: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and async client + crc64downloadfileasyncsm: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and multi-block payload + crc64downloadfilemd: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation, async client, and multi-block payload + crc64downloadfileasyncmd: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation + crc64inputstreamsm: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation and large payload + crc64inputstreamlg: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation + crc64bytechannelreadsm: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation and large payload + crc64bytechannelreadlg: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + # this test uploads 1KB (1024 bytes) to append blob, no chunking appendblocksmall: testScenario: appendblock @@ -323,3 +467,243 @@ matrix: uploadFaults: true durationMin: 60 imageBuildDir: "../../.." + + crc64appendblock-sm: + testScenario: appendblockwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendblock-lg: + testScenario: appendblockwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 30 + imageBuildDir: "../../.." + + crc64appendblockasync-sm: + testScenario: appendblockwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendblockasync-lg: + testScenario: appendblockwithcrc64 + sync: false + sizeBytes: "26214400" + uploadFaults: true + durationMin: 30 + imageBuildDir: "../../.." + + crc64appendoutputstream-sm: + testScenario: appendbloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendoutputstream-lg: + testScenario: appendbloboutputstreamwithcrc64 + sync: true + sizeBytes: "10240" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64blockblobupload-sm: + testScenario: blockblobuploadwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64blockblobupload-lg: + testScenario: blockblobuploadwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64blockoutputstream-sm: + testScenario: blockbloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64blockoutputstream-lg: + testScenario: blockbloboutputstreamwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64bytechannelwrite-sm: + testScenario: seekablebytechannelwritewithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 10 + imageBuildDir: "../../.." + + crc64bytechannelwrite-lg: + testScenario: seekablebytechannelwritewithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64stageblock-sm: + testScenario: stageblockwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64stageblock-lg: + testScenario: stageblockwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64stageblockasync-sm: + testScenario: stageblockwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64stageblockasync-lg: + testScenario: stageblockwithcrc64 + sync: false + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64pageoutputstream-sm: + testScenario: pagebloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64pageoutputstream-lg: + testScenario: pagebloboutputstreamwithcrc64 + sync: true + sizeBytes: "10240" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadpages-sm: + testScenario: uploadpageswithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadpages-lg: + testScenario: uploadpageswithcrc64 + sync: true + sizeBytes: "4194304" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadpagesasync-sm: + testScenario: uploadpageswithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadpagesasync-lg: + testScenario: uploadpageswithcrc64 + sync: false + sizeBytes: "4194304" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64upload-sm: + testScenario: uploadwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64upload-lg: + testScenario: uploadwithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadasync-sm: + testScenario: uploadwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadasync-lg: + testScenario: uploadwithcrc64 + sync: false + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadfromfile-sm: + testScenario: uploadfromfilewithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadfromfile-lg: + testScenario: uploadfromfilewithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadfromfileasync-sm: + testScenario: uploadfromfilewithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadfromfileasync-lg: + testScenario: uploadfromfilewithcrc64 + sync: false + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java index e38bd16791ca..f146945db423 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java @@ -15,6 +15,21 @@ public static void main(String[] args) { BlockBlobOutputStream.class, BlockBlobUpload.class, CommitBlockList.class, + DownloadContentWithCRC64.class, + DownloadStreamWithCRC64.class, + DownloadToFileWithCRC64.class, + OpenInputStreamWithCRC64.class, + OpenSeekableByteChannelReadWithCRC64.class, + AppendBlobOutputStreamWithCRC64.class, + AppendBlockWithCRC64.class, + BlockBlobOutputStreamWithCRC64.class, + BlockBlobUploadWithCRC64.class, + PageBlobOutputStreamWithCRC64.class, + StageBlockWithCRC64.class, + SeekableByteChannelWriteWithCRC64.class, + UploadWithCRC64.class, + UploadFromFileWithCRC64.class, + UploadPagesWithCRC64.class, DownloadToFile.class, DownloadStream.class, DownloadContent.class, diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..7ea8a3841f32 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.core.util.Context; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Append blob output stream with CRC64 enabled (sync only). + */ +public class AppendBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(AppendBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Separate blob used to upload reference content for {@link OriginalContent} checksum (block blob). */ + private final BlobAsyncClient tempSetupBlobClient; + + public AppendBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + // Reset the append blob at the start of each iteration. The boolean overload + // getBlobOutputStream(true) does this implicitly via create(true); the options overload + // does not, so we replicate that behavior here. Without this reset, fault-injection + // sequences that commit a block server-side but drop the response leave the cached + // appendPosition stale, causing subsequent retries to fail with 412 AppendPositionConditionNotMet, + // which combined with non-retriable Crc64Mismatch on truncated-body faults collapses the pass rate. + appendBlobClient.create(true); + + AppendBlobOutputStreamOptions streamOptions = new AppendBlobOutputStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = appendBlobClient.getBlobOutputStream(streamOptions)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java new file mode 100644 index 000000000000..0eafab84e609 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.specialized.AppendBlobAsyncClient; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Append block with CRC64 enabled. + */ +public class AppendBlockWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final BlobAsyncClient tempSetupBlobClient; + + public AppendBlockWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + appendBlobClient.appendBlockWithResponse(inputStream, options.getSize(), + new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + AppendBlobAsyncClient appendBlobAsyncClient = asyncClient.getAppendBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return appendBlobAsyncClient.appendBlockWithResponse(byteBufferFlux, options.getSize(), + new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getAppendBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..1873ac740d72 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Block blob output stream with CRC64 enabled (sync only). + */ +public class BlockBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(BlockBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public BlockBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions().setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobOutputStreamOptions streamOptions = new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = blockBlobClient.getBlobOutputStream(streamOptions, span)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java new file mode 100644 index 000000000000..b88b9b5c808d --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Single-shot block blob upload with request content validation with CRC64 enabled. + */ +public class BlockBlobUploadWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public BlockBlobUploadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + blockBlobClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(inputStream, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + return blockBlobAsyncClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(byteBufferFlux, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java new file mode 100644 index 000000000000..6c92e1563c60 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +/** + * Download content with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadContentWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadContentWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + originalContent.checkMatch( + syncClient.downloadContentWithResponse( + new BlobDownloadContentOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span).getValue(), + span).block(); + } + + @Override + protected Mono runInternalAsync(Context span) { + // TODO return downloadContent once it stops buffering. + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> { + long contentLength = Long.valueOf(response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + return BinaryData.fromFlux(response.getValue(), contentLength, false); + }) + .flatMap(bd -> originalContent.checkMatch(bd, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java new file mode 100644 index 000000000000..64cccb2c7dd4 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcOutputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +/** + * Streaming blob download with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadStreamWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (CrcOutputStream outputStream = new CrcOutputStream()) { + syncClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + outputStream.close(); + originalContent.checkMatch(outputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> originalContent.checkMatch(response.getValue(), span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java new file mode 100644 index 000000000000..ceb2faa8b153 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.UUID; + +/** + * Download to file with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadToFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(DownloadToFileWithCRC64.class); + private final Path directoryPath; + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public DownloadToFileWithCRC64(StorageStressOptions options) { + super(options); + this.directoryPath = getTempPath("test"); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = directoryPath.resolve(UUID.randomUUID() + ".txt"); + BlobDownloadToFileOptions blobOptions = new BlobDownloadToFileOptions(downloadPath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try { + syncClient.downloadToFileWithResponse(blobOptions, Duration.ofSeconds(options.getDuration()), span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return Mono.using( + () -> directoryPath.resolve(UUID.randomUUID() + ".txt"), + path -> asyncClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(path.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(path), span)), + DownloadToFileWithCRC64::deleteFile); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java new file mode 100644 index 000000000000..eaae1570560b --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.InputStream; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Open input stream with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenInputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenInputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenInputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (InputStream stream = syncClient.openInputStream( + new BlobInputStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span)) { + try (CrcInputStream crcStream = new CrcInputStream(stream)) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("openInputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java new file mode 100644 index 000000000000..e6376238fe92 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.channels.Channels; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Seekable byte channel read with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenSeekableByteChannelReadWithCRC64 + extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenSeekableByteChannelReadWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenSeekableByteChannelReadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlobSeekableByteChannelReadResult result = syncClient.openSeekableByteChannelRead( + new BlobSeekableByteChannelReadOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span); + try (CrcInputStream crcStream = new CrcInputStream(Channels.newInputStream(result.getChannel()))) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, + new RuntimeException("openSeekableByteChannelRead() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..f4dda3295b18 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Page blob output stream with CRC64 enabled (sync only). + */ +public class PageBlobOutputStreamWithCRC64 extends PageBlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(PageBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Page blob used only to seed {@link OriginalContent} (same pattern as {@link PageBlobOutputStream}). */ + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public PageBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) throws IOException { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageBlobOutputStreamOptions streamOptions = new PageBlobOutputStreamOptions( + new PageRange().setStart(0).setEnd(options.getSize() - 1)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = pageBlobClient.getBlobOutputStream(streamOptions)) { + ByteArrayOutputStream bufferStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[512]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + // Always accumulate into bufferStream to avoid dropping or reordering bytes + bufferStream.write(buffer, 0, bytesRead); + // Flush all full 512-byte pages from the accumulator + if (bufferStream.size() >= buffer.length) { + byte[] toWrite = bufferStream.toByteArray(); + int length = toWrite.length - (toWrite.length % buffer.length); + if (length > 0) { + outputStream.write(toWrite, 0, length); + bufferStream.reset(); + // Keep any remaining partial page bytes in the accumulator + bufferStream.write(toWrite, length, toWrite.length - length); + } + } + } + // For page blobs, total content size must be a multiple of 512 bytes. + // Any remaining bytes here indicate misalignment and would result in silent truncation. + if (bufferStream.size() != 0) { + throw new IOException("Remaining bytes in buffer that do not align to 512-byte page size."); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java new file mode 100644 index 000000000000..5f345bbfd5f1 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageSeekableByteChannel; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static com.azure.core.util.FluxUtil.monoError; +import static com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE; + +/** + * Block-blob seekable byte channel write with CRC64 enabled (sync only). + */ +public class SeekableByteChannelWriteWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(SeekableByteChannelWriteWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public SeekableByteChannelWriteWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobSeekableByteChannelWriteOptions writeOptions = new BlockBlobSeekableByteChannelWriteOptions(OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream crcInput = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + Flux byteBufferFlux = crcInput.convertStreamToByteBuffer(); + try (StorageSeekableByteChannel channel = (StorageSeekableByteChannel) blockBlobClient.openSeekableByteChannelWrite( + writeOptions)) { + Mono writeOperation = byteBufferFlux + .doOnNext(buffer -> { + try { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new RuntimeException(e)); + } + }).then(); + writeOperation.block(); + channel.getWriteBehavior().commit(options.getSize()); + } + originalContent.checkMatch(byteBufferFlux, span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException( + "openSeekableByteChannelWrite() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java new file mode 100644 index 000000000000..c06bbe6a44b6 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; + +/** + * Stage block with CRC64 enabled on the faulted client, then commit via the non-faulted client. + */ +public class StageBlockWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + + public StageBlockWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobClient blockBlobClientNoFault = syncNoFaultClient.getBlockBlobClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + BinaryData data = BinaryData.fromStream(inputStream, options.getSize()); + blockBlobClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + blockBlobClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + BlockBlobAsyncClient blockBlobAsyncClientNoFault = asyncNoFaultClient.getBlockBlobAsyncClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return BinaryData.fromFlux(byteBufferFlux, options.getSize(), false) + .flatMap(binaryData -> blockBlobAsyncClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) + .then(blockBlobAsyncClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)))) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java new file mode 100644 index 000000000000..b9594326e6ad --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.UUID; + +/** + * Upload from file with CRC64 enabled. + */ +public class UploadFromFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(UploadFromFileWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public UploadFromFileWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = getTempPath("test"); + Path uploadFilePath = null; + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + uploadFilePath = generateFile(inputStream); + downloadPath = downloadPath.resolve(CoreUtils.randomUuid() + ".txt"); + syncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + syncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadPath.toString()), null, span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + deleteFile(uploadFilePath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Path downloadPath = getTempPath("test"); + // This is written differently than the other runInternalAsync methods because uploadFromFile requires a file + // path, so we need to generate the temp file. + return Mono.using( + () -> new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()), + inputStream -> uploadAndVerifyAsync(inputStream, downloadPath, span), + CrcInputStream::close); + } + + private Mono uploadAndVerifyAsync(CrcInputStream inputStream, Path downloadDir, Context span) { + Path uploadFilePath = generateFile(inputStream); + Path downloadFilePath = downloadDir.resolve(UUID.randomUUID() + ".txt"); + + return asyncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(ignored -> asyncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadFilePath.toString()))) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(downloadFilePath), span)) + .doFinally(signal -> { + deleteFile(uploadFilePath); + deleteFile(downloadFilePath); + }); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } + + private static Path generateFile(InputStream inputStream) { + try { + File file = Files.createTempFile(CoreUtils.randomUuid().toString(), ".txt").toFile(); + file.deleteOnExit(); + Files.copy(inputStream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + return file.toPath(); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java new file mode 100644 index 000000000000..a7d1f887117b --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Page blob upload pages with CRC64 enabled. + */ +public class UploadPagesWithCRC64 extends PageBlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public UploadPagesWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + pageBlobClient.uploadPagesWithResponse(range, inputStream, + new PageBlobUploadPagesOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + PageBlobAsyncClient pageBlobAsyncClient = asyncClient.getPageBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + return pageBlobAsyncClient.uploadPagesWithResponse(range, byteBufferFlux, + new PageBlobUploadPagesOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java new file mode 100644 index 000000000000..55340ba93946 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Parallel blob upload with CRC64 enabled. + */ +public class UploadWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public UploadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + syncClient.uploadWithResponse(new BlobParallelUploadOptions(inputStream) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return asyncClient.uploadWithResponse(new BlobParallelUploadOptions(byteBufferFlux) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index 1a4ec57e5034..24064885f7ed 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -3,6 +3,11 @@ ## 12.36.0-beta.1 (Unreleased) ### Features Added +- Added support for retrieving `AccessTier` headers to `BlobDownloadHeaders`. +- Added support for `PutBlob` operations returning both the existing MD5 content hash and the new CRC64 checksum. +- Added support for Apache Arrow response format for `ListBlobs` flat and `ListBlobs` hierarchical options. +- Added support for content validation with structured message and CRC64 via `ContentValidationAlgorithm`. Supported for +`GetBlob`, `PutBlob`, `PutBlock`, `PutPage`, and `AppendBlock` operations. ### Breaking Changes @@ -17,6 +22,12 @@ logs a warning and signals end-of-file when such an error occurs at or past the known end of the resource. ### Other Changes +- Deprecated `AppendBlobAsyncClient.appendBlockWithResponse(Flux, long, byte[], AppendBlobRequestConditions)` and + `AppendBlobClient.appendBlockWithResponse(InputStream, long, byte[], AppendBlobRequestConditions, Duration, Context)` + in favor of the new `appendBlockWithResponse` overloads that accept an `AppendBlobAppendBlockOptions` options bag. +- Deprecated `PageBlobAsyncClient.uploadPagesWithResponse(PageRange, Flux, byte[], PageBlobRequestConditions)` and + `PageBlobClient.uploadPagesWithResponse(PageRange, InputStream, byte[], PageBlobRequestConditions, Duration, Context)` + in favor of the new `uploadPagesWithResponse` overloads that accept a `PageBlobUploadPagesOptions` options bag. ## 12.35.0 (2026-06-11) diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 6b2f467873bf..222d2cf55571 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "java", "TagPrefix": "java/storage/azure-storage-blob", - "Tag": "java/storage/azure-storage-blob_447cc62a15" + "Tag": "java/storage/azure-storage-blob_018b1a065c" } diff --git a/sdk/storage/azure-storage-blob/cgmanifest.json b/sdk/storage/azure-storage-blob/cgmanifest.json new file mode 100644 index 000000000000..ebf3303b2cf6 --- /dev/null +++ b/sdk/storage/azure-storage-blob/cgmanifest.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "version": 1, + "registrations": [ + { + "developmentDependency": false, + "component": { + "type": "maven", + "maven": { + "groupId": "org.apache.arrow", + "artifactId": "arrow-format", + "version": "19.0.0" + } + } + }, + { + "developmentDependency": false, + "component": { + "type": "maven", + "maven": { + "groupId": "com.google.flatbuffers", + "artifactId": "flatbuffers-java", + "version": "25.2.10" + } + } + } + ] +} diff --git a/sdk/storage/azure-storage-blob/pom.xml b/sdk/storage/azure-storage-blob/pom.xml index 4c591a7d5304..788e9790c31d 100644 --- a/sdk/storage/azure-storage-blob/pom.xml +++ b/sdk/storage/azure-storage-blob/pom.xml @@ -59,6 +59,10 @@ concurrent + + **/implementation/util/apachearrow/**/*.java,**/implementation/util/flatbuffers/**/*.java + **/implementation/util/apachearrow/**/*.java,**/implementation/util/flatbuffers/**/*.java + false @@ -138,8 +142,21 @@ 1.17.7 test - + + + inject-sas-service-version @@ -182,5 +199,103 @@ + + + + + + + shade-apache-arrow-format + + + shade-apache-arrow-format + + + + + + org.apache.arrow + arrow-format + 19.0.0 + + + + com.google.flatbuffers + flatbuffers-java + 25.2.10 + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + + shade + package + + shade + + + + + + true + true + true + shade-apache-arrow-format + false + + + + org.apache.arrow:arrow-format + com.google.flatbuffers:flatbuffers-java + + + + + + org.apache.arrow.flatbuf + com.azure.storage.blob.implementation.util.apachearrow + + + com.google.flatbuffers + com.azure.storage.blob.implementation.util.flatbuffers + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.6.1 + + + + + com.google.flatbuffers:flatbuffers-java + + + + + org.apache.arrow:arrow-format:[19.0.0] + com.google.flatbuffers:flatbuffers-java:[25.2.10] + + + + + + + + diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java index 5c4b16b3b9e9..6f4a32f70dda 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java @@ -37,10 +37,14 @@ import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.specialized.PageBlobAsyncClient; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; +import com.azure.storage.common.ContentValidationAlgorithm; + import com.azure.storage.common.implementation.BufferStagingArea; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.UploadUtils; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -695,16 +699,23 @@ public Mono> uploadWithResponse(BlobParallelUploadOption ? new BlobImmutabilityPolicy() : options.getImmutabilityPolicy(); final Boolean legalHold = options.isLegalHold(); + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + + ContentValidationModeResolver.validateTransactionalChecksumOptions(computeMd5, contentValidationAlgorithm); + ContentValidationModeResolver.validateProgressWithContentValidation( + parallelTransferOptions.getProgressListener(), contentValidationAlgorithm); BlockBlobAsyncClient blockBlobAsyncClient = getBlockBlobAsyncClient(); Function, Mono>> uploadInChunksFunction = (stream) -> uploadInChunks(blockBlobAsyncClient, stream, parallelTransferOptions, headers, metadata, - tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold); + tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold, + contentValidationAlgorithm); BiFunction, Long, Mono>> uploadFullBlobFunction = (stream, length) -> uploadFullBlob(blockBlobAsyncClient, stream, length, parallelTransferOptions, - headers, metadata, tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold); + headers, metadata, tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold, + contentValidationAlgorithm); Flux data = options.getDataFlux(); data = UploadUtils.extractByteBuffer(data, options.getOptionalLength(), @@ -721,7 +732,7 @@ private Mono> uploadFullBlob(BlockBlobAsyncClient blockB Flux data, long length, ParallelTransferOptions parallelTransferOptions, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions requestConditions, boolean computeMd5, BlobImmutabilityPolicy immutabilityPolicy, - Boolean legalHold) { + Boolean legalHold, ContentValidationAlgorithm contentValidationAlgorithm) { /* * Note that there is no need to buffer here as the flux returned by the size gate in this case is created @@ -738,7 +749,8 @@ private Mono> uploadFullBlob(BlockBlobAsyncClient blockB .setImmutabilityPolicy(immutabilityPolicy) .setLegalHold(legalHold)) .flatMap(options -> { - Mono> responseMono = blockBlobAsyncClient.uploadWithResponse(options); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.uploadWithResponse(options), contentValidationAlgorithm, length, false); if (parallelTransferOptions.getProgressListener() != null) { ProgressReporter progressReporter = ProgressReporter.withProgressListener(parallelTransferOptions.getProgressListener()); @@ -753,7 +765,7 @@ private Mono> uploadInChunks(BlockBlobAsyncClient blockB Flux data, ParallelTransferOptions parallelTransferOptions, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions requestConditions, boolean computeMd5, BlobImmutabilityPolicy immutabilityPolicy, - Boolean legalHold) { + Boolean legalHold, ContentValidationAlgorithm contentValidationAlgorithm) { // TODO: Sample/api reference ProgressListener progressListener = parallelTransferOptions.getProgressListener(); @@ -777,12 +789,12 @@ private Mono> uploadInChunks(BlockBlobAsyncClient blockB .concatWith(Flux.defer(stagingArea::flush)) .flatMapSequential(bufferAggregator -> { Flux chunkData = bufferAggregator.asFlux(); - String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString().getBytes(UTF_8)); return UploadUtils.computeMd5(chunkData, computeMd5, LOGGER).flatMap(fluxMd5Wrapper -> { - Mono> responseMono - = blockBlobAsyncClient.stageBlockWithResponse(blockId, fluxMd5Wrapper.getData(), - bufferAggregator.length(), fluxMd5Wrapper.getMd5(), requestConditions.getLeaseId()); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.stageBlockWithResponse(blockId, fluxMd5Wrapper.getData(), + bufferAggregator.length(), fluxMd5Wrapper.getMd5(), requestConditions.getLeaseId()), + contentValidationAlgorithm, 0, true); if (progressReporter != null) { responseMono = responseMono.contextWrite(FluxUtil.toReactorContext(Contexts.empty() .setHttpRequestProgressReporter(progressReporter.createChild()) @@ -968,7 +980,12 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi : options.getParallelTransferOptions().getBlockSizeLong(); final ParallelTransferOptions finalParallelTransferOptions = ModelHelper.populateAndApplyDefaults(options.getParallelTransferOptions()); + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + try { + ContentValidationModeResolver.validateProgressWithContentValidation( + finalParallelTransferOptions.getProgressListener(), contentValidationAlgorithm); + Path filePath = Paths.get(options.getFilePath()); BlockBlobAsyncClient blockBlobAsyncClient = getBlockBlobAsyncClient(); // This will retrieve file length but won't read file body. @@ -980,15 +997,17 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi if (fileSize > finalParallelTransferOptions.getMaxSingleUploadSizeLong()) { return uploadFileChunks(fileSize, finalParallelTransferOptions, originalBlockSize, options.getHeaders(), options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), - filePath, blockBlobAsyncClient); + filePath, blockBlobAsyncClient, contentValidationAlgorithm); } else { // Otherwise, we know it can be sent in a single request reducing network overhead. - Mono> responseMono = blockBlobAsyncClient - .uploadWithResponse(new BlockBlobSimpleUploadOptions(fullFileData).setHeaders(options.getHeaders()) - .setMetadata(options.getMetadata()) - .setTags(options.getTags()) - .setTier(options.getTier()) - .setRequestConditions(options.getRequestConditions())); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(fullFileData).setHeaders(options.getHeaders()) + .setMetadata(options.getMetadata()) + .setTags(options.getTags()) + .setTier(options.getTier()) + .setRequestConditions(options.getRequestConditions())), + contentValidationAlgorithm, fileSize, false); if (finalParallelTransferOptions.getProgressListener() != null) { ProgressReporter progressReporter = ProgressReporter.withProgressListener(finalParallelTransferOptions.getProgressListener()); @@ -1005,7 +1024,8 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi private Mono> uploadFileChunks(long fileSize, ParallelTransferOptions parallelTransferOptions, Long originalBlockSize, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, - BlobRequestConditions requestConditions, Path filePath, BlockBlobAsyncClient client) { + BlobRequestConditions requestConditions, Path filePath, BlockBlobAsyncClient client, + ContentValidationAlgorithm contentValidationAlgorithm) { final BlobRequestConditions finalRequestConditions = (requestConditions == null) ? new BlobRequestConditions() : requestConditions; // parallelTransferOptions are finalized in the calling method. @@ -1023,8 +1043,10 @@ private Mono> uploadFileChunks(long fileSize, BinaryData data = BinaryData.fromFile(filePath, chunk.getOffset(), chunk.getCount(), DEFAULT_FILE_READ_CHUNK_SIZE); - Mono> responseMono = client.stageBlockWithResponse( - new BlockBlobStageBlockOptions(blockId, data).setLeaseId(finalRequestConditions.getLeaseId())); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + client.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, data).setLeaseId(finalRequestConditions.getLeaseId())), + contentValidationAlgorithm, 0, true); if (progressReporter != null) { responseMono = responseMono.contextWrite(FluxUtil.toReactorContext( Contexts.empty().setHttpRequestProgressReporter(progressReporter.createChild()).getContext())); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerAsyncClient.java index b86fe4e76b2f..6f3a12dd90a5 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerAsyncClient.java @@ -28,6 +28,8 @@ import com.azure.storage.blob.implementation.models.EncryptionScope; import com.azure.storage.blob.implementation.models.ListBlobsFlatSegmentResponse; import com.azure.storage.blob.implementation.models.ListBlobsHierarchySegmentResponse; +import com.azure.storage.blob.implementation.util.ArrowBlobListDeserializer; +import com.azure.storage.blob.implementation.util.ArrowBlobListDeserializer.ArrowListBlobsResult; import com.azure.storage.blob.implementation.util.BlobConstants; import com.azure.storage.blob.implementation.util.BlobSasImplUtil; import com.azure.storage.blob.implementation.util.ModelHelper; @@ -42,6 +44,7 @@ import com.azure.storage.blob.models.CustomerProvidedKey; import com.azure.storage.blob.models.ListBlobsIncludeItem; import com.azure.storage.blob.models.ListBlobsOptions; +import com.azure.storage.blob.models.StorageResponseSerializationFormat; import com.azure.storage.blob.models.PublicAccessType; import com.azure.storage.blob.models.StorageAccountInfo; import com.azure.storage.blob.models.TaggedBlobItem; @@ -51,6 +54,7 @@ import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; +import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; import reactor.core.publisher.Mono; @@ -1116,20 +1120,38 @@ public PagedFlux listBlobs(ListBlobsOptions options, String continuati */ PagedFlux listBlobsFlatWithOptionalTimeout(ListBlobsOptions options, String continuationToken, Duration timeout) { + ModelHelper.validateListBlobsOptions(options); BiFunction>> func = (marker, pageSize) -> { ListBlobsOptions finalOptions; + /* + If pageSize was not set in a .byPage(int) method, the page size from options will be preserved. + Otherwise, prefer the new value. + */ if (pageSize != null) { if (options == null) { finalOptions = new ListBlobsOptions().setMaxResultsPerPage(pageSize); } else { + // Note that this prefers the value passed to .byPage(int) over the value on the options finalOptions = new ListBlobsOptions().setMaxResultsPerPage(pageSize) .setPrefix(options.getPrefix()) - .setDetails(options.getDetails()); + .setDetails(options.getDetails()) + .setStartFrom(options.getStartFrom()); + if (ModelHelper.resolveSerializationFormat(options.getStorageResponseSerializationFormat()) + == StorageResponseSerializationFormat.ARROW) { + finalOptions.setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setEndBefore(options.getEndBefore()); + } } } else { finalOptions = options; } + if (finalOptions != null + && ModelHelper.resolveSerializationFormat(finalOptions.getStorageResponseSerializationFormat()) + == StorageResponseSerializationFormat.ARROW) { + return listBlobsFlatSegmentArrow(marker, finalOptions, timeout); + } + return listBlobsFlatSegment(marker, finalOptions, timeout).map(response -> { List value = response.getValue().getSegment() == null ? Collections.emptyList() @@ -1177,6 +1199,61 @@ PagedFlux listBlobsFlatWithOptionalTimeout(ListBlobsOptions options, S timeout); } + private Mono> listBlobsFlatSegmentArrow(String marker, ListBlobsOptions options, + Duration timeout) { + options = options == null ? new ListBlobsOptions() : options; + + ArrayList include = options.getDetails().toList(); + include = include.isEmpty() ? null : include; + + return StorageImplUtils.applyOptionalTimeout(this.azureBlobStorage.getContainers() + .listBlobFlatSegmentApacheArrowWithResponseAsync(containerName, options.getPrefix(), marker, + options.getMaxResultsPerPage(), include, null, options.getStartFrom(), options.getEndBefore(), null, + Context.NONE), + timeout).flatMap(response -> { + String contentType = response.getHeaders().getValue(com.azure.core.http.HttpHeaderName.CONTENT_TYPE); + + return FluxUtil.collectBytesInByteBufferStream(response.getValue()).map(bytes -> { + java.io.ByteArrayInputStream inputStream = new java.io.ByteArrayInputStream(bytes); + + if (StorageImplUtils.hasMatchingHeaderValue(contentType, + Constants.ContentTypeConstants.APPLICATION_VND_APACHE_ARROW_STREAM)) { + ArrowListBlobsResult arrowResult = ArrowBlobListDeserializer.deserialize(inputStream); + + List value = arrowResult.getBlobItems() + .stream() + .map(ModelHelper::populateBlobItem) + .collect(Collectors.toList()); + + return (PagedResponse) new PagedResponseBase<>(response.getRequest(), + response.getStatusCode(), response.getHeaders(), value, arrowResult.getNextMarker(), + response.getDeserializedHeaders()); + } else { + // XML fallback + try { + ListBlobsFlatSegmentResponse xmlResponse + = ListBlobsFlatSegmentResponse.fromXml(com.azure.xml.XmlReader.fromStream(inputStream)); + + List value = xmlResponse.getSegment() == null + ? Collections.emptyList() + : xmlResponse.getSegment() + .getBlobItems() + .stream() + .map(ModelHelper::populateBlobItem) + .collect(Collectors.toList()); + + return (PagedResponse) new PagedResponseBase<>(response.getRequest(), + response.getStatusCode(), response.getHeaders(), value, xmlResponse.getNextMarker(), + null); + } catch (javax.xml.stream.XMLStreamException e) { + throw LOGGER + .logExceptionAsError(new RuntimeException("Failed to parse XML fallback response", e)); + } + } + }); + }); + } + /** * Returns a reactive Publisher emitting all the blobs and directories (prefixes) under the given directory * (prefix). Directories will have {@link BlobItem#isPrefix()} set to true. @@ -1287,6 +1364,7 @@ public PagedFlux listBlobsByHierarchy(String delimiter, ListBlobsOptio */ PagedFlux listBlobsHierarchyWithOptionalTimeout(String delimiter, ListBlobsOptions options, Duration timeout) { + ModelHelper.validateListBlobsOptions(options); BiFunction>> func = (marker, pageSize) -> { ListBlobsOptions finalOptions; /* @@ -1302,10 +1380,22 @@ PagedFlux listBlobsHierarchyWithOptionalTimeout(String delimiter, List .setPrefix(options.getPrefix()) .setDetails(options.getDetails()) .setStartFrom(options.getStartFrom()); + if (ModelHelper.resolveSerializationFormat(options.getStorageResponseSerializationFormat()) + == StorageResponseSerializationFormat.ARROW) { + finalOptions.setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setEndBefore(options.getEndBefore()); + } } } else { finalOptions = options; } + + if (finalOptions != null + && ModelHelper.resolveSerializationFormat(finalOptions.getStorageResponseSerializationFormat()) + == StorageResponseSerializationFormat.ARROW) { + return listBlobsHierarchySegmentArrow(marker, delimiter, finalOptions, timeout); + } + return listBlobsHierarchySegment(marker, delimiter, finalOptions, timeout).map(response -> { BlobHierarchyListSegment segment = response.getValue().getSegment(); List value; @@ -1344,6 +1434,71 @@ PagedFlux listBlobsHierarchyWithOptionalTimeout(String delimiter, List timeout); } + private Mono> listBlobsHierarchySegmentArrow(String marker, String delimiter, + ListBlobsOptions options, Duration timeout) { + options = options == null ? new ListBlobsOptions() : options; + if (options.getDetails().getRetrieveSnapshots()) { + throw LOGGER.logExceptionAsError( + new UnsupportedOperationException("Including snapshots in a hierarchical listing is not supported.")); + } + + ArrayList include + = options.getDetails().toList().isEmpty() ? null : options.getDetails().toList(); + + ListBlobsOptions finalOptions = options; + return StorageImplUtils + .applyOptionalTimeout(this.azureBlobStorage.getContainers() + .listBlobHierarchySegmentApacheArrowWithResponseAsync(containerName, delimiter, + finalOptions.getPrefix(), marker, finalOptions.getMaxResultsPerPage(), include, null, + finalOptions.getStartFrom(), finalOptions.getEndBefore(), null, Context.NONE), + timeout) + .flatMap(response -> { + String contentType = response.getHeaders().getValue(com.azure.core.http.HttpHeaderName.CONTENT_TYPE); + + return FluxUtil.collectBytesInByteBufferStream(response.getValue()).map(bytes -> { + java.io.ByteArrayInputStream inputStream = new java.io.ByteArrayInputStream(bytes); + + if (StorageImplUtils.hasMatchingHeaderValue(contentType, + Constants.ContentTypeConstants.APPLICATION_VND_APACHE_ARROW_STREAM)) { + ArrowListBlobsResult arrowResult = ArrowBlobListDeserializer.deserialize(inputStream); + + List value = arrowResult.getBlobItems() + .stream() + .map(ModelHelper::populateBlobItem) + .collect(Collectors.toList()); + + return (PagedResponse) new PagedResponseBase<>(response.getRequest(), + response.getStatusCode(), response.getHeaders(), value, arrowResult.getNextMarker(), + response.getDeserializedHeaders()); + } else { + // XML fallback + try { + ListBlobsHierarchySegmentResponse xmlResponse = ListBlobsHierarchySegmentResponse + .fromXml(com.azure.xml.XmlReader.fromStream(inputStream)); + + BlobHierarchyListSegment segment = xmlResponse.getSegment(); + List value = new ArrayList<>(); + if (segment != null) { + segment.getBlobItems() + .forEach(item -> value.add(BlobItemConstructorProxy.create(item))); + segment.getBlobPrefixes() + .forEach(prefix -> value + .add(new BlobItem().setName(ModelHelper.toBlobNameString(prefix.getName())) + .setIsPrefix(true))); + } + + return (PagedResponse) new PagedResponseBase<>(response.getRequest(), + response.getStatusCode(), response.getHeaders(), value, xmlResponse.getNextMarker(), + null); + } catch (javax.xml.stream.XMLStreamException e) { + throw LOGGER + .logExceptionAsError(new RuntimeException("Failed to parse XML fallback response", e)); + } + } + }); + }); + } + /** * Returns a reactive Publisher emitting the blobs in this container whose tags match the query expression. For more * information, including information on the query syntax, see the Azure Docs. diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerClient.java index 64de81617f9c..67dc02480be4 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerClient.java @@ -25,12 +25,16 @@ import com.azure.storage.blob.implementation.models.ContainersGetAccessPolicyHeaders; import com.azure.storage.blob.implementation.models.ContainersGetAccountInfoHeaders; import com.azure.storage.blob.implementation.models.ContainersGetPropertiesHeaders; +import com.azure.storage.blob.implementation.models.ContainersListBlobFlatSegmentApacheArrowHeaders; import com.azure.storage.blob.implementation.models.ContainersListBlobFlatSegmentHeaders; +import com.azure.storage.blob.implementation.models.ContainersListBlobHierarchySegmentApacheArrowHeaders; import com.azure.storage.blob.implementation.models.ContainersListBlobHierarchySegmentHeaders; import com.azure.storage.blob.implementation.models.EncryptionScope; import com.azure.storage.blob.implementation.models.FilterBlobSegment; import com.azure.storage.blob.implementation.models.ListBlobsFlatSegmentResponse; import com.azure.storage.blob.implementation.models.ListBlobsHierarchySegmentResponse; +import com.azure.storage.blob.implementation.util.ArrowBlobListDeserializer; +import com.azure.storage.blob.implementation.util.ArrowBlobListDeserializer.ArrowListBlobsResult; import com.azure.storage.blob.implementation.util.BlobConstants; import com.azure.storage.blob.implementation.util.BlobSasImplUtil; import com.azure.storage.blob.implementation.util.ModelHelper; @@ -47,6 +51,7 @@ import com.azure.storage.blob.models.ListBlobsOptions; import com.azure.storage.blob.models.PublicAccessType; import com.azure.storage.blob.models.StorageAccountInfo; +import com.azure.storage.blob.models.StorageResponseSerializationFormat; import com.azure.storage.blob.models.TaggedBlobItem; import com.azure.storage.blob.models.UserDelegationKey; import com.azure.storage.blob.options.BlobContainerCreateOptions; @@ -54,9 +59,16 @@ import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; +import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; +import com.azure.xml.XmlReader; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import javax.xml.stream.XMLStreamException; import java.net.URI; import java.time.Duration; import java.time.OffsetDateTime; @@ -1021,6 +1033,7 @@ public PagedIterable listBlobs(ListBlobsOptions options, Duration time */ @ServiceMethod(returns = ReturnType.COLLECTION) public PagedIterable listBlobs(ListBlobsOptions options, String continuationToken, Duration timeout) { + ModelHelper.validateListBlobsOptions(options); BiFunction> retriever = (nextMarker, pageSize) -> { ListBlobsOptions finalOptions = new ListBlobsOptions(); if (options != null) { @@ -1029,6 +1042,11 @@ public PagedIterable listBlobs(ListBlobsOptions options, String contin .setStartFrom(options.getStartFrom()) .setDetails(options.getDetails()); + if (options.getStorageResponseSerializationFormat() == StorageResponseSerializationFormat.ARROW) { + finalOptions.setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setEndBefore(options.getEndBefore()); + } + } /* If pageSize was not set in a .byPage(int) method, the page size from options will be preserved. @@ -1040,26 +1058,79 @@ public PagedIterable listBlobs(ListBlobsOptions options, String contin ArrayList include = finalOptions.getDetails().toList().isEmpty() ? null : finalOptions.getDetails().toList(); - Callable> operation - = () -> this.azureBlobStorage.getContainers() - .listBlobFlatSegmentWithResponse(containerName, finalOptions.getPrefix(), nextMarker, - finalOptions.getMaxResultsPerPage(), include, finalOptions.getStartFrom(), null, null, - Context.NONE); - - ResponseBase response - = StorageImplUtils.sendRequest(operation, timeout, BlobStorageException.class); - - List value = response.getValue().getSegment() == null - ? Collections.emptyList() - : response.getValue() - .getSegment() - .getBlobItems() - .stream() - .map(ModelHelper::populateBlobItem) - .collect(Collectors.toList()); - - return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), - value, response.getValue().getNextMarker(), response.getDeserializedHeaders()); + if (finalOptions.getStorageResponseSerializationFormat() == StorageResponseSerializationFormat.ARROW) { + Callable> operation + = () -> this.azureBlobStorage.getContainers() + .listBlobFlatSegmentApacheArrowWithResponse(containerName, finalOptions.getPrefix(), nextMarker, + finalOptions.getMaxResultsPerPage(), include, null, finalOptions.getStartFrom(), + finalOptions.getEndBefore(), null, Context.NONE); + ResponseBase response + = StorageImplUtils.sendRequest(operation, timeout, BlobStorageException.class); + + String contentType = response.getHeaders().getValue(com.azure.core.http.HttpHeaderName.CONTENT_TYPE); + + // The response body is an InputStream backed by the network buffer. It must be closed to release the + // underlying buffer, otherwise the transport (e.g. Netty) will report a resource leak. + try (InputStream responseBody = response.getValue()) { + if (StorageImplUtils.hasMatchingHeaderValue(contentType, + Constants.ContentTypeConstants.APPLICATION_VND_APACHE_ARROW_STREAM)) { + // Arrow response — parse with Arrow parser entrypoint + ArrowListBlobsResult arrowResult = ArrowBlobListDeserializer.deserialize(responseBody); + + List value = arrowResult.getBlobItems() + .stream() + .map(ModelHelper::populateBlobItem) + .collect(Collectors.toList()); + + return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), + response.getHeaders(), value, arrowResult.getNextMarker(), + response.getDeserializedHeaders()); + } else { + // XML fallback — service returned XML instead of Arrow + try { + ListBlobsFlatSegmentResponse xmlResponse + = ListBlobsFlatSegmentResponse.fromXml(XmlReader.fromStream(responseBody)); + + List value = xmlResponse.getSegment() == null + ? Collections.emptyList() + : xmlResponse.getSegment() + .getBlobItems() + .stream() + .map(ModelHelper::populateBlobItem) + .collect(Collectors.toList()); + + return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), + response.getHeaders(), value, xmlResponse.getNextMarker(), null); + } catch (XMLStreamException e) { + throw LOGGER + .logExceptionAsError(new RuntimeException("Failed to parse XML fallback response", e)); + } + } + } catch (IOException e) { + throw LOGGER + .logExceptionAsError(new UncheckedIOException("Failed to close ListBlobs response stream.", e)); + } + } else { + Callable> operation + = () -> this.azureBlobStorage.getContainers() + .listBlobFlatSegmentWithResponse(containerName, finalOptions.getPrefix(), nextMarker, + finalOptions.getMaxResultsPerPage(), include, finalOptions.getStartFrom(), null, null, + Context.NONE); + ResponseBase response + = StorageImplUtils.sendRequest(operation, timeout, BlobStorageException.class); + + List value = response.getValue().getSegment() == null + ? Collections.emptyList() + : response.getValue() + .getSegment() + .getBlobItems() + .stream() + .map(ModelHelper::populateBlobItem) + .collect(Collectors.toList()); + + return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), + value, response.getValue().getNextMarker(), response.getDeserializedHeaders()); + } }; return new PagedIterable<>(pageSize -> retriever.apply(continuationToken, pageSize), retriever); @@ -1157,6 +1228,7 @@ public PagedIterable listBlobsByHierarchy(String directory) { */ @ServiceMethod(returns = ReturnType.COLLECTION) public PagedIterable listBlobsByHierarchy(String delimiter, ListBlobsOptions options, Duration timeout) { + ModelHelper.validateListBlobsOptions(options); BiFunction> func = (marker, pageSize) -> { ListBlobsOptions finalOptions = new ListBlobsOptions(); if (options != null) { @@ -1164,6 +1236,11 @@ public PagedIterable listBlobsByHierarchy(String delimiter, ListBlobsO .setPrefix(options.getPrefix()) .setDetails(options.getDetails()) .setStartFrom(options.getStartFrom()); + if (ModelHelper.resolveSerializationFormat(options.getStorageResponseSerializationFormat()) + == StorageResponseSerializationFormat.ARROW) { + finalOptions.setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setEndBefore(options.getEndBefore()); + } } /* If pageSize was not set in a .byPage(int) method, the page size from options will be preserved. @@ -1186,25 +1263,80 @@ private PagedResponse listBlobsHierarchySegment(String marker, String ArrayList include = options.getDetails().toList().isEmpty() ? null : options.getDetails().toList(); - Callable> operation - = () -> azureBlobStorage.getContainers() - .listBlobHierarchySegmentWithResponse(containerName, delimiter, options.getPrefix(), marker, - options.getMaxResultsPerPage(), include, options.getStartFrom(), null, null, Context.NONE); + if (ModelHelper.resolveSerializationFormat(options.getStorageResponseSerializationFormat()) + == StorageResponseSerializationFormat.ARROW) { + Callable> operation + = () -> azureBlobStorage.getContainers() + .listBlobHierarchySegmentApacheArrowWithResponse(containerName, delimiter, options.getPrefix(), + marker, options.getMaxResultsPerPage(), include, null, options.getStartFrom(), + options.getEndBefore(), null, Context.NONE); + ResponseBase response + = StorageImplUtils.sendRequest(operation, timeout, BlobStorageException.class); + + String contentType = response.getHeaders().getValue(com.azure.core.http.HttpHeaderName.CONTENT_TYPE); + + // The response body is an InputStream backed by the network buffer. It must be closed to release the + // underlying buffer, otherwise the transport (e.g. Netty) will report a resource leak. + try (InputStream responseBody = response.getValue()) { + if (StorageImplUtils.hasMatchingHeaderValue(contentType, + Constants.ContentTypeConstants.APPLICATION_VND_APACHE_ARROW_STREAM)) { + ArrowListBlobsResult arrowResult = ArrowBlobListDeserializer.deserialize(responseBody); + + List value = arrowResult.getBlobItems() + .stream() + .map(ModelHelper::populateBlobItem) + .collect(Collectors.toList()); + + return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), + response.getHeaders(), value, arrowResult.getNextMarker(), response.getDeserializedHeaders()); + } else { + // XML fallback — service returned XML instead of Arrow + try { + ListBlobsHierarchySegmentResponse xmlResponse + = ListBlobsHierarchySegmentResponse.fromXml(XmlReader.fromStream(responseBody)); + + BlobHierarchyListSegment segment = xmlResponse.getSegment(); + List value = new ArrayList<>(); + if (segment != null) { + segment.getBlobItems().forEach(item -> value.add(BlobItemConstructorProxy.create(item))); + segment.getBlobPrefixes() + .forEach(prefix -> value + .add(new BlobItem().setName(ModelHelper.toBlobNameString(prefix.getName())) + .setIsPrefix(true))); + } + + return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), + response.getHeaders(), value, xmlResponse.getNextMarker(), null); + } catch (XMLStreamException e) { + throw LOGGER + .logExceptionAsError(new RuntimeException("Failed to parse XML fallback response", e)); + } + } + } catch (IOException e) { + throw LOGGER + .logExceptionAsError(new UncheckedIOException("Failed to close ListBlobs response stream.", e)); + } + } else { + Callable> operation + = () -> azureBlobStorage.getContainers() + .listBlobHierarchySegmentWithResponse(containerName, delimiter, options.getPrefix(), marker, + options.getMaxResultsPerPage(), include, options.getStartFrom(), null, null, Context.NONE); - ResponseBase response - = StorageImplUtils.sendRequest(operation, timeout, BlobStorageException.class); + ResponseBase response + = StorageImplUtils.sendRequest(operation, timeout, BlobStorageException.class); - BlobHierarchyListSegment segment = response.getValue().getSegment(); - List value = new ArrayList<>(); - if (segment != null) { - segment.getBlobItems().forEach(item -> value.add(BlobItemConstructorProxy.create(item))); - segment.getBlobPrefixes() - .forEach(prefix -> value - .add(new BlobItem().setName(ModelHelper.toBlobNameString(prefix.getName())).setIsPrefix(true))); - } + BlobHierarchyListSegment segment = response.getValue().getSegment(); + List value = new ArrayList<>(); + if (segment != null) { + segment.getBlobItems().forEach(item -> value.add(BlobItemConstructorProxy.create(item))); + segment.getBlobPrefixes() + .forEach(prefix -> value + .add(new BlobItem().setName(ModelHelper.toBlobNameString(prefix.getName())).setIsPrefix(true))); + } - return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), value, - response.getValue().getNextMarker(), response.getDeserializedHeaders()); + return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), + value, response.getValue().getNextMarker(), response.getDeserializedHeaders()); + } } /** diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java index 75fb74a59a5d..7ddf558eefb5 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java @@ -162,7 +162,12 @@ public enum BlobServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"); private final String version; @@ -184,6 +189,6 @@ public String getVersion() { * @return the latest {@link BlobServiceVersion} */ public static BlobServiceVersion getLatest() { - return V2026_06_06; + return V2026_10_06; } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/ContainersImpl.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/ContainersImpl.java index 7fd2af96e4df..af54bda3cb4b 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/ContainersImpl.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/ContainersImpl.java @@ -37,7 +37,9 @@ import com.azure.storage.blob.implementation.models.ContainersGetAccessPolicyHeaders; import com.azure.storage.blob.implementation.models.ContainersGetAccountInfoHeaders; import com.azure.storage.blob.implementation.models.ContainersGetPropertiesHeaders; +import com.azure.storage.blob.implementation.models.ContainersListBlobFlatSegmentApacheArrowHeaders; import com.azure.storage.blob.implementation.models.ContainersListBlobFlatSegmentHeaders; +import com.azure.storage.blob.implementation.models.ContainersListBlobHierarchySegmentApacheArrowHeaders; import com.azure.storage.blob.implementation.models.ContainersListBlobHierarchySegmentHeaders; import com.azure.storage.blob.implementation.models.ContainersReleaseLeaseHeaders; import com.azure.storage.blob.implementation.models.ContainersRenameHeaders; @@ -853,6 +855,56 @@ Response listBlobFlatSegmentNoCustomHeadersSync(@H @HeaderParam("x-ms-version") String version, @HeaderParam("x-ms-client-request-id") String requestId, @HeaderParam("Accept") String accept, Context context); + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + Mono>> + listBlobFlatSegmentApacheArrow(@HostParam("url") String url, + @PathParam("containerName") String containerName, @QueryParam("restype") String restype, + @QueryParam("comp") String comp, @HeaderParam("Accept") String accept, + @QueryParam("prefix") String prefix, @QueryParam("marker") String marker, + @QueryParam("maxresults") Integer maxresults, @QueryParam("include") String include, + @QueryParam("timeout") Integer timeout, @QueryParam("startFrom") String startFrom, + @QueryParam("endBefore") String endBefore, @HeaderParam("x-ms-version") String version, + @HeaderParam("x-ms-client-request-id") String requestId, Context context); + + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + Mono listBlobFlatSegmentApacheArrowNoCustomHeaders(@HostParam("url") String url, + @PathParam("containerName") String containerName, @QueryParam("restype") String restype, + @QueryParam("comp") String comp, @HeaderParam("Accept") String accept, @QueryParam("prefix") String prefix, + @QueryParam("marker") String marker, @QueryParam("maxresults") Integer maxresults, + @QueryParam("include") String include, @QueryParam("timeout") Integer timeout, + @QueryParam("startFrom") String startFrom, @QueryParam("endBefore") String endBefore, + @HeaderParam("x-ms-version") String version, @HeaderParam("x-ms-client-request-id") String requestId, + Context context); + + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + ResponseBase listBlobFlatSegmentApacheArrowSync( + @HostParam("url") String url, @PathParam("containerName") String containerName, + @QueryParam("restype") String restype, @QueryParam("comp") String comp, + @HeaderParam("Accept") String accept, @QueryParam("prefix") String prefix, + @QueryParam("marker") String marker, @QueryParam("maxresults") Integer maxresults, + @QueryParam("include") String include, @QueryParam("timeout") Integer timeout, + @QueryParam("startFrom") String startFrom, @QueryParam("endBefore") String endBefore, + @HeaderParam("x-ms-version") String version, @HeaderParam("x-ms-client-request-id") String requestId, + Context context); + + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + Response listBlobFlatSegmentApacheArrowNoCustomHeadersSync(@HostParam("url") String url, + @PathParam("containerName") String containerName, @QueryParam("restype") String restype, + @QueryParam("comp") String comp, @HeaderParam("Accept") String accept, @QueryParam("prefix") String prefix, + @QueryParam("marker") String marker, @QueryParam("maxresults") Integer maxresults, + @QueryParam("include") String include, @QueryParam("timeout") Integer timeout, + @QueryParam("startFrom") String startFrom, @QueryParam("endBefore") String endBefore, + @HeaderParam("x-ms-version") String version, @HeaderParam("x-ms-client-request-id") String requestId, + Context context); + @Get("/{containerName}") @ExpectedResponses({ 200 }) @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) @@ -903,6 +955,58 @@ Response listBlobHierarchySegmentNoCustomHead @HeaderParam("x-ms-version") String version, @HeaderParam("x-ms-client-request-id") String requestId, @HeaderParam("Accept") String accept, Context context); + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + Mono>> + listBlobHierarchySegmentApacheArrow(@HostParam("url") String url, + @PathParam("containerName") String containerName, @QueryParam("restype") String restype, + @QueryParam("comp") String comp, @HeaderParam("Accept") String accept, + @QueryParam("prefix") String prefix, @QueryParam("delimiter") String delimiter, + @QueryParam("marker") String marker, @QueryParam("maxresults") Integer maxresults, + @QueryParam("include") String include, @QueryParam("timeout") Integer timeout, + @QueryParam("startFrom") String startFrom, @QueryParam("endBefore") String endBefore, + @HeaderParam("x-ms-version") String version, @HeaderParam("x-ms-client-request-id") String requestId, + Context context); + + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + Mono listBlobHierarchySegmentApacheArrowNoCustomHeaders(@HostParam("url") String url, + @PathParam("containerName") String containerName, @QueryParam("restype") String restype, + @QueryParam("comp") String comp, @HeaderParam("Accept") String accept, @QueryParam("prefix") String prefix, + @QueryParam("delimiter") String delimiter, @QueryParam("marker") String marker, + @QueryParam("maxresults") Integer maxresults, @QueryParam("include") String include, + @QueryParam("timeout") Integer timeout, @QueryParam("startFrom") String startFrom, + @QueryParam("endBefore") String endBefore, @HeaderParam("x-ms-version") String version, + @HeaderParam("x-ms-client-request-id") String requestId, Context context); + + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + ResponseBase + listBlobHierarchySegmentApacheArrowSync(@HostParam("url") String url, + @PathParam("containerName") String containerName, @QueryParam("restype") String restype, + @QueryParam("comp") String comp, @HeaderParam("Accept") String accept, + @QueryParam("prefix") String prefix, @QueryParam("delimiter") String delimiter, + @QueryParam("marker") String marker, @QueryParam("maxresults") Integer maxresults, + @QueryParam("include") String include, @QueryParam("timeout") Integer timeout, + @QueryParam("startFrom") String startFrom, @QueryParam("endBefore") String endBefore, + @HeaderParam("x-ms-version") String version, @HeaderParam("x-ms-client-request-id") String requestId, + Context context); + + @Get("/{containerName}") + @ExpectedResponses({ 200 }) + @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) + Response listBlobHierarchySegmentApacheArrowNoCustomHeadersSync(@HostParam("url") String url, + @PathParam("containerName") String containerName, @QueryParam("restype") String restype, + @QueryParam("comp") String comp, @HeaderParam("Accept") String accept, @QueryParam("prefix") String prefix, + @QueryParam("delimiter") String delimiter, @QueryParam("marker") String marker, + @QueryParam("maxresults") Integer maxresults, @QueryParam("include") String include, + @QueryParam("timeout") Integer timeout, @QueryParam("startFrom") String startFrom, + @QueryParam("endBefore") String endBefore, @HeaderParam("x-ms-version") String version, + @HeaderParam("x-ms-client-request-id") String requestId, Context context); + @Get("/{containerName}") @ExpectedResponses({ 200 }) @UnexpectedResponseExceptionType(BlobStorageExceptionInternal.class) @@ -6073,12 +6177,10 @@ public Response listBlobFlatSegmentNoCustomHeaders } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6091,36 +6193,35 @@ public Response listBlobFlatSegmentNoCustomHeaders * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs along with {@link ResponseBase} on successful completion of {@link Mono}. + * @return the response body along with {@link ResponseBase} on successful completion of {@link Mono}. */ @ServiceMethod(returns = ReturnType.SINGLE) - public Mono> - listBlobHierarchySegmentWithResponseAsync(String containerName, String delimiter, String prefix, String marker, - Integer maxresults, List include, String startFrom, Integer timeout, + public Mono>> + listBlobFlatSegmentApacheArrowWithResponseAsync(String containerName, String prefix, String marker, + Integer maxresults, List include, Integer timeout, String startFrom, String endBefore, String requestId) { return FluxUtil - .withContext(context -> listBlobHierarchySegmentWithResponseAsync(containerName, delimiter, prefix, marker, - maxresults, include, startFrom, timeout, requestId, context)) + .withContext(context -> listBlobFlatSegmentApacheArrowWithResponseAsync(containerName, prefix, marker, + maxresults, include, timeout, startFrom, endBefore, requestId, context)) .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6133,45 +6234,45 @@ public Response listBlobFlatSegmentNoCustomHeaders * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs along with {@link ResponseBase} on successful completion of {@link Mono}. + * @return the response body along with {@link ResponseBase} on successful completion of {@link Mono}. */ @ServiceMethod(returns = ReturnType.SINGLE) - public Mono> - listBlobHierarchySegmentWithResponseAsync(String containerName, String delimiter, String prefix, String marker, - Integer maxresults, List include, String startFrom, Integer timeout, String requestId, - Context context) { + public Mono>> + listBlobFlatSegmentApacheArrowWithResponseAsync(String containerName, String prefix, String marker, + Integer maxresults, List include, Integer timeout, String startFrom, String endBefore, + String requestId, Context context) { final String restype = "container"; final String comp = "list"; - final String accept = "application/xml"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; String includeConverted = (include == null) ? null : include.stream() .map(paramItemValue -> Objects.toString(paramItemValue, "")) .collect(Collectors.joining(",")); return service - .listBlobHierarchySegment(this.client.getUrl(), containerName, restype, comp, prefix, delimiter, marker, - maxresults, includeConverted, startFrom, timeout, this.client.getVersion(), requestId, accept, context) + .listBlobFlatSegmentApacheArrow(this.client.getUrl(), containerName, restype, comp, accept, prefix, marker, + maxresults, includeConverted, timeout, startFrom, endBefore, this.client.getVersion(), requestId, + context) .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6184,35 +6285,34 @@ public Response listBlobFlatSegmentNoCustomHeaders * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs on successful completion of {@link Mono}. + * @return the response. */ @ServiceMethod(returns = ReturnType.SINGLE) - public Mono listBlobHierarchySegmentAsync(String containerName, String delimiter, - String prefix, String marker, Integer maxresults, List include, String startFrom, - Integer timeout, String requestId) { - return listBlobHierarchySegmentWithResponseAsync(containerName, delimiter, prefix, marker, maxresults, include, - startFrom, timeout, requestId) + public Flux listBlobFlatSegmentApacheArrowAsync(String containerName, String prefix, String marker, + Integer maxresults, List include, Integer timeout, String startFrom, String endBefore, + String requestId) { + return listBlobFlatSegmentApacheArrowWithResponseAsync(containerName, prefix, marker, maxresults, include, + timeout, startFrom, endBefore, requestId) .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException) - .flatMap(res -> Mono.justOrEmpty(res.getValue())); + .flatMapMany(fluxByteBufferResponse -> fluxByteBufferResponse.getValue()); } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6225,36 +6325,35 @@ public Mono listBlobHierarchySegmentAsync(Str * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs on successful completion of {@link Mono}. + * @return the response. */ @ServiceMethod(returns = ReturnType.SINGLE) - public Mono listBlobHierarchySegmentAsync(String containerName, String delimiter, - String prefix, String marker, Integer maxresults, List include, String startFrom, - Integer timeout, String requestId, Context context) { - return listBlobHierarchySegmentWithResponseAsync(containerName, delimiter, prefix, marker, maxresults, include, - startFrom, timeout, requestId, context) + public Flux listBlobFlatSegmentApacheArrowAsync(String containerName, String prefix, String marker, + Integer maxresults, List include, Integer timeout, String startFrom, String endBefore, + String requestId, Context context) { + return listBlobFlatSegmentApacheArrowWithResponseAsync(containerName, prefix, marker, maxresults, include, + timeout, startFrom, endBefore, requestId, context) .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException) - .flatMap(res -> Mono.justOrEmpty(res.getValue())); + .flatMapMany(fluxByteBufferResponse -> fluxByteBufferResponse.getValue()); } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6267,35 +6366,34 @@ public Mono listBlobHierarchySegmentAsync(Str * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs along with {@link Response} on successful completion of {@link Mono}. + * @return the response body on successful completion of {@link Mono}. */ @ServiceMethod(returns = ReturnType.SINGLE) - public Mono> listBlobHierarchySegmentNoCustomHeadersWithResponseAsync( - String containerName, String delimiter, String prefix, String marker, Integer maxresults, - List include, String startFrom, Integer timeout, String requestId) { + public Mono listBlobFlatSegmentApacheArrowNoCustomHeadersWithResponseAsync(String containerName, + String prefix, String marker, Integer maxresults, List include, Integer timeout, + String startFrom, String endBefore, String requestId) { return FluxUtil - .withContext(context -> listBlobHierarchySegmentNoCustomHeadersWithResponseAsync(containerName, delimiter, - prefix, marker, maxresults, include, startFrom, timeout, requestId, context)) + .withContext(context -> listBlobFlatSegmentApacheArrowNoCustomHeadersWithResponseAsync(containerName, + prefix, marker, maxresults, include, timeout, startFrom, endBefore, requestId, context)) .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6308,45 +6406,44 @@ public Mono> listBlobHierarchySegmen * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs along with {@link Response} on successful completion of {@link Mono}. + * @return the response body on successful completion of {@link Mono}. */ @ServiceMethod(returns = ReturnType.SINGLE) - public Mono> listBlobHierarchySegmentNoCustomHeadersWithResponseAsync( - String containerName, String delimiter, String prefix, String marker, Integer maxresults, - List include, String startFrom, Integer timeout, String requestId, Context context) { + public Mono listBlobFlatSegmentApacheArrowNoCustomHeadersWithResponseAsync(String containerName, + String prefix, String marker, Integer maxresults, List include, Integer timeout, + String startFrom, String endBefore, String requestId, Context context) { final String restype = "container"; final String comp = "list"; - final String accept = "application/xml"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; String includeConverted = (include == null) ? null : include.stream() .map(paramItemValue -> Objects.toString(paramItemValue, "")) .collect(Collectors.joining(",")); return service - .listBlobHierarchySegmentNoCustomHeaders(this.client.getUrl(), containerName, restype, comp, prefix, - delimiter, marker, maxresults, includeConverted, startFrom, timeout, this.client.getVersion(), - requestId, accept, context) + .listBlobFlatSegmentApacheArrowNoCustomHeaders(this.client.getUrl(), containerName, restype, comp, accept, + prefix, marker, maxresults, includeConverted, timeout, startFrom, endBefore, this.client.getVersion(), + requestId, context) .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6359,48 +6456,47 @@ public Mono> listBlobHierarchySegmen * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs along with {@link ResponseBase}. + * @return the response body along with {@link ResponseBase}. */ @ServiceMethod(returns = ReturnType.SINGLE) - public ResponseBase - listBlobHierarchySegmentWithResponse(String containerName, String delimiter, String prefix, String marker, - Integer maxresults, List include, String startFrom, Integer timeout, String requestId, - Context context) { + public ResponseBase + listBlobFlatSegmentApacheArrowWithResponse(String containerName, String prefix, String marker, + Integer maxresults, List include, Integer timeout, String startFrom, String endBefore, + String requestId, Context context) { try { final String restype = "container"; final String comp = "list"; - final String accept = "application/xml"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; String includeConverted = (include == null) ? null : include.stream() .map(paramItemValue -> Objects.toString(paramItemValue, "")) .collect(Collectors.joining(",")); - return service.listBlobHierarchySegmentSync(this.client.getUrl(), containerName, restype, comp, prefix, - delimiter, marker, maxresults, includeConverted, startFrom, timeout, this.client.getVersion(), - requestId, accept, context); + return service.listBlobFlatSegmentApacheArrowSync(this.client.getUrl(), containerName, restype, comp, + accept, prefix, marker, maxresults, includeConverted, timeout, startFrom, endBefore, + this.client.getVersion(), requestId, context); } catch (BlobStorageExceptionInternal internalException) { throw ModelHelper.mapToBlobStorageException(internalException); } } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6413,37 +6509,36 @@ public Mono> listBlobHierarchySegmen * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs. + * @return the response. */ @ServiceMethod(returns = ReturnType.SINGLE) - public ListBlobsHierarchySegmentResponse listBlobHierarchySegment(String containerName, String delimiter, - String prefix, String marker, Integer maxresults, List include, String startFrom, - Integer timeout, String requestId) { + public InputStream listBlobFlatSegmentApacheArrow(String containerName, String prefix, String marker, + Integer maxresults, List include, Integer timeout, String startFrom, String endBefore, + String requestId) { try { - return listBlobHierarchySegmentWithResponse(containerName, delimiter, prefix, marker, maxresults, include, - startFrom, timeout, requestId, Context.NONE).getValue(); + return listBlobFlatSegmentApacheArrowWithResponse(containerName, prefix, marker, maxresults, include, + timeout, startFrom, endBefore, requestId, Context.NONE).getValue(); } catch (BlobStorageExceptionInternal internalException) { throw ModelHelper.mapToBlobStorageException(internalException); } } /** - * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * The List Blobs operation returns a list of the blobs under the specified container. This operation is for Apache + * Arrow use case so response is returned as raw to be deserialized by the client. * * @param containerName The container name. - * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the - * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the - * appearance of the delimiter character. The delimiter may be a single character or a string. * @param prefix Filters the results to return only containers whose name begins with the specified prefix. * @param marker A string value that identifies the portion of the list of containers to be returned with the next * listing operation. The operation returns the NextMarker value within the response body if the listing operation @@ -6456,35 +6551,892 @@ public ListBlobsHierarchySegmentResponse listBlobHierarchySegment(String contain * the remainder of the results. For this reason, it is possible that the service will return fewer results than * specified by maxresults, or than the default of 5000. * @param include Include this parameter to specify one or more datasets to include in the response. - * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is - * supported; For recursive list, multiple entity levels are supported. (Inclusive). * @param timeout The timeout parameter is expressed in seconds. For more information, see <a * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the * analytics logs when storage analytics logging is enabled. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. - * @return an enumeration of blobs along with {@link Response}. + * @return the response body along with {@link Response}. */ @ServiceMethod(returns = ReturnType.SINGLE) - public Response listBlobHierarchySegmentNoCustomHeadersWithResponse( - String containerName, String delimiter, String prefix, String marker, Integer maxresults, - List include, String startFrom, Integer timeout, String requestId, Context context) { + public Response listBlobFlatSegmentApacheArrowNoCustomHeadersWithResponse(String containerName, + String prefix, String marker, Integer maxresults, List include, Integer timeout, + String startFrom, String endBefore, String requestId, Context context) { try { final String restype = "container"; final String comp = "list"; - final String accept = "application/xml"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; String includeConverted = (include == null) ? null : include.stream() .map(paramItemValue -> Objects.toString(paramItemValue, "")) .collect(Collectors.joining(",")); - return service.listBlobHierarchySegmentNoCustomHeadersSync(this.client.getUrl(), containerName, restype, - comp, prefix, delimiter, marker, maxresults, includeConverted, startFrom, timeout, - this.client.getVersion(), requestId, accept, context); + return service.listBlobFlatSegmentApacheArrowNoCustomHeadersSync(this.client.getUrl(), containerName, + restype, comp, accept, prefix, marker, maxresults, includeConverted, timeout, startFrom, endBefore, + this.client.getVersion(), requestId, context); + } catch (BlobStorageExceptionInternal internalException) { + throw ModelHelper.mapToBlobStorageException(internalException); + } + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs along with {@link ResponseBase} on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> + listBlobHierarchySegmentWithResponseAsync(String containerName, String delimiter, String prefix, String marker, + Integer maxresults, List include, String startFrom, Integer timeout, + String requestId) { + return FluxUtil + .withContext(context -> listBlobHierarchySegmentWithResponseAsync(containerName, delimiter, prefix, marker, + maxresults, include, startFrom, timeout, requestId, context)) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs along with {@link ResponseBase} on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> + listBlobHierarchySegmentWithResponseAsync(String containerName, String delimiter, String prefix, String marker, + Integer maxresults, List include, String startFrom, Integer timeout, String requestId, + Context context) { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service + .listBlobHierarchySegment(this.client.getUrl(), containerName, restype, comp, prefix, delimiter, marker, + maxresults, includeConverted, startFrom, timeout, this.client.getVersion(), requestId, accept, context) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono listBlobHierarchySegmentAsync(String containerName, String delimiter, + String prefix, String marker, Integer maxresults, List include, String startFrom, + Integer timeout, String requestId) { + return listBlobHierarchySegmentWithResponseAsync(containerName, delimiter, prefix, marker, maxresults, include, + startFrom, timeout, requestId) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException) + .flatMap(res -> Mono.justOrEmpty(res.getValue())); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono listBlobHierarchySegmentAsync(String containerName, String delimiter, + String prefix, String marker, Integer maxresults, List include, String startFrom, + Integer timeout, String requestId, Context context) { + return listBlobHierarchySegmentWithResponseAsync(containerName, delimiter, prefix, marker, maxresults, include, + startFrom, timeout, requestId, context) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException) + .flatMap(res -> Mono.justOrEmpty(res.getValue())); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs along with {@link Response} on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> listBlobHierarchySegmentNoCustomHeadersWithResponseAsync( + String containerName, String delimiter, String prefix, String marker, Integer maxresults, + List include, String startFrom, Integer timeout, String requestId) { + return FluxUtil + .withContext(context -> listBlobHierarchySegmentNoCustomHeadersWithResponseAsync(containerName, delimiter, + prefix, marker, maxresults, include, startFrom, timeout, requestId, context)) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs along with {@link Response} on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> listBlobHierarchySegmentNoCustomHeadersWithResponseAsync( + String containerName, String delimiter, String prefix, String marker, Integer maxresults, + List include, String startFrom, Integer timeout, String requestId, Context context) { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service + .listBlobHierarchySegmentNoCustomHeaders(this.client.getUrl(), containerName, restype, comp, prefix, + delimiter, marker, maxresults, includeConverted, startFrom, timeout, this.client.getVersion(), + requestId, accept, context) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs along with {@link ResponseBase}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public ResponseBase + listBlobHierarchySegmentWithResponse(String containerName, String delimiter, String prefix, String marker, + Integer maxresults, List include, String startFrom, Integer timeout, String requestId, + Context context) { + try { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service.listBlobHierarchySegmentSync(this.client.getUrl(), containerName, restype, comp, prefix, + delimiter, marker, maxresults, includeConverted, startFrom, timeout, this.client.getVersion(), + requestId, accept, context); + } catch (BlobStorageExceptionInternal internalException) { + throw ModelHelper.mapToBlobStorageException(internalException); + } + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public ListBlobsHierarchySegmentResponse listBlobHierarchySegment(String containerName, String delimiter, + String prefix, String marker, Integer maxresults, List include, String startFrom, + Integer timeout, String requestId) { + try { + return listBlobHierarchySegmentWithResponse(containerName, delimiter, prefix, marker, maxresults, include, + startFrom, timeout, requestId, Context.NONE).getValue(); + } catch (BlobStorageExceptionInternal internalException) { + throw ModelHelper.mapToBlobStorageException(internalException); + } + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return an enumeration of blobs along with {@link Response}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response listBlobHierarchySegmentNoCustomHeadersWithResponse( + String containerName, String delimiter, String prefix, String marker, Integer maxresults, + List include, String startFrom, Integer timeout, String requestId, Context context) { + try { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service.listBlobHierarchySegmentNoCustomHeadersSync(this.client.getUrl(), containerName, restype, + comp, prefix, delimiter, marker, maxresults, includeConverted, startFrom, timeout, + this.client.getVersion(), requestId, accept, context); + } catch (BlobStorageExceptionInternal internalException) { + throw ModelHelper.mapToBlobStorageException(internalException); + } + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response body along with {@link ResponseBase} on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono>> + listBlobHierarchySegmentApacheArrowWithResponseAsync(String containerName, String delimiter, String prefix, + String marker, Integer maxresults, List include, Integer timeout, String startFrom, + String endBefore, String requestId) { + return FluxUtil + .withContext(context -> listBlobHierarchySegmentApacheArrowWithResponseAsync(containerName, delimiter, + prefix, marker, maxresults, include, timeout, startFrom, endBefore, requestId, context)) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response body along with {@link ResponseBase} on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono>> + listBlobHierarchySegmentApacheArrowWithResponseAsync(String containerName, String delimiter, String prefix, + String marker, Integer maxresults, List include, Integer timeout, String startFrom, + String endBefore, String requestId, Context context) { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service + .listBlobHierarchySegmentApacheArrow(this.client.getUrl(), containerName, restype, comp, accept, prefix, + delimiter, marker, maxresults, includeConverted, timeout, startFrom, endBefore, + this.client.getVersion(), requestId, context) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Flux listBlobHierarchySegmentApacheArrowAsync(String containerName, String delimiter, + String prefix, String marker, Integer maxresults, List include, Integer timeout, + String startFrom, String endBefore, String requestId) { + return listBlobHierarchySegmentApacheArrowWithResponseAsync(containerName, delimiter, prefix, marker, + maxresults, include, timeout, startFrom, endBefore, requestId) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException) + .flatMapMany(fluxByteBufferResponse -> fluxByteBufferResponse.getValue()); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Flux listBlobHierarchySegmentApacheArrowAsync(String containerName, String delimiter, + String prefix, String marker, Integer maxresults, List include, Integer timeout, + String startFrom, String endBefore, String requestId, Context context) { + return listBlobHierarchySegmentApacheArrowWithResponseAsync(containerName, delimiter, prefix, marker, + maxresults, include, timeout, startFrom, endBefore, requestId, context) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException) + .flatMapMany(fluxByteBufferResponse -> fluxByteBufferResponse.getValue()); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response body on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono listBlobHierarchySegmentApacheArrowNoCustomHeadersWithResponseAsync( + String containerName, String delimiter, String prefix, String marker, Integer maxresults, + List include, Integer timeout, String startFrom, String endBefore, String requestId) { + return FluxUtil + .withContext(context -> listBlobHierarchySegmentApacheArrowNoCustomHeadersWithResponseAsync(containerName, + delimiter, prefix, marker, maxresults, include, timeout, startFrom, endBefore, requestId, context)) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response body on successful completion of {@link Mono}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono listBlobHierarchySegmentApacheArrowNoCustomHeadersWithResponseAsync( + String containerName, String delimiter, String prefix, String marker, Integer maxresults, + List include, Integer timeout, String startFrom, String endBefore, String requestId, + Context context) { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service + .listBlobHierarchySegmentApacheArrowNoCustomHeaders(this.client.getUrl(), containerName, restype, comp, + accept, prefix, delimiter, marker, maxresults, includeConverted, timeout, startFrom, endBefore, + this.client.getVersion(), requestId, context) + .onErrorMap(BlobStorageExceptionInternal.class, ModelHelper::mapToBlobStorageException); + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response body along with {@link ResponseBase}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public ResponseBase + listBlobHierarchySegmentApacheArrowWithResponse(String containerName, String delimiter, String prefix, + String marker, Integer maxresults, List include, Integer timeout, String startFrom, + String endBefore, String requestId, Context context) { + try { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service.listBlobHierarchySegmentApacheArrowSync(this.client.getUrl(), containerName, restype, comp, + accept, prefix, delimiter, marker, maxresults, includeConverted, timeout, startFrom, endBefore, + this.client.getVersion(), requestId, context); + } catch (BlobStorageExceptionInternal internalException) { + throw ModelHelper.mapToBlobStorageException(internalException); + } + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public InputStream listBlobHierarchySegmentApacheArrow(String containerName, String delimiter, String prefix, + String marker, Integer maxresults, List include, Integer timeout, String startFrom, + String endBefore, String requestId) { + try { + return listBlobHierarchySegmentApacheArrowWithResponse(containerName, delimiter, prefix, marker, maxresults, + include, timeout, startFrom, endBefore, requestId, Context.NONE).getValue(); + } catch (BlobStorageExceptionInternal internalException) { + throw ModelHelper.mapToBlobStorageException(internalException); + } + } + + /** + * [Update] The List Blobs operation returns a list of the blobs under the specified container. This operation is + * for Apache Arrow use case so response is returned as raw to be deserialized by the client. + * + * @param containerName The container name. + * @param delimiter When the request includes this parameter, the operation returns a BlobPrefix element in the + * response body that acts as a placeholder for all blobs whose names begin with the same substring up to the + * appearance of the delimiter character. The delimiter may be a single character or a string. + * @param prefix Filters the results to return only containers whose name begins with the specified prefix. + * @param marker A string value that identifies the portion of the list of containers to be returned with the next + * listing operation. The operation returns the NextMarker value within the response body if the listing operation + * did not return all containers remaining to be listed with the current page. The NextMarker value can be used as + * the value for the marker parameter in a subsequent call to request the next page of list items. The marker value + * is opaque to the client. + * @param maxresults Specifies the maximum number of containers to return. If the request does not specify + * maxresults, or specifies a value greater than 5000, the server will return up to 5000 items. Note that if the + * listing operation crosses a partition boundary, then the service will return a continuation token for retrieving + * the remainder of the results. For this reason, it is possible that the service will return fewer results than + * specified by maxresults, or than the default of 5000. + * @param include Include this parameter to specify one or more datasets to include in the response. + * @param timeout The timeout parameter is expressed in seconds. For more information, see <a + * href="https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations">Setting + * Timeouts for Blob Service Operations.</a>. + * @param startFrom Specifies the relative path to list paths from. For non-recursive list, only one entity level is + * supported; For recursive list, multiple entity levels are supported. (Inclusive). + * @param endBefore Specifies the relative path to end before list paths. (Exclusive). + * @param requestId Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the + * analytics logs when storage analytics logging is enabled. + * @param context The context to associate with this operation. + * @throws IllegalArgumentException thrown if parameters fail the validation. + * @throws BlobStorageExceptionInternal thrown if the request is rejected by server. + * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. + * @return the response body along with {@link Response}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response listBlobHierarchySegmentApacheArrowNoCustomHeadersWithResponse(String containerName, + String delimiter, String prefix, String marker, Integer maxresults, List include, + Integer timeout, String startFrom, String endBefore, String requestId, Context context) { + try { + final String restype = "container"; + final String comp = "list"; + final String accept = "application/vnd.apache.arrow.stream,application/xml"; + String includeConverted = (include == null) + ? null + : include.stream() + .map(paramItemValue -> Objects.toString(paramItemValue, "")) + .collect(Collectors.joining(",")); + return service.listBlobHierarchySegmentApacheArrowNoCustomHeadersSync(this.client.getUrl(), containerName, + restype, comp, accept, prefix, delimiter, marker, maxresults, includeConverted, timeout, startFrom, + endBefore, this.client.getVersion(), requestId, context); } catch (BlobStorageExceptionInternal internalException) { throw ModelHelper.mapToBlobStorageException(internalException); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/AppendBlobItemConstructorProxy.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/AppendBlobItemConstructorProxy.java new file mode 100644 index 000000000000..b62fc2ecf1da --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/AppendBlobItemConstructorProxy.java @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.accesshelpers; + +import com.azure.storage.blob.models.AppendBlobItem; + +import java.time.OffsetDateTime; + +/** + * Helper class to access private values of {@link AppendBlobItem} across package boundaries. + */ +public final class AppendBlobItemConstructorProxy { + private static AppendBlobItemConstructorAccessor accessor; + + private AppendBlobItemConstructorProxy() { + } + + /** + * Type defining the methods to set the non-public properties of an {@link AppendBlobItem}. + */ + public interface AppendBlobItemConstructorAccessor { + /** + * Creates a new instance of {@link AppendBlobItem}. + * + * @param eTag ETag of the append blob. + * @param lastModified Last modified time of the append blob. + * @param contentMd5 Content MD5 of the append blob. + * @param isServerEncrypted Flag indicating if the append blob is encrypted on the server. + * @param encryptionKeySha256 The encryption key used to encrypt the append blob. + * @param encryptionScope The encryption scope used to encrypt the append blob. + * @param blobAppendOffset The offset at which the block was committed to the append blob. + * @param blobCommittedBlockCount The number of committed blocks in the append blob. + * @param versionId The version identifier of the append blob. + * @param contentCrc64 Content CRC64 of the append blob. + * @return A new instance of {@link AppendBlobItem}. + */ + AppendBlobItem create(String eTag, OffsetDateTime lastModified, byte[] contentMd5, boolean isServerEncrypted, + String encryptionKeySha256, String encryptionScope, String blobAppendOffset, + Integer blobCommittedBlockCount, String versionId, byte[] contentCrc64); + } + + /** + * The method called from {@link AppendBlobItem} to set its accessor. + * + * @param accessor The accessor. + */ + public static void setAccessor(final AppendBlobItemConstructorAccessor accessor) { + AppendBlobItemConstructorProxy.accessor = accessor; + } + + /** + * Creates a new instance of {@link AppendBlobItem}. + * + * @param eTag ETag of the append blob. + * @param lastModified Last modified time of the append blob. + * @param contentMd5 Content MD5 of the append blob. + * @param isServerEncrypted Flag indicating if the append blob is encrypted on the server. + * @param encryptionKeySha256 The encryption key used to encrypt the append blob. + * @param encryptionScope The encryption scope used to encrypt the append blob. + * @param blobAppendOffset The offset at which the block was committed to the append blob. + * @param blobCommittedBlockCount The number of committed blocks in the append blob. + * @param versionId The version identifier of the append blob. + * @param contentCrc64 Content CRC64 of the append blob. + * @return A new instance of {@link AppendBlobItem}. + */ + public static AppendBlobItem create(String eTag, OffsetDateTime lastModified, byte[] contentMd5, + boolean isServerEncrypted, String encryptionKeySha256, String encryptionScope, String blobAppendOffset, + Integer blobCommittedBlockCount, String versionId, byte[] contentCrc64) { + // This looks odd but is necessary, it is possible to engage the access helper before anywhere else in the + // application accesses AppendBlobItem which triggers the accessor to be configured. So, if the accessor is null + // this effectively pokes the class to set up the accessor. + if (accessor == null) { + new AppendBlobItem(null, null, null, false, null, null, null, null, null); + } + + assert accessor != null; + return accessor.create(eTag, lastModified, contentMd5, isServerEncrypted, encryptionKeySha256, encryptionScope, + blobAppendOffset, blobCommittedBlockCount, versionId, contentCrc64); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlockBlobItemConstructorProxy.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlockBlobItemConstructorProxy.java new file mode 100644 index 000000000000..0d973e425bea --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/BlockBlobItemConstructorProxy.java @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.accesshelpers; + +import com.azure.storage.blob.models.BlockBlobItem; + +import java.time.OffsetDateTime; + +/** + * Helper class to access private values of {@link BlockBlobItem} across package boundaries. + */ +public final class BlockBlobItemConstructorProxy { + private static BlockBlobItemConstructorAccessor accessor; + + private BlockBlobItemConstructorProxy() { + } + + /** + * Type defining the methods to set the non-public properties of a {@link BlockBlobItem}. + */ + public interface BlockBlobItemConstructorAccessor { + /** + * Creates a new instance of {@link BlockBlobItem}. + * + * @param eTag ETag of the block blob. + * @param lastModified Last modified time of the block blob. + * @param contentMd5 Content MD5 of the block blob. + * @param isServerEncrypted Flag indicating if the block blob is encrypted on the server. + * @param encryptionKeySha256 The encryption key used to encrypt the block blob. + * @param encryptionScope The encryption scope used to encrypt the block blob. + * @param versionId The version identifier of the block blob. + * @param contentCrc64 Content CRC64 of the block blob. + * @return A new instance of {@link BlockBlobItem}. + */ + BlockBlobItem create(String eTag, OffsetDateTime lastModified, byte[] contentMd5, Boolean isServerEncrypted, + String encryptionKeySha256, String encryptionScope, String versionId, byte[] contentCrc64); + } + + /** + * The method called from {@link BlockBlobItem} to set its accessor. + * + * @param accessor The accessor. + */ + public static void setAccessor(final BlockBlobItemConstructorAccessor accessor) { + BlockBlobItemConstructorProxy.accessor = accessor; + } + + /** + * Creates a new instance of {@link BlockBlobItem}. + * + * @param eTag ETag of the block blob. + * @param lastModified Last modified time of the block blob. + * @param contentMd5 Content MD5 of the block blob. + * @param isServerEncrypted Flag indicating if the block blob is encrypted on the server. + * @param encryptionKeySha256 The encryption key used to encrypt the block blob. + * @param encryptionScope The encryption scope used to encrypt the block blob. + * @param versionId The version identifier of the block blob. + * @param contentCrc64 Content CRC64 of the block blob. + * @return A new instance of {@link BlockBlobItem}. + */ + public static BlockBlobItem create(String eTag, OffsetDateTime lastModified, byte[] contentMd5, + Boolean isServerEncrypted, String encryptionKeySha256, String encryptionScope, String versionId, + byte[] contentCrc64) { + + if (accessor == null) { + new BlockBlobItem(null, null, null, (Boolean) null, null, null, null); + } + + assert accessor != null; + return accessor.create(eTag, lastModified, contentMd5, isServerEncrypted, encryptionKeySha256, encryptionScope, + versionId, contentCrc64); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/PageBlobItemConstructorProxy.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/PageBlobItemConstructorProxy.java new file mode 100644 index 000000000000..fb241d47a268 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/accesshelpers/PageBlobItemConstructorProxy.java @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.accesshelpers; + +import com.azure.storage.blob.models.PageBlobItem; + +import java.time.OffsetDateTime; + +/** + * Helper class to access private values of {@link PageBlobItem} across package boundaries. + */ +public final class PageBlobItemConstructorProxy { + private static PageBlobItemConstructorAccessor accessor; + + private PageBlobItemConstructorProxy() { + } + + /** + * Type defining the methods to set the non-public properties of a {@link PageBlobItem}. + */ + public interface PageBlobItemConstructorAccessor { + /** + * Creates a new instance of {@link PageBlobItem}. + * + * @param eTag ETag of the page blob. + * @param lastModified Last modified time of the page blob. + * @param contentMd5 Content MD5 of the page blob. + * @param isServerEncrypted Flag indicating if the page blob is encrypted on the server. + * @param encryptionKeySha256 The encryption key used to encrypt the page blob. + * @param encryptionScope The encryption scope used to encrypt the page blob. + * @param blobSequenceNumber The current sequence number for the page blob. + * @param versionId The version identifier of the page blob. + * @param contentCrc64 Content CRC64 of the page blob. + * @return A new instance of {@link PageBlobItem}. + */ + PageBlobItem create(String eTag, OffsetDateTime lastModified, byte[] contentMd5, Boolean isServerEncrypted, + String encryptionKeySha256, String encryptionScope, Long blobSequenceNumber, String versionId, + byte[] contentCrc64); + } + + /** + * The method called from {@link PageBlobItem} to set its accessor. + * + * @param accessor The accessor. + */ + public static void setAccessor(final PageBlobItemConstructorAccessor accessor) { + PageBlobItemConstructorProxy.accessor = accessor; + } + + /** + * Creates a new instance of {@link PageBlobItem}. + * + * @param eTag ETag of the page blob. + * @param lastModified Last modified time of the page blob. + * @param contentMd5 Content MD5 of the page blob. + * @param isServerEncrypted Flag indicating if the page blob is encrypted on the server. + * @param encryptionKeySha256 The encryption key used to encrypt the page blob. + * @param encryptionScope The encryption scope used to encrypt the page blob. + * @param blobSequenceNumber The current sequence number for the page blob. + * @param versionId The version identifier of the page blob. + * @param contentCrc64 Content CRC64 of the page blob. + * @return A new instance of {@link PageBlobItem}. + */ + public static PageBlobItem create(String eTag, OffsetDateTime lastModified, byte[] contentMd5, + Boolean isServerEncrypted, String encryptionKeySha256, String encryptionScope, Long blobSequenceNumber, + String versionId, byte[] contentCrc64) { + + if (accessor == null) { + new PageBlobItem(null, null, null, false, null, null, null, null); + } + + assert accessor != null; + return accessor.create(eTag, lastModified, contentMd5, isServerEncrypted, encryptionKeySha256, encryptionScope, + blobSequenceNumber, versionId, contentCrc64); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobListArrowParseException.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobListArrowParseException.java new file mode 100644 index 000000000000..6d2465d26bff --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobListArrowParseException.java @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.models; + +/** + * Exception thrown when parsing Arrow ListBlobs payloads fails or encounters unsupported content. + */ +public final class BlobListArrowParseException extends RuntimeException { + private static final long serialVersionUID = 1L; + + /** + * Creates an exception with a message. + * + * @param message parse failure details. + */ + public BlobListArrowParseException(String message) { + super(message); + } + + /** + * Creates an exception with a message and cause. + * + * @param message parse failure details. + * @param cause originating cause. + */ + public BlobListArrowParseException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsPutBlobFromUrlHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsPutBlobFromUrlHeaders.java index 9e8fb72f68db..02c212f1e3fa 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsPutBlobFromUrlHeaders.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsPutBlobFromUrlHeaders.java @@ -36,6 +36,12 @@ public final class BlockBlobsPutBlobFromUrlHeaders { @Generated private byte[] contentMD5; + /* + * The x-ms-content-crc64 property. + */ + @Generated + private byte[] xMsContentCrc64; + /* * The x-ms-client-request-id property. */ @@ -84,6 +90,8 @@ public final class BlockBlobsPutBlobFromUrlHeaders { @Generated private String xMsEncryptionScope; + private static final HttpHeaderName X_MS_CONTENT_CRC64 = HttpHeaderName.fromString("x-ms-content-crc64"); + private static final HttpHeaderName X_MS_VERSION = HttpHeaderName.fromString("x-ms-version"); private static final HttpHeaderName X_MS_VERSION_ID = HttpHeaderName.fromString("x-ms-version-id"); @@ -116,6 +124,12 @@ public BlockBlobsPutBlobFromUrlHeaders(HttpHeaders rawHeaders) { } else { this.contentMD5 = null; } + String xMsContentCrc64 = rawHeaders.getValue(X_MS_CONTENT_CRC64); + if (xMsContentCrc64 != null) { + this.xMsContentCrc64 = Base64.getDecoder().decode(xMsContentCrc64); + } else { + this.xMsContentCrc64 = null; + } this.xMsClientRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_CLIENT_REQUEST_ID); this.xMsRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_REQUEST_ID); this.xMsVersion = rawHeaders.getValue(X_MS_VERSION); @@ -209,6 +223,28 @@ public BlockBlobsPutBlobFromUrlHeaders setContentMD5(byte[] contentMD5) { return this; } + /** + * Get the xMsContentCrc64 property: The x-ms-content-crc64 property. + * + * @return the xMsContentCrc64 value. + */ + @Generated + public byte[] getXMsContentCrc64() { + return CoreUtils.clone(this.xMsContentCrc64); + } + + /** + * Set the xMsContentCrc64 property: The x-ms-content-crc64 property. + * + * @param xMsContentCrc64 the xMsContentCrc64 value to set. + * @return the BlockBlobsPutBlobFromUrlHeaders object itself. + */ + @Generated + public BlockBlobsPutBlobFromUrlHeaders setXMsContentCrc64(byte[] xMsContentCrc64) { + this.xMsContentCrc64 = CoreUtils.clone(xMsContentCrc64); + return this; + } + /** * Get the xMsClientRequestId property: The x-ms-client-request-id property. * diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsUploadHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsUploadHeaders.java index fb8da8e12407..63fa2cd2212d 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsUploadHeaders.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlockBlobsUploadHeaders.java @@ -36,6 +36,12 @@ public final class BlockBlobsUploadHeaders { @Generated private byte[] contentMD5; + /* + * The x-ms-content-crc64 property. + */ + @Generated + private byte[] xMsContentCrc64; + /* * The x-ms-client-request-id property. */ @@ -90,6 +96,8 @@ public final class BlockBlobsUploadHeaders { @Generated private String xMsStructuredBody; + private static final HttpHeaderName X_MS_CONTENT_CRC64 = HttpHeaderName.fromString("x-ms-content-crc64"); + private static final HttpHeaderName X_MS_VERSION = HttpHeaderName.fromString("x-ms-version"); private static final HttpHeaderName X_MS_VERSION_ID = HttpHeaderName.fromString("x-ms-version-id"); @@ -124,6 +132,12 @@ public BlockBlobsUploadHeaders(HttpHeaders rawHeaders) { } else { this.contentMD5 = null; } + String xMsContentCrc64 = rawHeaders.getValue(X_MS_CONTENT_CRC64); + if (xMsContentCrc64 != null) { + this.xMsContentCrc64 = Base64.getDecoder().decode(xMsContentCrc64); + } else { + this.xMsContentCrc64 = null; + } this.xMsClientRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_CLIENT_REQUEST_ID); this.xMsRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_REQUEST_ID); this.xMsVersion = rawHeaders.getValue(X_MS_VERSION); @@ -218,6 +232,28 @@ public BlockBlobsUploadHeaders setContentMD5(byte[] contentMD5) { return this; } + /** + * Get the xMsContentCrc64 property: The x-ms-content-crc64 property. + * + * @return the xMsContentCrc64 value. + */ + @Generated + public byte[] getXMsContentCrc64() { + return CoreUtils.clone(this.xMsContentCrc64); + } + + /** + * Set the xMsContentCrc64 property: The x-ms-content-crc64 property. + * + * @param xMsContentCrc64 the xMsContentCrc64 value to set. + * @return the BlockBlobsUploadHeaders object itself. + */ + @Generated + public BlockBlobsUploadHeaders setXMsContentCrc64(byte[] xMsContentCrc64) { + this.xMsContentCrc64 = CoreUtils.clone(xMsContentCrc64); + return this; + } + /** * Get the xMsClientRequestId property: The x-ms-client-request-id property. * diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/ContainersListBlobFlatSegmentApacheArrowHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/ContainersListBlobFlatSegmentApacheArrowHeaders.java new file mode 100644 index 000000000000..d137f165b8de --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/ContainersListBlobFlatSegmentApacheArrowHeaders.java @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// Code generated by Microsoft (R) AutoRest Code Generator. + +package com.azure.storage.blob.implementation.models; + +import com.azure.core.annotation.Fluent; +import com.azure.core.annotation.Generated; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.DateTimeRfc1123; +import java.time.OffsetDateTime; + +/** + * The ContainersListBlobFlatSegmentApacheArrowHeaders model. + */ +@Fluent +public final class ContainersListBlobFlatSegmentApacheArrowHeaders { + /* + * The Content-Type property. + */ + @Generated + private String contentType; + + /* + * The x-ms-client-request-id property. + */ + @Generated + private String xMsClientRequestId; + + /* + * The x-ms-request-id property. + */ + @Generated + private String xMsRequestId; + + /* + * The x-ms-version property. + */ + @Generated + private String xMsVersion; + + /* + * The Date property. + */ + @Generated + private DateTimeRfc1123 date; + + private static final HttpHeaderName X_MS_VERSION = HttpHeaderName.fromString("x-ms-version"); + + // HttpHeaders containing the raw property values. + /** + * Creates an instance of ContainersListBlobFlatSegmentApacheArrowHeaders class. + * + * @param rawHeaders The raw HttpHeaders that will be used to create the property values. + */ + public ContainersListBlobFlatSegmentApacheArrowHeaders(HttpHeaders rawHeaders) { + this.contentType = rawHeaders.getValue(HttpHeaderName.CONTENT_TYPE); + this.xMsClientRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_CLIENT_REQUEST_ID); + this.xMsRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_REQUEST_ID); + this.xMsVersion = rawHeaders.getValue(X_MS_VERSION); + String date = rawHeaders.getValue(HttpHeaderName.DATE); + if (date != null) { + this.date = new DateTimeRfc1123(date); + } else { + this.date = null; + } + } + + /** + * Get the contentType property: The Content-Type property. + * + * @return the contentType value. + */ + @Generated + public String getContentType() { + return this.contentType; + } + + /** + * Set the contentType property: The Content-Type property. + * + * @param contentType the contentType value to set. + * @return the ContainersListBlobFlatSegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobFlatSegmentApacheArrowHeaders setContentType(String contentType) { + this.contentType = contentType; + return this; + } + + /** + * Get the xMsClientRequestId property: The x-ms-client-request-id property. + * + * @return the xMsClientRequestId value. + */ + @Generated + public String getXMsClientRequestId() { + return this.xMsClientRequestId; + } + + /** + * Set the xMsClientRequestId property: The x-ms-client-request-id property. + * + * @param xMsClientRequestId the xMsClientRequestId value to set. + * @return the ContainersListBlobFlatSegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobFlatSegmentApacheArrowHeaders setXMsClientRequestId(String xMsClientRequestId) { + this.xMsClientRequestId = xMsClientRequestId; + return this; + } + + /** + * Get the xMsRequestId property: The x-ms-request-id property. + * + * @return the xMsRequestId value. + */ + @Generated + public String getXMsRequestId() { + return this.xMsRequestId; + } + + /** + * Set the xMsRequestId property: The x-ms-request-id property. + * + * @param xMsRequestId the xMsRequestId value to set. + * @return the ContainersListBlobFlatSegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobFlatSegmentApacheArrowHeaders setXMsRequestId(String xMsRequestId) { + this.xMsRequestId = xMsRequestId; + return this; + } + + /** + * Get the xMsVersion property: The x-ms-version property. + * + * @return the xMsVersion value. + */ + @Generated + public String getXMsVersion() { + return this.xMsVersion; + } + + /** + * Set the xMsVersion property: The x-ms-version property. + * + * @param xMsVersion the xMsVersion value to set. + * @return the ContainersListBlobFlatSegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobFlatSegmentApacheArrowHeaders setXMsVersion(String xMsVersion) { + this.xMsVersion = xMsVersion; + return this; + } + + /** + * Get the date property: The Date property. + * + * @return the date value. + */ + @Generated + public OffsetDateTime getDate() { + if (this.date == null) { + return null; + } + return this.date.getDateTime(); + } + + /** + * Set the date property: The Date property. + * + * @param date the date value to set. + * @return the ContainersListBlobFlatSegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobFlatSegmentApacheArrowHeaders setDate(OffsetDateTime date) { + if (date == null) { + this.date = null; + } else { + this.date = new DateTimeRfc1123(date); + } + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/ContainersListBlobHierarchySegmentApacheArrowHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/ContainersListBlobHierarchySegmentApacheArrowHeaders.java new file mode 100644 index 000000000000..8c18650a8be5 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/ContainersListBlobHierarchySegmentApacheArrowHeaders.java @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// Code generated by Microsoft (R) AutoRest Code Generator. + +package com.azure.storage.blob.implementation.models; + +import com.azure.core.annotation.Fluent; +import com.azure.core.annotation.Generated; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.DateTimeRfc1123; +import java.time.OffsetDateTime; + +/** + * The ContainersListBlobHierarchySegmentApacheArrowHeaders model. + */ +@Fluent +public final class ContainersListBlobHierarchySegmentApacheArrowHeaders { + /* + * The Content-Type property. + */ + @Generated + private String contentType; + + /* + * The x-ms-client-request-id property. + */ + @Generated + private String xMsClientRequestId; + + /* + * The x-ms-request-id property. + */ + @Generated + private String xMsRequestId; + + /* + * The x-ms-version property. + */ + @Generated + private String xMsVersion; + + /* + * The Date property. + */ + @Generated + private DateTimeRfc1123 date; + + private static final HttpHeaderName X_MS_VERSION = HttpHeaderName.fromString("x-ms-version"); + + // HttpHeaders containing the raw property values. + /** + * Creates an instance of ContainersListBlobHierarchySegmentApacheArrowHeaders class. + * + * @param rawHeaders The raw HttpHeaders that will be used to create the property values. + */ + public ContainersListBlobHierarchySegmentApacheArrowHeaders(HttpHeaders rawHeaders) { + this.contentType = rawHeaders.getValue(HttpHeaderName.CONTENT_TYPE); + this.xMsClientRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_CLIENT_REQUEST_ID); + this.xMsRequestId = rawHeaders.getValue(HttpHeaderName.X_MS_REQUEST_ID); + this.xMsVersion = rawHeaders.getValue(X_MS_VERSION); + String date = rawHeaders.getValue(HttpHeaderName.DATE); + if (date != null) { + this.date = new DateTimeRfc1123(date); + } else { + this.date = null; + } + } + + /** + * Get the contentType property: The Content-Type property. + * + * @return the contentType value. + */ + @Generated + public String getContentType() { + return this.contentType; + } + + /** + * Set the contentType property: The Content-Type property. + * + * @param contentType the contentType value to set. + * @return the ContainersListBlobHierarchySegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobHierarchySegmentApacheArrowHeaders setContentType(String contentType) { + this.contentType = contentType; + return this; + } + + /** + * Get the xMsClientRequestId property: The x-ms-client-request-id property. + * + * @return the xMsClientRequestId value. + */ + @Generated + public String getXMsClientRequestId() { + return this.xMsClientRequestId; + } + + /** + * Set the xMsClientRequestId property: The x-ms-client-request-id property. + * + * @param xMsClientRequestId the xMsClientRequestId value to set. + * @return the ContainersListBlobHierarchySegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobHierarchySegmentApacheArrowHeaders setXMsClientRequestId(String xMsClientRequestId) { + this.xMsClientRequestId = xMsClientRequestId; + return this; + } + + /** + * Get the xMsRequestId property: The x-ms-request-id property. + * + * @return the xMsRequestId value. + */ + @Generated + public String getXMsRequestId() { + return this.xMsRequestId; + } + + /** + * Set the xMsRequestId property: The x-ms-request-id property. + * + * @param xMsRequestId the xMsRequestId value to set. + * @return the ContainersListBlobHierarchySegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobHierarchySegmentApacheArrowHeaders setXMsRequestId(String xMsRequestId) { + this.xMsRequestId = xMsRequestId; + return this; + } + + /** + * Get the xMsVersion property: The x-ms-version property. + * + * @return the xMsVersion value. + */ + @Generated + public String getXMsVersion() { + return this.xMsVersion; + } + + /** + * Set the xMsVersion property: The x-ms-version property. + * + * @param xMsVersion the xMsVersion value to set. + * @return the ContainersListBlobHierarchySegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobHierarchySegmentApacheArrowHeaders setXMsVersion(String xMsVersion) { + this.xMsVersion = xMsVersion; + return this; + } + + /** + * Get the date property: The Date property. + * + * @return the date value. + */ + @Generated + public OffsetDateTime getDate() { + if (this.date == null) { + return null; + } + return this.date.getDateTime(); + } + + /** + * Set the date property: The Date property. + * + * @param date the date value to set. + * @return the ContainersListBlobHierarchySegmentApacheArrowHeaders object itself. + */ + @Generated + public ContainersListBlobHierarchySegmentApacheArrowHeaders setDate(OffsetDateTime date) { + if (date == null) { + this.date = null; + } else { + this.date = new DateTimeRfc1123(date); + } + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ArrowBlobListDeserializer.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ArrowBlobListDeserializer.java new file mode 100644 index 000000000000..ad46d85ce175 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ArrowBlobListDeserializer.java @@ -0,0 +1,373 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.util; + +import com.azure.storage.blob.implementation.models.BlobItemInternal; +import com.azure.storage.blob.implementation.models.BlobItemPropertiesInternal; +import com.azure.storage.blob.implementation.models.BlobListArrowParseException; +import com.azure.storage.blob.implementation.models.BlobName; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.Batch; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.BoolColumn; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.Column; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.IntColumn; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.MapColumn; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.DecodedArrowStream; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.StringColumn; +import com.azure.storage.blob.implementation.util.BlobListArrowStreamReader.TimestampColumn; +import com.azure.storage.blob.models.AccessTier; +import com.azure.storage.blob.models.ArchiveStatus; +import com.azure.storage.blob.models.BlobImmutabilityPolicyMode; +import com.azure.storage.blob.models.BlobType; +import com.azure.storage.blob.models.CopyStatusType; +import com.azure.storage.blob.models.LeaseDurationType; +import com.azure.storage.blob.models.LeaseStateType; +import com.azure.storage.blob.models.LeaseStatusType; +import com.azure.storage.blob.models.RehydratePriority; + +import java.io.InputStream; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Deserializes an Apache Arrow IPC stream from the ListBlobs response into a list of {@link BlobItemInternal} objects. + */ +public final class ArrowBlobListDeserializer { + + /** + * The authoritative set of Arrow column names this deserializer knows how to read. It must mirror exactly the + * field names requested by {@link #readRow(Batch, int)}; the {@code alltypes} golden fixture drift-guard test + * asserts they stay in sync. Any column present in the response schema but absent from this set means the service + * has emitted a field this SDK version does not understand, which {@link #validateKnownColumns(Batch)} rejects. + */ + private static final Set KNOWN_COLUMNS = Collections.unmodifiableSet(new HashSet<>( + Arrays.asList("Name", "ResourceType", "Deleted", "Snapshot", "VersionId", "IsCurrentVersion", "HasVersionsOnly", + "Metadata", "OrMetadata", "Tags", "Creation-Time", "Last-Modified", "Etag", "Content-Length", + "Content-Type", "Content-Encoding", "Content-Language", "Content-Disposition", "Cache-Control", + "Content-MD5", "BlobType", "AccessTier", "AccessTierInferred", "AccessTierChangeTime", "SmartAccessTier", + "LeaseStatus", "LeaseState", "LeaseDuration", "ServerEncrypted", "CustomerProvidedKeySha256", + "EncryptionScope", "IncrementalCopy", "CopyId", "CopyStatus", "CopySource", "CopyProgress", + "CopyCompletionTime", "CopyStatusDescription", "CopyDestinationSnapshot", "x-ms-blob-sequence-number", + "Sealed", "LegalHold", "DeletedTime", "LastAccessTime", "ImmutabilityPolicyUntilDate", + "ImmutabilityPolicyMode", "ArchiveStatus", "RehydratePriority", "TagCount", "RemainingRetentionDays"))); + + private ArrowBlobListDeserializer() { + } + + /** + * Exposes the authoritative set of columns this deserializer understands, for the drift-guard test that asserts it + * stays in sync with the full-schema golden fixture. Package-private: not part of the public API. + * + * @return the immutable set of known column names + */ + static Set knownColumns() { + return KNOWN_COLUMNS; + } + + /** + * Deserializes an Arrow IPC stream into blob items and pagination metadata. + * + * @param arrowStream the Arrow IPC input stream from the service response + * @return the deserialized result containing blob items and next marker + * @throws BlobListArrowParseException if deserialization fails + */ + public static ArrowListBlobsResult deserialize(InputStream arrowStream) { + if (arrowStream == null) { + throw new BlobListArrowParseException("ListBlobs Arrow parse failure: input stream is null."); + } + + List results = new ArrayList<>(); + String nextMarker = null; + Integer numberOfRecords = null; + + DecodedArrowStream decodedArrowStream = BlobListArrowStreamReader.read(arrowStream); + + Map schemaMetadata = decodedArrowStream.getSchemaMetadata(); + if (schemaMetadata != null) { + nextMarker = schemaMetadata.get("NextMarker"); + if (nextMarker != null && nextMarker.isEmpty()) { + nextMarker = null; + } + + String numberOfRecordsStr = schemaMetadata.get("NumberOfRecords"); + if (numberOfRecordsStr != null && !numberOfRecordsStr.isEmpty()) { + try { + numberOfRecords = Integer.parseInt(numberOfRecordsStr); + } catch (NumberFormatException e) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: schema metadata 'NumberOfRecords' isn't a valid integer.", e); + } + } + } + + for (Batch batch : decodedArrowStream.getBatches()) { + validateKnownColumns(batch); + int rowCount = batch.getRowCount(); + for (int i = 0; i < rowCount; i++) { + results.add(readRow(batch, i)); + } + } + + return new ArrowListBlobsResult(results, nextMarker, numberOfRecords); + } + + /** + * Fails loudly if the batch schema contains any column this deserializer does not handle, so that a newly added + * service field surfaces as an error instead of being silently dropped. + * + * @param batch the record batch to validate + * @throws BlobListArrowParseException if the batch contains one or more columns absent from {@link #KNOWN_COLUMNS} + */ + private static void validateKnownColumns(Batch batch) { + List unknownColumns = null; + for (String columnName : batch.getColumnNames()) { + if (!KNOWN_COLUMNS.contains(columnName)) { + if (unknownColumns == null) { + unknownColumns = new ArrayList<>(); + } + unknownColumns.add(columnName); + } + } + + if (unknownColumns != null) { + Collections.sort(unknownColumns); + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: response contains unhandled column(s) " + unknownColumns + "."); + } + } + + private static BlobItemInternal readRow(Batch batch, int index) { + BlobItemInternal item = new BlobItemInternal(); + + String name = getVarChar(batch, "Name", index); + if (name != null) { + item.setName(new BlobName().setContent(name)); + } + + String resourceType = getVarChar(batch, "ResourceType", index); + if ("blobprefix".equals(resourceType)) { + item.setIsPrefix(true); + return item; + } + + BlobItemPropertiesInternal properties = new BlobItemPropertiesInternal(); + + Boolean deleted = getBit(batch, "Deleted", index); + if (deleted != null) { + item.setDeleted(deleted); + } + + item.setSnapshot(getVarChar(batch, "Snapshot", index)); + item.setVersionId(getVarChar(batch, "VersionId", index)); + item.setIsCurrentVersion(getBit(batch, "IsCurrentVersion", index)); + item.setHasVersionsOnly(getBit(batch, "HasVersionsOnly", index)); + + Map metadata = getMap(batch, "Metadata", index); + if (metadata != null) { + item.setMetadata(metadata); + } + + Map orMetadata = getMap(batch, "OrMetadata", index); + if (orMetadata != null) { + item.setObjectReplicationMetadata(orMetadata); + } + + Map tags = getMap(batch, "Tags", index); + if (tags != null) { + item.setBlobTags(ModelHelper.toBlobTags(tags)); + } + + properties.setCreationTime(getTimestamp(batch, "Creation-Time", index)); + properties.setLastModified(getTimestamp(batch, "Last-Modified", index)); + properties.setETag(getVarChar(batch, "Etag", index)); + properties.setContentLength(getUInt64(batch, "Content-Length", index)); + properties.setContentType(getVarChar(batch, "Content-Type", index)); + properties.setContentEncoding(getVarChar(batch, "Content-Encoding", index)); + properties.setContentLanguage(getVarChar(batch, "Content-Language", index)); + properties.setContentDisposition(getVarChar(batch, "Content-Disposition", index)); + properties.setCacheControl(getVarChar(batch, "Cache-Control", index)); + + String contentMd5 = getVarChar(batch, "Content-MD5", index); + if (contentMd5 != null) { + properties.setContentMd5(Base64.getDecoder().decode(contentMd5)); + } + + String blobType = getVarChar(batch, "BlobType", index); + if (blobType != null) { + properties.setBlobType(BlobType.fromString(blobType)); + } + + String accessTier = getVarChar(batch, "AccessTier", index); + if (accessTier != null) { + properties.setAccessTier(AccessTier.fromString(accessTier)); + } + properties.setAccessTierInferred(getBit(batch, "AccessTierInferred", index)); + properties.setAccessTierChangeTime(getTimestamp(batch, "AccessTierChangeTime", index)); + + String smartAccessTier = getVarChar(batch, "SmartAccessTier", index); + if (smartAccessTier != null) { + properties.setSmartAccessTier(AccessTier.fromString(smartAccessTier)); + } + + String leaseStatus = getVarChar(batch, "LeaseStatus", index); + if (leaseStatus != null) { + properties.setLeaseStatus(LeaseStatusType.fromString(leaseStatus)); + } + String leaseState = getVarChar(batch, "LeaseState", index); + if (leaseState != null) { + properties.setLeaseState(LeaseStateType.fromString(leaseState)); + } + String leaseDuration = getVarChar(batch, "LeaseDuration", index); + if (leaseDuration != null) { + properties.setLeaseDuration(LeaseDurationType.fromString(leaseDuration)); + } + + properties.setServerEncrypted(getBit(batch, "ServerEncrypted", index)); + properties.setCustomerProvidedKeySha256(getVarChar(batch, "CustomerProvidedKeySha256", index)); + properties.setEncryptionScope(getVarChar(batch, "EncryptionScope", index)); + properties.setIncrementalCopy(getBit(batch, "IncrementalCopy", index)); + + properties.setCopyId(getVarChar(batch, "CopyId", index)); + String copyStatus = getVarChar(batch, "CopyStatus", index); + if (copyStatus != null) { + properties.setCopyStatus(CopyStatusType.fromString(copyStatus)); + } + properties.setCopySource(getVarChar(batch, "CopySource", index)); + properties.setCopyProgress(getVarChar(batch, "CopyProgress", index)); + properties.setCopyCompletionTime(getTimestamp(batch, "CopyCompletionTime", index)); + properties.setCopyStatusDescription(getVarChar(batch, "CopyStatusDescription", index)); + properties.setDestinationSnapshot(getVarChar(batch, "CopyDestinationSnapshot", index)); + + properties.setBlobSequenceNumber(getUInt64(batch, "x-ms-blob-sequence-number", index)); + + properties.setIsSealed(getBit(batch, "Sealed", index)); + properties.setLegalHold(getBit(batch, "LegalHold", index)); + properties.setDeletedTime(getTimestamp(batch, "DeletedTime", index)); + properties.setLastAccessedOn(getTimestamp(batch, "LastAccessTime", index)); + properties.setImmutabilityPolicyExpiresOn(getTimestamp(batch, "ImmutabilityPolicyUntilDate", index)); + + String immutabilityMode = getVarChar(batch, "ImmutabilityPolicyMode", index); + if (immutabilityMode != null) { + properties.setImmutabilityPolicyMode(BlobImmutabilityPolicyMode.fromString(immutabilityMode)); + } + + String archiveStatus = getVarChar(batch, "ArchiveStatus", index); + if (archiveStatus != null) { + properties.setArchiveStatus(ArchiveStatus.fromString(archiveStatus)); + } + + String rehydratePriority = getVarChar(batch, "RehydratePriority", index); + if (rehydratePriority != null) { + properties.setRehydratePriority(RehydratePriority.fromString(rehydratePriority)); + } + + Long tagCount = getUInt64(batch, "TagCount", index); + if (tagCount != null) { + properties.setTagCount(tagCount.intValue()); + } + Long remainingRetentionDays = getUInt64(batch, "RemainingRetentionDays", index); + if (remainingRetentionDays != null) { + properties.setRemainingRetentionDays(remainingRetentionDays.intValue()); + } + + item.setProperties(properties); + return item; + } + + private static T getColumn(Batch batch, String name, int index, Class type, + String typeLabel) { + Column column = batch.getColumn(name); + if (column == null || column.isNull(index)) { + return null; + } + if (!type.isInstance(column)) { + throw new BlobListArrowParseException("ListBlobs Arrow parse failure: field '" + name + "' has unsupported " + + typeLabel + " column type '" + column.getClass().getSimpleName() + "'."); + } + return type.cast(column); + } + + // region Arrow helpers + + private static String getVarChar(Batch batch, String name, int index) { + StringColumn column = getColumn(batch, name, index, StringColumn.class, "string"); + return column == null ? null : column.get(index); + } + + private static Long getUInt64(Batch batch, String name, int index) { + IntColumn column = getColumn(batch, name, index, IntColumn.class, "integer"); + return column == null ? null : column.get(index); + } + + private static Boolean getBit(Batch batch, String name, int index) { + BoolColumn column = getColumn(batch, name, index, BoolColumn.class, "boolean"); + return column == null ? null : column.get(index); + } + + private static OffsetDateTime getTimestamp(Batch batch, String name, int index) { + TimestampColumn column = getColumn(batch, name, index, TimestampColumn.class, "timestamp"); + if (column == null) { + return null; + } + return OffsetDateTime.ofInstant(Instant.ofEpochSecond(column.getEpochSeconds(index)), ZoneOffset.UTC); + } + + private static Map getMap(Batch batch, String name, int index) { + MapColumn column = getColumn(batch, name, index, MapColumn.class, "map"); + return column == null ? null : column.get(index); + } + + /** + * Result of deserializing an Arrow ListBlobs response. + */ + public static final class ArrowListBlobsResult { + private final List blobItems; + private final String nextMarker; + private final Integer numberOfRecords; + + /** + * Creates an ArrowListBlobsResult. + * + * @param blobItems the deserialized blob items + * @param nextMarker the continuation token for the next page, or null if this is the last page + * @param numberOfRecords the total number of records reported by the service, or null if not present + */ + public ArrowListBlobsResult(List blobItems, String nextMarker, Integer numberOfRecords) { + this.blobItems = blobItems; + this.nextMarker = nextMarker; + this.numberOfRecords = numberOfRecords; + } + + /** + * @return the deserialized blob items + */ + public List getBlobItems() { + return blobItems; + } + + /** + * @return the continuation token for the next page, or null if this is the last page + */ + public String getNextMarker() { + return nextMarker; + } + + /** + * @return the total number of records reported by the service, or null if not present + */ + public Integer getNumberOfRecords() { + return numberOfRecords; + } + } + + //endregion +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobListArrowStreamReader.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobListArrowStreamReader.java new file mode 100644 index 000000000000..8db6c0b24556 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobListArrowStreamReader.java @@ -0,0 +1,570 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.util; + +import com.azure.storage.blob.implementation.models.BlobListArrowParseException; +import com.azure.storage.blob.implementation.util.apachearrow.Buffer; +import com.azure.storage.blob.implementation.util.apachearrow.Endianness; +import com.azure.storage.blob.implementation.util.apachearrow.Field; +import com.azure.storage.blob.implementation.util.apachearrow.FieldNode; +import com.azure.storage.blob.implementation.util.apachearrow.Int; +import com.azure.storage.blob.implementation.util.apachearrow.KeyValue; +import com.azure.storage.blob.implementation.util.apachearrow.Message; +import com.azure.storage.blob.implementation.util.apachearrow.MessageHeader; +import com.azure.storage.blob.implementation.util.apachearrow.RecordBatch; +import com.azure.storage.blob.implementation.util.apachearrow.Schema; +import com.azure.storage.blob.implementation.util.apachearrow.TimeUnit; +import com.azure.storage.blob.implementation.util.apachearrow.Timestamp; +import com.azure.storage.blob.implementation.util.apachearrow.Type; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Minimal Apache Arrow IPC stream reader scoped to the needs of the ListBlobs Arrow response. + *

+ * This reader intentionally supports only the subset of the Arrow IPC format emitted by the Storage ListBlobs + * endpoint: a single schema message followed by zero or more uncompressed, little-endian record batches whose + * columns are UTF-8 strings, booleans, integers, second-precision timestamps, and map<string,string>. Anything + * outside that subset (dictionaries, compression, big-endian, unsupported types) fails fast with + * {@link BlobListArrowParseException}. + *

+ * It depends only on the {@code arrow-format} flatbuffer definitions for metadata decoding and reads record batch + * bodies directly, so it does not require the {@code arrow-vector} runtime. + */ +final class BlobListArrowStreamReader { + // This is hex representation of the maximum value of an unsigned 32-bit integer. + private static final int CONTINUATION_MARKER = 0xFFFFFFFF; + + private BlobListArrowStreamReader() { + } + + /** + * The decoded contents of an Arrow IPC stream. + */ + static final class DecodedArrowStream { + private final Map schemaMetadata; + private final List batches; + + DecodedArrowStream(Map schemaMetadata, List batches) { + this.schemaMetadata = schemaMetadata; + this.batches = batches; + } + + Map getSchemaMetadata() { + return schemaMetadata; + } + + List getBatches() { + return batches; + } + } + + /** + * A single decoded record batch: a row count and columns addressable by field name. + */ + static final class Batch { + private final int rowCount; + private final Map columns; + + Batch(int rowCount, Map columns) { + this.rowCount = rowCount; + this.columns = columns; + } + + int getRowCount() { + return rowCount; + } + + Column getColumn(String name) { + return columns.get(name); + } + + Set getColumnNames() { + return Collections.unmodifiableSet(columns.keySet()); + } + } + + /** + * Reads and decodes an Arrow IPC stream. + * + * @param stream the Arrow IPC stream. + * @return the decoded schema metadata and record batches. + * @throws BlobListArrowParseException if the stream is malformed or uses an unsupported feature. + */ + static DecodedArrowStream read(InputStream stream) { + byte[] bytes; + try { + bytes = readAll(stream); + } catch (IOException e) { + throw new BlobListArrowParseException("ListBlobs Arrow parse failure: unable to read IPC stream.", e); + } + + ByteBuffer steamBuffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + + Map schemaMetadata = null; + List fields = null; + List batches = new ArrayList<>(); + + int pos = 0; + int length = bytes.length; + while (atLeastFourBytesRemaining(pos, length)) { + int marker = steamBuffer.getInt(pos); + pos += 4; + + int metadataLength; + if (isModernStream(marker)) { + if (pos + 4 > length) { + break; + } + metadataLength = steamBuffer.getInt(pos); + pos += 4; + } else { + // Pre-0.15 streams used a bare length prefix without the continuation marker. + metadataLength = marker; + } + + if (metadataLength == 0) { + // End-of-stream marker. + break; + } + if (isMessageOutOfBounds(metadataLength, pos, length)) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: message metadata length is out of bounds."); + } + + ByteBuffer messageBuffer + = ByteBuffer.wrap(bytes, pos, metadataLength).slice().order(ByteOrder.LITTLE_ENDIAN); + Message message = Message.getRootAsMessage(messageBuffer); + pos += metadataLength; + + long bodyLength = message.bodyLength(); + if (isMessageOutOfBounds(bodyLength, pos, length)) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: message body length is out of bounds."); + } + int bodyStart = pos; + pos += (int) bodyLength; + + byte headerType = message.headerType(); + if (headerType == MessageHeader.SCHEMA) { + Schema schema = requireHeader((Schema) message.header(new Schema()), "schema"); + if (schema.endianness() != Endianness.LITTLE) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: only little-endian streams are supported."); + } + schemaMetadata = readKeyValueMetadata(schema); + fields = readFields(schema); + } else if (headerType == MessageHeader.RECORD_BATCH) { + if (fields == null) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: record batch encountered before schema."); + } + RecordBatch recordBatch + = requireHeader((RecordBatch) message.header(new RecordBatch()), "record batch"); + if (recordBatch.compression() != null) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: compressed record batches are not supported."); + } + batches.add(buildBatch(fields, recordBatch, steamBuffer, bodyStart)); + } else if (headerType == MessageHeader.DICTIONARY_BATCH) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: dictionary-encoded streams are not supported."); + } + // Other header types (NONE, and the commented-out Tensor/SparseTensor members of the Arrow MessageHeader + // union) are not expected in a ListBlobs response and are ignored. See MessageHeader for why those two + // constants are kept (commented out) and how to revive them if the service ever starts emitting them. + } + + if (fields == null) { + throw new BlobListArrowParseException("ListBlobs Arrow parse failure: stream contained no schema."); + } + + Map finalSchemaMetadata = schemaMetadata == null ? new HashMap<>() : schemaMetadata; + return new DecodedArrowStream(finalSchemaMetadata, batches); + } + + private static boolean isMessageOutOfBounds(long bodyLength, int pos, int length) { + return bodyLength < 0 || pos + bodyLength > length; + } + + private static boolean isModernStream(int marker) { + return marker == CONTINUATION_MARKER; + } + + private static boolean atLeastFourBytesRemaining(int pos, int length) { + return pos + 4 <= length; + } + + private static T requireHeader(T header, String description) { + if (header == null) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: " + description + " message header is missing."); + } + return header; + } + + private static Batch buildBatch(List fields, RecordBatch recordBatch, ByteBuffer body, int bodyStart) { + BatchCursor cursor = new BatchCursor(recordBatch, body, bodyStart); + Map columns = new LinkedHashMap<>(); + for (ArrowField field : fields) { + columns.put(field.name, buildColumn(field, cursor)); + } + return new Batch((int) recordBatch.length(), columns); + } + + private static Column buildColumn(ArrowField field, BatchCursor cursor) { + FieldNode node = cursor.nextNode(); + int valueCount = (int) node.length(); + + switch (field.typeType) { + case Type.UTF8: + case Type.BINARY: + return new StringColumn(valueCount, cursor.nextBuffer(), cursor.nextBuffer(), cursor.nextBuffer(), + cursor.body, cursor.bodyStart); + + case Type.BOOL: + return new BoolColumn(valueCount, cursor.nextBuffer(), cursor.nextBuffer(), cursor.body, + cursor.bodyStart); + + case Type.INT: + return new IntColumn(valueCount, cursor.nextBuffer(), cursor.nextBuffer(), field.bitWidth, field.signed, + cursor.body, cursor.bodyStart); + + case Type.TIMESTAMP: + return new TimestampColumn(valueCount, cursor.nextBuffer(), cursor.nextBuffer(), cursor.body, + cursor.bodyStart); + + case Type.MAP: + BufferRegion mapValidity = cursor.nextBuffer(); + BufferRegion mapOffsets = cursor.nextBuffer(); + // Map has a single Struct child ("entries") with key and value children. + ArrowField mapEntries = field.children.get(0); + StructColumn mapStruct = (StructColumn) buildColumn(mapEntries, cursor); + Column mapKeyColumn = mapStruct.children.get(0); + Column mapValueColumn = mapStruct.children.get(1); + if (!(mapKeyColumn instanceof StringColumn) || !(mapValueColumn instanceof StringColumn)) { + throw new BlobListArrowParseException("ListBlobs Arrow parse failure: field '" + field.name + + "' map entries must be string keys and values."); + } + return new MapColumn(valueCount, mapValidity, mapOffsets, (StringColumn) mapKeyColumn, + (StringColumn) mapValueColumn, cursor.body, cursor.bodyStart); + + case Type.STRUCT: + cursor.nextBuffer(); // struct validity buffer + List structChildren = new ArrayList<>(field.children.size()); + for (ArrowField child : field.children) { + structChildren.add(buildColumn(child, cursor)); + } + return new StructColumn(structChildren); + + default: + throw new BlobListArrowParseException("ListBlobs Arrow parse failure: field '" + field.name + + "' has unsupported Arrow type '" + Type.name(field.typeType) + "'."); + } + } + + private static List readFields(Schema schema) { + int count = schema.fieldsLength(); + List fields = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + fields.add(readField(schema.fields(i))); + } + return fields; + } + + private static ArrowField readField(Field field) { + String name = field.name(); + byte typeType = field.typeType(); + int bitWidth = 0; + boolean signed = false; + + if (typeType == Type.INT) { + Int intType = (Int) field.type(new Int()); + if (intType != null) { + bitWidth = intType.bitWidth(); + signed = intType.isSigned(); + } + } else if (typeType == Type.TIMESTAMP) { + Timestamp timestamp = (Timestamp) field.type(new Timestamp()); + if (timestamp != null && timestamp.unit() != TimeUnit.SECOND) { + throw new BlobListArrowParseException("ListBlobs Arrow parse failure: field '" + name + + "' uses an unsupported timestamp unit '" + TimeUnit.name(timestamp.unit()) + "'."); + } + } + + int childCount = field.childrenLength(); + List children = new ArrayList<>(childCount); + for (int i = 0; i < childCount; i++) { + children.add(readField(field.children(i))); + } + return new ArrowField(name, typeType, bitWidth, signed, children); + } + + private static Map readKeyValueMetadata(Schema schema) { + int count = schema.customMetadataLength(); + if (count == 0) { + return new HashMap<>(); + } + Map metadata = new HashMap<>(); + for (int i = 0; i < count; i++) { + KeyValue keyValue = schema.customMetadata(i); + metadata.put(keyValue.key(), keyValue.value()); + } + return metadata; + } + + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + int read; + while ((read = stream.read(chunk)) != -1) { + buffer.write(chunk, 0, read); + } + return buffer.toByteArray(); + } + + /** + * A parsed schema field with the minimal type information required for decoding. + */ + private static final class ArrowField { + private final String name; + private final byte typeType; + private final int bitWidth; + private final boolean signed; + private final List children; + + ArrowField(String name, byte typeType, int bitWidth, boolean signed, List children) { + this.name = name; + this.typeType = typeType; + this.bitWidth = bitWidth; + this.signed = signed; + this.children = children; + } + } + + /** + * Sequentially hands out the field nodes and buffers of a record batch in pre-order. + */ + private static final class BatchCursor { + private final RecordBatch recordBatch; + private final ByteBuffer body; + private final int bodyStart; + private int nodeIndex; + private int bufferIndex; + + BatchCursor(RecordBatch recordBatch, ByteBuffer body, int bodyStart) { + this.recordBatch = recordBatch; + this.body = body; + this.bodyStart = bodyStart; + } + + FieldNode nextNode() { + if (nodeIndex >= recordBatch.nodesLength()) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: record batch is missing expected field nodes."); + } + return recordBatch.nodes(nodeIndex++); + } + + BufferRegion nextBuffer() { + if (bufferIndex >= recordBatch.buffersLength()) { + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: record batch is missing expected buffers."); + } + Buffer buffer = recordBatch.buffers(bufferIndex++); + return new BufferRegion(buffer.offset(), buffer.length()); + } + } + + /** + * Offset and length of a single buffer within the record batch body. + */ + private static final class BufferRegion { + private final long offset; + private final long length; + + BufferRegion(long offset, long length) { + this.offset = offset; + this.length = length; + } + } + + /** + * Base class for decoded columns. + */ + abstract static class Column { + final int valueCount; + final BufferRegion validity; + final ByteBuffer body; + final int bodyStart; + + Column(int valueCount, BufferRegion validity, ByteBuffer body, int bodyStart) { + this.valueCount = valueCount; + this.validity = validity; + this.body = body; + this.bodyStart = bodyStart; + } + + boolean isNull(int index) { + if (validity == null || validity.length == 0) { + return false; + } + int bytePosition = bodyStart + (int) validity.offset + (index >> 3); + int bit = (body.get(bytePosition) >> (index & 7)) & 1; + return bit == 0; + } + } + + /** + * UTF-8 string column (Arrow Utf8/Binary). + */ + static final class StringColumn extends Column { + private final BufferRegion offsets; + private final BufferRegion data; + + StringColumn(int valueCount, BufferRegion validity, BufferRegion offsets, BufferRegion data, ByteBuffer body, + int bodyStart) { + super(valueCount, validity, body, bodyStart); + this.offsets = offsets; + this.data = data; + } + + String get(int index) { + int start = body.getInt(bodyStart + (int) offsets.offset + index * 4); + int end = body.getInt(bodyStart + (int) offsets.offset + (index + 1) * 4); + int dataStart = bodyStart + (int) data.offset + start; + byte[] valueBytes = new byte[end - start]; + for (int i = 0; i < valueBytes.length; i++) { + valueBytes[i] = body.get(dataStart + i); + } + return new String(valueBytes, StandardCharsets.UTF_8); + } + } + + /** + * Boolean column stored as a bitmap (Arrow Bool). + */ + static final class BoolColumn extends Column { + private final BufferRegion data; + + BoolColumn(int valueCount, BufferRegion validity, BufferRegion data, ByteBuffer body, int bodyStart) { + super(valueCount, validity, body, bodyStart); + this.data = data; + } + + boolean get(int index) { + int bytePosition = bodyStart + (int) data.offset + (index >> 3); + int bit = (body.get(bytePosition) >> (index & 7)) & 1; + return bit == 1; + } + } + + /** + * Integer column (Arrow Int) of width 8/16/32/64, signed or unsigned, returned as a long. + */ + static final class IntColumn extends Column { + private final BufferRegion data; + private final int bitWidth; + private final boolean signed; + + IntColumn(int valueCount, BufferRegion validity, BufferRegion data, int bitWidth, boolean signed, + ByteBuffer body, int bodyStart) { + super(valueCount, validity, body, bodyStart); + this.data = data; + this.bitWidth = bitWidth; + this.signed = signed; + } + + long get(int index) { + int base = bodyStart + (int) data.offset; + switch (bitWidth) { + case 64: + return body.getLong(base + index * 8); + + case 32: + int value32 = body.getInt(base + index * 4); + return signed ? value32 : (value32 & 0xFFFFFFFFL); + + case 16: + short value16 = body.getShort(base + index * 2); + return signed ? value16 : (value16 & 0xFFFF); + + case 8: + byte value8 = body.get(base + index); + return signed ? value8 : (value8 & 0xFF); + + default: + throw new BlobListArrowParseException( + "ListBlobs Arrow parse failure: unsupported integer bit width '" + bitWidth + "'."); + } + } + } + + /** + * Second-precision timestamp column (Arrow Timestamp, SECOND unit). + */ + static final class TimestampColumn extends Column { + private final BufferRegion data; + + TimestampColumn(int valueCount, BufferRegion validity, BufferRegion data, ByteBuffer body, int bodyStart) { + super(valueCount, validity, body, bodyStart); + this.data = data; + } + + long getEpochSeconds(int index) { + return body.getLong(bodyStart + (int) data.offset + index * 8); + } + } + + /** + * Struct column holding ordered child columns (used internally for map entries). + */ + static final class StructColumn extends Column { + private final List children; + + StructColumn(List children) { + super(0, null, null, 0); + this.children = children; + } + } + + /** + * Map<string,string> column (Arrow Map of Struct<key:utf8,value:utf8>). + */ + static final class MapColumn extends Column { + private final BufferRegion offsets; + private final StringColumn keys; + private final StringColumn values; + + MapColumn(int valueCount, BufferRegion validity, BufferRegion offsets, StringColumn keys, StringColumn values, + ByteBuffer body, int bodyStart) { + super(valueCount, validity, body, bodyStart); + this.offsets = offsets; + this.keys = keys; + this.values = values; + } + + Map get(int index) { + int start = body.getInt(bodyStart + (int) offsets.offset + index * 4); + int end = body.getInt(bodyStart + (int) offsets.offset + (index + 1) * 4); + Map map = new HashMap<>(); + for (int entry = start; entry < end; entry++) { + map.put(keys.get(entry), values.get(entry)); + } + return map.isEmpty() ? null : map; + } + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index 0866d310981c..4d5da7f03116 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -39,6 +39,8 @@ import com.azure.storage.common.policy.ResponseValidationPolicyBuilder; import com.azure.storage.common.policy.ScrubEtagPolicy; import com.azure.storage.common.policy.StorageBearerTokenChallengeAuthorizationPolicy; +import com.azure.storage.common.policy.StorageContentValidationDecodingPolicy; +import com.azure.storage.common.policy.StorageContentValidationEncodingPolicy; import com.azure.storage.common.policy.StorageSharedKeyCredentialPolicy; import java.net.MalformedURLException; @@ -115,6 +117,9 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare } policies.add(new MetadataValidationPolicy()); + policies.add(new StorageContentValidationEncodingPolicy()); + policies.add(new StorageContentValidationDecodingPolicy()); + if (storageSharedKeyCredential != null) { policies.add(new StorageSharedKeyCredentialPolicy(storageSharedKeyCredential)); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java index cb859dfa595d..e5c79d643864 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/ModelHelper.java @@ -41,12 +41,14 @@ import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.CopyStatusType; import com.azure.storage.blob.models.ListBlobContainersIncludeType; +import com.azure.storage.blob.models.ListBlobsOptions; import com.azure.storage.blob.models.ObjectReplicationPolicy; import com.azure.storage.blob.models.ObjectReplicationRule; import com.azure.storage.blob.models.ObjectReplicationStatus; import com.azure.storage.blob.models.PageBlobCopyIncrementalRequestConditions; import com.azure.storage.blob.models.PageRange; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.models.StorageResponseSerializationFormat; import com.azure.storage.blob.models.TaggedBlobItem; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -85,6 +87,14 @@ public final class ModelHelper { */ public static final int PAGE_BYTES = 512; + /** + * The format that {@link StorageResponseSerializationFormat#AUTO} currently resolves to on the + * wire. Changing this constant is a behavioral change (and should be noted in the CHANGELOG) + * but it is not a public API change. + */ + private static final StorageResponseSerializationFormat DEFAULT_SERIALIZATION_FORMAT + = StorageResponseSerializationFormat.XML; + /** * Determines whether the passed authority is IP style, that is, it is of the format {@code :}. * @@ -663,6 +673,38 @@ public static BlobStorageException mapToBlobStorageException(BlobStorageExceptio internal.getResponse(), code, headerName), internal.getResponse(), internal.getValue()); } + /** + * Resolves a user-supplied {@link StorageResponseSerializationFormat} to the concrete value + * to send on the wire. Treats {@code null} and {@link StorageResponseSerializationFormat#AUTO} + * identically — both yield {@link #DEFAULT_SERIALIZATION_FORMAT}. + * + * @param format the format requested by the caller, or {@code null} if unset. + * @return the concrete {@link StorageResponseSerializationFormat} to send on the wire. + */ + public static StorageResponseSerializationFormat + resolveSerializationFormat(StorageResponseSerializationFormat format) { + if (format == null || format == StorageResponseSerializationFormat.AUTO) { + return DEFAULT_SERIALIZATION_FORMAT; + } + return format; + } + + /** + * Validates that {@code endBefore}, when specified, is used with the Apache Arrow response format. + * + * @param options the listing options to validate. + * @throws IllegalArgumentException if {@code endBefore} is specified for a non-Arrow response format. + */ + public static void validateListBlobsOptions(ListBlobsOptions options) { + if (options != null + && options.getEndBefore() != null + && resolveSerializationFormat(options.getStorageResponseSerializationFormat()) + != StorageResponseSerializationFormat.ARROW) { + throw new IllegalArgumentException( + "The endBefore option is only supported when storageResponseSerializationFormat is set to ARROW."); + } + } + private ModelHelper() { } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/BodyCompression.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/BodyCompression.java new file mode 100644 index 000000000000..bdda779d54fe --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/BodyCompression.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code BodyCompression} table. + *

+ * The ListBlobs reader only needs to detect the presence of this table to reject compressed record batches, so no + * fields are exposed. + */ +public final class BodyCompression extends Table { + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public BodyCompression __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Buffer.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Buffer.java new file mode 100644 index 000000000000..3e1f4f40ebdb --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Buffer.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Struct; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code Buffer} struct (offset/length of a buffer within a record batch body). + */ +public final class Buffer extends Struct { + /** + * Positions this accessor at the given struct offset. + * + * @param i the struct offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given struct offset. + * + * @param i the struct offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public Buffer __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the byte offset of the buffer relative to the start of the record batch body. + * + * @return the buffer offset. + */ + public long offset() { + return bb.getLong(bb_pos); + } + + /** + * Gets the length, in bytes, of the buffer. + * + * @return the buffer length. + */ + public long length() { + return bb.getLong(bb_pos + 8); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Endianness.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Endianness.java new file mode 100644 index 000000000000..af39bc7b6967 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Endianness.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +/** + * Values for the Arrow IPC {@code Endianness} enum. + */ +public final class Endianness { + private Endianness() { + } + + /** Little-endian byte order. */ + public static final short LITTLE = 0; + /** Big-endian byte order. */ + public static final short BIG = 1; +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Field.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Field.java new file mode 100644 index 000000000000..a62086809118 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Field.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code Field} table describing a single column. + */ +public final class Field extends Table { + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public Field __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the field name. + * + * @return the field name, or {@code null} when absent. + */ + public String name() { + int o = __offset(4); + return o != 0 ? __string(o + bb_pos) : null; + } + + /** + * Gets the discriminator identifying the field's {@code type} union (see {@link Type}). + * + * @return the type union discriminator, or {@code 0} when absent. + */ + public byte typeType() { + int o = __offset(8); + return o != 0 ? bb.get(o + bb_pos) : 0; + } + + /** + * Resolves the {@code type} union value into the supplied accessor. + * + * @param obj the accessor to assign to the union value. + * @return the assigned accessor, or {@code null} when absent. + */ + public Table type(Table obj) { + int o = __offset(10); + return o != 0 ? __union(obj, o + bb_pos) : null; + } + + /** + * Gets the child field at the given index. + * + * @param j the child index. + * @return the child field accessor. + */ + public Field children(int j) { + return children(new Field(), j); + } + + /** + * Gets the child field at the given index into the supplied accessor. + * + * @param obj the accessor to assign. + * @param j the child index. + * @return the assigned accessor, or {@code null} when absent. + */ + public Field children(Field obj, int j) { + int o = __offset(14); + return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; + } + + /** + * Gets the number of child fields. + * + * @return the child field count. + */ + public int childrenLength() { + int o = __offset(14); + return o != 0 ? __vector_len(o) : 0; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/FieldNode.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/FieldNode.java new file mode 100644 index 000000000000..cd3678cf22fa --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/FieldNode.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Struct; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code FieldNode} struct (per-column metadata within a record batch). + */ +public final class FieldNode extends Struct { + /** + * Positions this accessor at the given struct offset. + * + * @param i the struct offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given struct offset. + * + * @param i the struct offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public FieldNode __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the number of value slots in the column. + * + * @return the value count. + */ + public long length() { + return bb.getLong(bb_pos); + } + + /** + * Gets the number of null value slots in the column. + * + * @return the null count. + */ + public long nullCount() { + return bb.getLong(bb_pos + 8); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Int.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Int.java new file mode 100644 index 000000000000..e44c4904319f --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Int.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code Int} type table. + */ +public final class Int extends Table { + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public Int __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the bit width of the integer (8, 16, 32, or 64). + * + * @return the bit width, or {@code 0} when absent. + */ + public int bitWidth() { + int o = __offset(4); + return o != 0 ? bb.getInt(o + bb_pos) : 0; + } + + /** + * Gets whether the integer is signed. + * + * @return {@code true} if signed, otherwise {@code false}. + */ + public boolean isSigned() { + int o = __offset(6); + return o != 0 && 0 != bb.get(o + bb_pos); + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/KeyValue.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/KeyValue.java new file mode 100644 index 000000000000..d97e3b661fdc --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/KeyValue.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code KeyValue} table (a single custom metadata entry). + */ +public final class KeyValue extends Table { + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public KeyValue __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the metadata key. + * + * @return the key, or {@code null} when absent. + */ + public String key() { + int o = __offset(4); + return o != 0 ? __string(o + bb_pos) : null; + } + + /** + * Gets the metadata value. + * + * @return the value, or {@code null} when absent. + */ + public String value() { + int o = __offset(6); + return o != 0 ? __string(o + bb_pos) : null; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Message.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Message.java new file mode 100644 index 000000000000..432d05fc25d0 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Message.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Accessor for the Arrow IPC {@code Message} table (root of every encapsulated IPC message). + */ +public final class Message extends Table { + /** + * Reads the {@code Message} located at the root offset of the supplied buffer. + * + * @param bb the little-endian buffer positioned at the start of the message. + * @return the {@code Message} accessor. + */ + public static Message getRootAsMessage(ByteBuffer bb) { + return getRootAsMessage(bb, new Message()); + } + + /** + * Reads the {@code Message} located at the root offset of the supplied buffer into {@code obj}. + * + * @param bb the buffer positioned at the start of the message. + * @param obj the accessor instance to assign. + * @return the assigned {@code Message} accessor. + */ + public static Message getRootAsMessage(ByteBuffer bb, Message obj) { + bb.order(ByteOrder.LITTLE_ENDIAN); + // A FlatBuffer begins with a 4-byte offset (relative to here) pointing to the root table. + int rootTableOffset = bb.getInt(bb.position()) + bb.position(); + return obj.__assign(rootTableOffset, bb); + } + + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public Message __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the discriminator identifying the type of the {@code header} union (see {@link MessageHeader}). + * + * @return the header union type, or {@code 0} when absent. + */ + public byte headerType() { + int o = __offset(6); + return o != 0 ? bb.get(o + bb_pos) : 0; + } + + /** + * Resolves the {@code header} union value into the supplied accessor. + * + * @param obj the accessor to assign to the union value. + * @return the assigned accessor, or {@code null} when the header is absent. + */ + public Table header(Table obj) { + int o = __offset(8); + return o != 0 ? __union(obj, o + bb_pos) : null; + } + + /** + * Gets the length, in bytes, of the message body that follows the metadata. + * + * @return the body length, or {@code 0} when absent. + */ + public long bodyLength() { + int o = __offset(10); + return o != 0 ? bb.getLong(o + bb_pos) : 0L; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/MessageHeader.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/MessageHeader.java new file mode 100644 index 000000000000..8eca35389946 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/MessageHeader.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +/** + * Discriminator values for the Arrow IPC {@code MessageHeader} union. + *

+ * Only the subset that a ListBlobs response can contain is defined as active constants: {@link #SCHEMA} and + * {@link #RECORD_BATCH} (a ListBlobs payload is a tabular result, i.e. a schema followed by record batches), + * plus {@link #DICTIONARY_BATCH}, which the reader recognizes solely to reject it. The union's remaining members, + * {@code TENSOR} (4) and {@code SPARSE_TENSOR} (5), are valid in the Arrow format but are never emitted for a + * ListBlobs response, so they are intentionally kept commented out below rather than deleted. If a future service or + * format change ever starts sending them, uncomment those constants and add the corresponding handling in + * {@code BlobListArrowStreamReader}. + */ +public final class MessageHeader { + private MessageHeader() { + } + + /** No header. */ + public static final byte NONE = 0; + /** A {@link Schema} header. */ + public static final byte SCHEMA = 1; + /** A dictionary batch header. */ + public static final byte DICTIONARY_BATCH = 2; + /** A {@link RecordBatch} header. */ + public static final byte RECORD_BATCH = 3; + // TENSOR (4) and SPARSE_TENSOR (5) are omitted on purpose; see the class javadoc for why and how to revive them. + // public static final byte TENSOR = 4; + // public static final byte SPARSE_TENSOR = 5; +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/RecordBatch.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/RecordBatch.java new file mode 100644 index 000000000000..3ee2d7e8aca2 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/RecordBatch.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code RecordBatch} table. + */ +public final class RecordBatch extends Table { + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public RecordBatch __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the number of rows in the record batch. + * + * @return the row count, or {@code 0} when absent. + */ + public long length() { + int o = __offset(4); + return o != 0 ? bb.getLong(o + bb_pos) : 0L; + } + + /** + * Gets the field node at the given index. + * + * @param j the node index. + * @return the field node accessor. + */ + public FieldNode nodes(int j) { + return nodes(new FieldNode(), j); + } + + /** + * Gets the field node at the given index into the supplied accessor. + * + * @param obj the accessor to assign. + * @param j the node index. + * @return the assigned accessor, or {@code null} when absent. + */ + public FieldNode nodes(FieldNode obj, int j) { + int o = __offset(6); + return o != 0 ? obj.__assign(__vector(o) + j * 16, bb) : null; + } + + /** + * Gets the number of field nodes. + * + * @return the field node count. + */ + public int nodesLength() { + int o = __offset(6); + return o != 0 ? __vector_len(o) : 0; + } + + /** + * Gets the buffer region at the given index. + * + * @param j the buffer index. + * @return the buffer accessor. + */ + public Buffer buffers(int j) { + return buffers(new Buffer(), j); + } + + /** + * Gets the buffer region at the given index into the supplied accessor. + * + * @param obj the accessor to assign. + * @param j the buffer index. + * @return the assigned accessor, or {@code null} when absent. + */ + public Buffer buffers(Buffer obj, int j) { + int o = __offset(8); + return o != 0 ? obj.__assign(__vector(o) + j * 16, bb) : null; + } + + /** + * Gets the number of buffers. + * + * @return the buffer count. + */ + public int buffersLength() { + int o = __offset(8); + return o != 0 ? __vector_len(o) : 0; + } + + /** + * Gets the optional body compression descriptor. + * + * @return the body compression accessor, or {@code null} when the batch is uncompressed. + */ + public BodyCompression compression() { + return compression(new BodyCompression()); + } + + /** + * Gets the optional body compression descriptor into the supplied accessor. + * + * @param obj the accessor to assign. + * @return the assigned accessor, or {@code null} when the batch is uncompressed. + */ + public BodyCompression compression(BodyCompression obj) { + int o = __offset(10); + return o != 0 ? obj.__assign(__indirect(o + bb_pos), bb) : null; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Schema.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Schema.java new file mode 100644 index 000000000000..468ceec91979 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Schema.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code Schema} table. + */ +public final class Schema extends Table { + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public Schema __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the byte order of the schema's buffers (see {@link Endianness}). + * + * @return the endianness, or {@code 0} ({@link Endianness#LITTLE}) when absent. + */ + public short endianness() { + int o = __offset(4); + return o != 0 ? bb.getShort(o + bb_pos) : 0; + } + + /** + * Gets the field at the given index. + * + * @param j the field index. + * @return the field accessor. + */ + public Field fields(int j) { + return fields(new Field(), j); + } + + /** + * Gets the field at the given index into the supplied accessor. + * + * @param obj the accessor to assign. + * @param j the field index. + * @return the assigned accessor, or {@code null} when absent. + */ + public Field fields(Field obj, int j) { + int o = __offset(6); + return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; + } + + /** + * Gets the number of top-level fields in the schema. + * + * @return the field count. + */ + public int fieldsLength() { + int o = __offset(6); + return o != 0 ? __vector_len(o) : 0; + } + + /** + * Gets the custom metadata entry at the given index. + * + * @param j the entry index. + * @return the key/value accessor. + */ + public KeyValue customMetadata(int j) { + return customMetadata(new KeyValue(), j); + } + + /** + * Gets the custom metadata entry at the given index into the supplied accessor. + * + * @param obj the accessor to assign. + * @param j the entry index. + * @return the assigned accessor, or {@code null} when absent. + */ + public KeyValue customMetadata(KeyValue obj, int j) { + int o = __offset(8); + return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; + } + + /** + * Gets the number of custom metadata entries. + * + * @return the entry count. + */ + public int customMetadataLength() { + int o = __offset(8); + return o != 0 ? __vector_len(o) : 0; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/TimeUnit.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/TimeUnit.java new file mode 100644 index 000000000000..2ba6d3072d40 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/TimeUnit.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +/** + * Values for the Arrow IPC {@code TimeUnit} enum. + */ +public final class TimeUnit { + private TimeUnit() { + } + + /** Second resolution. */ + public static final short SECOND = 0; + /** Millisecond resolution. */ + public static final short MILLISECOND = 1; + /** Microsecond resolution. */ + public static final short MICROSECOND = 2; + /** Nanosecond resolution. */ + public static final short NANOSECOND = 3; + + private static final String[] NAMES = { "SECOND", "MILLISECOND", "MICROSECOND", "NANOSECOND" }; + + /** + * Gets the canonical Arrow name for a {@code TimeUnit} value, for diagnostic messages. + * + * @param e the time unit value. + * @return the canonical Arrow time unit name. + */ + public static String name(int e) { + return NAMES[e]; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Timestamp.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Timestamp.java new file mode 100644 index 000000000000..83fb712cf836 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Timestamp.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +import com.azure.storage.blob.implementation.util.flatbuffers.Table; + +import java.nio.ByteBuffer; + +/** + * Accessor for the Arrow IPC {@code Timestamp} type table. + */ +public final class Timestamp extends Table { + /** + * Positions this accessor at the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + */ + public void __init(int i, ByteBuffer bb) { + __reset(i, bb); + } + + /** + * Assigns this accessor to the given table offset. + * + * @param i the table offset. + * @param bb the backing buffer. + * @return this accessor. + */ + public Timestamp __assign(int i, ByteBuffer bb) { + __init(i, bb); + return this; + } + + /** + * Gets the timestamp resolution (see {@link TimeUnit}). + * + * @return the time unit, or {@code 0} ({@link TimeUnit#SECOND}) when absent. + */ + public short unit() { + int o = __offset(4); + return o != 0 ? bb.getShort(o + bb_pos) : 0; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Type.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Type.java new file mode 100644 index 000000000000..690eebf19562 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/apachearrow/Type.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Portions Copyright (c) Microsoft Corporation + */ + +package com.azure.storage.blob.implementation.util.apachearrow; + +/** + * Discriminator values for the Arrow IPC {@code Type} union, identifying a field's logical type. + */ +public final class Type { + private Type() { + } + + /** No type. */ + public static final byte NONE = 0; + /** Null type. */ + public static final byte NULL = 1; + /** Integer type (see {@link Int}). */ + public static final byte INT = 2; + /** Floating-point type. */ + public static final byte FLOATING_POINT = 3; + /** Variable-length binary type. */ + public static final byte BINARY = 4; + /** Variable-length UTF-8 string type. */ + public static final byte UTF8 = 5; + /** Boolean type. */ + public static final byte BOOL = 6; + /** Decimal type. */ + public static final byte DECIMAL = 7; + /** Date type. */ + public static final byte DATE = 8; + /** Time-of-day type. */ + public static final byte TIME = 9; + /** Timestamp type (see {@link Timestamp}). */ + public static final byte TIMESTAMP = 10; + /** Interval type. */ + public static final byte INTERVAL = 11; + /** List type. */ + public static final byte LIST = 12; + /** Struct type. */ + public static final byte STRUCT = 13; + /** Union type. */ + public static final byte UNION = 14; + /** Fixed-size binary type. */ + public static final byte FIXED_SIZE_BINARY = 15; + /** Fixed-size list type. */ + public static final byte FIXED_SIZE_LIST = 16; + /** Map type. */ + public static final byte MAP = 17; + /** Duration type. */ + public static final byte DURATION = 18; + /** Large variable-length binary type. */ + public static final byte LARGE_BINARY = 19; + /** Large variable-length UTF-8 string type. */ + public static final byte LARGE_UTF8 = 20; + /** Large list type. */ + public static final byte LARGE_LIST = 21; + /** Run-end encoded type. */ + public static final byte RUN_END_ENCODED = 22; + /** Binary view type. */ + public static final byte BINARY_VIEW = 23; + /** UTF-8 string view type. */ + public static final byte UTF8_VIEW = 24; + /** List view type. */ + public static final byte LIST_VIEW = 25; + /** Large list view type. */ + public static final byte LARGE_LIST_VIEW = 26; + + private static final String[] NAMES = { + "NONE", + "Null", + "Int", + "FloatingPoint", + "Binary", + "Utf8", + "Bool", + "Decimal", + "Date", + "Time", + "Timestamp", + "Interval", + "List", + "Struct_", + "Union", + "FixedSizeBinary", + "FixedSizeList", + "Map", + "Duration", + "LargeBinary", + "LargeUtf8", + "LargeList", + "RunEndEncoded", + "BinaryView", + "Utf8View", + "ListView", + "LargeListView" }; + + /** + * Gets the canonical Arrow name for a {@code Type} union discriminator, for diagnostic messages. + * + * @param e the discriminator value. + * @return the canonical Arrow type name. + */ + public static String name(int e) { + return NAMES[e]; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Constants.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Constants.java new file mode 100644 index 000000000000..d406f0239227 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Constants.java @@ -0,0 +1,43 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Adapted by Microsoft from FlatBuffers Java 25.2.10 for the internal Blob Storage Arrow reader. + +package com.azure.storage.blob.implementation.util.flatbuffers; + +/// @cond FLATBUFFERS_INTERNAL + +/** + * Class that holds shared constants. + */ +public class Constants { + static final int SIZEOF_BYTE = 1; + static final int SIZEOF_SHORT = 2; + static final int SIZEOF_INT = 4; + static final int SIZEOF_FLOAT = 4; + static final int SIZEOF_LONG = 8; + static final int SIZEOF_DOUBLE = 8; + static final int FILE_IDENTIFIER_LENGTH = 4; + public static final int SIZE_PREFIX_LENGTH = 4; + + /** + * Version identifier used by generated FlatBuffers code. + */ + public static void FLATBUFFERS_25_2_10() { + } +} + +/// @endcond diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Struct.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Struct.java new file mode 100644 index 000000000000..eda399adf18c --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Struct.java @@ -0,0 +1,45 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Adapted by Microsoft from FlatBuffers Java 25.2.10 for the internal Blob Storage Arrow reader. + +package com.azure.storage.blob.implementation.util.flatbuffers; + +import java.nio.ByteBuffer; + +/// @cond FLATBUFFERS_INTERNAL + +/** + * All structs in generated FlatBuffers code derive from this class. + */ +public class Struct { + protected int bb_pos; + protected ByteBuffer bb; + + protected void __reset(int index, ByteBuffer byteBuffer) { + bb = byteBuffer; + bb_pos = bb == null ? 0 : index; + } + + /** + * Resets internal state with a null buffer and zero position. + */ + public void __reset() { + __reset(0, null); + } +} + +/// @endcond diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Table.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Table.java new file mode 100644 index 000000000000..3e5ac1426719 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Table.java @@ -0,0 +1,196 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Adapted by Microsoft from FlatBuffers Java 25.2.10 for the internal Blob Storage Arrow reader. + +package com.azure.storage.blob.implementation.util.flatbuffers; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Comparator; + +import static com.azure.storage.blob.implementation.util.flatbuffers.Constants.FILE_IDENTIFIER_LENGTH; +import static com.azure.storage.blob.implementation.util.flatbuffers.Constants.SIZEOF_INT; + +/// @cond FLATBUFFERS_INTERNAL + +/** + * Base class for tables in generated FlatBuffers code. + */ +public class Table { + protected int bb_pos; + protected ByteBuffer bb; + private int vtableStart; + private int vtableSize; + private final Utf8 utf8 = Utf8.getDefault(); + + public ByteBuffer getByteBuffer() { + return bb; + } + + protected int __offset(int vtableOffset) { + return vtableOffset < vtableSize ? bb.getShort(vtableStart + vtableOffset) : 0; + } + + protected static int __offset(int vtableOffset, int offset, ByteBuffer byteBuffer) { + int vtable = byteBuffer.capacity() - offset; + return byteBuffer.getShort(vtable + vtableOffset - byteBuffer.getInt(vtable)) + vtable; + } + + protected int __indirect(int offset) { + return offset + bb.getInt(offset); + } + + protected static int __indirect(int offset, ByteBuffer byteBuffer) { + return offset + byteBuffer.getInt(offset); + } + + protected String __string(int offset) { + return __string(offset, bb, utf8); + } + + protected static String __string(int offset, ByteBuffer byteBuffer, Utf8 utf8) { + offset += byteBuffer.getInt(offset); + int length = byteBuffer.getInt(offset); + return utf8.decodeUtf8(byteBuffer, offset + SIZEOF_INT, length); + } + + protected int __vector_len(int offset) { + offset += bb_pos; + offset += bb.getInt(offset); + return bb.getInt(offset); + } + + protected int __vector(int offset) { + offset += bb_pos; + return offset + bb.getInt(offset) + SIZEOF_INT; + } + + protected ByteBuffer __vector_as_bytebuffer(int vectorOffset, int elementSize) { + int offset = __offset(vectorOffset); + if (offset == 0) { + return null; + } + ByteBuffer byteBuffer = bb.duplicate().order(ByteOrder.LITTLE_ENDIAN); + int vectorStart = __vector(offset); + byteBuffer.position(vectorStart); + byteBuffer.limit(vectorStart + __vector_len(offset) * elementSize); + return byteBuffer; + } + + protected ByteBuffer __vector_in_bytebuffer(ByteBuffer byteBuffer, int vectorOffset, int elementSize) { + int offset = __offset(vectorOffset); + if (offset == 0) { + return null; + } + int vectorStart = __vector(offset); + byteBuffer.rewind(); + byteBuffer.limit(vectorStart + __vector_len(offset) * elementSize); + byteBuffer.position(vectorStart); + return byteBuffer; + } + + protected Table __union(Table table, int offset) { + return __union(table, offset, bb); + } + + protected static Table __union(Table table, int offset, ByteBuffer byteBuffer) { + table.__reset(__indirect(offset, byteBuffer), byteBuffer); + return table; + } + + protected static boolean __has_identifier(ByteBuffer byteBuffer, String identifier) { + if (identifier.length() != FILE_IDENTIFIER_LENGTH) { + throw new AssertionError("FlatBuffers: file identifier must be length " + FILE_IDENTIFIER_LENGTH); + } + for (int index = 0; index < FILE_IDENTIFIER_LENGTH; index++) { + if (identifier.charAt(index) != (char) byteBuffer.get(byteBuffer.position() + SIZEOF_INT + index)) { + return false; + } + } + return true; + } + + protected void sortTables(int[] offsets, final ByteBuffer byteBuffer) { + Integer[] boxedOffsets = new Integer[offsets.length]; + for (int index = 0; index < offsets.length; index++) { + boxedOffsets[index] = offsets[index]; + } + Arrays.sort(boxedOffsets, new Comparator() { + @Override + public int compare(Integer first, Integer second) { + return keysCompare(first, second, byteBuffer); + } + }); + for (int index = 0; index < offsets.length; index++) { + offsets[index] = boxedOffsets[index]; + } + } + + protected int keysCompare(Integer first, Integer second, ByteBuffer byteBuffer) { + return 0; + } + + protected static int compareStrings(int firstOffset, int secondOffset, ByteBuffer byteBuffer) { + firstOffset += byteBuffer.getInt(firstOffset); + secondOffset += byteBuffer.getInt(secondOffset); + int firstLength = byteBuffer.getInt(firstOffset); + int secondLength = byteBuffer.getInt(secondOffset); + int firstStart = firstOffset + SIZEOF_INT; + int secondStart = secondOffset + SIZEOF_INT; + int length = Math.min(firstLength, secondLength); + for (int index = 0; index < length; index++) { + if (byteBuffer.get(index + firstStart) != byteBuffer.get(index + secondStart)) { + return byteBuffer.get(index + firstStart) - byteBuffer.get(index + secondStart); + } + } + return firstLength - secondLength; + } + + protected static int compareStrings(int offset, byte[] key, ByteBuffer byteBuffer) { + offset += byteBuffer.getInt(offset); + int valueLength = byteBuffer.getInt(offset); + int keyLength = key.length; + int start = offset + SIZEOF_INT; + int length = Math.min(valueLength, keyLength); + for (int index = 0; index < length; index++) { + if (byteBuffer.get(index + start) != key[index]) { + return byteBuffer.get(index + start) - key[index]; + } + } + return valueLength - keyLength; + } + + protected void __reset(int index, ByteBuffer byteBuffer) { + bb = byteBuffer; + if (bb == null) { + bb_pos = 0; + vtableStart = 0; + vtableSize = 0; + } else { + bb_pos = index; + vtableStart = bb_pos - bb.getInt(bb_pos); + vtableSize = bb.getShort(vtableStart); + } + } + + public void __reset() { + __reset(0, null); + } +} + +/// @endcond diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Utf8.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Utf8.java new file mode 100644 index 000000000000..ba9bf2231291 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Utf8.java @@ -0,0 +1,108 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Adapted by Microsoft from FlatBuffers Java 25.2.10 for the internal Blob Storage Arrow reader. + +package com.azure.storage.blob.implementation.util.flatbuffers; + +import java.nio.ByteBuffer; + +/** + * UTF-8 encoder and decoder used by the FlatBuffers table runtime. + */ +public abstract class Utf8 { + private static Utf8 defaultInstance; + + public abstract int encodedLength(CharSequence sequence); + + public abstract void encodeUtf8(CharSequence input, ByteBuffer output); + + public abstract String decodeUtf8(ByteBuffer buffer, int offset, int length); + + public static Utf8 getDefault() { + if (defaultInstance == null) { + defaultInstance = new Utf8Safe(); + } + return defaultInstance; + } + + public static void setDefault(Utf8 instance) { + defaultInstance = instance; + } + + static final class DecodeUtil { + static boolean isOneByte(byte value) { + return value >= 0; + } + + static boolean isTwoBytes(byte value) { + return value < (byte) 0xE0; + } + + static boolean isThreeBytes(byte value) { + return value < (byte) 0xF0; + } + + static void handleOneByte(byte byte1, char[] result, int resultPosition) { + result[resultPosition] = (char) byte1; + } + + static void handleTwoBytes(byte byte1, byte byte2, char[] result, int resultPosition) { + if (byte1 < (byte) 0xC2 || isNotTrailingByte(byte2)) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + result[resultPosition] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2)); + } + + static void handleThreeBytes(byte byte1, byte byte2, byte byte3, char[] result, int resultPosition) { + if (isNotTrailingByte(byte2) + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + || isNotTrailingByte(byte3)) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + result[resultPosition] + = (char) (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | trailingByteValue(byte3)); + } + + static void handleFourBytes(byte byte1, byte byte2, byte byte3, byte byte4, char[] result, int resultPosition) { + if (isNotTrailingByte(byte2) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + || isNotTrailingByte(byte3) + || isNotTrailingByte(byte4)) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + int codePoint = ((byte1 & 0x07) << 18) | (trailingByteValue(byte2) << 12) | (trailingByteValue(byte3) << 6) + | trailingByteValue(byte4); + result[resultPosition] = Character.highSurrogate(codePoint); + result[resultPosition + 1] = Character.lowSurrogate(codePoint); + } + + private static boolean isNotTrailingByte(byte value) { + return value > (byte) 0xBF; + } + + private static int trailingByteValue(byte value) { + return value & 0x3F; + } + } + + static final class UnpairedSurrogateException extends IllegalArgumentException { + UnpairedSurrogateException(int index, int length) { + super("Unpaired surrogate at index " + index + " of " + length); + } + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Utf8Safe.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Utf8Safe.java new file mode 100644 index 000000000000..0cd4061b501a --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/flatbuffers/Utf8Safe.java @@ -0,0 +1,153 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Adapted by Microsoft from FlatBuffers Java 25.2.10 for the internal Blob Storage Arrow reader. + +package com.azure.storage.blob.implementation.util.flatbuffers; + +import java.nio.ByteBuffer; + +/** + * Safe UTF-8 implementation used by the FlatBuffers table runtime. + */ +public final class Utf8Safe extends Utf8 { + private static int computeEncodedLength(CharSequence sequence) { + int utf16Length = sequence.length(); + int utf8Length = utf16Length; + int index = 0; + while (index < utf16Length && sequence.charAt(index) < 0x80) { + index++; + } + for (; index < utf16Length; index++) { + char value = sequence.charAt(index); + if (value < 0x800) { + utf8Length += (0x7f - value) >>> 31; + } else { + utf8Length += encodedLengthGeneral(sequence, index); + break; + } + } + if (utf8Length < utf16Length) { + throw new IllegalArgumentException("UTF-8 length does not fit in int: " + (utf8Length + (1L << 32))); + } + return utf8Length; + } + + private static int encodedLengthGeneral(CharSequence sequence, int start) { + int utf16Length = sequence.length(); + int utf8Length = 0; + for (int index = start; index < utf16Length; index++) { + char value = sequence.charAt(index); + if (value < 0x800) { + utf8Length += (0x7f - value) >>> 31; + } else { + utf8Length += 2; + if (Character.MIN_SURROGATE <= value && value <= Character.MAX_SURROGATE) { + int codePoint = Character.codePointAt(sequence, index); + if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT) { + throw new UnpairedSurrogateException(index, utf16Length); + } + index++; + } + } + } + return utf8Length; + } + + @Override + public int encodedLength(CharSequence input) { + return computeEncodedLength(input); + } + + @Override + public String decodeUtf8(ByteBuffer buffer, int offset, int length) { + if ((offset | length | buffer.limit() - offset - length) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), offset, length)); + } + char[] result = new char[length]; + int resultPosition = 0; + int limit = offset + length; + while (offset < limit) { + byte value = buffer.get(offset); + if (!DecodeUtil.isOneByte(value)) { + break; + } + offset++; + DecodeUtil.handleOneByte(value, result, resultPosition++); + } + while (offset < limit) { + byte byte1 = buffer.get(offset++); + if (DecodeUtil.isOneByte(byte1)) { + DecodeUtil.handleOneByte(byte1, result, resultPosition++); + while (offset < limit) { + byte value = buffer.get(offset); + if (!DecodeUtil.isOneByte(value)) { + break; + } + offset++; + DecodeUtil.handleOneByte(value, result, resultPosition++); + } + } else if (DecodeUtil.isTwoBytes(byte1)) { + if (offset >= limit) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleTwoBytes(byte1, buffer.get(offset++), result, resultPosition++); + } else if (DecodeUtil.isThreeBytes(byte1)) { + if (offset >= limit - 1) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleThreeBytes(byte1, buffer.get(offset++), buffer.get(offset++), result, + resultPosition++); + } else { + if (offset >= limit - 2) { + throw new IllegalArgumentException("Invalid UTF-8"); + } + DecodeUtil.handleFourBytes(byte1, buffer.get(offset++), buffer.get(offset++), buffer.get(offset++), + result, resultPosition++); + resultPosition++; + } + } + return new String(result, 0, resultPosition); + } + + @Override + public void encodeUtf8(CharSequence input, ByteBuffer output) { + int inputLength = input.length(); + for (int index = 0; index < inputLength; index++) { + char value = input.charAt(index); + if (value < 0x80) { + output.put((byte) value); + } else if (value < 0x800) { + output.put((byte) (0xC0 | (value >>> 6))); + output.put((byte) (0x80 | (0x3F & value))); + } else if (value < Character.MIN_SURROGATE || value > Character.MAX_SURROGATE) { + output.put((byte) (0xE0 | (value >>> 12))); + output.put((byte) (0x80 | (0x3F & (value >>> 6)))); + output.put((byte) (0x80 | (0x3F & value))); + } else { + if (index + 1 == inputLength || !Character.isSurrogatePair(value, input.charAt(index + 1))) { + throw new UnpairedSurrogateException(index, inputLength); + } + int codePoint = Character.toCodePoint(value, input.charAt(++index)); + output.put((byte) (0xF0 | (codePoint >>> 18))); + output.put((byte) (0x80 | (0x3F & (codePoint >>> 12)))); + output.put((byte) (0x80 | (0x3F & (codePoint >>> 6)))); + output.put((byte) (0x80 | (0x3F & codePoint))); + } + } + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/AppendBlobItem.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/AppendBlobItem.java index 8acbf7de2813..57f8c54faee4 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/AppendBlobItem.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/AppendBlobItem.java @@ -5,6 +5,7 @@ import com.azure.core.annotation.Immutable; import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.implementation.accesshelpers.AppendBlobItemConstructorProxy; import java.time.OffsetDateTime; @@ -22,6 +23,11 @@ public class AppendBlobItem { private final String blobAppendOffset; private final Integer blobCommittedBlockCount; private final String versionId; + private final byte[] contentCrc64; + + static { + AppendBlobItemConstructorProxy.setAccessor(AppendBlobItem::new); + } /** * Constructs an {@link AppendBlobItem}. @@ -76,6 +82,14 @@ public AppendBlobItem(final String eTag, final OffsetDateTime lastModified, fina public AppendBlobItem(final String eTag, final OffsetDateTime lastModified, final byte[] contentMd5, final boolean isServerEncrypted, final String encryptionKeySha256, final String encryptionScope, final String blobAppendOffset, final Integer blobCommittedBlockCount, final String versionId) { + this(eTag, lastModified, contentMd5, isServerEncrypted, encryptionKeySha256, encryptionScope, blobAppendOffset, + blobCommittedBlockCount, versionId, null); + } + + private AppendBlobItem(final String eTag, final OffsetDateTime lastModified, final byte[] contentMd5, + final boolean isServerEncrypted, final String encryptionKeySha256, final String encryptionScope, + final String blobAppendOffset, final Integer blobCommittedBlockCount, final String versionId, + final byte[] contentCrc64) { this.eTag = eTag; this.lastModified = lastModified; this.contentMd5 = CoreUtils.clone(contentMd5); @@ -85,6 +99,7 @@ public AppendBlobItem(final String eTag, final OffsetDateTime lastModified, fina this.blobAppendOffset = blobAppendOffset; this.blobCommittedBlockCount = blobCommittedBlockCount; this.versionId = versionId; + this.contentCrc64 = CoreUtils.clone(contentCrc64); } /** @@ -141,6 +156,15 @@ public byte[] getContentMd5() { return CoreUtils.clone(contentMd5); } + /** + * Gets the calculated CRC64 of the append blob. + * + * @return the calculated CRC64 of the append blob + */ + public byte[] getContentCrc64() { + return CoreUtils.clone(contentCrc64); + } + /** * Gets the offset of the append blob. * diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlockBlobItem.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlockBlobItem.java index bdad9c963208..519c5bbabf06 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlockBlobItem.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlockBlobItem.java @@ -5,6 +5,7 @@ import com.azure.core.annotation.Immutable; import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.implementation.accesshelpers.BlockBlobItemConstructorProxy; import java.time.OffsetDateTime; @@ -20,6 +21,11 @@ public class BlockBlobItem { private final String encryptionKeySha256; private final String encryptionScope; private final String versionId; + private final byte[] contentCrc64; + + static { + BlockBlobItemConstructorProxy.setAccessor(BlockBlobItem::new); + } /** * Constructs a {@link BlockBlobItem}. @@ -88,6 +94,12 @@ public BlockBlobItem(final String eTag, final OffsetDateTime lastModified, final public BlockBlobItem(final String eTag, final OffsetDateTime lastModified, final byte[] contentMd5, final Boolean isServerEncrypted, final String encryptionKeySha256, final String encryptionScope, final String versionId) { + this(eTag, lastModified, contentMd5, isServerEncrypted, encryptionKeySha256, encryptionScope, versionId, null); + } + + private BlockBlobItem(final String eTag, final OffsetDateTime lastModified, final byte[] contentMd5, + final Boolean isServerEncrypted, final String encryptionKeySha256, final String encryptionScope, + final String versionId, final byte[] contentCrc64) { this.eTag = eTag; this.lastModified = lastModified; this.contentMd5 = CoreUtils.clone(contentMd5); @@ -95,6 +107,7 @@ public BlockBlobItem(final String eTag, final OffsetDateTime lastModified, final this.encryptionKeySha256 = encryptionKeySha256; this.encryptionScope = encryptionScope; this.versionId = versionId; + this.contentCrc64 = CoreUtils.clone(contentCrc64); } /** @@ -151,6 +164,15 @@ public byte[] getContentMd5() { return CoreUtils.clone(contentMd5); } + /** + * Gets the calculated CRC64 of the block blob. + * + * @return the calculated CRC64 of the block blob + */ + public byte[] getContentCrc64() { + return CoreUtils.clone(contentCrc64); + } + /** * Gets the version identifier of the block blob. * diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ListBlobsOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ListBlobsOptions.java index 47f29c2ab2b3..e434681ac6e9 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ListBlobsOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/ListBlobsOptions.java @@ -19,6 +19,8 @@ public final class ListBlobsOptions { private String prefix; private String startFrom; private Integer maxResultsPerPage; + private StorageResponseSerializationFormat storageResponseSerializationFormat; + private String endBefore; /** * Constructs an unpopulated {@link ListBlobsOptions}. @@ -74,7 +76,7 @@ public ListBlobsOptions setPrefix(String prefix) { * This parameter is similar to the prefix filter: it allows listing blobs starting from the specified path, rather than from the beginning of the container. * For non-recursive lists, only one entity level is supported. * - * @return the marker indicating where to start listing blobs + * @return the marker indicating where to start listing blobs (inclusive) */ public String getStartFrom() { return startFrom; @@ -84,7 +86,7 @@ public String getStartFrom() { * Sets an optional parameter that specifies an absolute path within the container. This parameter is similar to the prefix filter: it allows listing blobs starting from the specified path, rather than from the beginning of the container. * For non-recursive lists, only one entity level is supported. * - * @param startFrom The marker indicating where to start listing blobs + * @param startFrom The marker indicating where to start listing blobs (inclusive) * @return the updated ListBlobsOptions object */ public ListBlobsOptions setStartFrom(String startFrom) { @@ -92,6 +94,49 @@ public ListBlobsOptions setStartFrom(String startFrom) { return this; } + /** + * Gets the endBefore value. Only supported with Arrow listings. The listing will end before this path (exclusive). + * + * @return the endBefore value. + */ + public String getEndBefore() { + return endBefore; + } + + /** + * Sets the endBefore value. Only supported with Arrow listings. The listing will end before this path (exclusive). + * + * @param endBefore the endBefore value to set. + * @return the updated ListBlobsOptions object. + */ + public ListBlobsOptions setEndBefore(String endBefore) { + this.endBefore = endBefore; + return this; + } + + /** + * Gets the response serialization format the service should use when listing blobs. + * + * @return the {@link StorageResponseSerializationFormat}, or {@code null} if unset + * (equivalent to {@link StorageResponseSerializationFormat#AUTO}). + */ + public StorageResponseSerializationFormat getStorageResponseSerializationFormat() { + return storageResponseSerializationFormat; + } + + /** + * Sets the response serialization format the service should use when listing blobs. + * + * @param storageResponseSerializationFormat the format to request. {@code null} and + * {@link StorageResponseSerializationFormat#AUTO} both let the SDK pick. + * @return the updated {@link ListBlobsOptions} object. + */ + public ListBlobsOptions + setStorageResponseSerializationFormat(StorageResponseSerializationFormat storageResponseSerializationFormat) { + this.storageResponseSerializationFormat = storageResponseSerializationFormat; + return this; + } + /** * Specifies the maximum number of blobs to return, including all BlobPrefix elements. If the request does not * specify maxResultsPerPage or specifies a value greater than 5,000, the server will return up to 5,000 items. diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/PageBlobItem.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/PageBlobItem.java index ef127458a57a..39783a094794 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/PageBlobItem.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/PageBlobItem.java @@ -5,6 +5,7 @@ import com.azure.core.annotation.Immutable; import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.implementation.accesshelpers.PageBlobItemConstructorProxy; import java.time.OffsetDateTime; @@ -16,12 +17,17 @@ public class PageBlobItem { private final String eTag; private final OffsetDateTime lastModified; private final byte[] contentMd5; + private final byte[] contentCrc64; private final Boolean isServerEncrypted; private final String encryptionKeySha256; private final String encryptionScope; private final Long blobSequenceNumber; private final String versionId; + static { + PageBlobItemConstructorProxy.setAccessor(PageBlobItem::new); + } + /** * Constructs a {@link PageBlobItem}. * @@ -70,9 +76,17 @@ public PageBlobItem(final String eTag, final OffsetDateTime lastModified, final public PageBlobItem(final String eTag, final OffsetDateTime lastModified, final byte[] contentMd5, final Boolean isServerEncrypted, final String encryptionKeySha256, final String encryptionScope, final Long blobSequenceNumber, final String versionId) { + this(eTag, lastModified, contentMd5, isServerEncrypted, encryptionKeySha256, encryptionScope, + blobSequenceNumber, versionId, null); + } + + private PageBlobItem(final String eTag, final OffsetDateTime lastModified, final byte[] contentMd5, + final Boolean isServerEncrypted, final String encryptionKeySha256, final String encryptionScope, + final Long blobSequenceNumber, final String versionId, final byte[] contentCrc64) { this.eTag = eTag; this.lastModified = lastModified; this.contentMd5 = CoreUtils.clone(contentMd5); + this.contentCrc64 = CoreUtils.clone(contentCrc64); this.isServerEncrypted = isServerEncrypted; this.encryptionKeySha256 = encryptionKeySha256; this.encryptionScope = encryptionScope; @@ -134,6 +148,15 @@ public byte[] getContentMd5() { return CoreUtils.clone(contentMd5); } + /** + * Gets the calculated CRC64 of the page blob. + * + * @return the calculated CRC64 of the page blob + */ + public byte[] getContentCrc64() { + return CoreUtils.clone(contentCrc64); + } + /** * Gets the current sequence number of the page blob. * diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/StorageResponseSerializationFormat.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/StorageResponseSerializationFormat.java new file mode 100644 index 000000000000..e5017accedbc --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/StorageResponseSerializationFormat.java @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.models; + +/** + * Defines the serialization format the service uses for list-blobs responses. + */ +public enum StorageResponseSerializationFormat { + /** + * Let the SDK choose the serialization format that is most appropriate for the request. + *

+ * The exact format selected by {@code AUTO} is an implementation detail and may change + * between SDK releases. Choose {@link #XML} or {@link #ARROW} explicitly if you require + * a specific format. + */ + AUTO, + + /** + * XML response format. + */ + XML, + + /** + * Apache Arrow response format. + */ + ARROW +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java new file mode 100644 index 000000000000..d8f8a526628e --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.models.AppendBlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when appending a block to an append blob. + */ +@Fluent +public final class AppendBlobAppendBlockOptions { + private byte[] contentMd5; + private AppendBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link AppendBlobAppendBlockOptions}. + */ + public AppendBlobAppendBlockOptions() { + } + + /** + * Gets the MD5 hash of the block content. + * + * @return An MD5 hash of the content, or null. + */ + public byte[] getContentMd5() { + return CoreUtils.clone(contentMd5); + } + + /** + * Sets the MD5 hash of the block content for transactional verification. + * + * @param contentMd5 An MD5 hash of the block content. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions setContentMd5(byte[] contentMd5) { + this.contentMd5 = CoreUtils.clone(contentMd5); + return this; + } + + /** + * Gets the {@link AppendBlobRequestConditions}. + * + * @return The request conditions. + */ + public AppendBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link AppendBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions setRequestConditions(AppendBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java new file mode 100644 index 000000000000..df12b00c32d1 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.AppendBlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when opening an output stream to an append blob. + */ +@Fluent +public final class AppendBlobOutputStreamOptions { + private AppendBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link AppendBlobOutputStreamOptions}. + */ + public AppendBlobOutputStreamOptions() { + } + + /** + * Gets the {@link AppendBlobRequestConditions}. + * + * @return The request conditions. + */ + public AppendBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link AppendBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public AppendBlobOutputStreamOptions setRequestConditions(AppendBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public AppendBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java new file mode 100644 index 000000000000..df63b74238fc --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when downloading blob content (full blob or range in memory). + */ +@Fluent +public final class BlobDownloadContentOptions { + private BlobRange range; + private DownloadRetryOptions downloadRetryOptions; + private BlobRequestConditions requestConditions; + private boolean retrieveContentRangeMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link BlobDownloadContentOptions}. + */ + public BlobDownloadContentOptions() { + } + + /** + * Gets the {@link BlobRange}. + * + * @return The blob range. + */ + public BlobRange getRange() { + return range; + } + + /** + * Sets the {@link BlobRange}. + * + * @param range The blob range. + * @return The updated options. + */ + public BlobDownloadContentOptions setRange(BlobRange range) { + this.range = range; + return this; + } + + /** + * Gets the {@link DownloadRetryOptions}. + * + * @return The download retry options. + */ + public DownloadRetryOptions getDownloadRetryOptions() { + return downloadRetryOptions; + } + + /** + * Sets the {@link DownloadRetryOptions}. + * + * @param downloadRetryOptions The download retry options. + * @return The updated options. + */ + public BlobDownloadContentOptions setDownloadRetryOptions(DownloadRetryOptions downloadRetryOptions) { + this.downloadRetryOptions = downloadRetryOptions; + return this; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public BlobDownloadContentOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets whether the content MD5 for the specified blob range should be returned. + * + * @return Whether to retrieve content range MD5. + */ + public boolean isRetrieveContentRangeMd5() { + return retrieveContentRangeMd5; + } + + /** + * Sets whether the content MD5 for the specified blob range should be returned. + * + * @param retrieveContentRangeMd5 Whether to retrieve content range MD5. + * @return The updated options. + */ + public BlobDownloadContentOptions setRetrieveContentRangeMd5(boolean retrieveContentRangeMd5) { + this.retrieveContentRangeMd5 = retrieveContentRangeMd5; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadContentOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java new file mode 100644 index 000000000000..c8bf93b7ae03 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when downloading a blob range to an output stream. + */ +@Fluent +public final class BlobDownloadStreamOptions { + private BlobRange range; + private DownloadRetryOptions downloadRetryOptions; + private BlobRequestConditions requestConditions; + private boolean retrieveContentRangeMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link BlobDownloadStreamOptions}. + */ + public BlobDownloadStreamOptions() { + } + + /** + * Gets the {@link BlobRange}. + * + * @return The blob range. + */ + public BlobRange getRange() { + return range; + } + + /** + * Sets the {@link BlobRange}. + * + * @param range The blob range. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRange(BlobRange range) { + this.range = range; + return this; + } + + /** + * Gets the {@link DownloadRetryOptions}. + * + * @return The download retry options. + */ + public DownloadRetryOptions getDownloadRetryOptions() { + return downloadRetryOptions; + } + + /** + * Sets the {@link DownloadRetryOptions}. + * + * @param downloadRetryOptions The download retry options. + * @return The updated options. + */ + public BlobDownloadStreamOptions setDownloadRetryOptions(DownloadRetryOptions downloadRetryOptions) { + this.downloadRetryOptions = downloadRetryOptions; + return this; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets whether the content MD5 for the specified blob range should be returned. + * + * @return Whether to retrieve content range MD5. + */ + public boolean isRetrieveContentRangeMd5() { + return retrieveContentRangeMd5; + } + + /** + * Sets whether the content MD5 for the specified blob range should be returned. + * + * @param retrieveContentRangeMd5 Whether to retrieve content range MD5. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRetrieveContentRangeMd5(boolean retrieveContentRangeMd5) { + this.retrieveContentRangeMd5 = retrieveContentRangeMd5; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java index 434844645f58..0214ca4b8613 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java @@ -8,6 +8,7 @@ import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.DownloadRetryOptions; import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; import java.nio.file.OpenOption; @@ -25,6 +26,7 @@ public class BlobDownloadToFileOptions { private BlobRequestConditions requestConditions; private boolean retrieveContentRangeMd5; private Set openOptions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a {@link BlobDownloadToFileOptions}. @@ -165,4 +167,27 @@ public BlobDownloadToFileOptions setOpenOptions(Set openOptions) { this.openOptions = openOptions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadToFileOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java index d98d744f3dd3..bb2e2f240215 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java @@ -7,6 +7,7 @@ import com.azure.storage.blob.models.BlobRange; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ConsistentReadControl; +import com.azure.storage.common.ContentValidationAlgorithm; /** * Extended options that may be passed when opening a blob input stream. @@ -17,6 +18,7 @@ public class BlobInputStreamOptions { private BlobRequestConditions requestConditions; private Integer blockSize; private ConsistentReadControl consistentReadControl; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlobInputStreamOptions}. @@ -111,4 +113,26 @@ public BlobInputStreamOptions setConsistentReadControl(ConsistentReadControl con this.consistentReadControl = consistentReadControl; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobInputStreamOptions setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java index 681ff994ae62..e4aac1781f61 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java @@ -12,7 +12,9 @@ import com.azure.storage.blob.models.BlobImmutabilityPolicy; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import java.io.InputStream; @@ -38,6 +40,7 @@ public class BlobParallelUploadOptions { private Duration timeout; private BlobImmutabilityPolicy immutabilityPolicy; private Boolean legalHold; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a new {@link BlobParallelUploadOptions}. @@ -366,4 +369,27 @@ public BlobParallelUploadOptions setLegalHold(Boolean legalHold) { this.legalHold = legalHold; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobParallelUploadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java index ac92a1a022b3..91f61b9b509e 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java @@ -6,6 +6,7 @@ import com.azure.core.annotation.Fluent; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ConsistentReadControl; +import com.azure.storage.common.ContentValidationAlgorithm; import java.nio.channels.SeekableByteChannel; @@ -18,6 +19,7 @@ public final class BlobSeekableByteChannelReadOptions { private BlobRequestConditions requestConditions; private Integer readSizeInBytes; private ConsistentReadControl consistentReadControl; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlobSeekableByteChannelReadOptions}. @@ -108,4 +110,27 @@ public BlobSeekableByteChannelReadOptions setConsistentReadControl(ConsistentRea this.consistentReadControl = consistentReadControl; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobSeekableByteChannelReadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java index 65ca9b0ab052..a9e283466af6 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java @@ -8,6 +8,7 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; import java.util.Map; @@ -24,6 +25,7 @@ public class BlobUploadFromFileOptions { private Map tags; private AccessTier tier; private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a {@link BlobUploadFromFileOptions}. @@ -164,4 +166,27 @@ public BlobUploadFromFileOptions setRequestConditions(BlobRequestConditions requ this.requestConditions = requestConditions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobUploadFromFileOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java index e6385dfe5207..d02b6fd11681 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java @@ -7,6 +7,7 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import java.util.Map; @@ -20,6 +21,7 @@ public class BlockBlobOutputStreamOptions { private Map tags; private AccessTier tier; private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobOutputStreamOptions}. @@ -146,4 +148,27 @@ public BlockBlobOutputStreamOptions setRequestConditions(BlobRequestConditions r this.requestConditions = requestConditions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java index 1234ea04c738..3ab7ff719ead 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java @@ -6,6 +6,7 @@ import com.azure.storage.blob.models.AccessTier; import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; import java.util.Collection; import java.util.Map; @@ -60,6 +61,7 @@ public static Collection values() { private Map tags; private AccessTier tier; private BlobRequestConditions conditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Options constructor. @@ -199,4 +201,26 @@ public BlockBlobSeekableByteChannelWriteOptions setRequestConditions(BlobRequest return this; } + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated instance. + */ + public BlockBlobSeekableByteChannelWriteOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java index bef1780c2f59..47212d6033d8 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java @@ -9,7 +9,9 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobImmutabilityPolicy; import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import java.io.InputStream; @@ -32,6 +34,7 @@ public class BlockBlobSimpleUploadOptions { private BlobRequestConditions requestConditions; private BlobImmutabilityPolicy immutabilityPolicy; private Boolean legalHold; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobSimpleUploadOptions}. @@ -293,4 +296,27 @@ public BlockBlobSimpleUploadOptions setLegalHold(Boolean legalHold) { this.legalHold = legalHold; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobSimpleUploadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java index 37445b450a28..9c9868d57ed4 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java @@ -6,6 +6,7 @@ import com.azure.core.annotation.Fluent; import com.azure.core.util.BinaryData; import com.azure.core.util.CoreUtils; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; /** @@ -17,6 +18,7 @@ public final class BlockBlobStageBlockOptions { private final BinaryData data; private String leaseId; private byte[] contentMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobStageBlockOptions}. @@ -97,4 +99,27 @@ public BlockBlobStageBlockOptions setContentMd5(byte[] contentMd5) { this.contentMd5 = CoreUtils.clone(contentMd5); return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobStageBlockOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java new file mode 100644 index 000000000000..5a9ce4b49af4 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageImplUtils; + +/** + * Extended options that may be passed when opening an output stream to a page blob. + */ +@Fluent +public final class PageBlobOutputStreamOptions { + private final PageRange pageRange; + private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link PageBlobOutputStreamOptions}. + * + * @param pageRange The {@link PageRange} for the write. Pages must be aligned with 512-byte boundaries. + * @throws NullPointerException If {@code pageRange} is null. + */ + public PageBlobOutputStreamOptions(PageRange pageRange) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + this.pageRange = pageRange; + } + + /** + * Gets the page range. + * + * @return The page range. + */ + public PageRange getPageRange() { + return pageRange; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public PageBlobOutputStreamOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public PageBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java new file mode 100644 index 000000000000..1fff642df5ad --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.models.PageBlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when uploading pages to a page blob. + */ +@Fluent +public final class PageBlobUploadPagesOptions { + private byte[] contentMd5; + private PageBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link PageBlobUploadPagesOptions}. + */ + public PageBlobUploadPagesOptions() { + } + + /** + * Gets the MD5 hash of the page content. + * + * @return An MD5 hash of the content, or null. + */ + public byte[] getContentMd5() { + return CoreUtils.clone(contentMd5); + } + + /** + * Sets the MD5 hash of the page content for transactional verification. + * + * @param contentMd5 An MD5 hash of the page content. + * @return The updated options. + */ + public PageBlobUploadPagesOptions setContentMd5(byte[] contentMd5) { + this.contentMd5 = CoreUtils.clone(contentMd5); + return this; + } + + /** + * Gets the {@link PageBlobRequestConditions}. + * + * @return The request conditions. + */ + public PageBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link PageBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public PageBlobUploadPagesOptions setRequestConditions(PageBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public PageBlobUploadPagesOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java index 42b7b3f76f1c..4c0146889f7e 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java @@ -18,6 +18,7 @@ import com.azure.storage.blob.BlobContainerAsyncClient; import com.azure.storage.blob.BlobServiceAsyncClient; import com.azure.storage.blob.BlobServiceVersion; +import com.azure.storage.blob.implementation.accesshelpers.AppendBlobItemConstructorProxy; import com.azure.storage.blob.implementation.models.AppendBlobsAppendBlockFromUrlHeaders; import com.azure.storage.blob.implementation.models.AppendBlobsAppendBlockHeaders; import com.azure.storage.blob.implementation.models.AppendBlobsCreateHeaders; @@ -34,10 +35,13 @@ import com.azure.storage.blob.models.CpkInfo; import com.azure.storage.blob.models.CustomerProvidedKey; import com.azure.storage.blob.models.EncryptionAlgorithmType; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.AppendBlobAppendBlockFromUrlOptions; import com.azure.storage.blob.options.AppendBlobCreateOptions; import com.azure.storage.blob.options.AppendBlobSealOptions; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -447,13 +451,33 @@ public Mono appendBlock(Flux data, long length) { * @param appendBlobRequestConditions {@link AppendBlobRequestConditions} * @return A {@link Mono} containing {@link Response} whose {@link Response#getValue() value} contains the append * blob operation. + * @deprecated Use {@link #appendBlockWithResponse(Flux, long, AppendBlobAppendBlockOptions)}. The optional + * parameters are now carried by {@link AppendBlobAppendBlockOptions}, which is also forward-compatible with future + * optional settings. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public Mono> appendBlockWithResponse(Flux data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions) { + return appendBlockWithResponse(data, length, new AppendBlobAppendBlockOptions().setContentMd5(contentMd5) + .setRequestConditions(appendBlobRequestConditions)); + } + + /** + * Commits a new block of data to the end of the existing append blob with options. + * + * @param data The data to write to the blob. Note that this {@code Flux} must be replayable if retries are enabled + * (the default). In other words, the Flux must produce the same data each time it is subscribed to. + * @param length The exact length of the data. It is important that this value match precisely the length of the + * data emitted by the {@code Flux}. + * @param options Optional parameters for the request. Pass {@code null} to use defaults. + * @return A {@link Mono} containing {@link Response} whose value contains the append blob operation. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> appendBlockWithResponse(Flux data, long length, + AppendBlobAppendBlockOptions options) { try { - return withContext( - context -> appendBlockWithResponse(data, length, contentMd5, appendBlobRequestConditions, context)); + return withContext(context -> appendBlockWithResponseInternal(data, length, options, context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -461,27 +485,37 @@ public Mono> appendBlockWithResponse(Flux d Mono> appendBlockWithResponse(Flux data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, Context context) { + // Prevents revapi visibility increased error + return appendBlockWithResponseInternal(data, length, + new AppendBlobAppendBlockOptions().setContentMd5(contentMd5) + .setRequestConditions(appendBlobRequestConditions), + context); + } + Mono> appendBlockWithResponseInternal(Flux data, long length, + AppendBlobAppendBlockOptions options, Context context) { if (data == null) { - return Mono.error(new NullPointerException("'data' cannot be null.")); + return monoError(LOGGER, new NullPointerException("'data' cannot be null.")); } - - appendBlobRequestConditions - = appendBlobRequestConditions == null ? new AppendBlobRequestConditions() : appendBlobRequestConditions; - context = context == null ? Context.NONE : context; + AppendBlobAppendBlockOptions opts = options == null ? new AppendBlobAppendBlockOptions() : options; + AppendBlobRequestConditions requestConditions + = opts.getRequestConditions() == null ? new AppendBlobRequestConditions() : opts.getRequestConditions(); + context = ContentValidationModeResolver.addContentValidationMode(context == null ? Context.NONE : context, + opts.getContentValidationAlgorithm(), length, false); return this.azureBlobStorage.getAppendBlobs() - .appendBlockWithResponseAsync(containerName, blobName, length, data, null, contentMd5, null, - appendBlobRequestConditions.getLeaseId(), appendBlobRequestConditions.getMaxSize(), - appendBlobRequestConditions.getAppendPosition(), appendBlobRequestConditions.getIfModifiedSince(), - appendBlobRequestConditions.getIfUnmodifiedSince(), appendBlobRequestConditions.getIfMatch(), - appendBlobRequestConditions.getIfNoneMatch(), appendBlobRequestConditions.getTagsConditions(), null, - null, null, getCustomerProvidedKey(), encryptionScope, context) + .appendBlockWithResponseAsync(containerName, blobName, length, data, null, opts.getContentMd5(), null, + requestConditions.getLeaseId(), requestConditions.getMaxSize(), requestConditions.getAppendPosition(), + requestConditions.getIfModifiedSince(), requestConditions.getIfUnmodifiedSince(), + requestConditions.getIfMatch(), requestConditions.getIfNoneMatch(), + requestConditions.getTagsConditions(), null, null, null, getCustomerProvidedKey(), encryptionScope, + context) .map(rb -> { AppendBlobsAppendBlockHeaders hd = rb.getDeserializedHeaders(); - AppendBlobItem item = new AppendBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsBlobAppendOffset(), hd.getXMsBlobCommittedBlockCount()); + AppendBlobItem item + = AppendBlobItemConstructorProxy.create(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), + hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), + hd.getXMsBlobAppendOffset(), hd.getXMsBlobCommittedBlockCount(), null, hd.getXMsContentCrc64()); return new SimpleResponse<>(rb, item); }); } @@ -627,9 +661,10 @@ Mono> appendBlockFromUrlWithResponse(AppendBlobAppendBl getCustomerProvidedKey(), encryptionScope, context) .map(rb -> { AppendBlobsAppendBlockFromUrlHeaders hd = rb.getDeserializedHeaders(); - AppendBlobItem item = new AppendBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsBlobAppendOffset(), hd.getXMsBlobCommittedBlockCount()); + AppendBlobItem item + = AppendBlobItemConstructorProxy.create(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), + hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), + hd.getXMsBlobAppendOffset(), hd.getXMsBlobCommittedBlockCount(), null, hd.getXMsContentCrc64()); return new SimpleResponse<>(rb, item); }); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java index 0fd58f5ef575..24feedadebf2 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java @@ -6,7 +6,6 @@ import com.azure.core.annotation.ReturnType; import com.azure.core.annotation.ServiceClient; import com.azure.core.annotation.ServiceMethod; -import com.azure.core.exception.UnexpectedLengthException; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpResponse; import com.azure.core.http.rest.Response; @@ -32,8 +31,10 @@ import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.CpkInfo; import com.azure.storage.blob.models.CustomerProvidedKey; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.AppendBlobAppendBlockFromUrlOptions; import com.azure.storage.blob.options.AppendBlobCreateOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; import com.azure.storage.blob.options.AppendBlobSealOptions; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -45,7 +46,6 @@ import java.nio.ByteBuffer; import java.time.Duration; import java.util.Map; -import java.util.Objects; import java.util.concurrent.Callable; import static com.azure.storage.common.implementation.StorageImplUtils.sendRequest; @@ -182,7 +182,7 @@ public AppendBlobClient getCustomerProvidedKeyClient(CustomerProvidedKey custome * @throws BlobStorageException If a storage service error occurred. */ public BlobOutputStream getBlobOutputStream() { - return getBlobOutputStream(null); + return getBlobOutputStream((AppendBlobRequestConditions) null); } /** @@ -218,6 +218,18 @@ public BlobOutputStream getBlobOutputStream(AppendBlobRequestConditions requestC return BlobOutputStream.appendBlobOutputStream(appendBlobAsyncClient, requestConditions); } + /** + * Creates and opens an output stream to write data to the append blob. + * + * @param options {@link AppendBlobOutputStreamOptions} + * @return A {@link BlobOutputStream} object used to write data to the blob. + */ + public BlobOutputStream getBlobOutputStream(AppendBlobOutputStreamOptions options) { + options = options == null ? new AppendBlobOutputStreamOptions() : options; + return BlobOutputStream.appendBlobOutputStream(appendBlobAsyncClient, options.getRequestConditions(), + options.getContentValidationAlgorithm()); + } + /** * Creates a 0-length append blob. Call appendBlock to append data to an append blob. By default this method will * not overwrite an existing blob. @@ -458,7 +470,7 @@ public Response createIfNotExistsWithResponse(AppendBlobCreateOp */ @ServiceMethod(returns = ReturnType.SINGLE) public AppendBlobItem appendBlock(InputStream data, long length) { - return appendBlockWithResponse(data, length, null, null, null, Context.NONE).getValue(); + return appendBlockWithResponse(data, length, null, null, Context.NONE).getValue(); } /** @@ -499,21 +511,44 @@ public AppendBlobItem appendBlock(InputStream data, long length) { * @param timeout An optional timeout value beyond which a {@link RuntimeException} will be raised. * @param context Additional context that is passed through the Http pipeline during the service call. * @return A {@link Response} whose {@link Response#getValue() value} contains the append blob operation. - * @throws UnexpectedLengthException when the length of data does not match the input {@code length}. * @throws NullPointerException if the input data is null. + * @deprecated Use {@link #appendBlockWithResponse(InputStream, long, AppendBlobAppendBlockOptions, Duration, + * Context)}. The optional parameters are now carried by {@link AppendBlobAppendBlockOptions}, which is also + * forward-compatible with future optional settings. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public Response appendBlockWithResponse(InputStream data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, Duration timeout, Context context) { - Objects.requireNonNull(data, "'data' cannot be null."); - Flux fbb; + return appendBlockWithResponse(data, length, new AppendBlobAppendBlockOptions().setContentMd5(contentMd5) + .setRequestConditions(appendBlobRequestConditions), timeout, context); + } + + /** + * Commits a new block of data to the end of the existing append blob with options. + * + * @param data The data to write to the blob. The data must be markable. This is in order to support retries. If + * the data is not markable, consider using {@link #getBlobOutputStream()} and writing to the returned OutputStream. + * Alternatively, consider wrapping your data source in a {@link java.io.BufferedInputStream} to add mark support. + * @param length The exact length of the data. It is important that this value match precisely the length of the + * data. + * @param options Optional parameters for the request. Pass {@code null} to use defaults. + * @param timeout An optional timeout value. + * @param context Additional context. + * @return The information of the append blob operation. + * @throws NullPointerException if {@code data} is null. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response appendBlockWithResponse(InputStream data, long length, + AppendBlobAppendBlockOptions options, Duration timeout, Context context) { + StorageImplUtils.assertNotNull("data", data); // service versions 2022-11-02 and above support uploading block bytes up to 100MB, all older service versions // support up to 4MB - fbb = Utility.convertStreamToByteBuffer(data, length, getMaxAppendBlockBytes(), true); + Flux fbb = Utility.convertStreamToByteBuffer(data, length, getMaxAppendBlockBytes(), true); - Mono> response = appendBlobAsyncClient.appendBlockWithResponse(fbb, length, contentMd5, - appendBlobRequestConditions, context); + Mono> response + = appendBlobAsyncClient.appendBlockWithResponseInternal(fbb, length, options, context); return StorageImplUtils.blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index ef8ec9d2d4d8..c3392bfef4f9 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -73,16 +73,21 @@ import com.azure.storage.blob.models.UserDelegationKey; import com.azure.storage.blob.options.BlobBeginCopyOptions; import com.azure.storage.blob.options.BlobCopyFromUrlOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.blob.options.BlobGetTagsOptions; import com.azure.storage.blob.options.BlobQueryOptions; import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; @@ -529,7 +534,7 @@ Mono> existsWithResponse(Context context) { return Mono.just(new SimpleResponse<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), false)); } else { - return Mono.error(e); + return monoError(LOGGER, e); } }); } @@ -722,8 +727,8 @@ public PollerFlux beginCopy(BlobBeginCopyOptions options) { } }, (pollingContext, firstResponse) -> { if (firstResponse == null || firstResponse.getValue() == null) { - return Mono.error(LOGGER.logExceptionAsError( - new IllegalArgumentException("Cannot cancel a poll response that never started."))); + return monoError(LOGGER, + new IllegalArgumentException("Cannot cancel a poll response that never started.")); } final String copyIdentifier = firstResponse.getValue().getCopyId(); @@ -1166,9 +1171,25 @@ public Mono downloadWithResponse(BlobRange range, Dow @ServiceMethod(returns = ReturnType.SINGLE) public Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5) { + return downloadStreamWithResponse(new BlobDownloadStreamOptions().setRange(range) + .setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRetrieveContentRangeMd5(getRangeContentMd5)); + } + + /** + * Reads a range of bytes from a blob with options. + * + * @param options {@link BlobDownloadStreamOptions} + * @return A reactive response containing the blob data. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono downloadStreamWithResponse(BlobDownloadStreamOptions options) { try { - return withContext( - context -> downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context)); + BlobDownloadStreamOptions finalOptions = options == null ? new BlobDownloadStreamOptions() : options; + return withContext(context -> downloadStreamWithResponseInternal(finalOptions.getRange(), + finalOptions.getDownloadRetryOptions(), finalOptions.getRequestConditions(), + finalOptions.isRetrieveContentRangeMd5(), finalOptions.getContentValidationAlgorithm(), context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -1205,11 +1226,26 @@ public Mono downloadStreamWithResponse(BlobRange rang @ServiceMethod(returns = ReturnType.SINGLE) public Mono downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions) { + return downloadContentWithResponse( + new BlobDownloadContentOptions().setDownloadRetryOptions(options).setRequestConditions(requestConditions)); + } + + /** + * Reads blob content (full blob or range) with options. + * + * @param options {@link BlobDownloadContentOptions} + * @return A reactive response containing the blob content. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono downloadContentWithResponse(BlobDownloadContentOptions options) { try { - return withContext(context -> downloadStreamWithResponse(null, options, requestConditions, false, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders())))); + BlobDownloadContentOptions finalOptions = options == null ? new BlobDownloadContentOptions() : options; + return withContext(context -> downloadStreamWithResponseInternal(finalOptions.getRange(), + finalOptions.getDownloadRetryOptions(), finalOptions.getRequestConditions(), + finalOptions.isRetrieveContentRangeMd5(), finalOptions.getContentValidationAlgorithm(), context) + .flatMap(r -> BinaryData.fromFlux(r.getValue()) + .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), + r.getHeaders(), data, r.getDeserializedHeaders())))); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -1217,17 +1253,27 @@ public Mono downloadContentWithResponse(Downlo Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context) { + // Prevents revapi visibility increased error + return downloadStreamWithResponseInternal(range, options, requestConditions, getRangeContentMd5, null, context); + } + + Mono downloadStreamWithResponseInternal(BlobRange range, DownloadRetryOptions options, + BlobRequestConditions requestConditions, boolean getRangeContentMd5, + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; Boolean getMD5 = getRangeContentMd5 ? getRangeContentMd5 : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; + context + = ContentValidationModeResolver.addStructuredMessageDecodingToContext(context, contentValidationAlgorithm); + // The first range should eagerly convert headers as they'll be used to create response types. Context firstRangeContext = context == null ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - + Context nextRangeContext = context; return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, firstRangeContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); @@ -1272,7 +1318,7 @@ Mono downloadStreamWithResponse(BlobRange range, Down try { return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, - eTag, getMD5, context); + eTag, getMD5, nextRangeContext); } catch (Exception e) { return Mono.error(e); } @@ -1504,7 +1550,8 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti AsynchronousFileChannel channel = downloadToFileResourceSupplier(options.getFilePath(), openOptions); return Mono.just(channel) .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, - options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), context)) + options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context)) .doFinally(signalType -> this.downloadToFileCleanup(channel, options.getFilePath(), signalType)); } @@ -1519,7 +1566,7 @@ private AsynchronousFileChannel downloadToFileResourceSupplier(String filePath, private Mono> downloadToFileImpl(AsynchronousFileChannel file, BlobRange finalRange, com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, DownloadRetryOptions downloadRetryOptions, BlobRequestConditions requestConditions, boolean rangeGetContentMd5, - Context context) { + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { // See ProgressReporter for an explanation on why this lock is necessary and why we use AtomicLong. ProgressListener progressReceiver = finalParallelTransferOptions.getProgressListener(); ProgressReporter progressReporter @@ -1529,8 +1576,8 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne * Downloads the first chunk and gets the size of the data and etag if not specified by the user. */ BiFunction> downloadFunc - = (range, conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, - rangeGetContentMd5, context); + = (range, conditions) -> this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, + rangeGetContentMd5, contentValidationAlgorithm, context); return ChunkedDownloadUtils .downloadFirstChunk(finalRange, finalParallelTransferOptions, requestConditions, downloadFunc, true, diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java index 9c44f4cc8e84..b35c654a1132 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java @@ -76,6 +76,8 @@ import com.azure.storage.blob.models.UserDelegationKey; import com.azure.storage.blob.options.BlobBeginCopyOptions; import com.azure.storage.blob.options.BlobCopyFromUrlOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.blob.options.BlobGetTagsOptions; import com.azure.storage.blob.options.BlobInputStreamOptions; @@ -84,6 +86,7 @@ import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -91,6 +94,7 @@ import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.StorageSeekableByteChannel; + import reactor.core.publisher.Mono; import java.io.IOException; @@ -500,6 +504,7 @@ public BlobInputStream openInputStream(BlobInputStreamOptions options) { public BlobInputStream openInputStream(BlobInputStreamOptions options, Context context) { Context contextFinal = context == null ? Context.NONE : context; options = options == null ? new BlobInputStreamOptions() : options; + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); ConsistentReadControl consistentReadControl = options.getConsistentReadControl() == null ? ConsistentReadControl.ETAG : options.getConsistentReadControl(); @@ -511,8 +516,9 @@ public BlobInputStream openInputStream(BlobInputStreamOptions options, Context c com.azure.storage.common.ParallelTransferOptions parallelTransferOptions = new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong((long) chunkSize); - BiFunction> downloadFunc = (chunkRange, - conditions) -> client.downloadStreamWithResponse(chunkRange, null, conditions, false, contextFinal); + BiFunction> downloadFunc + = (chunkRange, conditions) -> client.downloadStreamWithResponseInternal(chunkRange, null, conditions, false, + contentValidationAlgorithm, contextFinal); return ChunkedDownloadUtils .downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloadFunc, true) .flatMap(tuple3 -> { @@ -588,8 +594,12 @@ public BlobSeekableByteChannelReadResult openSeekableByteChannelRead(BlobSeekabl BlobDownloadResponse response; try (ByteBufferBackedOutputStreamUtil dstStream = new ByteBufferBackedOutputStreamUtil(initialRange)) { response = this.downloadStreamWithResponse(dstStream, - new BlobRange(initialPosition, (long) initialRange.remaining()), null /*downloadRetryOptions*/, - options.getRequestConditions(), false, null, context); + new BlobDownloadStreamOptions() + .setRange(new BlobRange(initialPosition, (long) initialRange.remaining())) + .setRequestConditions(options.getRequestConditions()) + .setRetrieveContentRangeMd5(false) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, context); properties = ModelHelper.buildBlobPropertiesResponse(response).getValue(); } catch (IOException e) { throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); @@ -1266,12 +1276,35 @@ public BlobDownloadResponse downloadWithResponse(OutputStream stream, BlobRange @ServiceMethod(returns = ReturnType.SINGLE) public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5, + Duration timeout, Context context) { + return downloadStreamWithResponse(stream, + new BlobDownloadStreamOptions().setRange(range) + .setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRetrieveContentRangeMd5(getRangeContentMd5), + timeout, context); + } + + /** + * Downloads a range of bytes from a blob into an output stream with options. + * + * @param stream The output stream where the downloaded data will be written. + * @param options {@link BlobDownloadStreamOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return A response containing status code and HTTP headers. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, BlobDownloadStreamOptions options, Duration timeout, Context context) { StorageImplUtils.assertNotNull("stream", stream); - Mono download - = client.downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context) - .flatMap(response -> FluxUtil.writeToOutputStream(response.getValue(), stream) - .thenReturn(new BlobDownloadResponse(response))); + options = options == null ? new BlobDownloadStreamOptions() : options; + Mono download = client + .downloadStreamWithResponseInternal(options.getRange(), options.getDownloadRetryOptions(), + options.getRequestConditions(), options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context) + .flatMap(response -> FluxUtil.writeToOutputStream(response.getValue(), stream) + .thenReturn(new BlobDownloadResponse(response))); return blockWithOptionalTimeout(download, timeout); } @@ -1310,14 +1343,9 @@ public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, Blob @ServiceMethod(returns = ReturnType.SINGLE) public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions, Duration timeout, Context context) { - Mono download - = client.downloadStreamWithResponse(null, options, requestConditions, false, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders()))) - .map(BlobDownloadContentResponse::new); - - return blockWithOptionalTimeout(download, timeout); + return downloadContentWithResponse( + new BlobDownloadContentOptions().setDownloadRetryOptions(options).setRequestConditions(requestConditions), + timeout, context); } /** @@ -1358,12 +1386,32 @@ public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOpti public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions, BlobRange range, boolean getRangeContentMd5, Duration timeout, Context context) { - Mono download - = client.downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders()))) - .map(BlobDownloadContentResponse::new); + return downloadContentWithResponse(new BlobDownloadContentOptions().setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRange(range) + .setRetrieveContentRangeMd5(getRangeContentMd5), timeout, context); + } + + /** + * Downloads blob content (full blob or range) with options. + * + * @param options {@link BlobDownloadContentOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return A response containing status code and HTTP headers. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobDownloadContentResponse downloadContentWithResponse(BlobDownloadContentOptions options, Duration timeout, + Context context) { + options = options == null ? new BlobDownloadContentOptions() : options; + Mono download = client + .downloadStreamWithResponseInternal(options.getRange(), options.getDownloadRetryOptions(), + options.getRequestConditions(), options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context) + .flatMap(r -> BinaryData.fromFlux(r.getValue()) + .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), + data, r.getDeserializedHeaders()))) + .map(BlobDownloadContentResponse::new); return blockWithOptionalTimeout(download, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java index c43277ff30de..dcb2fb036acf 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java @@ -16,10 +16,14 @@ import com.azure.storage.blob.models.PageBlobRequestConditions; import com.azure.storage.blob.models.PageRange; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.BlobParallelUploadOptions; import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageOutputStream; import com.azure.storage.common.implementation.Constants; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -47,7 +51,13 @@ public abstract class BlobOutputStream extends StorageOutputStream { static BlobOutputStream appendBlobOutputStream(final AppendBlobAsyncClient client, final AppendBlobRequestConditions appendBlobRequestConditions) { - return new AppendBlobOutputStream(client, appendBlobRequestConditions); + return new AppendBlobOutputStream(client, appendBlobRequestConditions, null); + } + + static BlobOutputStream appendBlobOutputStream(final AppendBlobAsyncClient client, + final AppendBlobRequestConditions appendBlobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { + return new AppendBlobOutputStream(client, appendBlobRequestConditions, contentValidationAlgorithm); } /** @@ -104,12 +114,18 @@ public static BlobOutputStream blockBlobOutputStream(final BlobAsyncClient clien BlockBlobOutputStreamOptions options, Context context) { options = options == null ? new BlockBlobOutputStreamOptions() : options; return new BlockBlobOutputStream(client, options.getParallelTransferOptions(), options.getHeaders(), - options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), context); + options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), + options.getContentValidationAlgorithm(), context); } static BlobOutputStream pageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, final BlobRequestConditions requestConditions) { - return new PageBlobOutputStream(client, pageRange, requestConditions); + return pageBlobOutputStream(client, pageRange, requestConditions, null); + } + + static BlobOutputStream pageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, + final BlobRequestConditions requestConditions, final ContentValidationAlgorithm contentValidationAlgorithm) { + return new PageBlobOutputStream(client, pageRange, requestConditions, contentValidationAlgorithm); } abstract void commit(); @@ -157,9 +173,11 @@ private static final class AppendBlobOutputStream extends BlobOutputStream { private final AppendBlobRequestConditions appendBlobRequestConditions; private final AppendBlobAsyncClient client; + private final ContentValidationAlgorithm contentValidationAlgorithm; private AppendBlobOutputStream(final AppendBlobAsyncClient client, - final AppendBlobRequestConditions appendBlobRequestConditions) { + final AppendBlobRequestConditions appendBlobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { // service versions 2022-11-02 and above support uploading block bytes up to 100MB, all older service // versions support up to 4MB super(client.getServiceVersion().ordinal() < BlobServiceVersion.V2022_11_02.ordinal() @@ -170,6 +188,7 @@ private AppendBlobOutputStream(final AppendBlobAsyncClient client, this.appendBlobRequestConditions = (appendBlobRequestConditions == null) ? new AppendBlobRequestConditions() : appendBlobRequestConditions; + this.contentValidationAlgorithm = contentValidationAlgorithm; if (this.appendBlobRequestConditions.getAppendPosition() == null) { this.appendBlobRequestConditions.setAppendPosition(client.getProperties().block().getBlobSize()); @@ -178,7 +197,10 @@ private AppendBlobOutputStream(final AppendBlobAsyncClient client, private Mono appendBlock(Flux blockData, long writeLength) { long newAppendOffset = appendBlobRequestConditions.getAppendPosition() + writeLength; - return client.appendBlockWithResponse(blockData, writeLength, null, appendBlobRequestConditions) + AppendBlobAppendBlockOptions opts + = new AppendBlobAppendBlockOptions().setRequestConditions(appendBlobRequestConditions) + .setContentValidationAlgorithm(contentValidationAlgorithm); + return client.appendBlockWithResponse(blockData, writeLength, opts) .doOnNext(ignored -> appendBlobRequestConditions.setAppendPosition(newAppendOffset)) .then() .onErrorResume(t -> t instanceof IOException || t instanceof BlobStorageException, e -> { @@ -223,7 +245,8 @@ private static final class BlockBlobOutputStream extends BlobOutputStream { private BlockBlobOutputStream(final BlobAsyncClient client, final ParallelTransferOptions parallelTransferOptions, final BlobHttpHeaders headers, final Map metadata, Map tags, final AccessTier tier, - final BlobRequestConditions requestConditions, Context context) { + final BlobRequestConditions requestConditions, final ContentValidationAlgorithm contentValidationAlgorithm, + Context context) { super(Integer.MAX_VALUE); // writeThreshold is effectively not used by BlockBlobOutputStream. // There is a bug in reactor core that does not handle converting Context.NONE to a reactor context. context = context == null || context.equals(Context.NONE) ? null : context; @@ -241,7 +264,8 @@ private BlockBlobOutputStream(final BlobAsyncClient client, .setMetadata(metadata) .setTags(tags) .setTier(tier) - .setRequestConditions(requestConditions)) + .setRequestConditions(requestConditions) + .setContentValidationAlgorithm(contentValidationAlgorithm)) // This allows the operation to continue while maintaining the error that occurred. .onErrorResume(e -> { if (e instanceof IOException) { @@ -319,12 +343,15 @@ private static final class PageBlobOutputStream extends BlobOutputStream { private final PageBlobAsyncClient client; private final PageBlobRequestConditions pageBlobRequestConditions; private final PageRange pageRange; + private final ContentValidationAlgorithm contentValidationAlgorithm; private PageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, - final BlobRequestConditions blobRequestConditions) { + final BlobRequestConditions blobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { super(PageBlobClient.MAX_PUT_PAGES_BYTES); this.client = client; this.pageRange = pageRange; + this.contentValidationAlgorithm = contentValidationAlgorithm; if (blobRequestConditions != null) { this.pageBlobRequestConditions @@ -340,8 +367,10 @@ private PageBlobOutputStream(final PageBlobAsyncClient client, final PageRange p private Mono writePages(Flux pageData, int length, long offset) { return client - .uploadPagesWithResponse(new PageRange().setStart(offset).setEnd(offset + length - 1), pageData, null, - pageBlobRequestConditions) + .uploadPagesWithResponseInternal(new PageRange().setStart(offset).setEnd(offset + length - 1), pageData, + new PageBlobUploadPagesOptions().setRequestConditions(pageBlobRequestConditions) + .setContentValidationAlgorithm(contentValidationAlgorithm), + Context.NONE) .then() .onErrorResume(BlobStorageException.class, e -> { this.lastError = new IOException(e); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java index f938e21b4793..60abeff87774 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java @@ -15,6 +15,7 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.storage.blob.BlobAsyncClient; import com.azure.storage.blob.BlobServiceVersion; +import com.azure.storage.blob.implementation.accesshelpers.BlockBlobItemConstructorProxy; import com.azure.storage.blob.implementation.models.BlockBlobsCommitBlockListHeaders; import com.azure.storage.blob.implementation.models.BlockBlobsPutBlobFromUrlHeaders; import com.azure.storage.blob.implementation.models.BlockBlobsUploadHeaders; @@ -42,6 +43,8 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -415,6 +418,9 @@ public Mono> uploadWithResponse(BlockBlobSimpleUploadOpt Mono> uploadWithResponse(BlockBlobSimpleUploadOptions options, Context context) { StorageImplUtils.assertNotNull("options", options); + + ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + Mono dataMono; BinaryData binaryData = options.getData(); if (binaryData == null) { @@ -428,10 +434,14 @@ Mono> uploadWithResponse(BlockBlobSimpleUploadOptions op } BlobRequestConditions requestConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); - Context finalContext = context == null ? Context.NONE : context; BlobImmutabilityPolicy immutabilityPolicy = options.getImmutabilityPolicy() == null ? new BlobImmutabilityPolicy() : options.getImmutabilityPolicy(); + context = ContentValidationModeResolver.addContentValidationMode(context, contentValidationAlgorithm, + options.getLength(), false); + + Context finalContext = context; + return dataMono.flatMap(data -> this.azureBlobStorage.getBlockBlobs() .uploadWithResponseAsync(containerName, blobName, options.getLength(), data, null, options.getContentMd5(), options.getMetadata(), requestConditions.getLeaseId(), options.getTier(), @@ -442,9 +452,9 @@ Mono> uploadWithResponse(BlockBlobSimpleUploadOptions op null, null, options.getHeaders(), getCustomerProvidedKey(), encryptionScope, finalContext) .map(rb -> { BlockBlobsUploadHeaders hd = rb.getDeserializedHeaders(); - BlockBlobItem item = new BlockBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsVersionId()); + BlockBlobItem item = BlockBlobItemConstructorProxy.create(hd.getETag(), hd.getLastModified(), + hd.getContentMD5(), hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), + hd.getXMsEncryptionScope(), hd.getXMsVersionId(), hd.getXMsContentCrc64()); return new SimpleResponse<>(rb, item); })); } @@ -601,9 +611,9 @@ Mono> uploadFromUrlWithResponse(BlobUploadFromUrlOptions options.getHeaders(), getCustomerProvidedKey(), encryptionScope, context) .map(rb -> { BlockBlobsPutBlobFromUrlHeaders hd = rb.getDeserializedHeaders(); - BlockBlobItem item = new BlockBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsVersionId()); + BlockBlobItem item = BlockBlobItemConstructorProxy.create(hd.getETag(), hd.getLastModified(), + hd.getContentMD5(), hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), + hd.getXMsEncryptionScope(), hd.getXMsVersionId(), hd.getXMsContentCrc64()); return new SimpleResponse<>(rb, item); }); } @@ -707,12 +717,10 @@ public Mono stageBlock(String base64BlockId, BinaryData data) { @ServiceMethod(returns = ReturnType.SINGLE) public Mono> stageBlockWithResponse(String base64BlockId, Flux data, long length, byte[] contentMd5, String leaseId) { - try { - return withContext( - context -> stageBlockWithResponse(base64BlockId, data, length, contentMd5, leaseId, context)); - } catch (RuntimeException ex) { - return monoError(LOGGER, ex); - } + return BinaryData.fromFlux(data, length, false) + .flatMap(binaryData -> stageBlockWithResponse( + new BlockBlobStageBlockOptions(base64BlockId, binaryData).setContentMd5(contentMd5) + .setLeaseId(leaseId))); } /** @@ -745,27 +753,24 @@ public Mono> stageBlockWithResponse(String base64BlockId, Flux> stageBlockWithResponse(BlockBlobStageBlockOptions options) { Objects.requireNonNull(options, "options must not be null"); try { - return withContext(context -> stageBlockWithResponse(options.getBase64BlockId(), options.getData(), - options.getContentMd5(), options.getLeaseId(), context)); + return withContext(context -> stageBlockWithResponseInternal(options, context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } } - Mono> stageBlockWithResponse(String base64BlockId, Flux data, long length, - byte[] contentMd5, String leaseId, Context context) { - return BinaryData.fromFlux(data, length, false) - .flatMap(binaryData -> stageBlockWithResponse(base64BlockId, binaryData, contentMd5, leaseId, context)); - } + Mono> stageBlockWithResponseInternal(BlockBlobStageBlockOptions options, Context context) { + Objects.requireNonNull(options.getData(), "data must not be null"); + Objects.requireNonNull(options.getData().getLength(), "data must have defined length"); + + context = ContentValidationModeResolver.addContentValidationMode(context, + options.getContentValidationAlgorithm(), options.getData().getLength(), false); - Mono> stageBlockWithResponse(String base64BlockId, BinaryData data, byte[] contentMd5, - String leaseId, Context context) { - Objects.requireNonNull(data, "data must not be null"); - Objects.requireNonNull(data.getLength(), "data must have defined length"); - context = context == null ? Context.NONE : context; return this.azureBlobStorage.getBlockBlobs() - .stageBlockNoCustomHeadersWithResponseAsync(containerName, blobName, base64BlockId, data.getLength(), data, - contentMd5, null, null, leaseId, null, null, null, getCustomerProvidedKey(), encryptionScope, context); + .stageBlockNoCustomHeadersWithResponseAsync(containerName, blobName, options.getBase64BlockId(), + options.getData().getLength(), options.getData(), options.getContentMd5(), null, null, + options.getLeaseId(), null, null, null, getCustomerProvidedKey(), encryptionScope, context); + } /** @@ -1178,9 +1183,9 @@ Mono> commitBlockListWithResponse(BlockBlobCommitBlockLi getCustomerProvidedKey(), encryptionScope, context) .map(rb -> { BlockBlobsCommitBlockListHeaders hd = rb.getDeserializedHeaders(); - BlockBlobItem item = new BlockBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsVersionId()); + BlockBlobItem item = BlockBlobItemConstructorProxy.create(hd.getETag(), hd.getLastModified(), + hd.getContentMD5(), hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), + hd.getXMsEncryptionScope(), hd.getXMsVersionId(), hd.getXMsContentCrc64()); return new SimpleResponse<>(rb, item); }); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java index 8667e7afe28c..2ab1287cbe63 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java @@ -314,9 +314,7 @@ public SeekableByteChannel openSeekableByteChannelWrite(BlockBlobSeekableByteCha options.getBlockSizeInBytes() != null ? options.getBlockSizeInBytes().intValue() : BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, - new StorageSeekableByteChannelBlockBlobWriteBehavior(this, options.getHeaders(), options.getMetadata(), - options.getTags(), options.getTier(), options.getRequestConditions(), internalMode, null), - startingPosition); + new StorageSeekableByteChannelBlockBlobWriteBehavior(this, options, internalMode, null), startingPosition); } private BlobClientBuilder prepareBuilder() { @@ -787,11 +785,14 @@ public void stageBlock(String base64BlockId, BinaryData data) { public Response stageBlockWithResponse(String base64BlockId, InputStream data, long length, byte[] contentMd5, String leaseId, Duration timeout, Context context) { StorageImplUtils.assertNotNull("data", data); + Flux fbb = Utility.convertStreamToByteBuffer(data, length, BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, true); - Mono> response - = client.stageBlockWithResponse(base64BlockId, fbb, length, contentMd5, leaseId, context); + Mono> response = BinaryData.fromFlux(fbb, length, false) + .flatMap(binaryData -> client.stageBlockWithResponseInternal( + new BlockBlobStageBlockOptions(base64BlockId, binaryData).setContentMd5(contentMd5).setLeaseId(leaseId), + context)); return blockWithOptionalTimeout(response, timeout); } @@ -827,8 +828,8 @@ public Response stageBlockWithResponse(String base64BlockId, InputStream d public Response stageBlockWithResponse(BlockBlobStageBlockOptions options, Duration timeout, Context context) { Objects.requireNonNull(options, "options must not be null"); - Mono> response = client.stageBlockWithResponse(options.getBase64BlockId(), options.getData(), - options.getContentMd5(), options.getLeaseId(), context); + + Mono> response = client.stageBlockWithResponseInternal(options, context); return blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java index 5943047cf49b..b1758c507a54 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java @@ -22,6 +22,7 @@ import com.azure.core.util.logging.ClientLogger; import com.azure.storage.blob.BlobAsyncClient; import com.azure.storage.blob.BlobServiceVersion; +import com.azure.storage.blob.implementation.accesshelpers.PageBlobItemConstructorProxy; import com.azure.storage.blob.implementation.models.EncryptionScope; import com.azure.storage.blob.implementation.models.PageBlobsClearPagesHeaders; import com.azure.storage.blob.implementation.models.PageBlobsCreateHeaders; @@ -56,8 +57,11 @@ import com.azure.storage.blob.options.PageBlobCopyIncrementalOptions; import com.azure.storage.blob.options.PageBlobCreateOptions; import com.azure.storage.blob.options.PageBlobUploadPagesFromUrlOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -450,7 +454,7 @@ Mono> createIfNotExistsWithResponse(PageBlobCreateOptions */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono uploadPages(PageRange pageRange, Flux body) { - return uploadPagesWithResponse(pageRange, body, null, null).flatMap(FluxUtil::toMono); + return uploadPagesWithResponse(pageRange, body, null).flatMap(FluxUtil::toMono); } /** @@ -488,15 +492,14 @@ public Mono uploadPages(PageRange pageRange, Flux body * operation will fail. * @param pageBlobRequestConditions {@link PageBlobRequestConditions} * @return A reactive response containing the information of the uploaded pages. - * - * @throws IllegalArgumentException If {@code pageRange} is {@code null} + * @deprecated Use {@link #uploadPagesWithResponse(PageRange, Flux, PageBlobUploadPagesOptions)}. The optional + * parameters are now carried by {@link PageBlobUploadPagesOptions}, which is also forward-compatible with + * future optional settings. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public Mono> uploadPagesWithResponse(PageRange pageRange, Flux body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions) { - if (body == null) { - return Mono.error(new NullPointerException("'body' cannot be null.")); - } try { return withContext( context -> uploadPagesWithResponse(pageRange, body, contentMd5, pageBlobRequestConditions, context)); @@ -505,22 +508,55 @@ public Mono> uploadPagesWithResponse(PageRange pageRange, } } + /** + * Writes one or more pages to the page blob with options. + * + * @param pageRange A {@link PageRange} object. Given that pages must be aligned with 512-byte boundaries, the start + * offset must be a modulus of 512 and the end offset must be a modulus of 512 - 1. Examples of valid byte ranges + * are 0-511, 512-1023, etc. + * @param body The data to upload. Note that this {@code Flux} must be replayable if retries are enabled (the + * default). In other words, the {@code Flux} must produce the same data each time it is subscribed to. + * @param options Optional parameters for the request. Pass {@code null} to use defaults. + * @return A reactive response containing the information of the uploaded pages. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> uploadPagesWithResponse(PageRange pageRange, Flux body, + PageBlobUploadPagesOptions options) { + try { + return withContext(context -> uploadPagesWithResponseInternal(pageRange, body, options, context)); + } catch (RuntimeException ex) { + return monoError(LOGGER, ex); + } + } + Mono> uploadPagesWithResponse(PageRange pageRange, Flux body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, Context context) { - pageBlobRequestConditions - = pageBlobRequestConditions == null ? new PageBlobRequestConditions() : pageBlobRequestConditions; + // Prevents revapi visibility increased error + return uploadPagesWithResponseInternal(pageRange, body, + new PageBlobUploadPagesOptions().setContentMd5(contentMd5).setRequestConditions(pageBlobRequestConditions), + context); + } + Mono> uploadPagesWithResponseInternal(PageRange pageRange, Flux body, + PageBlobUploadPagesOptions options, Context context) { if (pageRange == null) { - // Throwing is preferred to Single.error because this will error out immediately instead of waiting until - // subscription. - throw LOGGER.logExceptionAsError(new IllegalArgumentException("pageRange cannot be null.")); + return monoError(LOGGER, new NullPointerException("'pageRange' cannot be null.")); + } + if (body == null) { + return monoError(LOGGER, new NullPointerException("'body' cannot be null.")); } + PageBlobUploadPagesOptions opts = options == null ? new PageBlobUploadPagesOptions() : options; + PageBlobRequestConditions pageBlobRequestConditions + = opts.getRequestConditions() == null ? new PageBlobRequestConditions() : opts.getRequestConditions(); + String pageRangeStr = ModelHelper.pageRangeToString(pageRange); - context = context == null ? Context.NONE : context; + long length = pageRange.getEnd() - pageRange.getStart() + 1; + context = ContentValidationModeResolver.addContentValidationMode(context == null ? Context.NONE : context, + opts.getContentValidationAlgorithm(), length, false); return this.azureBlobStorage.getPageBlobs() - .uploadPagesWithResponseAsync(containerName, blobName, pageRange.getEnd() - pageRange.getStart() + 1, body, - contentMd5, null, null, pageRangeStr, pageBlobRequestConditions.getLeaseId(), + .uploadPagesWithResponseAsync(containerName, blobName, length, body, opts.getContentMd5(), null, null, + pageRangeStr, pageBlobRequestConditions.getLeaseId(), pageBlobRequestConditions.getIfSequenceNumberLessThanOrEqualTo(), pageBlobRequestConditions.getIfSequenceNumberLessThan(), pageBlobRequestConditions.getIfSequenceNumberEqualTo(), pageBlobRequestConditions.getIfModifiedSince(), @@ -529,9 +565,9 @@ Mono> uploadPagesWithResponse(PageRange pageRange, Flux { PageBlobsUploadPagesHeaders hd = rb.getDeserializedHeaders(); - PageBlobItem item = new PageBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsBlobSequenceNumber()); + PageBlobItem item = PageBlobItemConstructorProxy.create(hd.getETag(), hd.getLastModified(), + hd.getContentMD5(), hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), + hd.getXMsEncryptionScope(), hd.getXMsBlobSequenceNumber(), null, hd.getXMsContentCrc64()); return new SimpleResponse<>(rb, item); }); } @@ -720,8 +756,9 @@ sourceCpkKey, sourceCpkKeySha256, sourceCpkAlgorithm, getCustomerProvidedKey(), context) .map(rb -> { PageBlobsUploadPagesFromURLHeaders hd = rb.getDeserializedHeaders(); - PageBlobItem item = new PageBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), null); + PageBlobItem item = PageBlobItemConstructorProxy.create(hd.getETag(), hd.getLastModified(), + hd.getContentMD5(), hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), + hd.getXMsEncryptionScope(), hd.getXMsBlobSequenceNumber(), null, hd.getXMsContentCrc64()); return new SimpleResponse<>(rb, item); }); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobClient.java index 693c02dbf6eb..23e520ed956e 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobClient.java @@ -53,7 +53,9 @@ import com.azure.storage.blob.options.ListPageRangesOptions; import com.azure.storage.blob.options.PageBlobCopyIncrementalOptions; import com.azure.storage.blob.options.PageBlobCreateOptions; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; import com.azure.storage.blob.options.PageBlobUploadPagesFromUrlOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; @@ -207,6 +209,21 @@ public BlobOutputStream getBlobOutputStream(PageRange pageRange, BlobRequestCond return BlobOutputStream.pageBlobOutputStream(pageBlobAsyncClient, pageRange, requestConditions); } + /** + * Creates and opens an output stream to write data to the page blob. + * + * @param options {@link PageBlobOutputStreamOptions} + * @return A {@link BlobOutputStream} object used to write data to the blob. + * @throws BlobStorageException If a storage service error occurred. + */ + public BlobOutputStream getBlobOutputStream(PageBlobOutputStreamOptions options) { + if (options == null) { + throw LOGGER.logExceptionAsError(new NullPointerException("'options' cannot be null.")); + } + return BlobOutputStream.pageBlobOutputStream(pageBlobAsyncClient, options.getPageRange(), + options.getRequestConditions(), options.getContentValidationAlgorithm()); + } + /** * Creates a page blob of the specified length. By default this method will not overwrite an existing blob. * Call PutPage to upload data data to a page blob. For more information, see the @@ -485,7 +502,7 @@ public Response createIfNotExistsWithResponse(PageBlobCreateOption */ @ServiceMethod(returns = ReturnType.SINGLE) public PageBlobItem uploadPages(PageRange pageRange, InputStream body) { - return uploadPagesWithResponse(pageRange, body, null, null, null, Context.NONE).getValue(); + return uploadPagesWithResponse(pageRange, body, null, null, Context.NONE).getValue(); } /** @@ -531,16 +548,45 @@ public PageBlobItem uploadPages(PageRange pageRange, InputStream body) { * @return The information of the uploaded pages. * @throws UnexpectedLengthException when the length of data does not match the input {@code length}. * @throws NullPointerException if the input data is null. + * @deprecated Use {@link #uploadPagesWithResponse(PageRange, InputStream, PageBlobUploadPagesOptions, Duration, + * Context)}. The optional parameters are now carried by {@link PageBlobUploadPagesOptions}, which is also + * forward-compatible with future optional settings. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public Response uploadPagesWithResponse(PageRange pageRange, InputStream body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, Duration timeout, Context context) { - Objects.requireNonNull(body, "'body' cannot be null."); + return uploadPagesWithResponse(pageRange, body, + new PageBlobUploadPagesOptions().setContentMd5(contentMd5).setRequestConditions(pageBlobRequestConditions), + timeout, context); + } + + /** + * Writes one or more pages to the page blob with options. + * + * @param pageRange A {@link PageRange} object. Given that pages must be aligned with 512-byte boundaries, the start + * offset must be a modulus of 512 and the end offset must be a modulus of 512 - 1. Examples of valid byte ranges + * are 0-511, 512-1023, etc. + * @param body The data to upload. The data must be markable. This is in order to support retries. If the data is + * not markable, consider using {@link #getBlobOutputStream(PageRange)} and writing to the returned OutputStream. + * Alternatively, consider wrapping your data source in a {@link java.io.BufferedInputStream} to add mark support. + * @param options {@link PageBlobUploadPagesOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return The information of the uploaded pages. + * @throws UnexpectedLengthException If the length of the data read from the provided stream does not match the + * expected length based on the specified page range. + * @throws NullPointerException if {@code pageRange} or {@code body} is null. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response uploadPagesWithResponse(PageRange pageRange, InputStream body, + PageBlobUploadPagesOptions options, Duration timeout, Context context) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + StorageImplUtils.assertNotNull("body", body); final long length = pageRange.getEnd() - pageRange.getStart() + 1; Flux fbb = Utility.convertStreamToByteBuffer(body, length, PAGE_BYTES, true); - - Mono> response = pageBlobAsyncClient.uploadPagesWithResponse(pageRange, fbb, contentMd5, - pageBlobRequestConditions, context); + Mono> response + = pageBlobAsyncClient.uploadPagesWithResponseInternal(pageRange, fbb, options, context); return StorageImplUtils.blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java index 71eb080a48cc..d98a78072519 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java @@ -10,7 +10,9 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageSeekableByteChannel; import java.io.IOException; @@ -43,6 +45,7 @@ enum WriteMode { private final WriteMode mode; private final List existingBlockIds; private final List newBlockIds = new ArrayList<>(); + private final ContentValidationAlgorithm contentValidationAlgorithm; StorageSeekableByteChannelBlockBlobWriteBehavior(BlockBlobClient client, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions conditions, @@ -55,6 +58,21 @@ enum WriteMode { this.conditions = conditions; this.mode = Objects.requireNonNull(mode); this.existingBlockIds = existingBlockIds != null ? existingBlockIds : Collections.emptyList(); + this.contentValidationAlgorithm = null; + } + + StorageSeekableByteChannelBlockBlobWriteBehavior(BlockBlobClient client, + BlockBlobSeekableByteChannelWriteOptions options, WriteMode mode, List existingBlockIds) { + this.client = Objects.requireNonNull(client); + Objects.requireNonNull(options); + this.headers = options.getHeaders(); + this.metadata = options.getMetadata(); + this.tags = options.getTags(); + this.tier = options.getTier(); + this.conditions = options.getRequestConditions(); + this.mode = Objects.requireNonNull(mode); + this.existingBlockIds = existingBlockIds != null ? existingBlockIds : Collections.emptyList(); + this.contentValidationAlgorithm = options.getContentValidationAlgorithm(); } BlockBlobClient getClient() { @@ -100,6 +118,9 @@ public void write(ByteBuffer src, long destOffset) throws IOException { if (conditions != null) { options.setLeaseId(conditions.getLeaseId()); } + if (contentValidationAlgorithm != null) { + options.setContentValidationAlgorithm(contentValidationAlgorithm); + } client.stageBlockWithResponse(options, null, null); newBlockIds.add(blockId); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java index 50a9eb63ef21..44a32b6a420f 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java @@ -129,6 +129,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; public class BlobApiTests extends BlobTestBase { private BlobClient bc; @@ -2153,12 +2154,24 @@ public void copyDestACFail(OffsetDateTime modified, OffsetDateTime unmodified, S () -> bu2.copyFromUrlWithResponse(bc.getBlobUrl(), null, null, null, bac, null, null)); } + // Abort-copy tests race the service: the cross-account copy must still be pending when the abort request + // lands. A larger source widens that window (A1); combined with an assumeTrue guard on the poll status (B), + // the tests assert abort behavior when a copy is genuinely pending and skip (rather than fail) if the + // service finished the copy first. Server-to-server copy, so the larger size costs only the initial upload. + // + // The source is capped just under the test proxy's 30,000,000-byte request-body limit: a body at/above that + // limit cannot be ingested by the proxy in record or playback (it fails with HTTP 500 "Request body too large" + // or a premature connection close, and also destabilizes the shared proxy connection pool for concurrent + // tests). 28 MiB (29,360,128 bytes) stays under the cap while keeping the window wide enough to exercise abort. + private static final int ABORT_COPY_SOURCE_SIZE_BYTES = 28 * Constants.MB; + @Test public void abortCopyBaseSimple() { // Data has to be large enough and copied between accounts to give us enough time to abort new SpecializedBlobClientBuilder().blobClient(bc) .buildBlockBlobClient() - .upload(new ByteArrayInputStream(getRandomByteArray(8 * 1024 * 1024)), 8 * 1024 * 1024, true); + .upload(new ByteArrayInputStream(getRandomByteArray(ABORT_COPY_SOURCE_SIZE_BYTES)), + ABORT_COPY_SOURCE_SIZE_BYTES, true); BlobContainerClient cu2 = alternateBlobServiceClient.getBlobContainerClient(generateBlobName()); cu2.create(); @@ -2171,6 +2184,8 @@ public void abortCopyBaseSimple() { PollResponse lastResponse = poller.poll(); assertNotNull(lastResponse); assertNotNull(lastResponse.getValue()); + assumeTrue(lastResponse.getStatus() == LongRunningOperationStatus.IN_PROGRESS, + "Copy already completed before abort could be issued; nothing pending to abort."); bu2.abortCopyFromUrl(lastResponse.getValue().getCopyId()); BlobStorageException e = assertThrows(BlobStorageException.class, () -> bu2.abortCopyFromUrl(lastResponse.getValue().getCopyId())); @@ -2186,7 +2201,8 @@ public void abortCopyLeaseFail() { // Data has to be large enough and copied between accounts to give us enough time to abort new SpecializedBlobClientBuilder().blobClient(bc) .buildBlockBlobClient() - .upload(new ByteArrayInputStream(getRandomByteArray(8 * 1024 * 1024)), 8 * 1024 * 1024, true); + .upload(new ByteArrayInputStream(getRandomByteArray(ABORT_COPY_SOURCE_SIZE_BYTES)), + ABORT_COPY_SOURCE_SIZE_BYTES, true); BlobContainerClient cu2 = alternateBlobServiceClient.getBlobContainerClient(generateBlobName()); cu2.create(); @@ -2202,6 +2218,8 @@ public void abortCopyLeaseFail() { bu2.beginCopy(bc.getBlobUrl() + "?" + sas, null, null, null, null, blobRequestConditions, null)); PollResponse response = poller.poll(); assertNotEquals(LongRunningOperationStatus.FAILED, response.getStatus()); + assumeTrue(response.getStatus() == LongRunningOperationStatus.IN_PROGRESS, + "Copy already completed before abort could be issued; cannot observe the 412 lease mismatch."); BlobCopyInfo blobCopyInfo = response.getValue(); BlobStorageException e = assertThrows(BlobStorageException.class, @@ -2217,7 +2235,8 @@ public void abortCopy() { // Data has to be large enough and copied between accounts to give us enough time to abort new SpecializedBlobClientBuilder().blobClient(bc) .buildBlockBlobClient() - .upload(new ByteArrayInputStream(getRandomByteArray(8 * 1024 * 1024)), 8 * 1024 * 1024, true); + .upload(new ByteArrayInputStream(getRandomByteArray(ABORT_COPY_SOURCE_SIZE_BYTES)), + ABORT_COPY_SOURCE_SIZE_BYTES, true); BlobContainerClient cu2 = alternateBlobServiceClient.getBlobContainerClient(generateBlobName()); cu2.create(); @@ -2230,6 +2249,8 @@ public void abortCopy() { PollResponse lastResponse = poller.poll(); assertNotNull(lastResponse); assertNotNull(lastResponse.getValue()); + assumeTrue(lastResponse.getStatus() == LongRunningOperationStatus.IN_PROGRESS, + "Copy already completed before abort could be issued; nothing pending to abort."); Response response = bu2.abortCopyFromUrlWithResponse(lastResponse.getValue().getCopyId(), null, null, null); HttpHeaders headers = response.getHeaders(); @@ -2247,7 +2268,8 @@ public void abortCopyLease() { // Data has to be large enough and copied between accounts to give us enough time to abort new SpecializedBlobClientBuilder().blobClient(bc) .buildBlockBlobClient() - .upload(new ByteArrayInputStream(getRandomByteArray(8 * 1024 * 1024)), 8 * 1024 * 1024, true); + .upload(new ByteArrayInputStream(getRandomByteArray(ABORT_COPY_SOURCE_SIZE_BYTES)), + ABORT_COPY_SOURCE_SIZE_BYTES, true); BlobContainerClient cu2 = alternateBlobServiceClient.getBlobContainerClient(generateContainerName()); cu2.create(); @@ -2264,6 +2286,8 @@ public void abortCopyLease() { assertNotNull(lastResponse); assertNotNull(lastResponse.getValue()); + assumeTrue(lastResponse.getStatus() == LongRunningOperationStatus.IN_PROGRESS, + "Copy already completed before abort could be issued; nothing pending to abort."); String copyId = lastResponse.getValue().getCopyId(); assertResponseStatusCode(bu2.abortCopyFromUrlWithResponse(copyId, leaseId, null, null), 204); // cleanup: diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java new file mode 100644 index 000000000000..5c3cf7816c9d --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java @@ -0,0 +1,707 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.ProgressListener; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; +import reactor.util.function.Tuples; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Async tests for structured message decoding during blob downloads using StorageContentValidationDecodingPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobContentValidationAsyncDownloadTests extends BlobTestBase { + // Generic "large enough to exercise multi-segment / multi-block content validation" payload. 5 MiB > the fixed + // 4 MiB structured-message segment / block size, so it still spans two segments/blocks while keeping the + // recordings (and heap/disk/CPU cost) roughly half of the previous 10-16 MiB payloads. + private static final int FIVE_MB = 5 * Constants.MB; + private static final int BLOCK_SIZE = 4 * Constants.MB; + /** + * {@link BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases()} starts at ~96 MiB; above this threshold fuzzy + * parallel download helpers use temp files + {@link BlobTestBase#compareFiles(File, File, long, long)} so the full + * payload never lives twice in heap. + */ + private static final int FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES = 96 * Constants.MB; + + /** + * Live-only random payload band for the dedicated random-size parallel-download fuzzy test + * ({@link #fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm)}): each run draws a per-run + * payload size in {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) so the structured-message + * decoder is exercised against payloads whose size varies per run in addition to the random byte contents. + */ + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + private File createRandomFile(Path tempDir, int size) throws IOException { + File file = Files.createTempFile(tempDir, "blob-cv-source", ".bin").toFile(); + + if (size > Constants.MB) { + try (OutputStream outputStream = Files.newOutputStream(file.toPath())) { + byte[] data = getRandomByteArray(Constants.MB); + int mbChunks = size / Constants.MB; + int remaining = size % Constants.MB; + for (int i = 0; i < mbChunks; i++) { + outputStream.write(data); + } + if (remaining > 0) { + outputStream.write(data, 0, remaining); + } + } + } else { + Files.write(file.toPath(), getRandomByteArray(size)); + } + + return file; + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.upload(BinaryData.fromBytes(data)) + .then(blobClient.downloadStreamWithResponse(options).flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + }))).assertNext(result -> TestUtils.assertArraysEqual(data, result)).verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + BlobDownloadContentOptions options + = new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier + .create(blobClient.upload(BinaryData.fromBytes(data)).then(blobClient.downloadContentWithResponse(options))) + .assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + TestUtils.assertArraysEqual(data, r.getValue().toBytes()); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation. + */ + @ParameterizedTest + @ValueSource( + ints = { + 0, // empty file + 20, // small file + FIVE_MB, // medium file spanning multiple blocks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize, @TempDir Path tempDir) throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.uploadFromFile(file.toPath().toString(), true) + .then(blobClient.downloadToFileWithResponse(options))).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + assertNotNull(r.getValue()); + }).verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize, @TempDir Path tempDir) + throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.uploadFromFile(file.toPath().toString(), true) + .then(blobClient.downloadToFileWithResponse(options))).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + assertNotNull(r.getValue()); + }).verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + StepVerifier.create(blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true) + .then(blobClient.downloadStreamWithResponse(new BlobDownloadStreamOptions()) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue())))) + .assertNext(result -> { + assertNotNull(result); + assertEquals(data.length, result.length); + }) + .verifyComplete(); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + @Test + public void downloadStreamWithResponseContentValidationRange() { + byte[] data = getRandomByteArray(4 * Constants.KB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setRange(new BlobRange(0, 512L)); + + StepVerifier.create(blobClient.upload(BinaryData.fromBytes(data)) + .then(blobClient.downloadStreamWithResponse(options).flatMap(r -> { + assertFalse(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + }))).assertNext(result -> assertEquals(512, result.length)).verifyComplete(); + + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + StepVerifier.create(blobClient.upload(BinaryData.fromBytes(data)) + .then(blobClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO)) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + }))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + StepVerifier + .create(blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true) + .then(blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE)))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + StepVerifier + .create(blobClient.upload(BinaryData.fromBytes(data)) + .then(blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO)))) + .assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + TestUtils.assertArraysEqual(data, r.getValue().toBytes()); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(2 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier.create(blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true) + .then(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .doFinally( + signalType -> assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked")) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + }))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] data = getRandomByteArray(dataSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier + .create(blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true) + .then( + downloadClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + }))) + .assertNext(result -> { + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(data, result); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * After consuming the response stream with CRC64 validation, decoded payload preserves the expected CRC64. + */ + @Test + public void structuredMessageVerifiesDecodedCrc64DownloadStreaming() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + long expectedCrc = StorageCrc64Calculator.compute(data, 0); + + StepVerifier.create(blobClient.upload(BinaryData.fromBytes(data)) + .then(blobClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes))))) + .assertNext(tuple -> { + TestUtils.assertArraysEqual(data, tuple.getT2()); + long actualCrc = StorageCrc64Calculator.compute(tuple.getT2(), 0); + assertEquals(expectedCrc, actualCrc); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Single interrupt with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(4 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier + .create(blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true) + .then(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue())))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + } + + /** + * Multiple interrupts with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptMultipleTimesWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(4 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier + .create(blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true) + .then(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue())))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + } + + @Test + public void verifyProgressListenerIsCompatibleWithContentValidation(@TempDir Path tempDir) throws IOException { + byte[] data = getRandomByteArray(FIVE_MB); + + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + MockProgressListener mockListenerWithContentVal = new MockProgressListener(); + MockProgressListener mockListenerWithoutContentVal = new MockProgressListener(); + + ParallelTransferOptions parallelOptionsWithContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithContentVal); + ParallelTransferOptions parallelOptionsWithoutContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithoutContentVal); + + createRandomFile(tempDir, FIVE_MB); + File outFileWithContentVal = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + createRandomFile(tempDir, FIVE_MB); + File outFileWithoutContentVal = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + + Files.deleteIfExists(outFileWithContentVal.toPath()); + Files.deleteIfExists(outFileWithoutContentVal.toPath()); + + BlobDownloadToFileOptions optionsWithContentVal + = new BlobDownloadToFileOptions(outFileWithContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithContentVal) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadToFileOptions optionsWithoutContentVal + = new BlobDownloadToFileOptions(outFileWithoutContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithoutContentVal); + + StepVerifier.create(client.upload(BinaryData.fromBytes(data)) + .then(client.downloadToFileWithResponse(optionsWithContentVal)) + .then(client.downloadToFileWithResponse(optionsWithoutContentVal))).assertNext(ignored -> { + long expectedBytes = data.length; + assertEquals(expectedBytes, mockListenerWithContentVal.getReportedByteCount()); + assertEquals(expectedBytes, mockListenerWithoutContentVal.getReportedByteCount()); + }).verifyComplete(); + } + + private static final class MockProgressListener implements ProgressListener { + private final AtomicLong reportedByteCount = new AtomicLong(0L); + + @Override + public void handleProgress(long bytesTransferred) { + this.reportedByteCount.updateAndGet(current -> Math.max(current, bytesTransferred)); + } + + long getReportedByteCount() { + return this.reportedByteCount.get(); + } + } + + // ---------- Fuzzy parallel download (deterministic grids) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadReplayableCases") + public void fuzzyParallelDownloadReplayableRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("replayable", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize with tiny totals; many small range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSmallMultiPartCases") + public void fuzzyParallelDownloadSmallMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("smallMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // sub-4 MiB chunked range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSubFourMiBCases") + public void fuzzyParallelDownloadSubFourMiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("subFourMiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // 4 MiB boundary tuples that fan out into chunked range GETs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadFourMiBBoundaryCases") + public void fuzzyParallelDownloadFourMiBBoundaryRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("fourMiBBoundary", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize for every tuple; chunked range GETs across many requests. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadMediumMultiPartCases") + public void fuzzyParallelDownloadMediumMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("mediumMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload >> blockSize; ~96-320 MiB downloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases") + public void fuzzyParallelDownloadLargeMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("largeMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // ~1 GiB single case; far too large for the test proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadOneGiBCases") + public void fuzzyParallelDownloadOneGiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("oneGiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + /** + * Live-only random-size parallel download fuzzy round-trip. Each run draws a per-run payload size in + * {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) and exercises both CRC64 and AUTO + * content-validation algorithms. The structured-message decoder is tested against payloads whose total size + * varies per run in addition to the random byte contents that the deterministic grids already exercise. Kept + * separate from the parameterized {@link #fuzzyParallelDownloadLargeMultiPartRoundTrip(int, long, int)} so the + * deterministic per-grid round-trips and the randomized round-trip don't share work or cost. + */ + @LiveOnly + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm algorithm) throws IOException { + int sizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + assertParallelDownloadFuzzyRoundTripAsync("liveRandom", sizeBytes, 8L * Constants.MB, 8, algorithm); + } + + private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency) throws IOException { + assertParallelDownloadFuzzyRoundTripAsync(caseKind, payloadBytes, blockSizeBytes, maxConcurrency, + ContentValidationAlgorithm.CRC64); + } + + private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency, ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + ParallelTransferOptions parallelOptions + = new ParallelTransferOptions().setBlockSizeLong(blockSizeBytes).setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel download [" + caseKind + "] payloadBytes=" + payloadBytes + + ", blockSize=" + blockSizeBytes + ", maxConcurrency=" + maxConcurrency + ", algorithm=" + algorithm; + + if (payloadBytes >= FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async", ".bin").toFile(); + Files.deleteIfExists(outFile.toPath()); + try { + BlobUploadFromFileOptions uploadOptions + = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()).setParallelTransferOptions( + new com.azure.storage.blob.models.ParallelTransferOptions().setBlockSizeLong(blockSizeBytes) + .setMaxConcurrency(maxConcurrency)); + + BlobDownloadToFileOptions downloadOptions = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + + StepVerifier + .create(client.uploadFromFileWithResponse(uploadOptions) + .then(client.downloadToFileWithResponse(downloadOptions))) + .assertNext(r -> assertNotNull(r.getValue(), assertionMessage)) + .verifyComplete(); + + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + // Delete eagerly (not deleteOnExit) so large fuzzy payloads don't accumulate for the whole JVM and + // exhaust agent disk space when many cases run in the same live-test process. + deleteFileQuietly(sourceFile); + deleteFileQuietly(outFile); + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + + if (payloadBytes > blockSizeBytes) { + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async-mp", ".bin").toFile(); + Files.deleteIfExists(outFile.toPath()); + try { + BlobDownloadToFileOptions downloadOptions + = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + + StepVerifier + .create(client.upload(BinaryData.fromBytes(randomData), true) + .then(client.downloadToFileWithResponse(downloadOptions))) + .assertNext(r -> assertNotNull(r.getValue(), assertionMessage)) + .verifyComplete(); + + byte[] downloaded = Files.readAllBytes(outFile.toPath()); + assertArrayEquals(randomData, downloaded, assertionMessage); + } finally { + deleteFileQuietly(outFile); + } + } else { + BlobDownloadContentOptions downloadOptions + = new BlobDownloadContentOptions().setContentValidationAlgorithm(algorithm); + + StepVerifier + .create(client.upload(BinaryData.fromBytes(randomData), true) + .then(client.downloadContentWithResponse(downloadOptions))) + .assertNext(r -> assertArrayEquals(randomData, r.getValue().toBytes(), assertionMessage)) + .verifyComplete(); + + BlobDownloadStreamOptions streamOptions + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(algorithm); + StepVerifier + .create(client.downloadStreamWithResponse(streamOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(bytes -> assertArrayEquals(randomData, bytes, assertionMessage)) + .verifyComplete(); + } + } + assertTrue(hasStructuredMessageDownloadRequestHeaders(recorded, false), assertionMessage); + } + + /** + * Deletes a temp file eagerly, falling back to {@link File#deleteOnExit()} only if the immediate delete fails. + * Prevents large fuzzy round-trip payloads from accumulating on disk across the many cases that share a JVM. + */ + private static void deleteFileQuietly(File file) { + if (file != null && file.exists() && !file.delete()) { + file.deleteOnExit(); + } + } + +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java new file mode 100644 index 000000000000..382644018489 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java @@ -0,0 +1,1112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.FluxUtil; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.AppendBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.AsynchronousFileChannel; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests content validation (CRC64 / structured message) for upload operations using async clients. + * Upload types that have no async counterpart (e.g. OutputStream, SeekableByteChannel) are tested in + * {@link BlobContentValidationUploadTests}. + */ +public class BlobContentValidationAsyncUploadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + // Generic ">= 4 MiB so the single-shot upload path uses a structured message" payload for the replayable + // (non-live) tests. 5 MiB keeps a single Put Blob / Put Block / Append Block request (still above the 4 MiB + // structured-message threshold) while roughly halving recording and heap/disk/CPU cost vs the old 10 MiB. + private static final int FIVE_MB = 5 * Constants.MB; + /* Single-part uploads with length < 4MB use CRC64 header; >= 4MB use structured message. */ + private static final int UNDER_4MB = 2 * Constants.MB; + + /** + * Live-only random payload band (256–500 MiB, inclusive upper bound via {@code randomLongFromNamer}+1) for + * {@code uploadWithResponse}, {@code uploadFromFileWithResponse}, and single-block {@code stageBlock}. + */ + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + /** + * Live-only random payload band for sequential append-block puts only ({@link + * #appendBlockLiveRandomRoundTripDataIntegrity()}): {@code Flux.concatMap} issues one append REST call per chunk in + * order (not parallel staging); use a smaller band than {@link #LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE}. + */ + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE = 32L * Constants.MB; + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE = 64L * Constants.MB; + + private static final String MD5_AND_CRC64_EXCLUSIVE_MESSAGE + = "Only one form of transactional content validation may be used."; + + // =========================================================================================== + // BlobAsyncClient.uploadWithResponse + // =========================================================================================== + + /** + * Single-part upload under 4MB: content validation uses CRC64 header only (no structured message). + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + /** + * Single-part upload >= 4MB: content validation uses structured message. + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(FIVE_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + /** + * Multipart (chunked) upload; content validation uses structured message on each stage block. + */ + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadWithoutContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(FIVE_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + /** + * Blob parallel upload rejects using both computeMd5 (SDK-computed MD5) and CRC64 (transfer validation checksum algorithm) at once. + */ + @Test + public void uploadWithComputeMd5AndCrc64Throws() { + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setComputeMd5(true) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options)) + .verifyErrorMatches(throwable -> throwable instanceof IllegalArgumentException + && throwable.getMessage().contains(MD5_AND_CRC64_EXCLUSIVE_MESSAGE)); + } + + // =========================================================================================== + // BlockBlobAsyncClient.uploadWithResponse (BlockBlobSimpleUpload / Put Blob) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void blockBlobSimpleUploadWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // BlockBlobAsyncClient.stageBlockWithResponse (Put Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasOnlyCrc64Headers(recorded))) + .verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasOnlyStructuredMessageHeaders(recorded))) + .verifyComplete(); + } + + @Test + public void stageBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasNoContentValidationHeaders(recorded))) + .verifyComplete(); + } + + // =========================================================================================== + // AppendBlobAsyncClient.appendBlockWithResponse (Append Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.create().then(client.appendBlockWithResponse(data, UNDER_4MB, options))) + .assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }) + .verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.create().then(client.appendBlockWithResponse(data, FIVE_MB, options))) + .assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }) + .verifyComplete(); + } + + @Test + public void appendBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.create().then(client.appendBlockWithResponse(data, FIVE_MB, options))) + .assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }) + .verifyComplete(); + } + + // =========================================================================================== + // PageBlobAsyncClient.uploadPagesWithResponse (Put Page) tests + // =========================================================================================== + + private static final int PAGE_BYTES = PageBlobClient.PAGE_BYTES; + private static final int UNDER_4MB_PAGE_ALIGNED = (UNDER_4MB / PAGE_BYTES) * PAGE_BYTES; + private static final int FOUR_MB_PAGE_ALIGNED = (4 * Constants.MB / PAGE_BYTES) * PAGE_BYTES; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(algorithm); + PageRange pageRange = new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1); + + StepVerifier + .create( + client.create(UNDER_4MB_PAGE_ALIGNED).then(client.uploadPagesWithResponse(pageRange, data, options))) + .assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }) + .verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(algorithm); + PageRange pageRange = new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1); + + StepVerifier + .create(client.create(FOUR_MB_PAGE_ALIGNED).then(client.uploadPagesWithResponse(pageRange, data, options))) + .assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }) + .verifyComplete(); + } + + @Test + public void uploadPagesWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + PageRange pageRange = new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1); + + StepVerifier + .create(client.create(FOUR_MB_PAGE_ALIGNED).then(client.uploadPagesWithResponse(pageRange, data, options))) + .assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }) + .verifyComplete(); + } + + // =========================================================================================== + // BlobAsyncClient.uploadFromFileWithResponse tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithCrc64Header(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(UNDER_4MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(FIVE_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadFromFileWithNoContentValidation() throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(FIVE_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // Exact 4MB boundary tests + // + // The cutoff between CRC64 header and structured message is exactly 4MB. + // Uploads of exactly 4MB should use structured message (>= threshold), not CRC64 header. + // =========================================================================================== + + private static final int EXACTLY_4MB = 4 * Constants.MB; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) EXACTLY_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // Progress reporting (transfer validation must be NONE/null when a progress listener is set) + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) { + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(FIVE_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB).setProgressListener(l -> { + })).setRequestConditions(new BlobRequestConditions()).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)) + .expectErrorSatisfies(ex -> assertEquals( + ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, ex.getMessage())) + .verify(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) + throws IOException { + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + File tempFile = getRandomFile(FIVE_MB); + + BlobUploadFromFileOptions options + = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB).setProgressListener(l -> { + })).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)) + .expectErrorSatisfies(ex -> assertEquals( + ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, ex.getMessage())) + .verify(); + } + + // =========================================================================================== + // Data integrity round-trip tests (upload with content validation, download, verify) + // + // Previous tests verify that the correct headers are sent. These tests verify end-to-end + // integrity: the data uploaded with CRC64/structured message can be downloaded and matches + // the original byte-for-byte. + // =========================================================================================== + + @Test + public void uploadWithCrc64RoundTripDataIntegrity() { + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void uploadWithStructuredMessageRoundTripDataIntegrity() { + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(FIVE_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void uploadChunkedWithStructuredMessageRoundTripDataIntegrity() { + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void blockBlobSimpleUploadRoundTripDataIntegrity() { + BlobAsyncClient blobClient = ccAsync.getBlobAsyncClient(generateBlobName()); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void appendBlockRoundTripDataIntegrity() { + BlobAsyncClient blobClient = ccAsync.getBlobAsyncClient(generateBlobName()); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create( + client.create().then(client.appendBlockWithResponse(data, FIVE_MB, options)).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void uploadPagesRoundTripDataIntegrity() { + BlobAsyncClient blobClient = ccAsync.getBlobAsyncClient(generateBlobName()); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + PageRange pageRange = new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1); + + StepVerifier + .create(client.create(FOUR_MB_PAGE_ALIGNED) + .then(client.uploadPagesWithResponse(pageRange, data, options)) + .then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + // =========================================================================================== + // Live-only random payload bands. + // - 256–500 MiB: parallelUpload, uploadFromFile, stageBlock — parallel staging / giant block; default transfer + // options where applicable. + // - 32–64 MiB (sequential append blocks only): appendBlockLiveRandom… — one append REST call per chunk in order. + // =========================================================================================== + + @LiveOnly // This test is too large for the test proxy. + @Test + public void parallelUploadLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-par-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + StepVerifier + .create(client.uploadWithResponse(options).then(client.downloadToFile(outFile.getPath(), true))) + .assertNext(Assertions::assertNotNull) + .verifyComplete(); + } + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void stageBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient blobClient = ccAsync.getBlobAsyncClient(generateBlobName()); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + String blockId = getBlockID(); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-stage-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BinaryData binaryData = BinaryData.fromFile(sourceFile.toPath()); + BlockBlobStageBlockOptions stageOptions = new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + StepVerifier + .create(client.stageBlockWithResponse(stageOptions) + .then(client.commitBlockList(Collections.singletonList(blockId))) + .then(client.downloadToFile(outFile.getPath(), true))) + .assertNext(Assertions::assertNotNull) + .verifyComplete(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void appendBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes + = (int) randomLongFromNamer(LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient blobClient = ccAsync.getBlobAsyncClient(generateBlobName()); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + + int maxAppendBlockBytes = client.getMaxAppendBlockBytes(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-append-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + + try { + // Append blobs must be created before any block can be appended; otherwise the first + // appendBlock call fails with 404 BlobNotFound. Create, append every block, then download, + // all chained into a single reactive pipeline (no blocking calls in the test). + StepVerifier.create(client.create() + .thenMany( + Flux.using(() -> AsynchronousFileChannel.open(sourceFile.toPath(), StandardOpenOption.READ), + channel -> FluxUtil.readFile(channel, maxAppendBlockBytes, 0, chosenPayloadSizeBytes) + .concatMap(bb -> { + AppendBlobAppendBlockOptions appendOptions = new AppendBlobAppendBlockOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + return client.appendBlockWithResponse(Flux.just(bb), bb.remaining(), appendOptions); + }), + channel -> { + try { + channel.close(); + } catch (IOException e) { + throw new java.io.UncheckedIOException(e); + } + })) + .then(blobClient.downloadToFile(outFile.getPath(), true))) + .assertNext(Assertions::assertNotNull) + .verifyComplete(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void uploadFromFileLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-uploadfromfile-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + StepVerifier.create(client.uploadFromFileWithResponse(options)) + .assertNext( + r -> assertNotNull(r.getValue().getETag(), prefix + "Missing E-Tag on upload-from-file.")) + .verifyComplete(); + StepVerifier.create(client.downloadToFile(outFile.getPath(), true)) + .assertNext(Assertions::assertNotNull) + .verifyComplete(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + // ---------- Deterministic parallel upload (async) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadPutBlobReplayableCases") + public void fuzzyParallelUploadPutBlobReplayableRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("putBlobReplay", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases: Put Block URLs include random IDs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSmallPayloadStagingCases") + public void fuzzyParallelUploadSmallPayloadRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTripAsync("smallPayloadStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment for every tuple; always staging/Put Block. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSub4MiBCases") + public void fuzzyParallelUploadSubFourMiBBlobRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("subFourMiB", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadFourMiBBoundaryStagingCases") + public void fuzzyParallelUploadFourMiBBoundaryRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTripAsync("fourMiBBoundaryStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Chunked uploads only. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadMediumMultiPartCases") + public void fuzzyParallelUploadMediumMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("mediumMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Large chunked uploads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadLargeMultiPartCases") + public void fuzzyParallelUploadLargeMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("largeMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + private void assertParallelUploadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(segmentBytes) + .setMaxSingleUploadSizeLong(segmentBytes) + .setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel upload [" + caseKind + "] payloadBytes=" + payloadBytes + + ", segmentBytes=" + segmentBytes + ", maxConcurrency=" + maxConcurrency; + + // above this threshold the fuzzy parallel upload helpers stream from a temp source file + // to avoid materializing the full payload twice in heap. + if (payloadBytes >= 96 * Constants.MB) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + int readChunkSize = (int) Math.min(8L * Constants.MB, Math.max(64 * Constants.KB, segmentBytes)); + AsynchronousFileChannel channel + = AsynchronousFileChannel.open(sourceFile.toPath(), StandardOpenOption.READ); + try { + try { + Flux data = FluxUtil.readFile(channel, readChunkSize, 0, payloadBytes); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + StepVerifier + .create(client.uploadWithResponse(options).then(client.downloadToFile(outFile.getPath(), true))) + .assertNext(Assertions::assertNotNull) + .verifyComplete(); + } finally { + channel.close(); + } + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes(), assertionMessage)) + .verifyComplete(); + } + } + + // =========================================================================================== + // Customer Provided MD5 Byte[] with Content Validation Algorithm + // =========================================================================================== + + private static final byte[] DEFAULT_MD5 = createDefaultMd5(); + private static final String MESSAGE = "Both x-ms-content-crc64 header and Content-MD5 header are present."; + + private static byte[] createDefaultMd5() { + try { + return Base64.getEncoder().encode(MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes())); + } catch (NoSuchAlgorithmException ex) { + throw LOGGER.logExceptionAsError(new RuntimeException("MD5 algorithm unavailable.", ex)); + } + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobUploadWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobAsyncClient(); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(DATA.getDefaultBinaryData()).setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.uploadWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobAsyncClient(); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), DATA.getDefaultBinaryData()) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.stageBlockWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + AppendBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getAppendBlobAsyncClient(); + + byte[] randomData = DATA.getDefaultBytes(); + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(algorithm).setContentMd5(DEFAULT_MD5); + + StepVerifier + .create( + client.create() + .then(client.appendBlockWithResponse(Flux.just(ByteBuffer.wrap(randomData)), randomData.length, + options))) + .verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) + throws NoSuchAlgorithmException { + PageBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getPageBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + byte[] md5 = MessageDigest.getInstance("MD5").digest(randomData); + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(algorithm).setContentMd5(md5); + PageRange pageRange = new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1); + + StepVerifier + .create(client.create(UNDER_4MB_PAGE_ALIGNED) + .then(client.uploadPagesWithResponse(pageRange, Flux.just(ByteBuffer.wrap(randomData)), options))) + .verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java new file mode 100644 index 000000000000..20d01d4185d1 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java @@ -0,0 +1,673 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.rest.Response; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.ProgressListener; +import com.azure.storage.blob.models.BlobDownloadContentResponse; +import com.azure.storage.blob.models.BlobDownloadResponse; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.specialized.BlobInputStream; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import static com.azure.storage.blob.specialized.BlobSeekableByteChannelTests.copy; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Sync tests for structured message decoding during blob downloads using StorageContentValidationDecodingPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobContentValidationDownloadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + // Generic "large enough to exercise multi-segment / multi-block content validation" payload. 5 MiB > the fixed + // 4 MiB structured-message segment / block size, so it still spans two segments/blocks while keeping the + // recordings (and heap/disk/CPU cost) roughly half of the previous 10-16 MiB payloads. + private static final int FIVE_MB = 5 * Constants.MB; + private static final int BLOCK_SIZE = 4 * Constants.MB; + /** + * {@link BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases()} starts at ~96 MiB; above this threshold fuzzy + * parallel download helpers use temp files + {@link BlobTestBase#compareFiles(File, File, long, long)} so the full + * payload never lives twice in heap. + */ + private static final int FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES = 96 * Constants.MB; + + /** + * Live-only random payload band for the dedicated random-size parallel-download fuzzy test + * ({@link #fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm)}): each run draws a per-run + * payload size in {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) so the structured-message + * decoder is exercised against payloads whose size varies per run in addition to the random byte contents. + */ + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + private File createRandomFile(Path tempDir, int size) throws IOException { + File file = Files.createTempFile(tempDir, "blob-cv-source", ".bin").toFile(); + + if (size > Constants.MB) { + try (OutputStream outputStream = Files.newOutputStream(file.toPath())) { + byte[] data = getRandomByteArray(Constants.MB); + int mbChunks = size / Constants.MB; + int remaining = size % Constants.MB; + for (int i = 0; i < mbChunks; i++) { + outputStream.write(data); + } + if (remaining > 0) { + outputStream.write(data, 0, remaining); + } + } + } else { + Files.write(file.toPath(), getRandomByteArray(size)); + } + + return file; + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadResponse response = blobClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, + Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobDownloadContentResponse response = blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, + Context.NONE); + byte[] result = response.getValue().toBytes(); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @ParameterizedTest + @ValueSource( + ints = { + 0, // empty file + 20, // small file + FIVE_MB, // medium file spanning multiple blocks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize, @TempDir Path tempDir) throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + Response response = blobClient.downloadToFileWithResponse(options, null, Context.NONE); + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + assertNotNull(response.getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize, @TempDir Path tempDir) + throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + Response response = blobClient.downloadToFileWithResponse(options, null, Context.NONE); + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + assertNotNull(response.getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + blobClient.downloadStreamWithResponse(outputStream, new BlobDownloadStreamOptions(), null, Context.NONE); + + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO); + BlobDownloadResponse response + = blobClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + byte[] result + = blobClient + .downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE), + null, Context.NONE) + .getValue() + .toBytes(); + + TestUtils.assertArraysEqual(data, result); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(FIVE_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobDownloadContentResponse response = blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO), null, + Context.NONE); + byte[] result = response.getValue().toBytes(); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(2 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadResponse response + = downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] data = getRandomByteArray(dataSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadResponse response + = downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + byte[] result = outputStream.toByteArray(); + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options + = new BlobInputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobInputStream inputStream = blobClient.openInputStream(options, Context.NONE); + + TestUtils.assertArraysEqual(data, convertInputStreamToByteArray(inputStream)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamRangeContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + int start = Constants.MB; + int count = 3 * Constants.MB + 257; + + blobClient.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options = new BlobInputStreamOptions().setRange(new BlobRange(start, (long) count)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setBlockSize(Constants.MB); + BlobInputStream inputStream = blobClient.openInputStream(options, Context.NONE); + + byte[] downloadedRange = convertInputStreamToByteArray(inputStream); + assertEquals(count, downloadedRange.length); + TestUtils.assertArraysEqual(data, start, downloadedRange, 0, count); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * openSeekableByteChannelRead with CRC64 content validation. + */ + @ParameterizedTest + @MethodSource("channelReadDataSupplier") + public void openSeekableByteChannelReadContentValidation(Integer streamBufferSize, int copyBufferSize, + int dataLength) throws IOException { + byte[] data = getRandomByteArray(dataLength); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + // when: "Channel initialized" + BlobSeekableByteChannelReadOptions options + = new BlobSeekableByteChannelReadOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setReadSizeInBytes(streamBufferSize); + BlobSeekableByteChannelReadResult result = blobClient.openSeekableByteChannelRead(options, Context.NONE); + SeekableByteChannel channel = result.getChannel(); + + // then: "Channel initialized to position zero" + assertEquals(0, channel.position()); + assertNotNull(result.getProperties()); + assertEquals(data.length, result.getProperties().getBlobSize()); + + // when: "read from channel" + ByteArrayOutputStream downloadedData = new ByteArrayOutputStream(); + int copied = copy(channel, downloadedData, copyBufferSize); + + // then: "channel position updated accordingly" + assertEquals(dataLength, copied); + assertEquals(dataLength, channel.position()); + + // and: "expected data downloaded" + TestUtils.assertArraysEqual(data, downloadedData.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + @Test + public void verifyProgressListenerIsCompatibleWithContentValidation(@TempDir Path tempDir) throws IOException { + byte[] data = getRandomByteArray(FIVE_MB); + + BlobClient client = cc.getBlobClient(generateBlobName()); + client.upload(BinaryData.fromBytes(data)); + + MockProgressListener mockListenerWithContentVal = new MockProgressListener(); + MockProgressListener mockListenerWithoutContentVal = new MockProgressListener(); + + ParallelTransferOptions parallelOptionsWithContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithContentVal); + ParallelTransferOptions parallelOptionsWithoutContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithoutContentVal); + + createRandomFile(tempDir, FIVE_MB); + File outFileWithContentVal = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + createRandomFile(tempDir, FIVE_MB); + File outFileWithoutContentVal = tempDir.resolve(generateResourceName(entityNo++) + ".bin").toFile(); + + Files.deleteIfExists(outFileWithContentVal.toPath()); + Files.deleteIfExists(outFileWithoutContentVal.toPath()); + + BlobDownloadToFileOptions optionsWithContentVal + = new BlobDownloadToFileOptions(outFileWithContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithContentVal) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadToFileOptions optionsWithoutContentVal + = new BlobDownloadToFileOptions(outFileWithoutContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithoutContentVal); + + client.downloadToFileWithResponse(optionsWithContentVal, null, Context.NONE); + client.downloadToFileWithResponse(optionsWithoutContentVal, null, Context.NONE); + + long expectedBytes = data.length; + assertEquals(expectedBytes, mockListenerWithContentVal.getReportedByteCount()); + assertEquals(expectedBytes, mockListenerWithoutContentVal.getReportedByteCount()); + } + + private static final class MockProgressListener implements ProgressListener { + private final AtomicLong reportedByteCount = new AtomicLong(0L); + + @Override + public void handleProgress(long bytesTransferred) { + this.reportedByteCount.updateAndGet(current -> Math.max(current, bytesTransferred)); + } + + long getReportedByteCount() { + return this.reportedByteCount.get(); + } + } + + // ---------- Fuzzy parallel download (deterministic grids) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadReplayableCases") + public void fuzzyParallelDownloadReplayableRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("replayable", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize with tiny totals; many small range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSmallMultiPartCases") + public void fuzzyParallelDownloadSmallMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("smallMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // sub-4 MiB chunked range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSubFourMiBCases") + public void fuzzyParallelDownloadSubFourMiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("subFourMiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // 4 MiB boundary tuples that fan out into chunked range GETs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadFourMiBBoundaryCases") + public void fuzzyParallelDownloadFourMiBBoundaryRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("fourMiBBoundary", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize for every tuple; chunked range GETs across many requests. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadMediumMultiPartCases") + public void fuzzyParallelDownloadMediumMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("mediumMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload >> blockSize; ~96-320 MiB downloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases") + public void fuzzyParallelDownloadLargeMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("largeMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // ~1 GiB single case; far too large for the test proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadOneGiBCases") + public void fuzzyParallelDownloadOneGiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("oneGiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + /** + * Live-only random-size parallel download fuzzy round-trip. Each run draws a per-run payload size in + * {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) and exercises both CRC64 and AUTO + * content-validation algorithms. The structured-message decoder is tested against payloads whose total size + * varies per run in addition to the random byte contents that the deterministic grids already exercise. Kept + * separate from the parameterized {@link #fuzzyParallelDownloadLargeMultiPartRoundTrip(int, long, int)} so the + * deterministic per-grid round-trips and the randomized round-trip don't share work or cost. + */ + @LiveOnly + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm algorithm) throws IOException { + int sizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + assertParallelDownloadFuzzyRoundTrip("liveRandom", sizeBytes, 8L * Constants.MB, 8, algorithm); + } + + private void assertParallelDownloadFuzzyRoundTrip(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency) throws IOException { + assertParallelDownloadFuzzyRoundTrip(caseKind, payloadBytes, blockSizeBytes, maxConcurrency, + ContentValidationAlgorithm.CRC64); + } + + private void assertParallelDownloadFuzzyRoundTrip(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency, ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + ParallelTransferOptions parallelOptions + = new ParallelTransferOptions().setBlockSizeLong(blockSizeBytes).setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel download [" + caseKind + "] payloadBytes=" + payloadBytes + + ", blockSize=" + blockSizeBytes + ", maxConcurrency=" + maxConcurrency + ", algorithm=" + algorithm; + + if (payloadBytes >= FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl", ".bin").toFile(); + Files.deleteIfExists(outFile.toPath()); + try { + BlobUploadFromFileOptions uploadOptions + = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()).setParallelTransferOptions( + new com.azure.storage.blob.models.ParallelTransferOptions().setBlockSizeLong(blockSizeBytes) + .setMaxConcurrency(maxConcurrency)); + assertNotNull(client.uploadFromFileWithResponse(uploadOptions, null, Context.NONE).getValue().getETag(), + assertionMessage); + + BlobDownloadToFileOptions downloadOptions = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + assertNotNull(client.downloadToFileWithResponse(downloadOptions, null, Context.NONE).getValue(), + assertionMessage); + + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + // Delete eagerly (not deleteOnExit) so large fuzzy payloads don't accumulate for the whole JVM and + // exhaust agent disk space when many cases run in the same live-test process. + deleteFileQuietly(sourceFile); + deleteFileQuietly(outFile); + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + client.upload(BinaryData.fromBytes(randomData), true); + + if (payloadBytes > blockSizeBytes) { + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-mp", ".bin").toFile(); + Files.deleteIfExists(outFile.toPath()); + try { + BlobDownloadToFileOptions downloadOptions + = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + assertNotNull(client.downloadToFileWithResponse(downloadOptions, null, Context.NONE).getValue(), + assertionMessage); + + byte[] downloaded = readAllBytesFromFile(outFile); + assertArrayEquals(randomData, downloaded, assertionMessage); + } finally { + deleteFileQuietly(outFile); + } + } else { + BlobDownloadContentOptions downloadOptions + = new BlobDownloadContentOptions().setContentValidationAlgorithm(algorithm); + byte[] downloaded + = client.downloadContentWithResponse(downloadOptions, null, Context.NONE).getValue().toBytes(); + assertArrayEquals(randomData, downloaded, assertionMessage); + } + } + assertTrue(hasStructuredMessageDownloadRequestHeaders(recorded, false), assertionMessage); + } + + private static byte[] readAllBytesFromFile(File file) throws IOException { + try (InputStream is = Files.newInputStream(file.toPath())) { + byte[] buffer = new byte[(int) file.length()]; + int offset = 0; + int read; + while (offset < buffer.length && (read = is.read(buffer, offset, buffer.length - offset)) != -1) { + offset += read; + } + return buffer; + } + } + + /** + * Deletes a temp file eagerly, falling back to {@link File#deleteOnExit()} only if the immediate delete fails. + * Prevents large fuzzy round-trip payloads from accumulating on disk across the many cases that share a JVM. + */ + private static void deleteFileQuietly(File file) { + if (file != null && file.exists() && !file.delete()) { + file.deleteOnExit(); + } + } + + static Stream channelReadDataSupplier() { + return Stream.of(Arguments.of(50, 40, Constants.KB), Arguments.of(Constants.KB + 50, 40, Constants.KB), + Arguments.of(null, Constants.MB, FIVE_MB)); + } + +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java new file mode 100644 index 000000000000..7c134aa2bc04 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java @@ -0,0 +1,1410 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests content validation (CRC64 / structured message) for upload operations using sync clients. + * Upload types that have no async counterpart (OutputStream, SeekableByteChannel) are tested only here. + * Async counterparts of the same operations are in {@link BlobContentValidationAsyncUploadTests}. + */ +public class BlobContentValidationUploadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + // Generic ">= 4 MiB so the single-shot upload path uses a structured message" payload for the replayable + // (non-live) tests. 5 MiB keeps a single Put Blob / Put Block / Append Block request (still above the 4 MiB + // structured-message threshold) while roughly halving recording and heap/disk/CPU cost vs the old 10 MiB. + private static final int FIVE_MB = 5 * Constants.MB; + /* single-shot uploads with length < 4MB use CRC64 header; >= 4MB use structured message. */ + private static final int UNDER_4MB = 2 * Constants.MB; + + /** + * Live-only random payload band (256–500 MiB, inclusive upper bound via {@code randomLongFromNamer}+1) for + * {@code uploadWithResponse}, {@code uploadFromFileWithResponse}, and single-block {@code stageBlock}. + */ + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + /** + * Live-only random payload band for sequential append-block puts only. + * {@code Flux.concatMap} issues one append REST call per chunk in + * order (not parallel staging); use a smaller band than {@link #LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE}. + */ + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE = 32L * Constants.MB; + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE = 64L * Constants.MB; + + private static final String MD5_AND_CRC64_EXCLUSIVE_MESSAGE + = "Only one form of transactional content validation may be used."; + + // =========================================================================================== + // BlobClient.uploadWithResponse + // =========================================================================================== + + /** + * Single-shot upload under 4MB: content validation uses CRC64 header only (no structured message). + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + /** + * Single-shot upload >= 4MB: content validation uses structured message. + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(FIVE_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + /** + * Multi-shot (chunked) upload; content validation uses structured message on each stage block. + */ + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadWithoutContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(FIVE_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + /** + * Blob parallel upload rejects using both computeMd5 (SDK-computed MD5) and CRC64 (transfer validation checksum algorithm) at once. + */ + @Test + public void uploadWithComputeMd5AndCrc64Throws() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setComputeMd5(true) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadWithResponse(options, null, Context.NONE)); + assertTrue(ex.getMessage().contains(MD5_AND_CRC64_EXCLUSIVE_MESSAGE)); + } + + // =========================================================================================== + // BlockBlobClient.uploadWithResponse (BlockBlobSimpleUpload / Put Blob) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void blockBlobSimpleUploadWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlockBlobClient.stageBlockWithResponse (Put Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void stageBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // AppendBlobClient.appendBlockWithResponse (Append Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(algorithm); + + assertNotNull( + client.appendBlockWithResponse(data, UNDER_4MB, options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(algorithm); + + assertNotNull(client.appendBlockWithResponse(data, FIVE_MB, options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void appendBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.appendBlockWithResponse(data, FIVE_MB, options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // PageBlobClient.uploadPagesWithResponse (Put Page) tests + // =========================================================================================== + + private static final int PAGE_BYTES = PageBlobClient.PAGE_BYTES; + private static final int UNDER_4MB_PAGE_ALIGNED = (UNDER_4MB / PAGE_BYTES) * PAGE_BYTES; + private static final int FOUR_MB_PAGE_ALIGNED = (4 * Constants.MB / PAGE_BYTES) * PAGE_BYTES; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(algorithm); + PageRange pageRange = new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1); + + assertNotNull( + client.uploadPagesWithResponse(pageRange, data, options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(algorithm); + PageRange pageRange = new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1); + + assertNotNull( + client.uploadPagesWithResponse(pageRange, data, options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadPagesWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + PageRange pageRange = new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1); + + assertNotNull( + client.uploadPagesWithResponse(pageRange, data, options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlobClient.uploadFromFileWithResponse tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithCrc64Header(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(UNDER_4MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(FIVE_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadFromFileWithNoContentValidation() throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(FIVE_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // Sync BlobOutputStream tests (getBlobOutputStream) + // =========================================================================================== + + // --- AppendBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (BlobOutputStream os = client + .getBlobOutputStream(new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + + try (BlobOutputStream os = client + .getBlobOutputStream(new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void appendBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + + try (BlobOutputStream os = client.getBlobOutputStream( + new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // --- BlockBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) + throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void blockBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // --- PageBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void pageBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void pageBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void pageBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlockBlobClient.openSeekableByteChannelWrite tests + // =========================================================================================== + + @LiveOnly // Seekable channel staging uses Put Block with random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void seekableByteChannelWriteWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(algorithm))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void seekableByteChannelWriteWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(algorithm))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void seekableByteChannelWriteWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // Exact 4MB boundary tests + // + // The cutoff between CRC64 header and structured message is exactly 4MB. + // Uploads of exactly 4MB should use structured message (>= threshold), not CRC64 header. + // =========================================================================================== + + private static final int EXACTLY_4MB = 4 * Constants.MB; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) EXACTLY_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + // =========================================================================================== + // Progress reporting (transfer validation must be NONE/null when a progress listener is set) + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) { + BlobClient client = cc.getBlobClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(FIVE_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB).setProgressListener(l -> { + })).setRequestConditions(new BlobRequestConditions()).setContentValidationAlgorithm(algorithm); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadWithResponse(options, null, Context.NONE)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) + throws IOException { + BlobClient client = cc.getBlobClient(generateBlobName()); + + File tempFile = getRandomFile(FIVE_MB); + + BlobUploadFromFileOptions options + = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB).setProgressListener(l -> { + })).setContentValidationAlgorithm(algorithm); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadFromFileWithResponse(options, null, Context.NONE)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + // =========================================================================================== + // Data integrity round-trip tests (upload with content validation, download, verify) + // + // Previous tests verify that the correct headers are sent. These tests verify end-to-end + // integrity: the data uploaded with CRC64/structured message can be downloaded and matches + // the original byte-for-byte. + // =========================================================================================== + + @Test + public void uploadWithCrc64RoundTripDataIntegrity() { + BlobClient client = cc.getBlobClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (CRC64 header path)"); + } + + @Test + public void uploadWithStructuredMessageRoundTripDataIntegrity() { + BlobClient client = cc.getBlobClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(FIVE_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (structured message path)"); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void uploadChunkedWithStructuredMessageRoundTripDataIntegrity() { + BlobClient client = cc.getBlobClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, + "Downloaded data must match uploaded data (chunked structured message path)"); + } + + @Test + public void blockBlobSimpleUploadRoundTripDataIntegrity() { + BlobClient blobClient = cc.getBlobClient(generateBlobName()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, + "Downloaded data must match uploaded data (block blob simple upload)"); + } + + @Test + public void appendBlockRoundTripDataIntegrity() { + BlobClient blobClient = cc.getBlobClient(generateBlobName()); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(FIVE_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.appendBlockWithResponse(data, FIVE_MB, options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (append block)"); + } + + @Test + public void uploadPagesRoundTripDataIntegrity() { + BlobClient blobClient = cc.getBlobClient(generateBlobName()); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + PageRange pageRange = new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1); + + client.uploadPagesWithResponse(pageRange, data, options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (page blob upload pages)"); + } + + @Test + public void uploadFromFileRoundTripDataIntegrity() throws IOException { + BlobClient client = cc.getBlobClient(generateBlobName()); + + byte[] randomData = getRandomByteArray(FIVE_MB); + File tempFile = File.createTempFile("blob-cv-roundtrip", ".bin"); + tempFile.deleteOnExit(); + try (java.io.FileOutputStream fos = new java.io.FileOutputStream(tempFile)) { + fos.write(randomData); + } + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) FIVE_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadFromFileWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded file data"); + } + + // =========================================================================================== + // Live-only random payload bands. + // - 256–500 MiB: parallelUpload, uploadFromFile, stageBlock, block BlobOutputStream, SeekableByteChannel — + // parallel staging or single giant block / default transfer options as applicable. + // - 32–64 MiB (sequential append blocks only): appendBlobAppendBlocksLiveRandom… — one append REST call per + // chunk in order. + // =========================================================================================== + + @LiveOnly // This test is too large for the test proxy. + @Test + public void parallelUploadLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient client = cc.getBlobClient(generateBlobName()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-par-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + } + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void stageBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = cc.getBlobClient(generateBlobName()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + String blockId = getBlockID(); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-stage-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BinaryData binaryData = BinaryData.fromFile(sourceFile.toPath()); + BlockBlobStageBlockOptions stageOptions = new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.stageBlockWithResponse(stageOptions, null, Context.NONE); + client.commitBlockList(Collections.singletonList(blockId)); + blobClient.downloadToFile(outFile.getPath(), true); + + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void appendBlobAppendBlocksLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes + = (int) randomLongFromNamer(LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = cc.getBlobClient(generateBlobName()); + AppendBlobClient client = blobClient.getAppendBlobClient(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-append-blocks-dl", ".bin").toFile(); + outFile.deleteOnExit(); + client.create(); + int maxAppendBlockBytes = client.getMaxAppendBlockBytes(); + try { + try (FileInputStream fis = new FileInputStream(sourceFile)) { + long remaining = chosenPayloadSizeBytes; + byte[] buf = new byte[maxAppendBlockBytes]; + while (remaining > 0) { + int chunk = (int) Math.min(maxAppendBlockBytes, remaining); + int totalRead = 0; + while (totalRead < chunk) { + int n = fis.read(buf, totalRead, chunk - totalRead); + if (n == -1) { + throw new EOFException( + prefix + "Unexpected EOF after " + totalRead + " bytes of chunk."); + } + totalRead += n; + } + ByteArrayInputStream chunkStream = new ByteArrayInputStream(buf, 0, chunk); + AppendBlobAppendBlockOptions appendOptions = new AppendBlobAppendBlockOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.appendBlockWithResponse(chunkStream, chunk, appendOptions, null, Context.NONE); + remaining -= chunk; + } + } + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void uploadFromFileLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient client = cc.getBlobClient(generateBlobName()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-uploadfromfile-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadFromFileWithResponse(options, null, Context.NONE); + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void blockBlobOutputStreamLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = cc.getBlobClient(generateBlobName()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + // Explicit parallel-transfer tuning (8 MiB blocks × concurrency 8) on the stream ingest path. + ParallelTransferOptions parallelTransferOptions + = new ParallelTransferOptions().setBlockSizeLong(8L * Constants.MB) + .setMaxSingleUploadSizeLong(8L * Constants.MB) + .setMaxConcurrency(8); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-block-os-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (BlobOutputStream outputStream = client.getBlobOutputStream( + new BlockBlobOutputStreamOptions().setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) { + Files.copy(sourceFile.toPath(), outputStream); + } + + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void seekableByteChannelWriteLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = cc.getBlobClient(generateBlobName()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-sbc-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (java.nio.channels.SeekableByteChannel seekableByteChannel + = client.openSeekableByteChannelWrite(new BlockBlobSeekableByteChannelWriteOptions( + BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) { + Files.copy(sourceFile.toPath(), Channels.newOutputStream(seekableByteChannel)); + } + + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + // ---------- Deterministic parallel upload ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadPutBlobReplayableCases") + public void fuzzyParallelUploadPutBlobReplayableRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("putBlobReplay", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases: Put Block URLs include random IDs + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSmallPayloadStagingCases") + public void fuzzyParallelUploadSmallPayloadRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTrip("smallPayloadStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment for every tuple; always staging/Put Block. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSub4MiBCases") + public void fuzzyParallelUploadSubFourMiBBlobRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("subFourMiB", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadFourMiBBoundaryStagingCases") + public void fuzzyParallelUploadFourMiBBoundaryRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTrip("fourMiBBoundaryStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment throughout; chunked upload. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadMediumMultiPartCases") + public void fuzzyParallelUploadMediumMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("mediumMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload >> segment throughout; chunked upload / large payloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadLargeMultiPartCases") + public void fuzzyParallelUploadLargeMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("largeMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + private void assertParallelUploadFuzzyRoundTrip(String caseKind, int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + BlobClient client = cc.getBlobClient(generateBlobName()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(segmentBytes) + .setMaxSingleUploadSizeLong(segmentBytes) + .setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel upload [" + caseKind + "] payloadBytes=" + payloadBytes + + ", segmentBytes=" + segmentBytes + ", maxConcurrency=" + maxConcurrency; + + // above this threshold the fuzzy parallel upload helpers stream from a temp source file + // to avoid materializing the full payload twice in heap. + if (payloadBytes >= 96 * Constants.MB) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + } + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + InputStream data = new ByteArrayInputStream(randomData); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, assertionMessage); + } + } + + // =========================================================================================== + // Customer Provided MD5 Byte[] with Content Validation Algorithm + // =========================================================================================== + + private static final byte[] DEFAULT_MD5 = createDefaultMd5(); + private static final String MESSAGE = "Both x-ms-content-crc64 header and Content-MD5 header are present."; + + private static byte[] createDefaultMd5() { + try { + return Base64.getEncoder().encode(MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes())); + } catch (NoSuchAlgorithmException ex) { + throw LOGGER.logExceptionAsError(new RuntimeException("MD5 algorithm unavailable.", ex)); + } + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobUploadWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobClient(); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(DATA.getDefaultBinaryData()).setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e + = assertThrows(BlobStorageException.class, () -> client.uploadWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobClient(); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), DATA.getDefaultBinaryData()) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.stageBlockWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + AppendBlobClient client + = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getAppendBlobClient(); + client.create(); + + byte[] randomData = DATA.getDefaultBytes(); + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions().setContentValidationAlgorithm(algorithm).setContentMd5(DEFAULT_MD5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.appendBlockWithResponse(new ByteArrayInputStream(randomData), randomData.length, options, null, + Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) + throws NoSuchAlgorithmException { + PageBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + byte[] md5 = MessageDigest.getInstance("MD5").digest(randomData); + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions().setContentValidationAlgorithm(algorithm).setContentMd5(md5); + PageRange pageRange = new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1); + + BlobStorageException e = assertThrows(BlobStorageException.class, () -> client + .uploadPagesWithResponse(pageRange, new ByteArrayInputStream(randomData), options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java index 99d925bff60c..71345472c393 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java @@ -12,6 +12,9 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.AddDatePolicy; @@ -61,6 +64,9 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.policy.RequestRetryOptions; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; import com.azure.storage.common.test.shared.StorageCommonTestUtils; import com.azure.storage.common.test.shared.TestAccount; import com.azure.storage.common.test.shared.TestDataFactory; @@ -91,7 +97,10 @@ import java.util.Map; import java.util.Objects; import java.util.Queue; +import java.util.Random; +import java.util.UUID; import java.util.concurrent.Callable; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -904,6 +913,37 @@ protected File getRandomFile(int size) throws IOException { return StorageCommonTestUtils.getRandomFile(size, testResourceNamer); } + /** + * Pseudorandom {@link Random} seeded from {@code testResourceNamer.randomUuid()} so values replay in playback + * mode (unlike {@link java.util.concurrent.ThreadLocalRandom}). + */ + protected Random newRandomFromNamer() { + long seed = UUID.fromString(testResourceNamer.randomUuid()).getMostSignificantBits() & Long.MAX_VALUE; + return new Random(seed); + } + + /** + * Pseudorandom int in {@code [origin, bound)} from the namer. Consumes one recorded UUID. + */ + protected int randomIntFromNamer(int origin, int bound) { + int span = bound - origin; + if (span <= 0) { + throw new IllegalArgumentException("bound must be greater than origin"); + } + return origin + newRandomFromNamer().nextInt(span); + } + + /** + * Pseudorandom long in {@code [origin, bound)} from the namer. Consumes one recorded UUID. + */ + protected long randomLongFromNamer(long origin, long bound) { + long span = bound - origin; + if (span <= 0) { + throw new IllegalArgumentException("bound must be greater than origin"); + } + return origin + Math.floorMod(newRandomFromNamer().nextLong(), span); + } + /*https://learn.microsoft.com/en-us/rest/api/storageservices/define-stored-access-policy#creating-or-modifying-a-stored-access-policy Second note, it can take up to 30 seconds to set/create an access policy and this was causing flakeyness in the live test pipeline */ @@ -1344,4 +1384,469 @@ public static HttpPipelinePolicy getAddHeadersAndQueryPolicy(Map return next.process(); }; } + + protected static boolean hasOnlyStructuredMessageHeaders(List recordedRequestHeaders) { + return hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, true); + } + + protected static boolean hasStructuredMessageDownloadRequestHeaders(HttpHeaders recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.getSize() == 0) { + return false; + } + return hasStructuredMessageDownloadRequestHeaders(Collections.singletonList(recordedRequestHeaders), false); + } + + protected static boolean hasStructuredMessageDownloadResponseHeaders(HttpHeaders headers) { + return validateBasicHeaders(headers) + && StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE + .equalsIgnoreCase(headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME)) + && hasValidStructuredContentLengthHeader(headers); + } + + protected static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + HttpHeaders recordedRequestHeaders, HttpHeaders recordedResponseHeaders) { + return getRequestAndResponseHeaderSniffer(targetUrlPrefix, headers -> { + synchronized (recordedRequestHeaders) { + recordedRequestHeaders.setAllHttpHeaders(headers); + } + }, recordedResponseHeaders); + } + + protected static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + List recordedRequestHeaders, HttpHeaders recordedResponseHeaders) { + return getRequestAndResponseHeaderSniffer(targetUrlPrefix, recordedRequestHeaders::add, + recordedResponseHeaders); + } + + private static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + Consumer requestRecorder, HttpHeaders recordedResponseHeaders) { + return new HttpPipelinePolicy() { + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + requestRecorder.accept(context.getHttpRequest().getHeaders()); + return next.process().map(response -> { + if (response.getRequest().getHttpMethod() == HttpMethod.GET + && response.getRequest().getUrl().toString().startsWith(targetUrlPrefix)) { + synchronized (recordedResponseHeaders) { + recordedResponseHeaders.setAllHttpHeaders(response.getHeaders()); + } + } + return response; + }); + } + }; + } + + protected static boolean hasStructuredMessageDownloadRequestHeaders(List recordedRequestHeaders, + boolean requireStructuredContentLength) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + // Only consider requests where any of the structured-message or CRC64-related headers is present. + List headersWithContentValidation = recordedRequestHeaders.stream().filter(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + return bodyType != null || contentLength != null || contentCrc64 != null; + }).collect(Collectors.toList()); + // If no requests had any content-validation headers at all, we cannot claim structured-message was applied. + if (headersWithContentValidation.isEmpty()) { + return false; + } + // All requests that used any content-validation header must be consistent structured-message requests. + return headersWithContentValidation.stream().allMatch(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + if (!StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE.equals(bodyType) || contentCrc64 != null) { + return false; + } + if (!requireStructuredContentLength) { + return true; + } + // Require non-blank content length that parses as non-negative long (same format as policy uses). + // Rejects empty string, whitespace, or non-numeric values so we never return true when + // structured message was not actually applied. + if (contentLength == null || contentLength.trim().isEmpty()) { + return false; + } + try { + return Long.parseLong(contentLength.trim()) >= 0; + } catch (NumberFormatException e) { + return false; + } + }); + } + + protected static boolean hasOnlyCrc64Headers(List recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + // Only consider requests where any of the structured-message or CRC64-related headers is present. + List headersWithContentValidation = recordedRequestHeaders.stream().filter(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + return bodyType != null || contentLength != null || contentCrc64 != null; + }).collect(Collectors.toList()); + // If no requests had any content-validation headers at all, we cannot claim CRC64 was applied. + if (headersWithContentValidation.isEmpty()) { + return false; + } + // All requests that used any content-validation header must be consistent CRC64-only requests. + return headersWithContentValidation.stream().allMatch(headers -> { + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + // Require CRC64 header to be present and non-blank (policy sets Base64-encoded 8 bytes). + // Reject empty/whitespace so we never return true when CRC64 was not actually applied. + if (contentCrc64 == null || contentCrc64.trim().isEmpty() || bodyType != null || contentLength != null) { + return false; + } + return true; + }); + } + + protected static boolean hasNoContentValidationHeaders(List recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + return recordedRequestHeaders.stream().allMatch(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + // All three must be absent (null). If any header is present, even with empty value, we return false. + return bodyType == null && contentLength == null && contentCrc64 == null; + }); + } + + /** + * Creates a BlobClient that records all outgoing request headers into the supplied list. + * Each test should use its own list so tests can run concurrently. + */ + protected BlobClient createBlobClientWithRequestSniffer(List recordedRequestHeaders) { + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recordedRequestHeaders.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + BlobServiceClient serviceClient = getServiceClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + ENVIRONMENT.getPrimaryAccount().getBlobEndpoint(), sniffPolicy); + return serviceClient.getBlobContainerClient(containerName).getBlobClient(generateBlobName()); + } + + /** + * Creates a BlobAsyncClient that records all outgoing request headers into the supplied list. + */ + protected BlobAsyncClient createBlobAsyncClientWithRequestSniffer(List recordedRequestHeaders) { + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recordedRequestHeaders.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + BlobServiceAsyncClient serviceClient = getServiceAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + ENVIRONMENT.getPrimaryAccount().getBlobEndpoint(), sniffPolicy); + return serviceClient.getBlobContainerAsyncClient(containerName).getBlobAsyncClient(generateBlobName()); + } + + protected static long expectedStructuredMessageEncodedLength(int unencodedContentBytes) { + return new StructuredMessageEncoder(unencodedContentBytes, + StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64) + .getEncodedMessageLength(); + } + + /** + * Sum of encoded lengths per block upload (each HTTP request carries its own structured message wrapper). + */ + protected static long expectedStructuredMessageEncodedLengthChunked(int totalUnencodedBytes, long blockSizeBytes) { + long sum = 0; + int remaining = totalUnencodedBytes; + while (remaining > 0) { + int chunk = (int) Math.min(remaining, blockSizeBytes); + sum += expectedStructuredMessageEncodedLength(chunk); + remaining -= chunk; + } + return sum; + } + + /** + * Every tuple keeps payloadBytes <= blockSizeBytes, so the parallel download path issues a single GET (no + * follow-on range requests for additional blocks), which replays under the test proxy. + *

+ * Sizes are deliberately non-power-of-two (e.g. 7 * KB + 3) and use mixed block ceilings (64 KiB through + * multi-MiB) to catch alignment and decoder edge cases at structural boundaries (message header, segment + * footer, message footer); the 4 MiB boundary row exercises the exact service-side default segment length. + */ + protected static Stream fuzzyParallelDownloadReplayableCases() { + return Stream.of(Arguments.of(1, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 128L * Constants.KB, 4), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 1), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 8), + Arguments.of(199 * Constants.KB + 5, 512L * Constants.KB, 2), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.MB, 8), + Arguments.of(896 * Constants.KB + 101, 1L * Constants.MB, 6), + Arguments.of(2 * Constants.MB - 1, 4L * Constants.MB, 4), + Arguments.of(2 * Constants.MB + 33, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB - 1, 4L * Constants.MB, 2), + Arguments.of(4 * Constants.MB, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB, 7L * Constants.MB + 919, 3)); + } + + /** + * Every tuple keeps payloadBytes <= segmentBytes, so the parallel upload path issues a single Put Blob (no + * Put Block IDs), which replays under the test proxy unlike staging-heavy cases. + *

+ * Sizes are deliberately non-power-of-two (e.g. 7 * KB + 3) and use mixed segment ceilings (64 KiB + * through multi-MiB) to catch alignment and buffer edge cases; rows include the exact 4 MiB service boundary + * and several concurrency values (1–8) to exercise parallel request fan-out without live-only staging. + */ + protected static Stream fuzzyParallelUploadPutBlobReplayableCases() { + return Stream.of(Arguments.of(7 * Constants.KB + 3, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 128L * Constants.KB, 4), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 1), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 8), + Arguments.of(199 * Constants.KB + 5, 512L * Constants.KB, 2), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.MB, 8), + Arguments.of(896 * Constants.KB + 101, 1L * Constants.MB, 6), + Arguments.of(4 * Constants.MB, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB, 7L * Constants.MB + 919, 3)); + } + + /** + * payloadBytes > blockSizeBytes but totals stay in the hundreds of KiB, so downloads issue many small ranged + * GETs even though the blob itself is small. Live-only because chunked range GETs across many tiny requests + * produce per-block proxy churn similar to the upload-side staging cases. + *

+ * One row pairs a ~200 KiB payload with a 64 KiB block size and moderate concurrency; the other uses a + * ~512 KiB payload with a 1 KiB block size to force many tiny range GETs (stress decoder framing and + * scheduling) without the cost of the large multi-part grids. + */ + protected static Stream fuzzyParallelDownloadSmallMultiPartCases() { + return Stream.of(Arguments.of(200 * Constants.KB, 64L * Constants.KB, 3), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.KB, 1)); + } + + /** + * payloadBytes > segmentBytes, so uploads still go through Put Block staging even though totals are only + * hundreds of KiB—too small for the proxy when block IDs vary per run. + *

+ * One row pairs a ~200 KiB payload with a 64 KiB segment and moderate concurrency; the other uses a + * ~512 KiB payload with a 1 KiB segment to force many tiny blocks (stress scheduling and per-block CRC64 + * framing) without the cost of the large multi-part grids. + */ + protected static Stream fuzzyParallelUploadSmallPayloadStagingCases() { + return Stream.of(Arguments.of(200 * Constants.KB, 64L * Constants.KB, 3), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.KB, 1)); + } + + /** + * payloadBytes > blockSizeBytes and payloadBytes <= 4 * Constants.MB - 1 (the ceiling field), so the blob + * stays strictly under the 4 MiB single-shot CRC64-header path while downloads remain chunked across + * multiple range GETs. + *

+ * Values mix MiB/KiB block sizes with offsets (e.g. + 19, - 903) so part counts and last-block lengths are + * not powers of two; the last rows hug ceiling with awkward divisors in blockSizeBytes to stress remainder + * handling near the sub-4 MiB limit. + */ + protected static Stream fuzzyParallelDownloadSubFourMiBCases() { + final int ceiling = (4 * Constants.MB) - 1; + return Stream.of(Arguments.of(1 * Constants.MB + 1, 1L * Constants.MB, 1), + Arguments.of(1 * Constants.MB + 1, 2L * Constants.KB, 8), + Arguments.of((5 * Constants.MB) / 4 + 19, 256L * Constants.KB, 4), + Arguments.of(2 * Constants.MB - 903, 1L * Constants.MB, 2), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.KB, 1), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.MB, 8), + Arguments.of((11 * Constants.MB) / 4 - 17, 512L * Constants.KB, 6), + Arguments.of(3 * Constants.MB - 777, 2L * Constants.MB, 8), + Arguments.of(3 * Constants.MB - 1, 1L * Constants.MB, 1), Arguments.of(ceiling - 511, 1L * Constants.MB, 4), + Arguments.of(ceiling - 511, 1L * Constants.MB + 511, 2), + Arguments.of(ceiling, (long) (ceiling / 7 + 17), 3), Arguments.of(ceiling, (long) (ceiling / 2 + 1), 8)); + } + + /** + * payloadBytes > segmentBytes and payloadBytes <= 4 * Constants.MB - 1 (the ceiling field), so the blob + * stays strictly under the 4 MiB transactional CRC64-header path while uploads remain + * chunked—live-only because of Put Block identity churn. + *

+ * Values mix MiB/KiB segment sizes with offsets (e.g. + 19, - 903) so part counts and last-block + * lengths are not powers of two; the last rows hug ceiling with awkward divisors in segmentBytes to + * stress remainder handling near the sub-4 MiB limit. + */ + protected static Stream fuzzyParallelUploadSub4MiBCases() { + final int ceiling = (4 * Constants.MB) - 1; + return Stream.of(Arguments.of(1 * Constants.MB + 1, 1L * Constants.MB, 1), + Arguments.of(1 * Constants.MB + 1, 2L * Constants.KB, 8), + Arguments.of((5 * Constants.MB) / 4 + 19, 256L * Constants.KB, 4), + Arguments.of(2 * Constants.MB - 903, 1L * Constants.MB, 2), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.KB, 1), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.MB, 8), + Arguments.of((11 * Constants.MB) / 4 - 17, 512L * Constants.KB, 6), + Arguments.of(3 * Constants.MB - 777, 2L * Constants.MB, 8), + Arguments.of(3 * Constants.MB - 1, 1L * Constants.MB, 1), Arguments.of(ceiling - 511, 1L * Constants.MB, 4), + Arguments.of(ceiling - 511, 1L * Constants.MB + 511, 2), + Arguments.of(ceiling, (long) (ceiling / 7 + 17), 3), Arguments.of(ceiling, (long) (ceiling / 2 + 1), 8)); + } + + /** + * Centers on 4 * Constants.MB - 1, exactly 4 * Constants.MB, and just above 4 MiB, with block sizes spanning + * KiB through multi-MiB - exercising the SDK/service boundary where single-shot vs chunked range GET and + * CRC64 header vs structured-message rules flip, while keeping deterministic single-GET coverage in the + * replayable supplier above. + *

+ * Includes near-boundary payloads (e.g. -8192, +31, +8191 from 4 MiB) so neither total size nor last segment + * length aligns to typical buffer multiples. + */ + protected static Stream fuzzyParallelDownloadFourMiBBoundaryCases() { + final int minus = (4 * Constants.MB) - 1; + final int plus = (4 * Constants.MB) + 1; + return Stream.of(Arguments.of(minus, 1L * Constants.MB, 1), Arguments.of(minus, 512L * Constants.KB, 6), + Arguments.of(minus, 2L * Constants.MB, 8), Arguments.of((4 * Constants.MB) - 8192, 1L * Constants.KB, 4), + Arguments.of(4 * Constants.MB, (long) (4 * Constants.MB / 2), 8), + Arguments.of(4 * Constants.MB, 256L * Constants.KB, 2), Arguments.of(plus, 1L * Constants.MB, 1), + Arguments.of(plus, 2L * Constants.MB, 8), Arguments.of(plus, 1L * Constants.KB, 7), + Arguments.of((4 * Constants.MB) + 31, 511L * Constants.KB + 409, 4), + Arguments.of((4 * Constants.MB) + 8191, 3L * Constants.MB - 413, 6)); + } + + /** + * Centers on 4 * Constants.MB - 1, exactly 4 * Constants.MB, and just above 4 MiB, with segment + * sizes spanning KiB through multi-MiB—exercising the SDK/service boundary where single-shot vs block staging and + * CRC64 header vs structured-message rules flip, while keeping deterministic Put Blob coverage in the replayable + * supplier above. + *

+ * Includes near-boundary payloads (e.g. -8192, +31, +8191 from 4 MiB) so neither total size nor last segment + * length aligns to typical buffer multiples. + */ + protected static Stream fuzzyParallelUploadFourMiBBoundaryStagingCases() { + final int minus = (4 * Constants.MB) - 1; + final int plus = (4 * Constants.MB) + 1; + return Stream.of(Arguments.of(minus, 1L * Constants.MB, 1), Arguments.of(minus, 512L * Constants.KB, 6), + Arguments.of(minus, 2L * Constants.MB, 8), Arguments.of((4 * Constants.MB) - 8192, 1L * Constants.KB, 4), + Arguments.of(4 * Constants.MB, (long) (4 * Constants.MB / 2), 8), + Arguments.of(4 * Constants.MB, 256L * Constants.KB, 2), Arguments.of(plus, 1L * Constants.MB, 1), + Arguments.of(plus, 2L * Constants.MB, 8), Arguments.of(plus, 1L * Constants.KB, 7), + Arguments.of((4 * Constants.MB) + 31, 511L * Constants.KB + 409, 4), + Arguments.of((4 * Constants.MB) + 8191, 3L * Constants.MB - 413, 6)); + } + + /** + * payloadBytes > blockSizeBytes, so downloads always go through multiple ranged GETs (parallel download + * fan-out) with totals roughly 6-80 MiB. Large enough to exercise the structured-message decoder over + * multiple HTTP responses, but cheaper than {@link #fuzzyParallelDownloadLargeMultiPartCases}. + *

+ * Block sizes step through common service limits (1-8 MiB, half-MiB tail values); concurrency 1-8 pairs + * with imbalanced payloads (e.g. 701, 333) to flush merge/retry edge cases. + */ + protected static Stream fuzzyParallelDownloadMediumMultiPartCases() { + return Stream.of(Arguments.of(6 * Constants.MB + 701, Constants.MB, 1), + Arguments.of(6 * Constants.MB + 701, 3L * Constants.MB + 271, 4), + Arguments.of(9 * Constants.MB + 333, 2L * Constants.MB, 1), + Arguments.of(9 * Constants.MB + 333, 3L * Constants.MB + 199, 8), + Arguments.of(12 * Constants.MB + 901, 4L * Constants.MB + 901, 2), + Arguments.of(14 * Constants.MB, 500L * Constants.KB + 13, 6), + Arguments.of(18 * Constants.MB - 4021, 5L * Constants.MB - 701, 3), + Arguments.of(24 * Constants.MB, 8L * Constants.MB, 8), + Arguments.of(28 * Constants.MB + 56789, 7L * Constants.MB + 13, 2), + Arguments.of(31 * Constants.MB, 1024L * Constants.KB + 17, 4), + Arguments.of(40 * Constants.MB + 12345, 7L * Constants.MB + 13, 3), + Arguments.of(48 * Constants.MB - 777, 5L * Constants.MB + 809L * Constants.KB, 6), + Arguments.of(56 * Constants.MB + 19, 9L * Constants.MB + 4096, 8), + Arguments.of(72 * Constants.MB, 4L * Constants.MB + 65536, 8), + Arguments.of(80 * Constants.MB + 321, 13L * Constants.MB - 3073, 1)); + } + + /** + * All rows keep payloadBytes > segmentBytes with totals roughly 6–80 MiB—large enough for meaningful parallel + * block fan-out and structured-message segments, but cheaper than {@link #fuzzyParallelUploadLargeMultiPartCases}. + *

+ * Block sizes step through common service limits (1–8 MiB, half-MiB tail values); concurrency 1–8 pairs with + * imbalanced payloads (e.g. 701, 333) to flush merge/retry edge cases. + */ + protected static Stream fuzzyParallelUploadMediumMultiPartCases() { + return Stream.of(Arguments.of(6 * Constants.MB + 701, Constants.MB, 1), + Arguments.of(6 * Constants.MB + 701, 3L * Constants.MB + 271, 4), + Arguments.of(9 * Constants.MB + 333, 2L * Constants.MB, 1), + Arguments.of(9 * Constants.MB + 333, 3L * Constants.MB + 199, 8), + Arguments.of(12 * Constants.MB + 901, 4L * Constants.MB + 901, 2), + Arguments.of(14 * Constants.MB, 500L * Constants.KB + 13, 6), + Arguments.of(18 * Constants.MB - 4021, 5L * Constants.MB - 701, 3), + Arguments.of(24 * Constants.MB, 8L * Constants.MB, 8), + Arguments.of(28 * Constants.MB + 56789, 7L * Constants.MB + 13, 2), + Arguments.of(31 * Constants.MB, 1024L * Constants.KB + 17, 4), + Arguments.of(40 * Constants.MB + 12345, 7L * Constants.MB + 13, 3), + Arguments.of(48 * Constants.MB - 777, 5L * Constants.MB + 809L * Constants.KB, 6), + Arguments.of(56 * Constants.MB + 19, 9L * Constants.MB + 4096, 8), + Arguments.of(72 * Constants.MB, 4L * Constants.MB + 65536, 8), + Arguments.of(80 * Constants.MB + 321, 13L * Constants.MB - 3073, 1)); + } + + /** + * Stresses high block counts and long-running parallel downloads (~96-320 MiB payloads) with service-realistic + * block sizes (8-61 MiB class) and heavy concurrency. + *

+ * The final rows use named near-256/288/320 MiB totals with irregular byte tails to keep total bytes and block + * remainders off common multiples while still bounding runtime for Live-only CI. + */ + protected static Stream fuzzyParallelDownloadLargeMultiPartCases() { + final int payload257MiBPlus = (int) (257L * Constants.MB + 18881); + final int payload288MiBPlus = (int) (288L * Constants.MB + 7777); + final int payload320MiBPlus = (int) (320L * Constants.MB + 1999); + return Stream.of(Arguments.of(96 * Constants.MB + 17, 8L * Constants.MB + 511, 2), + Arguments.of(112 * Constants.MB, 15L * Constants.MB + 4096, 8), + Arguments.of(128 * Constants.MB + 45673, 17L * Constants.MB - 11264 + 173, 4), + Arguments.of(160 * Constants.MB + 12345, 12L * Constants.MB + 8192, 8), + Arguments.of(192 * Constants.MB + 9876, 31L * Constants.MB - 513, 8), + Arguments.of(224 * Constants.MB, 23L * Constants.MB + 524288, 8), + Arguments.of(payload257MiBPlus, 61L * Constants.MB + 23L * Constants.KB, 6), + Arguments.of(payload288MiBPlus, 36L * Constants.MB + 513, 8), + Arguments.of(payload320MiBPlus, 16L * Constants.MB + 511, 8)); + } + + /** + * Stresses high block counts and long-running parallel uploads (~96–320 MiB payloads) with service-realistic segment + * sizes (8–61 MiB class) and heavy concurrency. + *

+ * The final rows use named near-256/288/320 MiB totals with irregular byte tails to keep total bytes and + * block remainders off common multiples while still bounding runtime for Live-only CI. + */ + protected static Stream fuzzyParallelUploadLargeMultiPartCases() { + final int payload257MiBPlus = (int) (257L * Constants.MB + 18881); + final int payload288MiBPlus = (int) (288L * Constants.MB + 7777); + final int payload320MiBPlus = (int) (320L * Constants.MB + 1999); + return Stream.of(Arguments.of(96 * Constants.MB + 17, 8L * Constants.MB + 511, 2), + Arguments.of(112 * Constants.MB, 15L * Constants.MB + 4096, 8), + Arguments.of(128 * Constants.MB + 45673, 17L * Constants.MB - 11264 + 173, 4), + Arguments.of(160 * Constants.MB + 12345, 12L * Constants.MB + 8192, 8), + Arguments.of(192 * Constants.MB + 9876, 31L * Constants.MB - 513, 8), + Arguments.of(224 * Constants.MB, 23L * Constants.MB + 524288, 8), + Arguments.of(payload257MiBPlus, 61L * Constants.MB + 23L * Constants.KB, 6), + Arguments.of(payload288MiBPlus, 36L * Constants.MB + 513, 8), + Arguments.of(payload320MiBPlus, 16L * Constants.MB + 511, 8)); + } + + /** + * Single ~1 GiB download with high concurrency and an awkward (non-aligned) tail to exercise the structured + * message decoder under a sustained, fan-out-heavy parallel download. Live-only and file-backed so payload + * never materializes twice in heap. + */ + protected static Stream fuzzyParallelDownloadOneGiBCases() { + return Stream.of(Arguments.of((int) (1L * Constants.GB + 1377), 16L * Constants.MB + 511, 8)); + } + + private static boolean hasValidStructuredContentLengthHeader(HttpHeaders headers) { + String structuredContentLength = headers.getValue("x-ms-structured-content-length"); + if (CoreUtils.isNullOrEmpty(structuredContentLength) + || CoreUtils.isNullOrEmpty(structuredContentLength.trim())) { + return false; + } + try { + return Long.parseLong(structuredContentLength.trim()) >= 0; + } catch (NumberFormatException ex) { + return false; + } + } } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerApiTests.java index f46116acdbb5..5bdf5cbd0c0e 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerApiTests.java @@ -34,6 +34,7 @@ import com.azure.storage.blob.models.PublicAccessType; import com.azure.storage.blob.models.RehydratePriority; import com.azure.storage.blob.models.StorageAccountInfo; +import com.azure.storage.blob.models.StorageResponseSerializationFormat; import com.azure.storage.blob.models.TaggedBlobItem; import com.azure.storage.blob.options.BlobContainerCreateOptions; import com.azure.storage.blob.options.BlobParallelUploadOptions; @@ -45,6 +46,7 @@ import com.azure.storage.blob.specialized.PageBlobClient; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.test.shared.TestHttpClientType; import com.azure.storage.common.test.shared.extensions.LiveOnly; import com.azure.storage.common.test.shared.extensions.PlaybackOnly; @@ -58,7 +60,17 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import com.azure.storage.blob.implementation.AzureBlobStorageImpl; +import com.azure.storage.blob.implementation.AzureBlobStorageImplBuilder; +import com.azure.storage.blob.implementation.models.ContainersListBlobFlatSegmentApacheArrowHeaders; +import com.azure.storage.blob.implementation.util.ArrowBlobListDeserializer; +import com.azure.storage.blob.implementation.util.ModelHelper; +import com.azure.storage.blob.models.ListBlobsIncludeItem; +import com.azure.core.http.rest.ResponseBase; + import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.ArrayList; import java.net.URL; import java.time.OffsetDateTime; import java.util.Arrays; @@ -1024,6 +1036,18 @@ public void listBlobsFlatOptionsFail() { assertThrows(IllegalArgumentException.class, () -> new ListBlobsOptions().setMaxResultsPerPage(0)); } + @Test + public void listBlobsFlatEndBeforeRequiresArrow() { + ListBlobsOptions options = new ListBlobsOptions().setEndBefore("blob") + .setStorageResponseSerializationFormat(StorageResponseSerializationFormat.XML); + + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> cc.listBlobs(options, null)); + + assertEquals("The endBefore option is only supported when storageResponseSerializationFormat is set to ARROW.", + exception.getMessage()); + } + @Test public void listBlobsFlatMarker() { int numBlobs = 10; @@ -1327,6 +1351,18 @@ private static Stream listBlobsHierOptionsFailSupplier() { Arguments.of(false, 0, IllegalArgumentException.class)); } + @Test + public void listBlobsHierEndBeforeRequiresArrow() { + ListBlobsOptions options = new ListBlobsOptions().setEndBefore("blob") + .setStorageResponseSerializationFormat(StorageResponseSerializationFormat.XML); + + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> cc.listBlobsByHierarchy("/", options, null)); + + assertEquals("The endBefore option is only supported when storageResponseSerializationFormat is set to ARROW.", + exception.getMessage()); + } + @Test public void listBlobsHierDelim() { List blobNames = Arrays.asList("a", "b/a", "c", "d/a", "e", "f", "g/a"); @@ -2128,4 +2164,315 @@ public void getBlobContainerUrlEncodesContainerName() { // then: // assertThrows(BlobStorageException.class, () -> // } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowBasic() { + // Upload a test blob + String blobName = generateBlobName(); + cc.getBlobClient(blobName).getBlockBlobClient().upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + List blobs = cc.listBlobs(options, null).stream().collect(Collectors.toList()); + + assertEquals(1, blobs.size()); + assertEquals(blobName, blobs.get(0).getName()); + assertNotNull(blobs.get(0).getProperties()); + assertEquals(DATA.getDefaultDataSize(), blobs.get(0).getProperties().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, blobs.get(0).getProperties().getBlobType()); + assertNotNull(blobs.get(0).getProperties().getLastModified()); + assertNotNull(blobs.get(0).getProperties().getETag()); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowWithTags() { + // Upload a blob and set tags + String blobName = generateBlobName(); + cc.getBlobClient(blobName).getBlockBlobClient().upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + + Map tags = new HashMap<>(); + tags.put("tagkey", "tagvalue"); + cc.getBlobClient(blobName).setTags(tags); + + // List with Arrow + retrieveTags + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setDetails(new BlobListDetails().setRetrieveTags(true)); + List blobs = cc.listBlobs(options, null).stream().collect(Collectors.toList()); + + assertEquals(1, blobs.size()); + assertEquals(blobName, blobs.get(0).getName()); + assertNotNull(blobs.get(0).getTags()); + assertEquals("tagvalue", blobs.get(0).getTags().get("tagkey")); + } + + @ParameterizedTest + @MethodSource("listBlobsFlatRehydratePrioritySupplier") + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowRehydratePriority(RehydratePriority rehydratePriority) { + String name = generateBlobName(); + BlockBlobClient bc = cc.getBlobClient(name).getBlockBlobClient(); + + bc.upload(DATA.getDefaultInputStream(), 7); + + if (rehydratePriority != null) { + bc.setAccessTier(AccessTier.ARCHIVE); + bc.setAccessTierWithResponse(new BlobSetAccessTierOptions(AccessTier.HOT).setPriority(rehydratePriority), + null, null); + } + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + BlobItem item = cc.listBlobs(options, null).iterator().next(); + + assertEquals(rehydratePriority, item.getProperties().getRehydratePriority()); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowWithMetadata() { + String blobName = generateBlobName(); + Map metadata = new HashMap<>(); + metadata.put("testkey", "testvalue"); + cc.getBlobClient(blobName) + .getBlockBlobClient() + .uploadWithResponse(DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), null, metadata, null, null, + null, null, null); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setDetails(new BlobListDetails().setRetrieveMetadata(true)); + List blobs = cc.listBlobs(options, null).stream().collect(Collectors.toList()); + + assertEquals(1, blobs.size()); + assertNotNull(blobs.get(0).getMetadata()); + assertEquals("testvalue", blobs.get(0).getMetadata().get("testkey")); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowPagination() { + // Upload 3 blobs + for (int i = 0; i < 4; i++) { + cc.getBlobClient("blob" + i) + .getBlockBlobClient() + .upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + } + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setMaxResultsPerPage(1); + List allBlobs = new ArrayList<>(); + for (PagedResponse page : cc.listBlobs(options, null).iterableByPage()) { + assertTrue(page.getValue().size() <= 1); + allBlobs.addAll(page.getValue()); + } + + cc.listBlobs().iterableByPage(2).forEach(page -> { + assertEquals(2, page.getValue().size()); + }); + + assertEquals(4, allBlobs.size()); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowNullUseArrowUsesXml() { + // Default apacheArrowEnabled is null — should use XML path without error + String blobName = generateBlobName(); + cc.getBlobClient(blobName).getBlockBlobClient().upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + + ListBlobsOptions options = new ListBlobsOptions(); + assertNull(options.getStorageResponseSerializationFormat()); + + List blobs = cc.listBlobs(options, null).stream().collect(Collectors.toList()); + assertEquals(1, blobs.size()); + assertEquals(blobName, blobs.get(0).getName()); + } + + @LiveOnly + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowEncryptedBlob() { + // Upload a blob with CPK (customer-provided key) + String blobName = generateBlobName(); + CustomerProvidedKey cpk = new CustomerProvidedKey(Base64.getEncoder().encodeToString(getRandomKey())); + BlobClient cpkClient = cc.getBlobClient(blobName).getCustomerProvidedKeyClient(cpk); + cpkClient.getBlockBlobClient().upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + List blobs = cc.listBlobs(options, null).stream().collect(Collectors.toList()); + + assertEquals(1, blobs.size()); + assertEquals(blobName, blobs.get(0).getName()); + // CPK blob should have server-encrypted = true + assertTrue(blobs.get(0).getProperties().isServerEncrypted()); + // Metadata should be null (no metadata was set) + assertNull(blobs.get(0).getMetadata()); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowDeserializer() throws Exception { + String blobName = generateBlobName(); + Map metadata = new HashMap<>(); + metadata.put("testkey", "testvalue"); + cc.getBlobClient(blobName) + .getBlockBlobClient() + .uploadWithResponse(DATA.getDefaultInputStream(), 7, null, metadata, null, null, null, null, null); + + AzureBlobStorageImpl impl = new AzureBlobStorageImplBuilder().pipeline(cc.getHttpPipeline()) + .url(cc.getAccountUrl()) + .version(BlobServiceVersion.getLatest().getVersion()) + .buildClient(); + + // Call the Arrow endpoint + ArrayList include = new ArrayList<>(); + include.add(ListBlobsIncludeItem.METADATA); + + ResponseBase response = impl.getContainers() + .listBlobFlatSegmentApacheArrowWithResponse(containerName, null, null, null, include, null, null, null, + null, com.azure.core.util.Context.NONE); + + // Verify Content-Type is Arrow + String contentType = response.getDeserializedHeaders().getContentType(); + assertTrue( + StorageImplUtils.hasMatchingHeaderValue(contentType, + Constants.ContentTypeConstants.APPLICATION_VND_APACHE_ARROW_STREAM), + "Expected Arrow content type but got: " + contentType); + + // Deserialize using ArrowBlobListDeserializer + ArrowBlobListDeserializer.ArrowListBlobsResult result + = ArrowBlobListDeserializer.deserialize(response.getValue()); + + // Verify pagination — single blob, no next page + assertNull(result.getNextMarker()); + + // Verify we got exactly one blob + assertEquals(1, result.getBlobItems().size()); + + com.azure.storage.blob.implementation.models.BlobItemInternal item = result.getBlobItems().get(0); + + // Name + assertNotNull(item.getName()); + assertEquals(blobName, item.getName().getContent()); + + // Properties + assertNotNull(item.getProperties()); + assertEquals(7L, (long) item.getProperties().getContentLength()); + assertEquals("application/octet-stream", item.getProperties().getContentType()); + assertNotNull(item.getProperties().getETag()); + assertNotNull(item.getProperties().getLastModified()); + assertNotNull(item.getProperties().getCreationTime()); + assertEquals(BlobType.BLOCK_BLOB, item.getProperties().getBlobType()); + assertEquals(AccessTier.HOT, item.getProperties().getAccessTier()); + assertTrue(item.getProperties().isAccessTierInferred()); + assertTrue(item.getProperties().isServerEncrypted()); + assertEquals(LeaseStateType.AVAILABLE, item.getProperties().getLeaseState()); + assertEquals(LeaseStatusType.UNLOCKED, item.getProperties().getLeaseStatus()); + assertNotNull(item.getProperties().getContentMd5()); + + // Metadata + assertNotNull(item.getMetadata()); + assertEquals("testvalue", item.getMetadata().get("testkey")); + + // Verify ModelHelper can convert to public BlobItem + BlobItem publicItem = ModelHelper.populateBlobItem(item); + assertEquals(blobName, publicItem.getName()); + assertEquals(7L, (long) publicItem.getProperties().getContentLength()); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsByHierarchyArrowBasic() { + // Upload blobs in a directory structure + cc.getBlobClient("dir/blob1") + .getBlockBlobClient() + .upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + cc.getBlobClient("dir/blob2") + .getBlockBlobClient() + .upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + cc.getBlobClient("topblob") + .getBlockBlobClient() + .upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + List items = cc.listBlobsByHierarchy("/", options, null).stream().collect(Collectors.toList()); + + // Root level: one prefix "dir/" and one blob "topblob" + assertEquals(2, items.size()); + + BlobItem prefixItem = items.stream().filter(BlobItem::isPrefix).findFirst().orElse(null); + BlobItem blobItem = items.stream().filter(i -> !i.isPrefix()).findFirst().orElse(null); + + assertNotNull(prefixItem); + assertEquals("dir/", prefixItem.getName()); + assertTrue(prefixItem.isPrefix()); + + assertNotNull(blobItem); + assertEquals("topblob", blobItem.getName()); + assertFalse(blobItem.isPrefix()); + assertNotNull(blobItem.getProperties()); + assertEquals(DATA.getDefaultDataSize(), blobItem.getProperties().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, blobItem.getProperties().getBlobType()); + assertNotNull(blobItem.getProperties().getLastModified()); + assertNotNull(blobItem.getProperties().getETag()); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsByHierarchyArrowWithMetadata() { + String blobName = generateBlobName(); + Map metadata = new HashMap<>(); + metadata.put("testkey", "testvalue"); + cc.getBlobClient("dir/" + blobName) + .getBlockBlobClient() + .uploadWithResponse(DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), null, metadata, null, null, + null, null, null); + cc.getBlobClient("topblob") + .getBlockBlobClient() + .upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setPrefix("dir/") + .setDetails(new BlobListDetails().setRetrieveMetadata(true)); + List blobs = cc.listBlobsByHierarchy("/", options, null).stream().collect(Collectors.toList()); + + assertEquals(1, blobs.size()); + assertFalse(blobs.get(0).isPrefix()); + assertNotNull(blobs.get(0).getMetadata()); + assertEquals("testvalue", blobs.get(0).getMetadata().get("testkey")); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsByHierarchyArrowPagination() { + // Upload blobs across multiple directories + for (int i = 0; i < 3; i++) { + cc.getBlobClient("dir" + i + "/blob") + .getBlockBlobClient() + .upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + } + cc.getBlobClient("topblob") + .getBlockBlobClient() + .upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setMaxResultsPerPage(1); + List allItems = new ArrayList<>(); + for (PagedResponse page : cc.listBlobsByHierarchy("/", options, null).iterableByPage()) { + assertTrue(page.getValue().size() <= 1); + allItems.addAll(page.getValue()); + } + + // 3 prefixes + 1 blob = 4 items + assertEquals(4, allItems.size()); + } + } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerAsyncApiTests.java index 04ebc06dc2b6..568603acdb9a 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerAsyncApiTests.java @@ -10,8 +10,14 @@ import com.azure.core.test.TestMode; import com.azure.core.test.utils.MockTokenCredential; import com.azure.core.util.Context; +import com.azure.core.util.FluxUtil; import com.azure.core.util.polling.PollerFlux; import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.storage.blob.implementation.AzureBlobStorageImpl; +import com.azure.storage.blob.implementation.AzureBlobStorageImplBuilder; +import com.azure.storage.blob.implementation.models.BlobItemInternal; +import com.azure.storage.blob.implementation.util.ArrowBlobListDeserializer; +import com.azure.storage.blob.implementation.util.ModelHelper; import com.azure.storage.blob.models.*; import com.azure.storage.blob.options.BlobContainerCreateOptions; import com.azure.storage.blob.options.BlobParallelUploadOptions; @@ -22,6 +28,8 @@ import com.azure.storage.blob.specialized.BlobAsyncClientBase; import com.azure.storage.blob.specialized.BlockBlobAsyncClient; import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.test.shared.TestHttpClientType; import com.azure.storage.common.test.shared.extensions.LiveOnly; import com.azure.storage.common.test.shared.extensions.PlaybackOnly; @@ -41,6 +49,7 @@ import reactor.util.function.Tuple2; import java.net.URL; +import java.io.ByteArrayInputStream; import java.time.Duration; import java.time.OffsetDateTime; import java.util.*; @@ -1020,6 +1029,18 @@ public void listBlobsFlatOptionsMaxResultsByPage() { .verifyComplete(); } + @Test + public void listBlobsFlatEndBeforeRequiresArrow() { + ListBlobsOptions options = new ListBlobsOptions().setEndBefore("blob") + .setStorageResponseSerializationFormat(StorageResponseSerializationFormat.XML); + + StepVerifier.create(ccAsync.listBlobs(options, null)) + .expectErrorMatches(error -> error instanceof IllegalArgumentException + && "The endBefore option is only supported when storageResponseSerializationFormat is set to ARROW." + .equals(error.getMessage())) + .verify(); + } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2020-10-02") @Test public void listBlobsFlatOptionsDeletedWithVersions() { @@ -1412,6 +1433,18 @@ public void listBlobsHierOptionsFail() { .verifyError(UnsupportedOperationException.class); } + @Test + public void listBlobsHierEndBeforeRequiresArrow() { + ListBlobsOptions options = new ListBlobsOptions().setEndBefore("blob") + .setStorageResponseSerializationFormat(StorageResponseSerializationFormat.XML); + + StepVerifier.create(ccAsync.listBlobsByHierarchy("/", options)) + .expectErrorMatches(error -> error instanceof IllegalArgumentException + && "The endBefore option is only supported when storageResponseSerializationFormat is set to ARROW." + .equals(error.getMessage())) + .verify(); + } + @Test public void listBlobsHierDelim() { List blobNames = Arrays.asList("a", "b/a", "c", "d/a", "e", "f", "g/a"); @@ -2142,4 +2175,336 @@ public void getBlobContainerUrlEncodesContainerName() { assertTrue(containerClient.getBlobContainerUrl().contains("my%20container")); } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowBasic() { + // Upload a test blob + String blobName = generateBlobName(); + BlockBlobAsyncClient bc = ccAsync.getBlobAsyncClient(blobName).getBlockBlobAsyncClient(); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + + StepVerifier + .create( + bc.upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize()).thenMany(ccAsync.listBlobs(options, null))) + .assertNext(item -> { + assertEquals(blobName, item.getName()); + assertNotNull(item.getProperties()); + assertEquals(DATA.getDefaultDataSize(), item.getProperties().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, item.getProperties().getBlobType()); + assertNotNull(item.getProperties().getLastModified()); + assertNotNull(item.getProperties().getETag()); + }) + .verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowWithMetadata() { + String blobName = generateBlobName(); + Map metadata = new HashMap<>(); + metadata.put("testkey", "testvalue"); + BlockBlobAsyncClient bc = ccAsync.getBlobAsyncClient(blobName).getBlockBlobAsyncClient(); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setDetails(new BlobListDetails().setRetrieveMetadata(true)); + + StepVerifier.create( + bc.uploadWithResponse(DATA.getDefaultFlux(), DATA.getDefaultDataSize(), null, metadata, null, null, null) + .thenMany(ccAsync.listBlobs(options, null))) + .assertNext(item -> { + assertNotNull(item.getMetadata()); + assertEquals("testvalue", item.getMetadata().get("testkey")); + }) + .verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowPagination() { + // Upload 4 blobs + Flux uploads = Flux.range(0, 4) + .flatMap(i -> ccAsync.getBlobAsyncClient("blob" + i) + .getBlockBlobAsyncClient() + .upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize())); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setMaxResultsPerPage(1); + + Mono> result = uploads.then(ccAsync.listBlobs(options, null).byPage().doOnNext(page -> { + assertTrue(page.getValue().size() <= 1); + }).flatMap(page -> Flux.fromIterable(page.getValue())).collectList()); + + StepVerifier.create(result).assertNext(allBlobs -> assertEquals(4, allBlobs.size())).verifyComplete(); + + // Mirror the sync test's secondary assertion: requesting page size 2 yields exactly 2 blobs per page. + StepVerifier.create(ccAsync.listBlobs().byPage(2)).thenConsumeWhile(page -> { + assertEquals(2, page.getValue().size()); + return true; + }).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowNullUseArrowUsesXml() { + // Default apacheArrowEnabled is null — should use XML path without error + String blobName = generateBlobName(); + BlockBlobAsyncClient bc = ccAsync.getBlobAsyncClient(blobName).getBlockBlobAsyncClient(); + + ListBlobsOptions options = new ListBlobsOptions(); + assertNull(options.getStorageResponseSerializationFormat()); + + StepVerifier + .create( + bc.upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize()).thenMany(ccAsync.listBlobs(options, null))) + .assertNext(item -> assertEquals(blobName, item.getName())) + .verifyComplete(); + } + + @LiveOnly + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowEncryptedBlob() { + // Upload a blob with CPK (customer-provided key) + String blobName = generateBlobName(); + CustomerProvidedKey cpk = new CustomerProvidedKey(Base64.getEncoder().encodeToString(getRandomKey())); + BlockBlobAsyncClient cpkClient + = ccAsync.getBlobAsyncClient(blobName).getCustomerProvidedKeyAsyncClient(cpk).getBlockBlobAsyncClient(); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + + StepVerifier.create(cpkClient.upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize()) + .thenMany(ccAsync.listBlobs(options, null))).assertNext(item -> { + assertEquals(blobName, item.getName()); + // CPK blob should have server-encrypted = true + assertTrue(item.getProperties().isServerEncrypted()); + // Metadata should be null (no metadata was set) + assertNull(item.getMetadata()); + }).verifyComplete(); + } + + @ParameterizedTest + @MethodSource("listBlobsFlatRehydratePrioritySupplier") + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowRehydratePriority(RehydratePriority rehydratePriority) { + String name = generateBlobName(); + BlockBlobAsyncClient bc = ccAsync.getBlobAsyncClient(name).getBlockBlobAsyncClient(); + + Mono> rehydrate = Mono.empty(); + + if (rehydratePriority != null) { + rehydrate = bc.setAccessTier(AccessTier.ARCHIVE) + .then(bc.setAccessTierWithResponse( + new BlobSetAccessTierOptions(AccessTier.HOT).setPriority(rehydratePriority))); + } + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + + Flux response = bc.upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize()) + .then(rehydrate) + .thenMany(ccAsync.listBlobs(options, null)); + + StepVerifier.create(response) + .assertNext(r -> assertEquals(rehydratePriority, r.getProperties().getRehydratePriority())) + .verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowDeserializer() { + String blobName = generateBlobName(); + Map metadata = new HashMap<>(); + metadata.put("testkey", "testvalue"); + + BlockBlobAsyncClient bc = ccAsync.getBlobAsyncClient(blobName).getBlockBlobAsyncClient(); + + AzureBlobStorageImpl impl = new AzureBlobStorageImplBuilder().pipeline(ccAsync.getHttpPipeline()) + .url(ccAsync.getAccountUrl()) + .version(BlobServiceVersion.getLatest().getVersion()) + .buildClient(); + + List include = new ArrayList<>(); + include.add(ListBlobsIncludeItem.METADATA); + + Mono testMono + = bc.uploadWithResponse(DATA.getDefaultFlux(), 7, null, metadata, null, null, null) + .then(impl.getContainers() + .listBlobFlatSegmentApacheArrowWithResponseAsync(containerName, null, null, null, include, null, + null, null, null)) + .flatMap(response -> { + // Verify Content-Type is Arrow + String contentType = response.getDeserializedHeaders().getContentType(); + assertTrue( + StorageImplUtils.hasMatchingHeaderValue(contentType, + Constants.ContentTypeConstants.APPLICATION_VND_APACHE_ARROW_STREAM), + "Expected Arrow content type but got: " + contentType); + + // Collect the Flux body into a byte[] and feed it to the deserializer. + return FluxUtil.collectBytesInByteBufferStream(response.getValue()) + .map(bytes -> ArrowBlobListDeserializer.deserialize(new ByteArrayInputStream(bytes))); + }); + + StepVerifier.create(testMono).assertNext(result -> { + // Verify pagination — single blob, no next page + assertNull(result.getNextMarker()); + + // Verify we got exactly one blob + assertEquals(1, result.getBlobItems().size()); + + BlobItemInternal item = result.getBlobItems().get(0); + + // Name + assertNotNull(item.getName()); + assertEquals(blobName, item.getName().getContent()); + + // Properties + assertNotNull(item.getProperties()); + assertEquals(7L, (long) item.getProperties().getContentLength()); + assertEquals("application/octet-stream", item.getProperties().getContentType()); + assertNotNull(item.getProperties().getETag()); + assertNotNull(item.getProperties().getLastModified()); + assertNotNull(item.getProperties().getCreationTime()); + assertEquals(BlobType.BLOCK_BLOB, item.getProperties().getBlobType()); + assertEquals(AccessTier.HOT, item.getProperties().getAccessTier()); + assertTrue(item.getProperties().isAccessTierInferred()); + assertTrue(item.getProperties().isServerEncrypted()); + assertEquals(LeaseStateType.AVAILABLE, item.getProperties().getLeaseState()); + assertEquals(LeaseStatusType.UNLOCKED, item.getProperties().getLeaseStatus()); + assertNotNull(item.getProperties().getContentMd5()); + + // Metadata + assertNotNull(item.getMetadata()); + assertEquals("testvalue", item.getMetadata().get("testkey")); + + // Verify ModelHelper can convert to public BlobItem + BlobItem publicItem = ModelHelper.populateBlobItem(item); + assertEquals(blobName, publicItem.getName()); + assertEquals(7L, (long) publicItem.getProperties().getContentLength()); + }).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsByHierarchyArrowBasic() { + // Upload blobs in a directory structure + Flux uploads = Flux.concat( + ccAsync.getBlobAsyncClient("dir/blob1") + .getBlockBlobAsyncClient() + .upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize()), + ccAsync.getBlobAsyncClient("dir/blob2") + .getBlockBlobAsyncClient() + .upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize()), + ccAsync.getBlobAsyncClient("topblob") + .getBlockBlobAsyncClient() + .upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize())); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW); + + Mono> items + = uploads.then(ccAsync.listBlobsByHierarchy("/", options).collect(Collectors.toList())); + + StepVerifier.create(items).assertNext(list -> { + // Root level: one prefix "dir/" and one blob "topblob" + assertEquals(2, list.size()); + + BlobItem prefixItem = list.stream().filter(BlobItem::isPrefix).findFirst().orElse(null); + BlobItem blobItem = list.stream().filter(i -> !i.isPrefix()).findFirst().orElse(null); + + assertNotNull(prefixItem); + assertEquals("dir/", prefixItem.getName()); + assertTrue(prefixItem.isPrefix()); + + assertNotNull(blobItem); + assertEquals("topblob", blobItem.getName()); + assertFalse(blobItem.isPrefix()); + assertNotNull(blobItem.getProperties()); + assertEquals(DATA.getDefaultDataSize(), blobItem.getProperties().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, blobItem.getProperties().getBlobType()); + assertNotNull(blobItem.getProperties().getLastModified()); + assertNotNull(blobItem.getProperties().getETag()); + }).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsByHierarchyArrowWithMetadata() { + String blobName = generateBlobName(); + Map metadata = new HashMap<>(); + metadata.put("testkey", "testvalue"); + + Mono uploads = ccAsync.getBlobAsyncClient("dir/" + blobName) + .getBlockBlobAsyncClient() + .uploadWithResponse(DATA.getDefaultFlux(), DATA.getDefaultDataSize(), null, metadata, null, null, null) + .then(ccAsync.getBlobAsyncClient("topblob") + .getBlockBlobAsyncClient() + .upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize())); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setPrefix("dir/") + .setDetails(new BlobListDetails().setRetrieveMetadata(true)); + + StepVerifier.create(uploads.thenMany(ccAsync.listBlobsByHierarchy("/", options))).assertNext(item -> { + assertFalse(item.isPrefix()); + assertNotNull(item.getMetadata()); + assertEquals("testvalue", item.getMetadata().get("testkey")); + }).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsByHierarchyArrowPagination() { + // Upload blobs across multiple directories + Flux uploads = Flux.concat( + Flux.range(0, 3) + .flatMap(i -> ccAsync.getBlobAsyncClient("dir" + i + "/blob") + .getBlockBlobAsyncClient() + .upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize())), + ccAsync.getBlobAsyncClient("topblob") + .getBlockBlobAsyncClient() + .upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize())); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setMaxResultsPerPage(1); + + Mono> result + = uploads.then(ccAsync.listBlobsByHierarchy("/", options).byPage().doOnNext(page -> { + assertTrue(page.getValue().size() <= 1); + }).flatMap(page -> Flux.fromIterable(page.getValue())).collectList()); + + // 3 prefixes + 1 blob = 4 items + StepVerifier.create(result).assertNext(allItems -> assertEquals(4, allItems.size())).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-06-06") + public void listBlobsArrowWithTags() { + // Upload a blob and set tags + String blobName = generateBlobName(); + BlockBlobAsyncClient bc = ccAsync.getBlobAsyncClient(blobName).getBlockBlobAsyncClient(); + + Map tags = new HashMap<>(); + tags.put("tagkey", "tagvalue"); + + ListBlobsOptions options + = new ListBlobsOptions().setStorageResponseSerializationFormat(StorageResponseSerializationFormat.ARROW) + .setDetails(new BlobListDetails().setRetrieveTags(true)); + + Mono upload = bc.upload(DATA.getDefaultFlux(), DATA.getDefaultDataSize()) + .then(ccAsync.getBlobAsyncClient(blobName).setTags(tags)); + + StepVerifier.create(upload.thenMany(ccAsync.listBlobs(options, null))).assertNext(item -> { + assertEquals(blobName, item.getName()); + assertNotNull(item.getTags()); + assertEquals("tagvalue", item.getTags().get("tagkey")); + }).verifyComplete(); + } } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/LargeBlobTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/LargeBlobTests.java index 37feec1a6b68..82a776ef38c3 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/LargeBlobTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/LargeBlobTests.java @@ -171,15 +171,23 @@ public void uploadLargeInputSyncNoLengthGiven() { public void uploadLargeFile() throws IOException { long tailSize = Constants.MB; File file = getRandomLargeFile(LARGE_BLOCK_SIZE + tailSize); - ParallelTransferOptions parallelTransferOptions - = new ParallelTransferOptions().setBlockSizeLong(LARGE_BLOCK_SIZE); - - blobClient.uploadFromFile(file.toPath().toString(), parallelTransferOptions, null, null, null, null, null); - BlockList blockList = blobClient.getBlockBlobClient().listBlocks(BlockListType.COMMITTED); - - assertEquals(2, blockList.getCommittedBlocks().size()); - assertEquals(LARGE_BLOCK_SIZE, blockList.getCommittedBlocks().get(0).getSizeLong()); - assertEquals(tailSize, blockList.getCommittedBlocks().get(1).getSizeLong()); + try { + ParallelTransferOptions parallelTransferOptions + = new ParallelTransferOptions().setBlockSizeLong(LARGE_BLOCK_SIZE); + + blobClient.uploadFromFile(file.toPath().toString(), parallelTransferOptions, null, null, null, null, null); + BlockList blockList = blobClient.getBlockBlobClient().listBlocks(BlockListType.COMMITTED); + + assertEquals(2, blockList.getCommittedBlocks().size()); + assertEquals(LARGE_BLOCK_SIZE, blockList.getCommittedBlocks().get(0).getSizeLong()); + assertEquals(tailSize, blockList.getCommittedBlocks().get(1).getSizeLong()); + } finally { + // Delete the ~2.5 GiB source eagerly instead of waiting for deleteOnExit() so it doesn't occupy agent + // disk for the remainder of the module's test JVM. + if (!file.delete()) { + file.deleteOnExit(); + } + } } private InputStream createLargeInputStream(long size) { diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/RetryTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/RetryTests.java index 44ef32fad304..b94b8f79daae 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/RetryTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/RetryTests.java @@ -9,6 +9,7 @@ import com.azure.storage.common.policy.RequestRetryOptions; import com.azure.storage.common.policy.RetryPolicyType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -26,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +@Isolated public class RetryTests extends BlobTestBase { private static URL retryTestURL; diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/implementation/util/BlobListArrowGoldenDecodeTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/implementation/util/BlobListArrowGoldenDecodeTests.java new file mode 100644 index 000000000000..ae5cedda007d --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/implementation/util/BlobListArrowGoldenDecodeTests.java @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.util; + +import com.azure.storage.blob.implementation.models.BlobItemInternal; +import com.azure.storage.blob.implementation.models.BlobListArrowParseException; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Golden-payload decode tests for the vendored {@link BlobListArrowStreamReader} / {@link ArrowBlobListDeserializer}. + *

+ * The fixture ({@code arrow/representative.arrow.base64}) is a real Arrow IPC stream that was produced once by the + * official {@code arrow-vector} writer (see {@code ArrowFixtureGenerator}) and committed as a frozen resource. This test + * decodes those exact bytes with the vendored reader and pins every field the ListBlobs path relies on, so any + * regression in the vendored FlatBuffer accessors surfaces here without requiring the Arrow library on the test + * classpath. It replaces the former differential parity test that re-decoded a freshly written payload with Apache + * Arrow's own accessors; the frozen fixture is the same shape but removes the runtime dependency. + */ +public class BlobListArrowGoldenDecodeTests { + + @Test + public void decodesRepresentativePayload() throws IOException { + ArrowBlobListDeserializer.ArrowListBlobsResult result + = ArrowBlobListDeserializer.deserialize(openFixture("representative.arrow.base64")); + + // Schema metadata. + assertEquals("nextPage", result.getNextMarker()); + assertEquals(Integer.valueOf(2), result.getNumberOfRecords()); + + // Two rows: one blob, one prefix. + List items = result.getBlobItems(); + assertEquals(2, items.size()); + + BlobItemInternal blob = items.get(0); + assertNotNull(blob.getName()); + assertEquals("blob1", blob.getName().getContent()); + assertNull(blob.isPrefix()); + assertEquals(Boolean.FALSE, blob.isDeleted()); + assertNotNull(blob.getProperties()); + assertEquals(7L, (long) blob.getProperties().getContentLength()); + assertEquals("application/octet-stream", blob.getProperties().getContentType()); + assertNotNull(blob.getProperties().getCreationTime()); + assertEquals(1000L, blob.getProperties().getCreationTime().toEpochSecond()); + + Map metadata = blob.getMetadata(); + assertNotNull(metadata); + assertEquals("v1", metadata.get("k1")); + assertEquals("v2", metadata.get("k2")); + + BlobItemInternal prefix = items.get(1); + assertNotNull(prefix.getName()); + assertEquals("dir/", prefix.getName().getContent()); + assertTrue(prefix.isPrefix()); + // The prefix row had no metadata entries; the reader must not surface an (empty) map for it. + assertNull(prefix.getMetadata()); + assertFalse(items.isEmpty()); + } + + @Test + public void allColumnsFixtureDecodesWithoutRejection() throws IOException { + // Every column the SDK claims to handle is present in this full-schema fixture and must decode without the + // unknown-column guard firing (no false positives) and without a type-mismatch error. + ArrowBlobListDeserializer.ArrowListBlobsResult result + = ArrowBlobListDeserializer.deserialize(openFixture("allcolumns.arrow.base64")); + + List items = result.getBlobItems(); + assertEquals(1, items.size()); + BlobItemInternal blob = items.get(0); + + // Spot-check one field from each Arrow type category to prove full-schema decode. + assertEquals("blob1", blob.getName().getContent()); + assertEquals(Boolean.FALSE, blob.isDeleted()); + assertEquals(7L, (long) blob.getProperties().getContentLength()); + assertEquals(1000L, blob.getProperties().getCreationTime().toEpochSecond()); + assertEquals("mv", blob.getMetadata().get("mk")); + } + + @Test + public void allColumnsFixtureSchemaMatchesKnownColumnSet() throws IOException { + // Drift guard: the frozen full-schema fixture and the deserializer's known-column set must stay identical. If a + // column is added to KNOWN_COLUMNS without regenerating the fixture (or vice versa), this fails loudly. + Set fixtureColumns; + try (InputStream stream = openFixture("allcolumns.arrow.base64")) { + BlobListArrowStreamReader.DecodedArrowStream decodedArrowStream = BlobListArrowStreamReader.read(stream); + fixtureColumns = decodedArrowStream.getBatches().get(0).getColumnNames(); + } + assertEquals(ArrowBlobListDeserializer.knownColumns(), fixtureColumns); + } + + @Test + public void rejectsUnknownColumn() throws IOException { + // The service added a column a future SDK understands; this SDK must reject loudly and name the offender. + try (InputStream stream = openFixture("reject-unknown-column.arrow.base64")) { + BlobListArrowParseException ex + = assertThrows(BlobListArrowParseException.class, () -> ArrowBlobListDeserializer.deserialize(stream)); + assertTrue(ex.getMessage().contains("FutureField"), "Unexpected message: " + ex.getMessage()); + } + } + + private static InputStream openFixture(String name) throws IOException { + byte[] base64; + try (InputStream resource = classpathResource("/arrow/" + name)) { + assertNotNull(resource, "missing test fixture: arrow/" + name); + base64 = readAll(resource); + } + return new ByteArrayInputStream(Base64.getDecoder().decode(base64)); + } + + /** + * Loads a resource from the test classpath. Calling {@code getResourceAsStream} on this test's class literal is + * equivalent to going through its class loader directly (like + * {@code Thread.currentThread().getContextClassLoader().getResourceAsStream(...)}); we use the class literal because + * this is a static context with no {@code this} on which to call {@code getClass()}. The leading {@code /} makes the + * path absolute from the classpath root. + * + * @param absolutePath the classpath-absolute resource path, beginning with {@code /} + * @return the resource stream, or null if the resource is not found + */ + private static InputStream classpathResource(String absolutePath) { + return BlobListArrowGoldenDecodeTests.class.getResourceAsStream(absolutePath); + } + + private static byte[] readAll(InputStream in) throws IOException { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = in.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/implementation/util/BlobListArrowStreamReaderRejectionTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/implementation/util/BlobListArrowStreamReaderRejectionTests.java new file mode 100644 index 000000000000..349db1cefa48 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/implementation/util/BlobListArrowStreamReaderRejectionTests.java @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.implementation.util; + +import com.azure.storage.blob.implementation.models.BlobListArrowParseException; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Base64; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Parity tests for Arrow IPC content the ListBlobs reader intentionally rejects or treats as an error. The payloads + * are committed golden fixtures (see {@code src/test/resources/arrow}) originally produced with the official + * {@code arrow-vector} writer, so these tests validate the vendored reader's rejection paths against genuine Arrow + * output with no Arrow dependency. The remaining cases cover malformed/edge inputs that the writer cannot produce + * (null and empty streams). + */ +public class BlobListArrowStreamReaderRejectionTests { + + @Test + public void rejectsNullStream() { + BlobListArrowParseException ex + = assertThrows(BlobListArrowParseException.class, () -> ArrowBlobListDeserializer.deserialize(null)); + assertTrue(ex.getMessage().contains("input stream is null"), "Unexpected message: " + ex.getMessage()); + } + + @Test + public void rejectsStreamWithNoSchema() { + InputStream stream = new ByteArrayInputStream(new byte[0]); + BlobListArrowParseException ex + = assertThrows(BlobListArrowParseException.class, () -> ArrowBlobListDeserializer.deserialize(stream)); + assertTrue(ex.getMessage().contains("stream contained no schema"), "Unexpected message: " + ex.getMessage()); + } + + @Test + public void rejectsDictionaryEncodedStreams() throws IOException { + assertRejected("reject-dictionary.arrow.base64", "dictionary-encoded streams are not supported"); + } + + @Test + public void rejectsUnsupportedColumnType() throws IOException { + assertRejected("reject-float.arrow.base64", "unsupported Arrow type 'FloatingPoint'"); + } + + @Test + public void rejectsUnsupportedTimestampUnit() throws IOException { + assertRejected("reject-timestamp-milli.arrow.base64", "unsupported timestamp unit 'MILLISECOND'"); + } + + @Test + public void rejectsMapWithNonStringValues() throws IOException { + assertRejected("reject-map-nonstring.arrow.base64", "map entries must be string keys and values"); + } + + // region helpers + + private static void assertRejected(String fixture, String expectedMessageFragment) throws IOException { + try (InputStream stream = openFixture(fixture)) { + BlobListArrowParseException ex + = assertThrows(BlobListArrowParseException.class, () -> ArrowBlobListDeserializer.deserialize(stream)); + assertTrue(ex.getMessage().contains(expectedMessageFragment), "Unexpected message: " + ex.getMessage()); + } + } + + private static InputStream openFixture(String name) throws IOException { + byte[] base64; + try (InputStream resource + = BlobListArrowStreamReaderRejectionTests.class.getResourceAsStream("/arrow/" + name)) { + assertNotNull(resource, "missing test fixture: arrow/" + name); + base64 = readAll(resource); + } + return new ByteArrayInputStream(Base64.getDecoder().decode(base64)); + } + + private static byte[] readAll(InputStream in) throws IOException { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int read; + while ((read = in.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + //endregion +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobApiTests.java index 5b65926014b6..7b9925cb808c 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobApiTests.java @@ -54,6 +54,7 @@ import java.util.stream.Stream; import static com.azure.storage.blob.specialized.AppendBlobClient.MAX_APPEND_BLOCKS; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -358,6 +359,26 @@ public void appendBlockDefaults() { assertEquals(1, bc.getProperties().getCommittedBlockCount()); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void appendBlockDefaultsWithCrc64() { + Response appendResponse = bc.appendBlockWithResponse(DATA.getDefaultInputStream(), + DATA.getDefaultDataSize(), null, null, null, null); + + ByteArrayOutputStream downloadStream = new ByteArrayOutputStream(); + bc.downloadStream(downloadStream); + TestUtils.assertArraysEqual(DATA.getDefaultBytes(), downloadStream.toByteArray()); + + validateBasicHeaders(appendResponse.getHeaders()); + assertNotNull(appendResponse.getHeaders().getValue(X_MS_CONTENT_CRC64)); + byte[] expectedContentCrc64 + = Base64.getDecoder().decode(appendResponse.getHeaders().getValue(X_MS_CONTENT_CRC64)); + TestUtils.assertArraysEqual(expectedContentCrc64, appendResponse.getValue().getContentCrc64()); + assertNotNull(appendResponse.getValue().getBlobAppendOffset()); + assertNotNull(appendResponse.getValue().getBlobCommittedBlockCount()); + assertEquals(1, bc.getProperties().getCommittedBlockCount()); + } + @Test public void appendBlockMin() { assertResponseStatusCode( @@ -575,6 +596,29 @@ public void appendBlockFromURLMD5() { } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void appendBlockFromUrlMd5Crc64() throws NoSuchAlgorithmException { + byte[] data = getRandomByteArray(1024); + byte[] expectedContentMd5 = MessageDigest.getInstance("MD5").digest(data); + bc.appendBlock(new ByteArrayInputStream(data), data.length); + + AppendBlobClient destURL = cc.getBlobClient(generateBlobName()).getAppendBlobClient(); + destURL.create(); + + String sas = bc.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1), + new BlobContainerSasPermission().setReadPermission(true))); + Response response = destURL.appendBlockFromUrlWithResponse(bc.getBlobUrl() + "?" + sas, null, + expectedContentMd5, null, null, null, Context.NONE); + + assertResponseStatusCode(response, 201); + validateBasicHeaders(response.getHeaders()); + assertArrayEquals(expectedContentMd5, response.getValue().getContentMd5()); + String contentCrc64 = response.getHeaders().getValue(X_MS_CONTENT_CRC64); + assertNotNull(contentCrc64); + assertArrayEquals(Base64.getDecoder().decode(contentCrc64), response.getValue().getContentCrc64()); + } + @Test public void appendBlockFromURLMD5Fail() { byte[] data = getRandomByteArray(1024); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobAsyncApiTests.java index cf7a17dade24..17d4a1e074da 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/AppendBlobAsyncApiTests.java @@ -54,6 +54,7 @@ import java.util.stream.Stream; import static com.azure.storage.blob.specialized.AppendBlobClient.MAX_APPEND_BLOCKS; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -364,6 +365,25 @@ public void appendBlockDefaults() { .verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void appendBlockDefaultsWithCrc64() { + StepVerifier.create(bc.appendBlockWithResponse(DATA.getDefaultFlux(), DATA.getDefaultDataSize(), null, null)) + .assertNext(r -> { + validateBasicHeaders(r.getHeaders()); + assertNotNull(r.getHeaders().getValue(X_MS_CONTENT_CRC64)); + byte[] expectedContentCrc64 = Base64.getDecoder().decode(r.getHeaders().getValue(X_MS_CONTENT_CRC64)); + TestUtils.assertArraysEqual(expectedContentCrc64, r.getValue().getContentCrc64()); + assertNotNull(r.getValue().getBlobAppendOffset()); + assertNotNull(r.getValue().getBlobCommittedBlockCount()); + }) + .verifyComplete(); + + StepVerifier.create(FluxUtil.collectBytesInByteBufferStream(bc.downloadStream())) + .assertNext(r -> TestUtils.assertArraysEqual(DATA.getDefaultBytes(), r)) + .verifyComplete(); + } + @Test public void appendBlockMin() { assertAsyncResponseStatusCode( @@ -591,6 +611,32 @@ public void appendBlockFromURLMD5() throws NoSuchAlgorithmException { StepVerifier.create(response).expectNextCount(1).verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void appendBlockFromUrlMd5Crc64() throws NoSuchAlgorithmException { + byte[] data = getRandomByteArray(1024); + byte[] expectedContentMd5 = MessageDigest.getInstance("MD5").digest(data); + + AppendBlobAsyncClient destURL = ccAsync.getBlobAsyncClient(generateBlobName()).getAppendBlobAsyncClient(); + + String sas = bc.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1), + new BlobSasPermission().setTagsPermission(true).setReadPermission(true))); + + Mono> response = bc.appendBlock(Flux.just(ByteBuffer.wrap(data)), data.length) + .then(destURL.create()) + .then(destURL.appendBlockFromUrlWithResponse(bc.getBlobUrl() + "?" + sas, null, expectedContentMd5, null, + null)); + + StepVerifier.create(response).assertNext(r -> { + assertResponseStatusCode(r, 201); + validateBasicHeaders(r.getHeaders()); + assertArrayEquals(expectedContentMd5, r.getValue().getContentMd5()); + String contentCrc64 = r.getHeaders().getValue(X_MS_CONTENT_CRC64); + assertNotNull(contentCrc64); + assertArrayEquals(Base64.getDecoder().decode(contentCrc64), r.getValue().getContentCrc64()); + }).verifyComplete(); + } + @Test public void appendBlockFromURLMD5Fail() throws NoSuchAlgorithmException { byte[] data = getRandomByteArray(1024); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java index 4ff8054894a6..24a9ca7e781a 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java @@ -106,7 +106,7 @@ static Stream channelReadDataSupplier() { * @param copySize Size of array to copy contents with. * @return Total number of bytes read from src. */ - private static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { + public static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { int read; int totalRead = 0; byte[] temp = new byte[copySize]; diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java index c2a903145440..0d7de17f26a0 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java @@ -210,7 +210,7 @@ public void stageBlockIllegalArguments(boolean getBlockId, InputStream stream, i private static Stream stageBlockIllegalArgumentsSupplier() { return Stream.of( - Arguments.of(false, DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), BlobStorageException.class), + Arguments.of(false, DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, null, DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, DATA.getDefaultInputStream(), DATA.getDefaultDataSize() + 1, UnexpectedLengthException.class), @@ -549,6 +549,26 @@ public void commitBlockList() { assertTrue(Boolean.parseBoolean(headers.getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void commitBlockListCrc64() { + String blockID = getBlockID(); + blockBlobClient.stageBlock(blockID, DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + List ids = Collections.singletonList(blockID); + + Response response + = blockBlobClient.commitBlockListWithResponse(ids, null, null, null, null, null, null); + HttpHeaders headers = response.getHeaders(); + + byte[] expectedCrc64Content = Base64.getDecoder().decode(headers.getValue(X_MS_CONTENT_CRC64)); + + assertResponseStatusCode(response, 201); + validateBasicHeaders(headers); + assertNotNull(headers.getValue(X_MS_CONTENT_CRC64)); + TestUtils.assertArraysEqual(expectedCrc64Content, response.getValue().getContentCrc64()); + assertTrue(Boolean.parseBoolean(headers.getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); + } + @Test public void commitBlockListmin() { blockBlobClient = cc.getBlobClient(generateBlobName()).getBlockBlobClient(); @@ -839,11 +859,31 @@ public void upload() { ByteArrayOutputStream outStream = new ByteArrayOutputStream(); blockBlobClient.downloadStream(outStream); TestUtils.assertArraysEqual(outStream.toByteArray(), DATA.getDefaultText().getBytes(StandardCharsets.UTF_8)); + validateBasicHeaders(response.getHeaders()); assertNotNull(response.getHeaders().getValue(HttpHeaderName.CONTENT_MD5)); assertTrue(Boolean.parseBoolean(response.getHeaders().getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadReturnsCrc64Content() { + Response response = blockBlobClient.uploadWithResponse(DATA.getDefaultInputStream(), + DATA.getDefaultDataSize(), null, null, null, null, null, null, null); + + assertResponseStatusCode(response, 201); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + blockBlobClient.downloadStream(outStream); + TestUtils.assertArraysEqual(outStream.toByteArray(), DATA.getDefaultText().getBytes(StandardCharsets.UTF_8)); + byte[] expectedCrc64Content = Base64.getDecoder().decode(response.getHeaders().getValue(X_MS_CONTENT_CRC64)); + + validateBasicHeaders(response.getHeaders()); + assertNotNull(response.getHeaders().getValue(HttpHeaderName.CONTENT_MD5)); + assertNotNull(response.getHeaders().getValue(X_MS_CONTENT_CRC64)); + TestUtils.assertArraysEqual(expectedCrc64Content, response.getValue().getContentCrc64()); + assertTrue(Boolean.parseBoolean(response.getHeaders().getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); + } + // Override name to prevent BinaryData.toString() invocation by test framework. @Test public void uploadBinaryData() { @@ -1505,12 +1545,35 @@ public void uploadFromUrlMax() throws NoSuchAlgorithmException { assertNotNull(blockBlobItem); assertNotNull(blockBlobItem.getETag()); assertNotNull(blockBlobItem.getLastModified()); + assertNotNull(blockBlobItem.getContentMd5()); + assertNotNull(blockBlobItem.getContentCrc64()); TestUtils.assertArraysEqual(DATA.getDefaultBytes(), os.toByteArray()); assertEquals("en-GB", destinationProperties.getContentLanguage()); assertEquals("text", destinationProperties.getContentType()); assertEquals(AccessTier.COOL, destinationProperties.getAccessTier()); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadFromUrlMaxReturnsCrc64() throws NoSuchAlgorithmException { + BlobClient sourceBlob + = primaryBlobServiceClient.getBlobContainerClient(containerName).getBlobClient(generateBlobName()); + sourceBlob.upload(DATA.getDefaultInputStream(), DATA.getDefaultDataSize()); + byte[] sourceBlobMD5 = MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes()); + String sas = sourceBlob.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1), + new BlobContainerSasPermission().setReadPermission(true))); + + BlobUploadFromUrlOptions options + = new BlobUploadFromUrlOptions(sourceBlob.getBlobUrl() + "?" + sas).setContentMd5(sourceBlobMD5); + Response response = blockBlobClient.uploadFromUrlWithResponse(options, null, null); + String contentCrc64 = response.getHeaders().getValue(X_MS_CONTENT_CRC64); + BlockBlobItem blockBlobItem = response.getValue(); + + assertNotNull(contentCrc64); + assertNotNull(blockBlobItem); + TestUtils.assertArraysEqual(Base64.getDecoder().decode(contentCrc64), blockBlobItem.getContentCrc64()); + } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2020-04-08") @Test public void uploadFromWithInvalidSourceMD5() throws NoSuchAlgorithmException { diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java index 664fe555846d..3a5939947d6d 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java @@ -250,7 +250,7 @@ public void stageBlockIllegalArguments(boolean getBlockId, Flux stre private static Stream stageBlockIllegalArgumentsSupplier() { return Stream.of( - Arguments.of(false, DATA.getDefaultFlux(), DATA.getDefaultDataSize(), BlobStorageException.class), + Arguments.of(false, DATA.getDefaultFlux(), DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, null, DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, DATA.getDefaultFlux(), DATA.getDefaultDataSize() + 1, UnexpectedLengthException.class), Arguments.of(true, DATA.getDefaultFlux(), DATA.getDefaultDataSize() - 1, UnexpectedLengthException.class)); @@ -625,12 +625,32 @@ public void commitBlockList() { .then(blockBlobAsyncClient.commitBlockListWithResponse(ids, null, null, null, null))).assertNext(r -> { assertResponseStatusCode(r, 201); HttpHeaders headers = r.getHeaders(); + validateBasicHeaders(headers); assertNotNull(headers.getValue(X_MS_CONTENT_CRC64)); assertTrue(Boolean.parseBoolean(headers.getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); }).verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void commitBlockListCrc64() { + String blockID = getBlockID(); + List ids = Collections.singletonList(blockID); + + StepVerifier.create(blockBlobAsyncClient.stageBlock(blockID, DATA.getDefaultFlux(), DATA.getDefaultDataSize()) + .then(blockBlobAsyncClient.commitBlockListWithResponse(ids, null, null, null, null))).assertNext(r -> { + assertResponseStatusCode(r, 201); + HttpHeaders headers = r.getHeaders(); + byte[] expectedCrc64Content = Base64.getDecoder().decode(headers.getValue(X_MS_CONTENT_CRC64)); + + validateBasicHeaders(headers); + assertNotNull(headers.getValue(X_MS_CONTENT_CRC64)); + TestUtils.assertArraysEqual(expectedCrc64Content, r.getValue().getContentCrc64()); + assertTrue(Boolean.parseBoolean(headers.getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); + }).verifyComplete(); + } + @Test public void commitBlockListmin() { blockBlobAsyncClient = ccAsync.getBlobAsyncClient(generateBlobName()).getBlockBlobAsyncClient(); @@ -956,7 +976,31 @@ public void upload() { .assertNext(r -> { assertResponseStatusCode(r, 201); validateBasicHeaders(r.getHeaders()); + + assertNotNull(r.getHeaders().getValue(HttpHeaderName.CONTENT_MD5)); + assertTrue(Boolean.parseBoolean(r.getHeaders().getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); + }) + .verifyComplete(); + + StepVerifier.create(FluxUtil.collectBytesInByteBufferStream(blockBlobAsyncClient.downloadStream())) + .assertNext(r -> TestUtils.assertArraysEqual(r, DATA.getDefaultBytes())) + .verifyComplete(); + } + + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadReturnsCrc64Content() { + StepVerifier + .create(blockBlobAsyncClient.uploadWithResponse(DATA.getDefaultFlux(), DATA.getDefaultDataSize(), null, + null, null, null, null)) + .assertNext(r -> { + assertResponseStatusCode(r, 201); + validateBasicHeaders(r.getHeaders()); + byte[] expectedCrc64Content = Base64.getDecoder().decode(r.getHeaders().getValue(X_MS_CONTENT_CRC64)); + assertNotNull(r.getHeaders().getValue(HttpHeaderName.CONTENT_MD5)); + assertNotNull(r.getHeaders().getValue(X_MS_CONTENT_CRC64)); + TestUtils.assertArraysEqual(expectedCrc64Content, r.getValue().getContentCrc64()); assertTrue(Boolean.parseBoolean(r.getHeaders().getValue(X_MS_REQUEST_SERVER_ENCRYPTED))); }) .verifyComplete(); @@ -2504,6 +2548,30 @@ public void uploadFromUrlMax() throws NoSuchAlgorithmException { .verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadFromUrlMaxReturnsCrc64() throws NoSuchAlgorithmException { + BlobAsyncClient sourceBlob = primaryBlobServiceAsyncClient.getBlobContainerAsyncClient(containerName) + .getBlobAsyncClient(generateBlobName()); + byte[] sourceBlobMD5 = MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes()); + String sas = sourceBlob.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1), + new BlobContainerSasPermission().setReadPermission(true))); + + BlobUploadFromUrlOptions options + = new BlobUploadFromUrlOptions(sourceBlob.getBlobUrl() + "?" + sas).setContentMd5(sourceBlobMD5); + Mono> response = sourceBlob.upload(DATA.getDefaultFlux(), null) + .then(blockBlobAsyncClient.uploadFromUrlWithResponse(options)); + + StepVerifier.create(response).assertNext(r -> { + String contentCrc64 = r.getHeaders().getValue(X_MS_CONTENT_CRC64); + BlockBlobItem blockBlobItem = r.getValue(); + + assertNotNull(contentCrc64); + assertNotNull(blockBlobItem); + TestUtils.assertArraysEqual(Base64.getDecoder().decode(contentCrc64), blockBlobItem.getContentCrc64()); + }).verifyComplete(); + } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2020-04-08") @Test public void uploadFromWithInvalidSourceMD5() throws NoSuchAlgorithmException { diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobApiTests.java index 5c2e32f9a962..ffae25810b3f 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobApiTests.java @@ -69,6 +69,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -379,6 +380,23 @@ public void uploadPage() { assertTrue(response.getValue().isServerEncrypted()); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadPageCrc64() { + Response response + = bc.uploadPagesWithResponse(new PageRange().setStart(0).setEnd(PageBlobClient.PAGE_BYTES - 1), + new ByteArrayInputStream(getRandomByteArray(PageBlobClient.PAGE_BYTES)), null, null, null, null); + + byte[] expectedContentCrc64 = Base64.getDecoder().decode(response.getHeaders().getValue(X_MS_CONTENT_CRC64)); + + assertResponseStatusCode(response, 201); + assertTrue(validateBasicHeaders(response.getHeaders())); + assertNotNull(response.getHeaders().getValue(X_MS_CONTENT_CRC64)); + assertArrayEquals(expectedContentCrc64, response.getValue().getContentCrc64()); + assertEquals(0, response.getValue().getBlobSequenceNumber()); + assertTrue(response.getValue().isServerEncrypted()); + } + @Test public void uploadPageMin() { assertResponseStatusCode( @@ -582,7 +600,26 @@ public void uploadPageFromURLIA() { } @Test - public void uploadPageFromURLMD5() { + public void uploadPageFromURLMD5() throws NoSuchAlgorithmException { + PageBlobClient destURL = cc.getBlobClient(generateBlobName()).getPageBlobClient(); + destURL.create(PageBlobClient.PAGE_BYTES); + byte[] data = getRandomByteArray(PageBlobClient.PAGE_BYTES); + PageRange pageRange = new PageRange().setStart(0).setEnd(PageBlobClient.PAGE_BYTES - 1); + bc.uploadPages(pageRange, new ByteArrayInputStream(data)); + + String sas = bc.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1), + new BlobContainerSasPermission().setReadPermission(true))); + Response response = destURL.uploadPagesFromUrlWithResponse(pageRange, bc.getBlobUrl() + "?" + sas, + null, MessageDigest.getInstance("MD5").digest(data), null, null, null, null); + + assertResponseStatusCode(response, 201); + assertTrue(validateBasicHeaders(response.getHeaders())); + assertNotNull(response.getHeaders().getValue(X_MS_CONTENT_CRC64)); + } + + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadPageFromURLMD5AndCrc64() throws NoSuchAlgorithmException { PageBlobClient destURL = cc.getBlobClient(generateBlobName()).getPageBlobClient(); destURL.create(PageBlobClient.PAGE_BYTES); byte[] data = getRandomByteArray(PageBlobClient.PAGE_BYTES); @@ -591,8 +628,14 @@ public void uploadPageFromURLMD5() { String sas = bc.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1), new BlobContainerSasPermission().setReadPermission(true))); - assertDoesNotThrow(() -> destURL.uploadPagesFromUrlWithResponse(pageRange, bc.getBlobUrl() + "?" + sas, null, - MessageDigest.getInstance("MD5").digest(data), null, null, null, null)); + Response response = destURL.uploadPagesFromUrlWithResponse(pageRange, bc.getBlobUrl() + "?" + sas, + null, MessageDigest.getInstance("MD5").digest(data), null, null, null, null); + byte[] expectedCrc64Content = Base64.getDecoder().decode(response.getHeaders().getValue(X_MS_CONTENT_CRC64)); + + assertResponseStatusCode(response, 201); + assertTrue(validateBasicHeaders(response.getHeaders())); + assertNotNull(response.getHeaders().getValue(X_MS_CONTENT_CRC64)); + assertArrayEquals(expectedCrc64Content, response.getValue().getContentCrc64()); } @Test diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobAsyncApiTests.java index 00f44de6b6d2..27839e45ee25 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/PageBlobAsyncApiTests.java @@ -72,6 +72,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -387,6 +388,25 @@ public void uploadPage() { .verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadPageCrc64() { + StepVerifier + .create(bc.uploadPagesWithResponse(new PageRange().setStart(0).setEnd(PageBlobClient.PAGE_BYTES - 1), + Flux.just(ByteBuffer.wrap(getRandomByteArray(PageBlobClient.PAGE_BYTES))), null, null)) + .assertNext(r -> { + byte[] expectedContentCrc64 = Base64.getDecoder().decode(r.getHeaders().getValue(X_MS_CONTENT_CRC64)); + + assertResponseStatusCode(r, 201); + assertTrue(validateBasicHeaders(r.getHeaders())); + assertNotNull(r.getHeaders().getValue(X_MS_CONTENT_CRC64)); + assertArrayEquals(expectedContentCrc64, r.getValue().getContentCrc64()); + assertEquals(0, r.getValue().getBlobSequenceNumber()); + assertTrue(r.getValue().isServerEncrypted()); + }) + .verifyComplete(); + } + @Test public void uploadPageMin() { assertAsyncResponseStatusCode( @@ -626,6 +646,28 @@ public void uploadPageFromURLMD5() throws NoSuchAlgorithmException { StepVerifier.create(response).expectNextCount(1).verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void uploadPageFromURLMD5AndCrc64() throws NoSuchAlgorithmException { + PageBlobAsyncClient destURL = ccAsync.getBlobAsyncClient(generateBlobName()).getPageBlobAsyncClient(); + byte[] data = getRandomByteArray(PageBlobClient.PAGE_BYTES); + PageRange pageRange = new PageRange().setStart(0).setEnd(PageBlobClient.PAGE_BYTES - 1); + String sas = bc.generateSas(new BlobServiceSasSignatureValues(testResourceNamer.now().plusDays(1), + new BlobSasPermission().setTagsPermission(true).setReadPermission(true))); + + Mono> response = destURL.create(PageBlobClient.PAGE_BYTES) + .then(bc.uploadPages(pageRange, Flux.just(ByteBuffer.wrap(data)))) + .then(destURL.uploadPagesFromUrlWithResponse(pageRange, bc.getBlobUrl() + "?" + sas, null, + MessageDigest.getInstance("MD5").digest(data), null, null)); + + StepVerifier.create(response).assertNext(r -> { + assertResponseStatusCode(r, 201); + assertTrue(validateBasicHeaders(r.getHeaders())); + assertArrayEquals(Base64.getDecoder().decode(r.getHeaders().getValue(X_MS_CONTENT_CRC64)), + r.getValue().getContentCrc64()); + }).verifyComplete(); + } + @Test public void uploadPageFromURLMD5Fail() throws NoSuchAlgorithmException { PageBlobAsyncClient destURL = ccAsync.getBlobAsyncClient(generateBlobName()).getPageBlobAsyncClient(); diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/README.md b/sdk/storage/azure-storage-blob/src/test/resources/arrow/README.md new file mode 100644 index 000000000000..747f82ae3a7f --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/README.md @@ -0,0 +1,13 @@ +# ListBlobs Arrow golden fixtures + +These `*.arrow.base64` files are committed golden fixtures for the vendored ListBlobs Arrow +deserializer tests (`BlobListArrowGoldenDecodeTests`, `BlobListArrowStreamReaderRejectionTests`). +Each one is an Apache Arrow IPC stream that has been **Base64-encoded** before being written to disk. + +The tests decode the Base64 back to the original Arrow bytes before parsing, so the on-disk +encoding is transparent to the deserializer under test. + +## Naming + +`.arrow.base64` — `.arrow` denotes the Apache Arrow IPC stream; `.base64` denotes the +Base64 text wrapper. diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/allcolumns.arrow.base64 b/sdk/storage/azure-storage-blob/src/test/resources/arrow/allcolumns.arrow.base64 new file mode 100644 index 000000000000..44eeb10f79cc --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/allcolumns.arrow.base64 @@ -0,0 +1 @@ +/////2gOAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAABkAAAAAgAAADgAAAAEAAAA2P///wgAAAAMAAAAAQAAADEAAAAPAAAATnVtYmVyT2ZSZWNvcmRzAAgADAAIAAQACAAAAAgAAAAMAAAAAAAAAAAAAAAKAAAATmV4dE1hcmtlcgAAMgAAAJgNAABMDQAAGA0AAOAMAACoDAAAaAwAACwMAABMCwAAfAoAALAJAAB0CQAAOAkAAAQJAAC4CAAAfAgAADwIAAD8BwAAvAcAAIAHAABIBwAAEAcAANgGAACYBgAAVAYAABgGAADgBQAAqAUAAGwFAAAwBQAA6AQAAKwEAABwBAAAPAQAAAQEAADMAwAAkAMAAFADAAAMAwAAyAIAAHgCAABEAgAADAIAANQBAACYAQAAUAEAAAwBAADQAAAAkAAAAFAAAAAEAAAAQvP//xQAAAAUAAAAFAAAAAAAAgEYAAAAAAAAAAAAAADc9///AAAAAUAAAAAWAAAAUmVtYWluaW5nUmV0ZW50aW9uRGF5cwAAivP//xQAAAAUAAAAFAAAAAAAAgEYAAAAAAAAAAAAAAAk+P//AAAAAUAAAAAIAAAAVGFnQ291bnQAAAAAxvP//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAAC08///EQAAAFJlaHlkcmF0ZVByaW9yaXR5AAAAAvT//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAADw8///DQAAAEFyY2hpdmVTdGF0dXMAAAA69P//FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAACj0//8WAAAASW1tdXRhYmlsaXR5UG9saWN5TW9kZQAAevT//xQAAAAUAAAAFAAAAAAACgEQAAAAAAAAAAAAAABo9P//GwAAAEltbXV0YWJpbGl0eVBvbGljeVVudGlsRGF0ZQC+9P//FAAAABQAAAAUAAAAAAAKARAAAAAAAAAAAAAAAKz0//8OAAAATGFzdEFjY2Vzc1RpbWUAAPb0//8UAAAAFAAAABQAAAAAAAoBEAAAAAAAAAAAAAAA5PT//wsAAABEZWxldGVkVGltZQAq9f//FAAAABQAAAAUAAAAAAAGARAAAAAAAAAAAAAAABj1//8JAAAATGVnYWxIb2xkAAAAXvX//xQAAAAUAAAAFAAAAAAABgEQAAAAAAAAAAAAAABM9f//BgAAAFNlYWxlZAAAjvX//xQAAAAUAAAAFAAAAAAAAgEYAAAAAAAAAAAAAAAo+v//AAAAAUAAAAAZAAAAeC1tcy1ibG9iLXNlcXVlbmNlLW51bWJlcgAAANr1//8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAyPX//xcAAABDb3B5RGVzdGluYXRpb25TbmFwc2hvdAAa9v//FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAAAj2//8VAAAAQ29weVN0YXR1c0Rlc2NyaXB0aW9uAAAAWvb//xQAAAAUAAAAFAAAAAAACgEQAAAAAAAAAAAAAABI9v//EgAAAENvcHlDb21wbGV0aW9uVGltZQAAlvb//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAACE9v//DAAAAENvcHlQcm9ncmVzcwAAAADO9v//FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAALz2//8KAAAAQ29weVNvdXJjZQAAAvf//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAADw9v//CgAAAENvcHlTdGF0dXMAADb3//8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAJPf//wYAAABDb3B5SWQAAGb3//8UAAAAFAAAABQAAAAAAAYBEAAAAAAAAAAAAAAAVPf//w8AAABJbmNyZW1lbnRhbENvcHkAnvf//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAACM9///DwAAAEVuY3J5cHRpb25TY29wZQDW9///FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAAMT3//8ZAAAAQ3VzdG9tZXJQcm92aWRlZEtleVNoYTI1NgAAABr4//8UAAAAFAAAABQAAAAAAAYBEAAAAAAAAAAAAAAACPj//w8AAABTZXJ2ZXJFbmNyeXB0ZWQAUvj//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAABA+P//DQAAAExlYXNlRHVyYXRpb24AAACK+P//FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAAHj4//8KAAAATGVhc2VTdGF0ZQAAvvj//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAACs+P//CwAAAExlYXNlU3RhdHVzAPL4//8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAA4Pj//w8AAABTbWFydEFjY2Vzc1RpZXIAKvn//xQAAAAUAAAAFAAAAAAACgEQAAAAAAAAAAAAAAAY+f//FAAAAEFjY2Vzc1RpZXJDaGFuZ2VUaW1lAAAAAGr5//8UAAAAFAAAABQAAAAAAAYBEAAAAAAAAAAAAAAAWPn//xIAAABBY2Nlc3NUaWVySW5mZXJyZWQAAKb5//8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAlPn//woAAABBY2Nlc3NUaWVyAADa+f//FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAAMj5//8IAAAAQmxvYlR5cGUAAAAADvr//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAAD8+f//CwAAAENvbnRlbnQtTUQ1AEL6//8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAMPr//w0AAABDYWNoZS1Db250cm9sAAAAevr//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAABo+v//EwAAAENvbnRlbnQtRGlzcG9zaXRpb24Atvr//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAACk+v//EAAAAENvbnRlbnQtTGFuZ3VhZ2UAAAAA8vr//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAADg+v//EAAAAENvbnRlbnQtRW5jb2RpbmcAAAAALvv//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAAAc+///DAAAAENvbnRlbnQtVHlwZQAAAABm+///FAAAABQAAAAcAAAAAAACASAAAAAAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADgAAAENvbnRlbnQtTGVuZ3RoAACu+///FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAAJz7//8EAAAARXRhZwAAAADe+///FAAAABQAAAAUAAAAAAAKARAAAAAAAAAAAAAAAMz7//8NAAAATGFzdC1Nb2RpZmllZAAAABb8//8UAAAAFAAAABQAAAAAAAoBEAAAAAAAAAAAAAAABPz//w0AAABDcmVhdGlvbi1UaW1lAAAATvz//xQAAAAUAAAArAAAAAAAEQGoAAAAAAAAAAEAAAAEAAAAFv7//xQAAAAUAAAAeAAAAAAAAA10AAAAAAAAAAIAAAA4AAAABAAAAJr8//8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAiPz//wUAAAB2YWx1ZQAAAG7+//8UAAAAFAAAABQAAAAAAAAFEAAAAAAAAAAAAAAAuPz//wMAAABrZXkAxPz//wcAAABlbnRyaWVzANT8//8EAAAAVGFncwAAAAAW/f//FAAAABQAAACsAAAAAAARAagAAAAAAAAAAQAAAAQAAADe/v//FAAAABQAAAB4AAAAAAAADXQAAAAAAAAAAgAAADgAAAAEAAAAYv3//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAABQ/f//BQAAAHZhbHVlAAAANv///xQAAAAUAAAAFAAAAAAAAAUQAAAAAAAAAAAAAACA/f//AwAAAGtleQCM/f//BwAAAGVudHJpZXMAnP3//woAAABPck1ldGFkYXRhAADi/f//FAAAABQAAAC8AAAAAAARAbgAAAAAAAAAAQAAAAQAAACq////FAAAABQAAACIAAAAAAAADYQAAAAAAAAAAgAAAEgAAAAEAAAALv7//xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAAAc/v//BQAAAHZhbHVlABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAAFAAAABQAAAAAAAAFEAAAAAAAAAAAAAAAXP7//wMAAABrZXkAaP7//wcAAABlbnRyaWVzAHj+//8IAAAATWV0YWRhdGEAAAAAvv7//xQAAAAUAAAAFAAAAAAABgEQAAAAAAAAAAAAAACs/v//DwAAAEhhc1ZlcnNpb25zT25seQD2/v//FAAAABQAAAAUAAAAAAAGARAAAAAAAAAAAAAAAOT+//8QAAAASXNDdXJyZW50VmVyc2lvbgAAAAAy////FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAACD///8JAAAAVmVyc2lvbklkAAAAZv///xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAABU////CAAAAFNuYXBzaG90AAAAAJr///8UAAAAFAAAABQAAAAAAAYBEAAAAAAAAAAAAAAAiP///wcAAABEZWxldGVkAMr///8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAuP///wwAAABSZXNvdXJjZVR5cGUAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAFAAAABgAAAAAAAUBFAAAAAAAAAAAAAAABAAEAAQAAAAEAAAATmFtZQAAAAAAAAAA/////1gNAAAUAAAAAAAAAAwAFgAOABUAEAAEAAwAAACwBQAAAAAAAAAABAAQAAAAAAMKABgADAAIAAQACgAAABQAAABoCQAAAQAAAAAAAAAAAAAAlQAAAAAAAAAAAAAAAQAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAEAAAAAAAAAAFAAAAAAAAABgAAAAAAAAAAQAAAAAAAAAgAAAAAAAAAAgAAAAAAAAAKAAAAAAAAAAEAAAAAAAAADAAAAAAAAAAAQAAAAAAAAA4AAAAAAAAAAEAAAAAAAAAQAAAAAAAAAABAAAAAAAAAEgAAAAAAAAACAAAAAAAAABQAAAAAAAAABwAAAAAAAAAcAAAAAAAAAABAAAAAAAAAHgAAAAAAAAACAAAAAAAAACAAAAAAAAAABwAAAAAAAAAoAAAAAAAAAABAAAAAAAAAKgAAAAAAAAAAQAAAAAAAACwAAAAAAAAAAEAAAAAAAAAuAAAAAAAAAABAAAAAAAAAMAAAAAAAAAAAQAAAAAAAADIAAAAAAAAAAgAAAAAAAAA0AAAAAAAAAABAAAAAAAAANgAAAAAAAAAAQAAAAAAAADgAAAAAAAAAAgAAAAAAAAA6AAAAAAAAAACAAAAAAAAAPAAAAAAAAAAAQAAAAAAAAD4AAAAAAAAAAgAAAAAAAAAAAEAAAAAAAACAAAAAAAAAAgBAAAAAAAAAQAAAAAAAAAQAQAAAAAAAAgAAAAAAAAAGAEAAAAAAAABAAAAAAAAACABAAAAAAAAAQAAAAAAAAAoAQAAAAAAAAgAAAAAAAAAMAEAAAAAAAACAAAAAAAAADgBAAAAAAAAAQAAAAAAAABAAQAAAAAAAAgAAAAAAAAASAEAAAAAAAACAAAAAAAAAFABAAAAAAAAAQAAAAAAAABYAQAAAAAAAAgAAAAAAAAAYAEAAAAAAAABAAAAAAAAAGgBAAAAAAAAAQAAAAAAAABwAQAAAAAAAAgAAAAAAAAAeAEAAAAAAAACAAAAAAAAAIABAAAAAAAAAQAAAAAAAACIAQAAAAAAAAgAAAAAAAAAkAEAAAAAAAACAAAAAAAAAJgBAAAAAAAAAQAAAAAAAACgAQAAAAAAAAgAAAAAAAAAqAEAAAAAAAABAAAAAAAAALABAAAAAAAACAAAAAAAAAC4AQAAAAAAAAEAAAAAAAAAwAEAAAAAAAAIAAAAAAAAAMgBAAAAAAAAEQAAAAAAAADgAQAAAAAAAAEAAAAAAAAA6AEAAAAAAAAIAAAAAAAAAPABAAAAAAAAAQAAAAAAAAD4AQAAAAAAAAgAAAAAAAAAAAIAAAAAAAAYAAAAAAAAABgCAAAAAAAAAQAAAAAAAAAgAgAAAAAAAAgAAAAAAAAAKAIAAAAAAAAEAAAAAAAAADACAAAAAAAAAQAAAAAAAAA4AgAAAAAAAAgAAAAAAAAAQAIAAAAAAAAFAAAAAAAAAEgCAAAAAAAAAQAAAAAAAABQAgAAAAAAAAgAAAAAAAAAWAIAAAAAAAAGAAAAAAAAAGACAAAAAAAAAQAAAAAAAABoAgAAAAAAAAgAAAAAAAAAcAIAAAAAAAAIAAAAAAAAAHgCAAAAAAAAAQAAAAAAAACAAgAAAAAAAAgAAAAAAAAAiAIAAAAAAAAIAAAAAAAAAJACAAAAAAAAAQAAAAAAAACYAgAAAAAAAAgAAAAAAAAAoAIAAAAAAAAJAAAAAAAAALACAAAAAAAAAQAAAAAAAAC4AgAAAAAAAAgAAAAAAAAAwAIAAAAAAAADAAAAAAAAAMgCAAAAAAAAAQAAAAAAAADQAgAAAAAAAAEAAAAAAAAA2AIAAAAAAAABAAAAAAAAAOACAAAAAAAACAAAAAAAAADoAgAAAAAAAAEAAAAAAAAA8AIAAAAAAAAIAAAAAAAAAPgCAAAAAAAABAAAAAAAAAAAAwAAAAAAAAEAAAAAAAAACAMAAAAAAAAIAAAAAAAAABADAAAAAAAACAAAAAAAAAAYAwAAAAAAAAEAAAAAAAAAIAMAAAAAAAAIAAAAAAAAACgDAAAAAAAACQAAAAAAAAA4AwAAAAAAAAEAAAAAAAAAQAMAAAAAAAAIAAAAAAAAAEgDAAAAAAAACAAAAAAAAABQAwAAAAAAAAEAAAAAAAAAWAMAAAAAAAABAAAAAAAAAGADAAAAAAAAAQAAAAAAAABoAwAAAAAAAAgAAAAAAAAAcAMAAAAAAAArAAAAAAAAAKADAAAAAAAAAQAAAAAAAACoAwAAAAAAAAgAAAAAAAAAsAMAAAAAAAAGAAAAAAAAALgDAAAAAAAAAQAAAAAAAADAAwAAAAAAAAEAAAAAAAAAyAMAAAAAAAABAAAAAAAAANADAAAAAAAACAAAAAAAAADYAwAAAAAAACQAAAAAAAAAAAQAAAAAAAABAAAAAAAAAAgEAAAAAAAACAAAAAAAAAAQBAAAAAAAAAcAAAAAAAAAGAQAAAAAAAABAAAAAAAAACAEAAAAAAAACAAAAAAAAAAoBAAAAAAAADYAAAAAAAAAYAQAAAAAAAABAAAAAAAAAGgEAAAAAAAACAAAAAAAAABwBAAAAAAAAAMAAAAAAAAAeAQAAAAAAAABAAAAAAAAAIAEAAAAAAAACAAAAAAAAACIBAAAAAAAAAEAAAAAAAAAkAQAAAAAAAAIAAAAAAAAAJgEAAAAAAAADQAAAAAAAACoBAAAAAAAAAEAAAAAAAAAsAQAAAAAAAAIAAAAAAAAALgEAAAAAAAAHAAAAAAAAADYBAAAAAAAAAEAAAAAAAAA4AQAAAAAAAAIAAAAAAAAAOgEAAAAAAAAAQAAAAAAAADwBAAAAAAAAAEAAAAAAAAA+AQAAAAAAAABAAAAAAAAAAAFAAAAAAAAAQAAAAAAAAAIBQAAAAAAAAEAAAAAAAAAEAUAAAAAAAAIAAAAAAAAABgFAAAAAAAAAQAAAAAAAAAgBQAAAAAAAAgAAAAAAAAAKAUAAAAAAAABAAAAAAAAADAFAAAAAAAACAAAAAAAAAA4BQAAAAAAAAEAAAAAAAAAQAUAAAAAAAAIAAAAAAAAAEgFAAAAAAAACAAAAAAAAABQBQAAAAAAAAEAAAAAAAAAWAUAAAAAAAAIAAAAAAAAAGAFAAAAAAAAGAAAAAAAAAB4BQAAAAAAAAEAAAAAAAAAgAUAAAAAAAAIAAAAAAAAAIgFAAAAAAAABAAAAAAAAACQBQAAAAAAAAEAAAAAAAAAmAUAAAAAAAAIAAAAAAAAAKAFAAAAAAAAAQAAAAAAAACoBQAAAAAAAAgAAAAAAAAAAAAAADsAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAUAAABibG9iMQAAAAEAAAAAAAAAAAAAAAQAAABibG9iAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAcAAAAMjAyMC0wMS0wMVQwMDowMDowMC4wMDAwMDAwWgAAAAABAAAAAAAAAAAAAAAcAAAAMjAyMC0wMS0wMVQwMDowMDowMC4wMDAwMDAxWgAAAAABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAIAAABtawAAAAAAAAEAAAAAAAAAAAAAAAIAAABtdgAAAAAAAAEAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAIAAABvawAAAAAAAAEAAAAAAAAAAAAAAAIAAABvdgAAAAAAAAEAAAAAAAAAAAAAAAEAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAIAAAB0awAAAAAAAAEAAAAAAAAAAAAAAAIAAAB0dgAAAAAAAAEAAAAAAAAA6AMAAAAAAAABAAAAAAAAANAHAAAAAAAAAQAAAAAAAAAAAAAAEQAAADB4OEQ4RDhEOEQ4RDhEOEQ4AAAAAAAAAAEAAAAAAAAABwAAAAAAAAABAAAAAAAAAAAAAAAYAAAAYXBwbGljYXRpb24vb2N0ZXQtc3RyZWFtAQAAAAAAAAAAAAAABAAAAGd6aXAAAAAAAQAAAAAAAAAAAAAABQAAAGVuLVVTAAAAAQAAAAAAAAAAAAAABgAAAGlubGluZQAAAQAAAAAAAAAAAAAACAAAAG5vLWNhY2hlAQAAAAAAAAAAAAAACAAAAEFRSURCQT09AQAAAAAAAAAAAAAACQAAAEJsb2NrQmxvYgAAAAAAAAABAAAAAAAAAAAAAAADAAAASG90AAAAAAABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAC4CwAAAAAAAAEAAAAAAAAAAAAAAAQAAABDb29sAAAAAAEAAAAAAAAAAAAAAAgAAAB1bmxvY2tlZAEAAAAAAAAAAAAAAAkAAABhdmFpbGFibGUAAAAAAAAAAQAAAAAAAAAAAAAACAAAAGluZmluaXRlAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAACsAAABQbXQwUjBrazJ2N3E1cUIxUW5ublE0eFEwUTBRMFEwUTBRMFEwUTBRMFE9AAAAAAABAAAAAAAAAAAAAAAGAAAAc2NvcGUxAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAJAAAADAwMDAwMDAwLTAwMDAtMDAwMC0wMDAwLTAwMDAwMDAwMDAwMQAAAAABAAAAAAAAAAAAAAAHAAAAc3VjY2VzcwABAAAAAAAAAAAAAAA2AAAAaHR0cHM6Ly9hY2NvdW50LmJsb2IuY29yZS53aW5kb3dzLm5ldC9jb250YWluZXIvc291cmNlAAABAAAAAAAAAAAAAAADAAAANy83AAAAAAABAAAAAAAAAKAPAAAAAAAAAQAAAAAAAAAAAAAADQAAAGNvcHkgY29tcGxldGUAAAABAAAAAAAAAAAAAAAcAAAAMjAyMC0wMS0wMlQwMDowMDowMC4wMDAwMDAwWgAAAAABAAAAAAAAACoAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAIgTAAAAAAAAAQAAAAAAAABwFwAAAAAAAAEAAAAAAAAAWBsAAAAAAAABAAAAAAAAAAAAAAAIAAAAdW5sb2NrZWQBAAAAAAAAAAAAAAAYAAAAcmVoeWRyYXRlLXBlbmRpbmctdG8taG90AQAAAAAAAAAAAAAABAAAAEhpZ2gAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAABQAAAAAAAAD/////AAAAAA== \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-dictionary.arrow.base64 b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-dictionary.arrow.base64 new file mode 100644 index 000000000000..a877f2b8d963 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-dictionary.arrow.base64 @@ -0,0 +1 @@ +/////7gAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAYAAAAAAASABwAGAAXABYAEAAEAAwACAASAAAAKAAAABQAAAAUAAAARAAAAAAABQFAAAAAAAAAAAAAAAAIABAACAAEAAgAAAAUAAAAAQAAAAAAAAAIAAwACAAHAAgAAAAAAAABIAAAAAQABAAEAAAABAAAAE5hbWUAAAAA/////7gAAAAUAAAAAAAAAAwAHAASABsAFAAEAAwAAAAoAAAAAAAAAAAAAAAAAAQAEAAAAAAAAAIIABIACAAEAAgAAAAYAAAAAQAAAAAAAAAAAAoAGAAMAAgABAAKAAAAFAAAAEgAAAACAAAAAAAAAAAAAAADAAAAAAAAAAAAAAABAAAAAAAAAAgAAAAAAAAADAAAAAAAAAAYAAAAAAAAAAoAAAAAAAAAAAAAAAEAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAABQAAAAoAAAAAAAAAYmxvYjFibG9iMgAAAAAAAP////+IAAAAFAAAAAAAAAAMABYADgAVABAABAAMAAAAEAAAAAAAAAAAAAQAEAAAAAADCgAYAAwACAAEAAoAAAAUAAAAOAAAAAIAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAEAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAABAAAAAgAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAEAAAD/////AAAAAA== \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-float.arrow.base64 b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-float.arrow.base64 new file mode 100644 index 000000000000..8065f28604cf --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-float.arrow.base64 @@ -0,0 +1 @@ +/////5AAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAADARwAAAAAAAAAAAAAAAAABgAIAAYABgAAAAAAAgAFAAAAU2NvcmUAAAD/////iAAAABQAAAAAAAAADAAWAA4AFQAQAAQADAAAABAAAAAAAAAAAAAEABAAAAAAAwoAGAAMAAgABAAKAAAAFAAAADgAAAABAAAAAAAAAAAAAAACAAAAAAAAAAAAAAABAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAQAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAPg//////wAAAAA= \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-map-nonstring.arrow.base64 b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-map-nonstring.arrow.base64 new file mode 100644 index 000000000000..9a26eb1d998a --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-map-nonstring.arrow.base64 @@ -0,0 +1 @@ +/////0gBAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAEAAAAsv///xQAAAAUAAAA5AAAAAAAEQHgAAAAAAAAAAEAAAAEAAAAhv///xQAAAAUAAAArAAAAAAAAA2oAAAAAAAAAAIAAABsAAAAGAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAAAUAAAAHAAAAAAAAgEgAAAAAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAUAAAB2YWx1ZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAABQAAAAUAAAAAAAABRAAAAAAAAAAAAAAAOT///8DAAAAa2V5APD///8HAAAAZW50cmllcwAEAAQABAAAAAgAAABNZXRhZGF0YQAAAAAAAAAA/////xgBAAAUAAAAAAAAAAwAFgAOABUAEAAEAAwAAABAAAAAAAAAAAAABAAQAAAAAAMKABgADAAIAAQACgAAABQAAACYAAAAAQAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAQAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAEAAAAAAAAAABAAAAAAAAABgAAAAAAAAAAQAAAAAAAAAgAAAAAAAAAAgAAAAAAAAAKAAAAAAAAAACAAAAAAAAADAAAAAAAAAAAQAAAAAAAAA4AAAAAAAAAAgAAAAAAAAAAAAAAAQAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAQAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAgAAAGsxAAAAAAAAAQAAAAAAAAAqAAAAAAAAAP////8AAAAA \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-timestamp-milli.arrow.base64 b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-timestamp-milli.arrow.base64 new file mode 100644 index 000000000000..58c6263aa995 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-timestamp-milli.arrow.base64 @@ -0,0 +1 @@ +/////5gAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAEAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAAKARwAAAAAAAAAAAAAAAAABgAIAAYABgAAAAAAAQANAAAAQ3JlYXRpb24tVGltZQAAAP////+IAAAAFAAAAAAAAAAMABYADgAVABAABAAMAAAAEAAAAAAAAAAAAAQAEAAAAAADCgAYAAwACAAEAAoAAAAUAAAAOAAAAAEAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAEAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAABAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAA6AMAAAAAAAD/////AAAAAA== \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-unknown-column.arrow.base64 b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-unknown-column.arrow.base64 new file mode 100644 index 000000000000..281e0d3be3da --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/reject-unknown-column.arrow.base64 @@ -0,0 +1 @@ +/////8AAAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAAAIAAAAAAAAAAIAAABQAAAABAAAAMr///8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAuP///wsAAABGdXR1cmVGaWVsZAAAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAFAAAABgAAAAAAAUBFAAAAAAAAAAAAAAABAAEAAQAAAAEAAAATmFtZQAAAAD/////2AAAABQAAAAAAAAADAAWAA4AFQAQAAQADAAAADAAAAAAAAAAAAAEABAAAAAAAwoAGAAMAAgABAAKAAAAFAAAAHgAAAABAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAABAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAQAAAAAAAAAAUAAAAAAAAAGAAAAAAAAAABAAAAAAAAACAAAAAAAAAACAAAAAAAAAAoAAAAAAAAAAgAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAUAAABibG9iMQAAAAEAAAAAAAAAAAAAAAgAAABzdXJwcmlzZf////8AAAAA \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/src/test/resources/arrow/representative.arrow.base64 b/sdk/storage/azure-storage-blob/src/test/resources/arrow/representative.arrow.base64 new file mode 100644 index 000000000000..7e43432b368e --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/resources/arrow/representative.arrow.base64 @@ -0,0 +1 @@ +/////wADAAAQAAAAAAAKAA4ABgANAAgACgAAAAAABAAQAAAAAAEKAAwAAAAIAAQACgAAAAgAAABsAAAAAgAAADgAAAAEAAAA2P///wgAAAAMAAAAAQAAADIAAAAPAAAATnVtYmVyT2ZSZWNvcmRzAAgADAAIAAQACAAAAAgAAAAUAAAACAAAAG5leHRQYWdlAAAAAAoAAABOZXh0TWFya2VyAAAHAAAAKAIAANwBAACQAQAAVAEAACABAADkAAAABAAAAAb+//8UAAAAFAAAALwAAAAAABEBuAAAAAAAAAABAAAABAAAAKr///8UAAAAFAAAAIgAAAAAAAANhAAAAAAAAAACAAAASAAAAAQAAABS/v//FAAAABQAAAAUAAAAAAAFARAAAAAAAAAAAAAAAED+//8FAAAAdmFsdWUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAAAUAAAAFAAAAAAAAAUQAAAAAAAAAAAAAACA/v//AwAAAGtleQCM/v//BwAAAGVudHJpZXMAnP7//wgAAABNZXRhZGF0YQAAAADi/v//FAAAABQAAAAUAAAAAAAKARAAAAAAAAAAAAAAAND+//8NAAAAQ3JlYXRpb24tVGltZQAAABr///8UAAAAFAAAABQAAAAAAAYBEAAAAAAAAAAAAAAACP///wcAAABEZWxldGVkAEr///8UAAAAFAAAABQAAAAAAAUBEAAAAAAAAAAAAAAAOP///wwAAABDb250ZW50LVR5cGUAAAAAgv///xQAAAAUAAAAHAAAAAAAAgEgAAAAAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAA4AAABDb250ZW50LUxlbmd0aAAAyv///xQAAAAUAAAAFAAAAAAABQEQAAAAAAAAAAAAAAC4////DAAAAFJlc291cmNlVHlwZQAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAAAUAAAAGAAAAAAABQEUAAAAAAAAAAAAAAAEAAQABAAAAAQAAABOYW1lAAAAAAAAAAD/////eAIAABQAAAAAAAAADAAWAA4AFQAQAAQADAAAACABAAAAAAAAAAAEABAAAAAAAwoAGAAMAAgABAAKAAAAFAAAAJgBAAACAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAABAAAAAAAAAAgAAAAAAAAADAAAAAAAAAAYAAAAAAAAAAkAAAAAAAAAKAAAAAAAAAABAAAAAAAAADAAAAAAAAAADAAAAAAAAABAAAAAAAAAAAoAAAAAAAAAUAAAAAAAAAABAAAAAAAAAFgAAAAAAAAAEAAAAAAAAABoAAAAAAAAAAEAAAAAAAAAcAAAAAAAAAAMAAAAAAAAAIAAAAAAAAAAGAAAAAAAAACYAAAAAAAAAAEAAAAAAAAAoAAAAAAAAAABAAAAAAAAAKgAAAAAAAAAAQAAAAAAAACwAAAAAAAAABAAAAAAAAAAwAAAAAAAAAABAAAAAAAAAMgAAAAAAAAADAAAAAAAAADYAAAAAAAAAAEAAAAAAAAA4AAAAAAAAAABAAAAAAAAAOgAAAAAAAAADAAAAAAAAAD4AAAAAAAAAAQAAAAAAAAAAAEAAAAAAAABAAAAAAAAAAgBAAAAAAAADAAAAAAAAAAYAQAAAAAAAAQAAAAAAAAAAAAAAAoAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAABAAAAAAAAAAIAAAAAAAAAAQAAAAAAAAACAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAABAAAAAAAAAAIAAAAAAAAAAQAAAAAAAAACAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAABQAAAAkAAAAAAAAAYmxvYjFkaXIvAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAKAAAAAAAAAGJsb2JwcmVmaXgAAAAAAAABAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAYAAAAGAAAAAAAAABhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0BAAAAAAAAAAAAAAAAAAAAAQAAAAAAAADoAwAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAgAAAAIAAAAAAAAAAwAAAAAAAAADAAAAAAAAAAAAAAACAAAABAAAAAAAAABrMWsyAAAAAAMAAAAAAAAAAAAAAAIAAAAEAAAAAAAAAHYxdjIAAAAA/////wAAAAA= \ No newline at end of file diff --git a/sdk/storage/azure-storage-blob/swagger/README.md b/sdk/storage/azure-storage-blob/swagger/README.md index 292d2f7c231d..0019d026a0d0 100644 --- a/sdk/storage/azure-storage-blob/swagger/README.md +++ b/sdk/storage/azure-storage-blob/swagger/README.md @@ -16,7 +16,7 @@ autorest ### Code generation settings ``` yaml use: '@autorest/java@4.1.63' -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/15d7f54a5389d5906ffb4e56bb2f38fe5525c0d3/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-06-06/blob.json +input-file: https://raw.githubusercontent.com/nickliu-msft/azure-rest-api-specs/f85584d452061985a5fc21a67b8fc0b46b75188a/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-10-06/blob.json java: true output-folder: ../ namespace: com.azure.storage.blob @@ -591,6 +591,24 @@ directive: delete $["x-ms-pageable"]; ``` +### Delete Container_ListBlobFlatSegment_ApacheArrow x-ms-pageable as response is raw Arrow stream +``` yaml +directive: +- from: swagger-document + where: $["x-ms-paths"]["/{containerName}?restype=container&comp=list&flat&arrow"].get + transform: > + delete $["x-ms-pageable"]; +``` + +### Delete Container_ListBlobHierarchySegment_ApacheArrow x-ms-pageable as response is raw Arrow stream +``` yaml +directive: +- from: swagger-document + where: $["x-ms-paths"]["/{containerName}?restype=container&comp=list&hierarchy&arrow"].get + transform: > + delete $["x-ms-pageable"]; +``` + ### BlobDeleteType expandable string enum ``` yaml directive: @@ -708,4 +726,3 @@ directive: ]; ``` - diff --git a/sdk/storage/azure-storage-common/ci.system.properties b/sdk/storage/azure-storage-common/ci.system.properties index 1d1c46cd13b4..5fa9f706dba3 100644 --- a/sdk/storage/azure-storage-common/ci.system.properties +++ b/sdk/storage/azure-storage-common/ci.system.properties @@ -1,2 +1,2 @@ -AZURE_LIVE_TEST_SERVICE_VERSION=V2026_06_06 -AZURE_STORAGE_SAS_SERVICE_VERSION=2026-06-06 +AZURE_LIVE_TEST_SERVICE_VERSION=V2026_10_06 +AZURE_STORAGE_SAS_SERVICE_VERSION=2026-10-06 diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java new file mode 100644 index 000000000000..0d1a7ea22b07 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common; + +/** + * Algorithm for content validation on upload and download operations. When enabled, the SDK computes checksums and + * validates content integrity using the selected algorithm. Content validation is off by default. + *

+ * Supported in Azure Storage Blob, Data Lake, and File Share packages for methods that use APIs supporting + * transactional CRC64, or structured message format. + */ +public enum ContentValidationAlgorithm { + + /** + * No content validation. This is the default; no checksum is computed or validated. + */ + NONE, + + /** + * Allow the SDK to choose the validation algorithm. Currently resolves to CRC64 where supported. Different + * library versions may make different choices. The resolution may change in the future. Please set an + * explicit algorithm if you need a specific behavior. + */ + AUTO, + + /** + * Azure Storage custom 64-bit CRC. The SDK computes and validates CRC64 checksums for the transfer. + */ + CRC64 +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index e3b88b661134..4490d2014d78 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -88,7 +88,7 @@ public final class Constants { public static final String PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION = "AZURE_STORAGE_SAS_SERVICE_VERSION"; public static final String SAS_SERVICE_VERSION - = Configuration.getGlobalConfiguration().get(PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION, "2026-06-06"); + = Configuration.getGlobalConfiguration().get(PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION, "2026-10-06"); public static final String ADJUSTED_BLOB_LENGTH_KEY = "adjustedBlobLength"; @@ -220,7 +220,7 @@ public static final class HeaderConstants { * @deprecated For SAS Service Version use {@link Constants#SAS_SERVICE_VERSION}. */ @Deprecated - public static final String TARGET_STORAGE_VERSION = "2026-06-06"; + public static final String TARGET_STORAGE_VERSION = "2026-10-06"; /** * Error code returned from the service. @@ -250,6 +250,20 @@ public static final class HeaderConstants { public static final String ETAG_WILDCARD = "*"; + public static final String CONTENT_CRC64 = "x-ms-content-crc64"; + + public static final HttpHeaderName CONTENT_CRC64_HEADER_NAME = HttpHeaderName.fromString(CONTENT_CRC64); + + public static final String STRUCTURED_BODY_TYPE = "x-ms-structured-body"; + + public static final HttpHeaderName STRUCTURED_BODY_TYPE_HEADER_NAME + = HttpHeaderName.fromString(STRUCTURED_BODY_TYPE); + + public static final String STRUCTURED_CONTENT_LENGTH = "x-ms-structured-content-length"; + + public static final HttpHeaderName STRUCTURED_CONTENT_LENGTH_HEADER_NAME + = HttpHeaderName.fromString(STRUCTURED_CONTENT_LENGTH); + /** * Metadata key ("hdi_isfolder") used to mark virtual directories in Azure Blob Storage. * @@ -277,6 +291,14 @@ private HeaderConstants() { } } + public static final class ContentTypeConstants { + public static final String APPLICATION_VND_APACHE_ARROW_STREAM = "application/vnd.apache.arrow.stream"; + + private ContentTypeConstants() { + // Private to prevent construction. + } + } + /** * Defines constants for use with URLs. * diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/StorageImplUtils.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/StorageImplUtils.java index 90aad19fffbb..081ea1f2df72 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/StorageImplUtils.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/StorageImplUtils.java @@ -397,6 +397,19 @@ public static String convertStorageExceptionMessage(String message, HttpResponse return message; } + /** + * Checks whether the provided header value is non-null, non-empty, and starts with the expected value. + * + * @param headerValue The header value to inspect. + * @param expectedHeaderValue The expected header value prefix. + * @return {@code true} if the header value is present and matches the expected value; otherwise {@code false}. + */ + public static boolean hasMatchingHeaderValue(String headerValue, String expectedHeaderValue) { + return !CoreUtils.isNullOrEmpty(headerValue) + && !CoreUtils.isNullOrEmpty(expectedHeaderValue) + && headerValue.startsWith(expectedHeaderValue); + } + /** * Given a String representing a date in a form of the ISO8601 pattern, generates a Date representing it with up to * millisecond precision. diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java index 32e10374b249..0b81cff41227 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java @@ -98,7 +98,9 @@ public static Flux chunkSource(Flux data, ParallelTransf } int numSplits = (int) Math.ceil(buffer.remaining() / (double) chunkSize); return Flux.range(0, numSplits).map(i -> { - ByteBuffer duplicate = buffer.duplicate().asReadOnlyBuffer(); + // While duplicate.asReadOnlyBuffer() is safer, it significantly slows down crc64 calculation because it forces a copy of the buffer. + // No downstream buffers should be modifying the buffer anyways. + ByteBuffer duplicate = buffer.duplicate(); duplicate.position(i * chunkSize); duplicate.limit(Math.min(duplicate.limit(), (i + 1) * chunkSize)); return duplicate; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java new file mode 100644 index 000000000000..5c3394117971 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY; + +import com.azure.core.util.Context; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.ProgressListener; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.ParallelTransferOptions; +import reactor.core.publisher.Mono; + +/** + * Determines the content validation mode string to pass in the pipeline context for upload operations. + * Callers put the returned value under {@link StructuredMessageConstants#CONTENT_VALIDATION_MODE_KEY}. + *

+ * Single-shot: use CRC64 header when length < 4 MiB, otherwise structured message. + * Chunked (multi-shot): always use structured message. + */ +public final class ContentValidationModeResolver { + + private ContentValidationModeResolver() { + } + + public static final String CONFLICTING_TRANSACTIONAL_CONTENT_VALIDATION_MESSAGE + = "Individual MD5 option and checksum algorithm option bag are both used. Only one form of transactional content validation may be used."; + + /** + * Progress reporting counts bytes on the wire; transfer validation (CRC64/AUTO) may use structured messages, so the + * two cannot be combined. Use {@link ContentValidationAlgorithm#NONE} or null, or omit the progress listener. + */ + public static final String PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE + = "Progress reporting cannot be combined with ContentValidationAlgorithm.CRC64 or ContentValidationAlgorithm.AUTO. " + + "Set ContentValidationAlgorithm to NONE or null, or remove the progress listener."; + + /** + * Resolves content validation mode and adds it to the Azure {@link Context} when non-null. + * + * @param context The request context; {@code null} is treated as {@link Context#NONE}. + * @param algorithm The transfer validation checksum algorithm. + * @param contentLength The upload length in bytes. + * @param chunkedUpload Whether this request is part of a multi-shot upload. + * @return The context, with {@link StructuredMessageConstants#CONTENT_VALIDATION_MODE_KEY} set when applicable. + */ + public static Context addContentValidationMode(Context context, ContentValidationAlgorithm algorithm, + long contentLength, boolean chunkedUpload) { + Context baseContext = context == null ? Context.NONE : context; + String mode + = chunkedUpload ? getModeForChunkedUpload(algorithm) : getModeForSingleShotUpload(algorithm, contentLength); + return mode == null ? baseContext : baseContext.addData(CONTENT_VALIDATION_MODE_KEY, mode); + } + + /** + * Resolves content validation mode and propagates it on the Reactor context for {@code mono} when non-null. + * + * @param mono The reactive sequence to augment. + * @param algorithm The transfer validation checksum algorithm. + * @param contentLength The upload length in bytes. + * @param chunkedUpload Whether this request is part of a multi-shot upload. + * @param The type of the elements in the reactive sequence. + * @return {@code mono}, possibly augmented with Reactor context writes. + */ + public static Mono addContentValidationMode(Mono mono, ContentValidationAlgorithm algorithm, + long contentLength, boolean chunkedUpload) { + String mode + = chunkedUpload ? getModeForChunkedUpload(algorithm) : getModeForSingleShotUpload(algorithm, contentLength); + if (mode == null) { + return mono; + } + return mono.contextWrite(FluxUtil.toReactorContext(new Context(CONTENT_VALIDATION_MODE_KEY, mode))); + } + + /** + * Mode for a single-shot upload. Use CRC64 header when length is less than 4MB, otherwise structured + * message. + */ + private static String getModeForSingleShotUpload(ContentValidationAlgorithm algorithm, long length) { + if (isCrc64OrAuto(algorithm)) { + return length < MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + ? USE_CRC64_CHECKSUM_HEADER_CONTEXT + : USE_STRUCTURED_MESSAGE_CONTEXT; + } + return null; + } + + /** + * Mode for a chunked (multi-shot) upload. Always use structured message. + */ + private static String getModeForChunkedUpload(ContentValidationAlgorithm algorithm) { + if (isCrc64OrAuto(algorithm)) { + return USE_STRUCTURED_MESSAGE_CONTEXT; + } + return null; + } + + /** + * Validates transactional checksum options when MD5 may be SDK-computed. Throws if {@code computeMd5} and a + * non-none {@code contentValidationAlgorithm} are both active. + * + * @param computeMd5 Whether the SDK will compute transactional MD5. + * @param contentValidationAlgorithm Transfer validation checksum algorithm from options. + * @throws IllegalArgumentException if options conflict. + */ + public static void validateTransactionalChecksumOptions(boolean computeMd5, + ContentValidationAlgorithm contentValidationAlgorithm) { + if (computeMd5 && contentValidationAlgorithm != null) { + throw new IllegalArgumentException(CONFLICTING_TRANSACTIONAL_CONTENT_VALIDATION_MESSAGE); + } + } + + /** + * @return {@code true} when {@code contentValidationAlgorithm} enables CRC64 or AUTO transfer validation (not + * {@code null} and not {@link ContentValidationAlgorithm#NONE}). + */ + public static boolean isContentValidationAlgorithmPresent(ContentValidationAlgorithm contentValidationAlgorithm) { + return contentValidationAlgorithm != null && contentValidationAlgorithm != ContentValidationAlgorithm.NONE; + } + + /** + * @return {@code true} when {@code algorithm} is {@link ContentValidationAlgorithm#CRC64} or + * {@link ContentValidationAlgorithm#AUTO}. Upload and download structured-message validation use this rule. + */ + public static boolean isCrc64OrAuto(ContentValidationAlgorithm algorithm) { + return algorithm == ContentValidationAlgorithm.CRC64 || algorithm == ContentValidationAlgorithm.AUTO; + } + + /** + * When the transfer validation mode is {@link ContentValidationAlgorithm#CRC64} or + * {@link ContentValidationAlgorithm#AUTO}, adds + * {@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY} so the HTTP + * pipeline can decode/validate the structured message response. For {@code null} or + * {@link ContentValidationAlgorithm#NONE}, returns the context unchanged (no key added), matching "no + * structured-message validation" for that download. + * + * @param context The base {@link Context}; null is treated as {@link Context#NONE}. + * @param contentValidationAlgorithm The algorithm from download options, or null. + * @return The same context, or a copy with the decoding key set when applicable. + */ + public static Context addStructuredMessageDecodingToContext(Context context, + ContentValidationAlgorithm contentValidationAlgorithm) { + Context base = context == null ? Context.NONE : context; + if (!isCrc64OrAuto(contentValidationAlgorithm)) { + return base; + } + return base.addData(STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + } + + /** + * Validates that progress reporting is not combined with CRC64/AUTO content validation. + * + * @param progressListener Progress listener from {@link ParallelTransferOptions} or equivalent; may be null. + * @param contentValidationAlgorithm Transfer validation algorithm from options. + * @throws IllegalArgumentException if {@code progressListener} is non-null and {@link #isContentValidationAlgorithmPresent} + * is true. + */ + public static void validateProgressWithContentValidation(ProgressListener progressListener, + ContentValidationAlgorithm contentValidationAlgorithm) { + if (progressListener != null && isContentValidationAlgorithmPresent(contentValidationAlgorithm)) { + throw new IllegalArgumentException(PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE); + } + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java new file mode 100644 index 000000000000..ed21ef153823 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java @@ -0,0 +1,2660 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * This class provides methods to compute and manipulate CRC64 checksums using the Azure Storage CRC64 polynomial. + * It includes methods for computing CRC64 checksums for byte arrays, updating CRC values using lookup tables, and + * concatenating CRC values. + *

+ * RESERVED FOR INTERNAL USE. + * + */ +public class StorageCrc64Calculator { + + private static final long POLY = 0x9A6C9329AC4BC9B5L; + + private static final long[] M_U1 = { + 0x0000000000000000L, + 0x7f6ef0c830358979L, + 0xfedde190606b12f2L, + 0x81b31158505e9b8bL, + 0xc962e5739841b68fL, + 0xb60c15bba8743ff6L, + 0x37bf04e3f82aa47dL, + 0x48d1f42bc81f2d04L, + 0xa61cecb46814fe75L, + 0xd9721c7c5821770cL, + 0x58c10d24087fec87L, + 0x27affdec384a65feL, + 0x6f7e09c7f05548faL, + 0x1010f90fc060c183L, + 0x91a3e857903e5a08L, + 0xeecd189fa00bd371L, + 0x78e0ff3b88be6f81L, + 0x078e0ff3b88be6f8L, + 0x863d1eabe8d57d73L, + 0xf953ee63d8e0f40aL, + 0xb1821a4810ffd90eL, + 0xceecea8020ca5077L, + 0x4f5ffbd87094cbfcL, + 0x30310b1040a14285L, + 0xdefc138fe0aa91f4L, + 0xa192e347d09f188dL, + 0x2021f21f80c18306L, + 0x5f4f02d7b0f40a7fL, + 0x179ef6fc78eb277bL, + 0x68f0063448deae02L, + 0xe943176c18803589L, + 0x962de7a428b5bcf0L, + 0xf1c1fe77117cdf02L, + 0x8eaf0ebf2149567bL, + 0x0f1c1fe77117cdf0L, + 0x7072ef2f41224489L, + 0x38a31b04893d698dL, + 0x47cdebccb908e0f4L, + 0xc67efa94e9567b7fL, + 0xb9100a5cd963f206L, + 0x57dd12c379682177L, + 0x28b3e20b495da80eL, + 0xa900f35319033385L, + 0xd66e039b2936bafcL, + 0x9ebff7b0e12997f8L, + 0xe1d10778d11c1e81L, + 0x606216208142850aL, + 0x1f0ce6e8b1770c73L, + 0x8921014c99c2b083L, + 0xf64ff184a9f739faL, + 0x77fce0dcf9a9a271L, + 0x08921014c99c2b08L, + 0x4043e43f0183060cL, + 0x3f2d14f731b68f75L, + 0xbe9e05af61e814feL, + 0xc1f0f56751dd9d87L, + 0x2f3dedf8f1d64ef6L, + 0x50531d30c1e3c78fL, + 0xd1e00c6891bd5c04L, + 0xae8efca0a188d57dL, + 0xe65f088b6997f879L, + 0x9931f84359a27100L, + 0x1882e91b09fcea8bL, + 0x67ec19d339c963f2L, + 0xd75adabd7a6e2d6fL, + 0xa8342a754a5ba416L, + 0x29873b2d1a053f9dL, + 0x56e9cbe52a30b6e4L, + 0x1e383fcee22f9be0L, + 0x6156cf06d21a1299L, + 0xe0e5de5e82448912L, + 0x9f8b2e96b271006bL, + 0x71463609127ad31aL, + 0x0e28c6c1224f5a63L, + 0x8f9bd7997211c1e8L, + 0xf0f5275142244891L, + 0xb824d37a8a3b6595L, + 0xc74a23b2ba0eececL, + 0x46f932eaea507767L, + 0x3997c222da65fe1eL, + 0xafba2586f2d042eeL, + 0xd0d4d54ec2e5cb97L, + 0x5167c41692bb501cL, + 0x2e0934dea28ed965L, + 0x66d8c0f56a91f461L, + 0x19b6303d5aa47d18L, + 0x980521650afae693L, + 0xe76bd1ad3acf6feaL, + 0x09a6c9329ac4bc9bL, + 0x76c839faaaf135e2L, + 0xf77b28a2faafae69L, + 0x8815d86aca9a2710L, + 0xc0c42c4102850a14L, + 0xbfaadc8932b0836dL, + 0x3e19cdd162ee18e6L, + 0x41773d1952db919fL, + 0x269b24ca6b12f26dL, + 0x59f5d4025b277b14L, + 0xd846c55a0b79e09fL, + 0xa72835923b4c69e6L, + 0xeff9c1b9f35344e2L, + 0x90973171c366cd9bL, + 0x1124202993385610L, + 0x6e4ad0e1a30ddf69L, + 0x8087c87e03060c18L, + 0xffe938b633338561L, + 0x7e5a29ee636d1eeaL, + 0x0134d92653589793L, + 0x49e52d0d9b47ba97L, + 0x368bddc5ab7233eeL, + 0xb738cc9dfb2ca865L, + 0xc8563c55cb19211cL, + 0x5e7bdbf1e3ac9decL, + 0x21152b39d3991495L, + 0xa0a63a6183c78f1eL, + 0xdfc8caa9b3f20667L, + 0x97193e827bed2b63L, + 0xe877ce4a4bd8a21aL, + 0x69c4df121b863991L, + 0x16aa2fda2bb3b0e8L, + 0xf86737458bb86399L, + 0x8709c78dbb8deae0L, + 0x06bad6d5ebd3716bL, + 0x79d4261ddbe6f812L, + 0x3105d23613f9d516L, + 0x4e6b22fe23cc5c6fL, + 0xcfd833a67392c7e4L, + 0xb0b6c36e43a74e9dL, + 0x9a6c9329ac4bc9b5L, + 0xe50263e19c7e40ccL, + 0x64b172b9cc20db47L, + 0x1bdf8271fc15523eL, + 0x530e765a340a7f3aL, + 0x2c608692043ff643L, + 0xadd397ca54616dc8L, + 0xd2bd67026454e4b1L, + 0x3c707f9dc45f37c0L, + 0x431e8f55f46abeb9L, + 0xc2ad9e0da4342532L, + 0xbdc36ec59401ac4bL, + 0xf5129aee5c1e814fL, + 0x8a7c6a266c2b0836L, + 0x0bcf7b7e3c7593bdL, + 0x74a18bb60c401ac4L, + 0xe28c6c1224f5a634L, + 0x9de29cda14c02f4dL, + 0x1c518d82449eb4c6L, + 0x633f7d4a74ab3dbfL, + 0x2bee8961bcb410bbL, + 0x548079a98c8199c2L, + 0xd53368f1dcdf0249L, + 0xaa5d9839ecea8b30L, + 0x449080a64ce15841L, + 0x3bfe706e7cd4d138L, + 0xba4d61362c8a4ab3L, + 0xc52391fe1cbfc3caL, + 0x8df265d5d4a0eeceL, + 0xf29c951de49567b7L, + 0x732f8445b4cbfc3cL, + 0x0c41748d84fe7545L, + 0x6bad6d5ebd3716b7L, + 0x14c39d968d029fceL, + 0x95708ccedd5c0445L, + 0xea1e7c06ed698d3cL, + 0xa2cf882d2576a038L, + 0xdda178e515432941L, + 0x5c1269bd451db2caL, + 0x237c997575283bb3L, + 0xcdb181ead523e8c2L, + 0xb2df7122e51661bbL, + 0x336c607ab548fa30L, + 0x4c0290b2857d7349L, + 0x04d364994d625e4dL, + 0x7bbd94517d57d734L, + 0xfa0e85092d094cbfL, + 0x856075c11d3cc5c6L, + 0x134d926535897936L, + 0x6c2362ad05bcf04fL, + 0xed9073f555e26bc4L, + 0x92fe833d65d7e2bdL, + 0xda2f7716adc8cfb9L, + 0xa54187de9dfd46c0L, + 0x24f29686cda3dd4bL, + 0x5b9c664efd965432L, + 0xb5517ed15d9d8743L, + 0xca3f8e196da80e3aL, + 0x4b8c9f413df695b1L, + 0x34e26f890dc31cc8L, + 0x7c339ba2c5dc31ccL, + 0x035d6b6af5e9b8b5L, + 0x82ee7a32a5b7233eL, + 0xfd808afa9582aa47L, + 0x4d364994d625e4daL, + 0x3258b95ce6106da3L, + 0xb3eba804b64ef628L, + 0xcc8558cc867b7f51L, + 0x8454ace74e645255L, + 0xfb3a5c2f7e51db2cL, + 0x7a894d772e0f40a7L, + 0x05e7bdbf1e3ac9deL, + 0xeb2aa520be311aafL, + 0x944455e88e0493d6L, + 0x15f744b0de5a085dL, + 0x6a99b478ee6f8124L, + 0x224840532670ac20L, + 0x5d26b09b16452559L, + 0xdc95a1c3461bbed2L, + 0xa3fb510b762e37abL, + 0x35d6b6af5e9b8b5bL, + 0x4ab846676eae0222L, + 0xcb0b573f3ef099a9L, + 0xb465a7f70ec510d0L, + 0xfcb453dcc6da3dd4L, + 0x83daa314f6efb4adL, + 0x0269b24ca6b12f26L, + 0x7d0742849684a65fL, + 0x93ca5a1b368f752eL, + 0xeca4aad306bafc57L, + 0x6d17bb8b56e467dcL, + 0x12794b4366d1eea5L, + 0x5aa8bf68aecec3a1L, + 0x25c64fa09efb4ad8L, + 0xa4755ef8cea5d153L, + 0xdb1bae30fe90582aL, + 0xbcf7b7e3c7593bd8L, + 0xc399472bf76cb2a1L, + 0x422a5673a732292aL, + 0x3d44a6bb9707a053L, + 0x759552905f188d57L, + 0x0afba2586f2d042eL, + 0x8b48b3003f739fa5L, + 0xf42643c80f4616dcL, + 0x1aeb5b57af4dc5adL, + 0x6585ab9f9f784cd4L, + 0xe436bac7cf26d75fL, + 0x9b584a0fff135e26L, + 0xd389be24370c7322L, + 0xace74eec0739fa5bL, + 0x2d545fb4576761d0L, + 0x523aaf7c6752e8a9L, + 0xc41748d84fe75459L, + 0xbb79b8107fd2dd20L, + 0x3acaa9482f8c46abL, + 0x45a459801fb9cfd2L, + 0x0d75adabd7a6e2d6L, + 0x721b5d63e7936bafL, + 0xf3a84c3bb7cdf024L, + 0x8cc6bcf387f8795dL, + 0x620ba46c27f3aa2cL, + 0x1d6554a417c62355L, + 0x9cd645fc4798b8deL, + 0xe3b8b53477ad31a7L, + 0xab69411fbfb21ca3L, + 0xd407b1d78f8795daL, + 0x55b4a08fdfd90e51L, + 0x2ada5047efec8728L, }; + + private static final long[] M_U32 = { + 0x0000000000000000L, + 0xb8c533c1177eb231L, + 0x455341d1766af709L, + 0xfd96721061144538L, + 0x8aa683a2ecd5ee12L, + 0x3263b063fbab5c23L, + 0xcff5c2739abf191bL, + 0x7730f1b28dc1ab2aL, + 0x21942116813c4f4fL, + 0x995112d79642fd7eL, + 0x64c760c7f756b846L, + 0xdc025306e0280a77L, + 0xab32a2b46de9a15dL, + 0x13f791757a97136cL, + 0xee61e3651b835654L, + 0x56a4d0a40cfde465L, + 0x4328422d02789e9eL, + 0xfbed71ec15062cafL, + 0x067b03fc74126997L, + 0xbebe303d636cdba6L, + 0xc98ec18feead708cL, + 0x714bf24ef9d3c2bdL, + 0x8cdd805e98c78785L, + 0x3418b39f8fb935b4L, + 0x62bc633b8344d1d1L, + 0xda7950fa943a63e0L, + 0x27ef22eaf52e26d8L, + 0x9f2a112be25094e9L, + 0xe81ae0996f913fc3L, + 0x50dfd35878ef8df2L, + 0xad49a14819fbc8caL, + 0x158c92890e857afbL, + 0x8650845a04f13d3cL, + 0x3e95b79b138f8f0dL, + 0xc303c58b729bca35L, + 0x7bc6f64a65e57804L, + 0x0cf607f8e824d32eL, + 0xb4333439ff5a611fL, + 0x49a546299e4e2427L, + 0xf16075e889309616L, + 0xa7c4a54c85cd7273L, + 0x1f01968d92b3c042L, + 0xe297e49df3a7857aL, + 0x5a52d75ce4d9374bL, + 0x2d6226ee69189c61L, + 0x95a7152f7e662e50L, + 0x6831673f1f726b68L, + 0xd0f454fe080cd959L, + 0xc578c6770689a3a2L, + 0x7dbdf5b611f71193L, + 0x802b87a670e354abL, + 0x38eeb467679de69aL, + 0x4fde45d5ea5c4db0L, + 0xf71b7614fd22ff81L, + 0x0a8d04049c36bab9L, + 0xb24837c58b480888L, + 0xe4ece76187b5ecedL, + 0x5c29d4a090cb5edcL, + 0xa1bfa6b0f1df1be4L, + 0x197a9571e6a1a9d5L, + 0x6e4a64c36b6002ffL, + 0xd68f57027c1eb0ceL, + 0x2b1925121d0af5f6L, + 0x93dc16d30a7447c7L, + 0x38782ee75175e913L, + 0x80bd1d26460b5b22L, + 0x7d2b6f36271f1e1aL, + 0xc5ee5cf73061ac2bL, + 0xb2dead45bda00701L, + 0x0a1b9e84aadeb530L, + 0xf78dec94cbcaf008L, + 0x4f48df55dcb44239L, + 0x19ec0ff1d049a65cL, + 0xa1293c30c737146dL, + 0x5cbf4e20a6235155L, + 0xe47a7de1b15de364L, + 0x934a8c533c9c484eL, + 0x2b8fbf922be2fa7fL, + 0xd619cd824af6bf47L, + 0x6edcfe435d880d76L, + 0x7b506cca530d778dL, + 0xc3955f0b4473c5bcL, + 0x3e032d1b25678084L, + 0x86c61eda321932b5L, + 0xf1f6ef68bfd8999fL, + 0x4933dca9a8a62baeL, + 0xb4a5aeb9c9b26e96L, + 0x0c609d78deccdca7L, + 0x5ac44ddcd23138c2L, + 0xe2017e1dc54f8af3L, + 0x1f970c0da45bcfcbL, + 0xa7523fccb3257dfaL, + 0xd062ce7e3ee4d6d0L, + 0x68a7fdbf299a64e1L, + 0x95318faf488e21d9L, + 0x2df4bc6e5ff093e8L, + 0xbe28aabd5584d42fL, + 0x06ed997c42fa661eL, + 0xfb7beb6c23ee2326L, + 0x43bed8ad34909117L, + 0x348e291fb9513a3dL, + 0x8c4b1adeae2f880cL, + 0x71dd68cecf3bcd34L, + 0xc9185b0fd8457f05L, + 0x9fbc8babd4b89b60L, + 0x2779b86ac3c62951L, + 0xdaefca7aa2d26c69L, + 0x622af9bbb5acde58L, + 0x151a0809386d7572L, + 0xaddf3bc82f13c743L, + 0x504949d84e07827bL, + 0xe88c7a195979304aL, + 0xfd00e89057fc4ab1L, + 0x45c5db514082f880L, + 0xb853a9412196bdb8L, + 0x00969a8036e80f89L, + 0x77a66b32bb29a4a3L, + 0xcf6358f3ac571692L, + 0x32f52ae3cd4353aaL, + 0x8a301922da3de19bL, + 0xdc94c986d6c005feL, + 0x6451fa47c1beb7cfL, + 0x99c78857a0aaf2f7L, + 0x2102bb96b7d440c6L, + 0x56324a243a15ebecL, + 0xeef779e52d6b59ddL, + 0x13610bf54c7f1ce5L, + 0xaba438345b01aed4L, + 0x70f05dcea2ebd226L, + 0xc8356e0fb5956017L, + 0x35a31c1fd481252fL, + 0x8d662fdec3ff971eL, + 0xfa56de6c4e3e3c34L, + 0x4293edad59408e05L, + 0xbf059fbd3854cb3dL, + 0x07c0ac7c2f2a790cL, + 0x51647cd823d79d69L, + 0xe9a14f1934a92f58L, + 0x14373d0955bd6a60L, + 0xacf20ec842c3d851L, + 0xdbc2ff7acf02737bL, + 0x6307ccbbd87cc14aL, + 0x9e91beabb9688472L, + 0x26548d6aae163643L, + 0x33d81fe3a0934cb8L, + 0x8b1d2c22b7edfe89L, + 0x768b5e32d6f9bbb1L, + 0xce4e6df3c1870980L, + 0xb97e9c414c46a2aaL, + 0x01bbaf805b38109bL, + 0xfc2ddd903a2c55a3L, + 0x44e8ee512d52e792L, + 0x124c3ef521af03f7L, + 0xaa890d3436d1b1c6L, + 0x571f7f2457c5f4feL, + 0xefda4ce540bb46cfL, + 0x98eabd57cd7aede5L, + 0x202f8e96da045fd4L, + 0xddb9fc86bb101aecL, + 0x657ccf47ac6ea8ddL, + 0xf6a0d994a61aef1aL, + 0x4e65ea55b1645d2bL, + 0xb3f39845d0701813L, + 0x0b36ab84c70eaa22L, + 0x7c065a364acf0108L, + 0xc4c369f75db1b339L, + 0x39551be73ca5f601L, + 0x819028262bdb4430L, + 0xd734f8822726a055L, + 0x6ff1cb4330581264L, + 0x9267b953514c575cL, + 0x2aa28a924632e56dL, + 0x5d927b20cbf34e47L, + 0xe55748e1dc8dfc76L, + 0x18c13af1bd99b94eL, + 0xa0040930aae70b7fL, + 0xb5889bb9a4627184L, + 0x0d4da878b31cc3b5L, + 0xf0dbda68d208868dL, + 0x481ee9a9c57634bcL, + 0x3f2e181b48b79f96L, + 0x87eb2bda5fc92da7L, + 0x7a7d59ca3edd689fL, + 0xc2b86a0b29a3daaeL, + 0x941cbaaf255e3ecbL, + 0x2cd9896e32208cfaL, + 0xd14ffb7e5334c9c2L, + 0x698ac8bf444a7bf3L, + 0x1eba390dc98bd0d9L, + 0xa67f0accdef562e8L, + 0x5be978dcbfe127d0L, + 0xe32c4b1da89f95e1L, + 0x48887329f39e3b35L, + 0xf04d40e8e4e08904L, + 0x0ddb32f885f4cc3cL, + 0xb51e0139928a7e0dL, + 0xc22ef08b1f4bd527L, + 0x7aebc34a08356716L, + 0x877db15a6921222eL, + 0x3fb8829b7e5f901fL, + 0x691c523f72a2747aL, + 0xd1d961fe65dcc64bL, + 0x2c4f13ee04c88373L, + 0x948a202f13b63142L, + 0xe3bad19d9e779a68L, + 0x5b7fe25c89092859L, + 0xa6e9904ce81d6d61L, + 0x1e2ca38dff63df50L, + 0x0ba03104f1e6a5abL, + 0xb36502c5e698179aL, + 0x4ef370d5878c52a2L, + 0xf636431490f2e093L, + 0x8106b2a61d334bb9L, + 0x39c381670a4df988L, + 0xc455f3776b59bcb0L, + 0x7c90c0b67c270e81L, + 0x2a34101270daeae4L, + 0x92f123d367a458d5L, + 0x6f6751c306b01dedL, + 0xd7a2620211ceafdcL, + 0xa09293b09c0f04f6L, + 0x1857a0718b71b6c7L, + 0xe5c1d261ea65f3ffL, + 0x5d04e1a0fd1b41ceL, + 0xced8f773f76f0609L, + 0x761dc4b2e011b438L, + 0x8b8bb6a28105f100L, + 0x334e8563967b4331L, + 0x447e74d11bbae81bL, + 0xfcbb47100cc45a2aL, + 0x012d35006dd01f12L, + 0xb9e806c17aaead23L, + 0xef4cd66576534946L, + 0x5789e5a4612dfb77L, + 0xaa1f97b40039be4fL, + 0x12daa47517470c7eL, + 0x65ea55c79a86a754L, + 0xdd2f66068df81565L, + 0x20b91416ecec505dL, + 0x987c27d7fb92e26cL, + 0x8df0b55ef5179897L, + 0x3535869fe2692aa6L, + 0xc8a3f48f837d6f9eL, + 0x7066c74e9403ddafL, + 0x075636fc19c27685L, + 0xbf93053d0ebcc4b4L, + 0x4205772d6fa8818cL, + 0xfac044ec78d633bdL, + 0xac649448742bd7d8L, + 0x14a1a789635565e9L, + 0xe937d599024120d1L, + 0x51f2e658153f92e0L, + 0x26c217ea98fe39caL, + 0x9e07242b8f808bfbL, + 0x6391563bee94cec3L, + 0xdb5465faf9ea7cf2L, + + 0x0000000000000000L, + 0xf6f734b768e04748L, + 0xd9374f3d89571dfbL, + 0x2fc07b8ae1b75ab3L, + 0x86b7b8284a39a89dL, + 0x70408c9f22d9efd5L, + 0x5f80f715c36eb566L, + 0xa977c3a2ab8ef22eL, + 0x39b65603cce4c251L, + 0xcf4162b4a4048519L, + 0xe081193e45b3dfaaL, + 0x16762d892d5398e2L, + 0xbf01ee2b86dd6accL, + 0x49f6da9cee3d2d84L, + 0x6636a1160f8a7737L, + 0x90c195a1676a307fL, + 0x736cac0799c984a2L, + 0x859b98b0f129c3eaL, + 0xaa5be33a109e9959L, + 0x5cacd78d787ede11L, + 0xf5db142fd3f02c3fL, + 0x032c2098bb106b77L, + 0x2cec5b125aa731c4L, + 0xda1b6fa53247768cL, + 0x4adafa04552d46f3L, + 0xbc2dceb33dcd01bbL, + 0x93edb539dc7a5b08L, + 0x651a818eb49a1c40L, + 0xcc6d422c1f14ee6eL, + 0x3a9a769b77f4a926L, + 0x155a0d119643f395L, + 0xe3ad39a6fea3b4ddL, + 0xe6d9580f33930944L, + 0x102e6cb85b734e0cL, + 0x3fee1732bac414bfL, + 0xc9192385d22453f7L, + 0x606ee02779aaa1d9L, + 0x9699d490114ae691L, + 0xb959af1af0fdbc22L, + 0x4fae9bad981dfb6aL, + 0xdf6f0e0cff77cb15L, + 0x29983abb97978c5dL, + 0x065841317620d6eeL, + 0xf0af75861ec091a6L, + 0x59d8b624b54e6388L, + 0xaf2f8293ddae24c0L, + 0x80eff9193c197e73L, + 0x7618cdae54f9393bL, + 0x95b5f408aa5a8de6L, + 0x6342c0bfc2bacaaeL, + 0x4c82bb35230d901dL, + 0xba758f824bedd755L, + 0x13024c20e063257bL, + 0xe5f5789788836233L, + 0xca35031d69343880L, + 0x3cc237aa01d47fc8L, + 0xac03a20b66be4fb7L, + 0x5af496bc0e5e08ffL, + 0x7534ed36efe9524cL, + 0x83c3d98187091504L, + 0x2ab41a232c87e72aL, + 0xdc432e944467a062L, + 0xf383551ea5d0fad1L, + 0x057461a9cd30bd99L, + 0xf96b964d3fb181e3L, + 0x0f9ca2fa5751c6abL, + 0x205cd970b6e69c18L, + 0xd6abedc7de06db50L, + 0x7fdc2e657588297eL, + 0x892b1ad21d686e36L, + 0xa6eb6158fcdf3485L, + 0x501c55ef943f73cdL, + 0xc0ddc04ef35543b2L, + 0x362af4f99bb504faL, + 0x19ea8f737a025e49L, + 0xef1dbbc412e21901L, + 0x466a7866b96ceb2fL, + 0xb09d4cd1d18cac67L, + 0x9f5d375b303bf6d4L, + 0x69aa03ec58dbb19cL, + 0x8a073a4aa6780541L, + 0x7cf00efdce984209L, + 0x533075772f2f18baL, + 0xa5c741c047cf5ff2L, + 0x0cb08262ec41addcL, + 0xfa47b6d584a1ea94L, + 0xd587cd5f6516b027L, + 0x2370f9e80df6f76fL, + 0xb3b16c496a9cc710L, + 0x454658fe027c8058L, + 0x6a862374e3cbdaebL, + 0x9c7117c38b2b9da3L, + 0x3506d46120a56f8dL, + 0xc3f1e0d6484528c5L, + 0xec319b5ca9f27276L, + 0x1ac6afebc112353eL, + 0x1fb2ce420c2288a7L, + 0xe945faf564c2cfefL, + 0xc685817f8575955cL, + 0x3072b5c8ed95d214L, + 0x9905766a461b203aL, + 0x6ff242dd2efb6772L, + 0x40323957cf4c3dc1L, + 0xb6c50de0a7ac7a89L, + 0x26049841c0c64af6L, + 0xd0f3acf6a8260dbeL, + 0xff33d77c4991570dL, + 0x09c4e3cb21711045L, + 0xa0b320698affe26bL, + 0x564414dee21fa523L, + 0x79846f5403a8ff90L, + 0x8f735be36b48b8d8L, + 0x6cde624595eb0c05L, + 0x9a2956f2fd0b4b4dL, + 0xb5e92d781cbc11feL, + 0x431e19cf745c56b6L, + 0xea69da6ddfd2a498L, + 0x1c9eeedab732e3d0L, + 0x335e95505685b963L, + 0xc5a9a1e73e65fe2bL, + 0x55683446590fce54L, + 0xa39f00f131ef891cL, + 0x8c5f7b7bd058d3afL, + 0x7aa84fccb8b894e7L, + 0xd3df8c6e133666c9L, + 0x2528b8d97bd62181L, + 0x0ae8c3539a617b32L, + 0xfc1ff7e4f2813c7aL, + 0xc60e0ac927f490adL, + 0x30f93e7e4f14d7e5L, + 0x1f3945f4aea38d56L, + 0xe9ce7143c643ca1eL, + 0x40b9b2e16dcd3830L, + 0xb64e8656052d7f78L, + 0x998efddce49a25cbL, + 0x6f79c96b8c7a6283L, + 0xffb85ccaeb1052fcL, + 0x094f687d83f015b4L, + 0x268f13f762474f07L, + 0xd07827400aa7084fL, + 0x790fe4e2a129fa61L, + 0x8ff8d055c9c9bd29L, + 0xa038abdf287ee79aL, + 0x56cf9f68409ea0d2L, + 0xb562a6cebe3d140fL, + 0x43959279d6dd5347L, + 0x6c55e9f3376a09f4L, + 0x9aa2dd445f8a4ebcL, + 0x33d51ee6f404bc92L, + 0xc5222a519ce4fbdaL, + 0xeae251db7d53a169L, + 0x1c15656c15b3e621L, + 0x8cd4f0cd72d9d65eL, + 0x7a23c47a1a399116L, + 0x55e3bff0fb8ecba5L, + 0xa3148b47936e8cedL, + 0x0a6348e538e07ec3L, + 0xfc947c525000398bL, + 0xd35407d8b1b76338L, + 0x25a3336fd9572470L, + 0x20d752c6146799e9L, + 0xd62066717c87dea1L, + 0xf9e01dfb9d308412L, + 0x0f17294cf5d0c35aL, + 0xa660eaee5e5e3174L, + 0x5097de5936be763cL, + 0x7f57a5d3d7092c8fL, + 0x89a09164bfe96bc7L, + 0x196104c5d8835bb8L, + 0xef963072b0631cf0L, + 0xc0564bf851d44643L, + 0x36a17f4f3934010bL, + 0x9fd6bced92baf325L, + 0x6921885afa5ab46dL, + 0x46e1f3d01bedeedeL, + 0xb016c767730da996L, + 0x53bbfec18dae1d4bL, + 0xa54cca76e54e5a03L, + 0x8a8cb1fc04f900b0L, + 0x7c7b854b6c1947f8L, + 0xd50c46e9c797b5d6L, + 0x23fb725eaf77f29eL, + 0x0c3b09d44ec0a82dL, + 0xfacc3d632620ef65L, + 0x6a0da8c2414adf1aL, + 0x9cfa9c7529aa9852L, + 0xb33ae7ffc81dc2e1L, + 0x45cdd348a0fd85a9L, + 0xecba10ea0b737787L, + 0x1a4d245d639330cfL, + 0x358d5fd782246a7cL, + 0xc37a6b60eac42d34L, + 0x3f659c841845114eL, + 0xc992a83370a55606L, + 0xe652d3b991120cb5L, + 0x10a5e70ef9f24bfdL, + 0xb9d224ac527cb9d3L, + 0x4f25101b3a9cfe9bL, + 0x60e56b91db2ba428L, + 0x96125f26b3cbe360L, + 0x06d3ca87d4a1d31fL, + 0xf024fe30bc419457L, + 0xdfe485ba5df6cee4L, + 0x2913b10d351689acL, + 0x806472af9e987b82L, + 0x76934618f6783ccaL, + 0x59533d9217cf6679L, + 0xafa409257f2f2131L, + 0x4c093083818c95ecL, + 0xbafe0434e96cd2a4L, + 0x953e7fbe08db8817L, + 0x63c94b09603bcf5fL, + 0xcabe88abcbb53d71L, + 0x3c49bc1ca3557a39L, + 0x1389c79642e2208aL, + 0xe57ef3212a0267c2L, + 0x75bf66804d6857bdL, + 0x83485237258810f5L, + 0xac8829bdc43f4a46L, + 0x5a7f1d0aacdf0d0eL, + 0xf308dea80751ff20L, + 0x05ffea1f6fb1b868L, + 0x2a3f91958e06e2dbL, + 0xdcc8a522e6e6a593L, + 0xd9bcc48b2bd6180aL, + 0x2f4bf03c43365f42L, + 0x008b8bb6a28105f1L, + 0xf67cbf01ca6142b9L, + 0x5f0b7ca361efb097L, + 0xa9fc4814090ff7dfL, + 0x863c339ee8b8ad6cL, + 0x70cb07298058ea24L, + 0xe00a9288e732da5bL, + 0x16fda63f8fd29d13L, + 0x393dddb56e65c7a0L, + 0xcfcae902068580e8L, + 0x66bd2aa0ad0b72c6L, + 0x904a1e17c5eb358eL, + 0xbf8a659d245c6f3dL, + 0x497d512a4cbc2875L, + 0xaad0688cb21f9ca8L, + 0x5c275c3bdaffdbe0L, + 0x73e727b13b488153L, + 0x8510130653a8c61bL, + 0x2c67d0a4f8263435L, + 0xda90e41390c6737dL, + 0xf5509f99717129ceL, + 0x03a7ab2e19916e86L, + 0x93663e8f7efb5ef9L, + 0x65910a38161b19b1L, + 0x4a5171b2f7ac4302L, + 0xbca645059f4c044aL, + 0x15d186a734c2f664L, + 0xe326b2105c22b12cL, + 0xcce6c99abd95eb9fL, + 0x3a11fd2dd575acd7L, + + 0x0000000000000000L, + 0x71b0c13da512335dL, + 0xe361827b4a2466baL, + 0x92d14346ef3655e7L, + 0xf21a22a5ccdf5e1fL, + 0x83aae39869cd6d42L, + 0x117ba0de86fb38a5L, + 0x60cb61e323e90bf8L, + 0xd0ed6318c1292f55L, + 0xa15da225643b1c08L, + 0x338ce1638b0d49efL, + 0x423c205e2e1f7ab2L, + 0x22f741bd0df6714aL, + 0x53478080a8e44217L, + 0xc196c3c647d217f0L, + 0xb02602fbe2c024adL, + 0x9503e062dac5cdc1L, + 0xe4b3215f7fd7fe9cL, + 0x7662621990e1ab7bL, + 0x07d2a32435f39826L, + 0x6719c2c7161a93deL, + 0x16a903fab308a083L, + 0x847840bc5c3ef564L, + 0xf5c88181f92cc639L, + 0x45ee837a1bece294L, + 0x345e4247befed1c9L, + 0xa68f010151c8842eL, + 0xd73fc03cf4dab773L, + 0xb7f4a1dfd733bc8bL, + 0xc64460e272218fd6L, + 0x549523a49d17da31L, + 0x2525e2993805e96cL, + 0x1edee696ed1c08e9L, + 0x6f6e27ab480e3bb4L, + 0xfdbf64eda7386e53L, + 0x8c0fa5d0022a5d0eL, + 0xecc4c43321c356f6L, + 0x9d74050e84d165abL, + 0x0fa546486be7304cL, + 0x7e158775cef50311L, + 0xce33858e2c3527bcL, + 0xbf8344b3892714e1L, + 0x2d5207f566114106L, + 0x5ce2c6c8c303725bL, + 0x3c29a72be0ea79a3L, + 0x4d99661645f84afeL, + 0xdf482550aace1f19L, + 0xaef8e46d0fdc2c44L, + 0x8bdd06f437d9c528L, + 0xfa6dc7c992cbf675L, + 0x68bc848f7dfda392L, + 0x190c45b2d8ef90cfL, + 0x79c72451fb069b37L, + 0x0877e56c5e14a86aL, + 0x9aa6a62ab122fd8dL, + 0xeb1667171430ced0L, + 0x5b3065ecf6f0ea7dL, + 0x2a80a4d153e2d920L, + 0xb851e797bcd48cc7L, + 0xc9e126aa19c6bf9aL, + 0xa92a47493a2fb462L, + 0xd89a86749f3d873fL, + 0x4a4bc532700bd2d8L, + 0x3bfb040fd519e185L, + 0x3dbdcd2dda3811d2L, + 0x4c0d0c107f2a228fL, + 0xdedc4f56901c7768L, + 0xaf6c8e6b350e4435L, + 0xcfa7ef8816e74fcdL, + 0xbe172eb5b3f57c90L, + 0x2cc66df35cc32977L, + 0x5d76accef9d11a2aL, + 0xed50ae351b113e87L, + 0x9ce06f08be030ddaL, + 0x0e312c4e5135583dL, + 0x7f81ed73f4276b60L, + 0x1f4a8c90d7ce6098L, + 0x6efa4dad72dc53c5L, + 0xfc2b0eeb9dea0622L, + 0x8d9bcfd638f8357fL, + 0xa8be2d4f00fddc13L, + 0xd90eec72a5efef4eL, + 0x4bdfaf344ad9baa9L, + 0x3a6f6e09efcb89f4L, + 0x5aa40feacc22820cL, + 0x2b14ced76930b151L, + 0xb9c58d918606e4b6L, + 0xc8754cac2314d7ebL, + 0x78534e57c1d4f346L, + 0x09e38f6a64c6c01bL, + 0x9b32cc2c8bf095fcL, + 0xea820d112ee2a6a1L, + 0x8a496cf20d0bad59L, + 0xfbf9adcfa8199e04L, + 0x6928ee89472fcbe3L, + 0x18982fb4e23df8beL, + 0x23632bbb3724193bL, + 0x52d3ea8692362a66L, + 0xc002a9c07d007f81L, + 0xb1b268fdd8124cdcL, + 0xd179091efbfb4724L, + 0xa0c9c8235ee97479L, + 0x32188b65b1df219eL, + 0x43a84a5814cd12c3L, + 0xf38e48a3f60d366eL, + 0x823e899e531f0533L, + 0x10efcad8bc2950d4L, + 0x615f0be5193b6389L, + 0x01946a063ad26871L, + 0x7024ab3b9fc05b2cL, + 0xe2f5e87d70f60ecbL, + 0x93452940d5e43d96L, + 0xb660cbd9ede1d4faL, + 0xc7d00ae448f3e7a7L, + 0x550149a2a7c5b240L, + 0x24b1889f02d7811dL, + 0x447ae97c213e8ae5L, + 0x35ca2841842cb9b8L, + 0xa71b6b076b1aec5fL, + 0xd6abaa3ace08df02L, + 0x668da8c12cc8fbafL, + 0x173d69fc89dac8f2L, + 0x85ec2aba66ec9d15L, + 0xf45ceb87c3feae48L, + 0x94978a64e017a5b0L, + 0xe5274b59450596edL, + 0x77f6081faa33c30aL, + 0x0646c9220f21f057L, + 0x7b7b9a5bb47023a4L, + 0x0acb5b66116210f9L, + 0x981a1820fe54451eL, + 0xe9aad91d5b467643L, + 0x8961b8fe78af7dbbL, + 0xf8d179c3ddbd4ee6L, + 0x6a003a85328b1b01L, + 0x1bb0fbb89799285cL, + 0xab96f94375590cf1L, + 0xda26387ed04b3facL, + 0x48f77b383f7d6a4bL, + 0x3947ba059a6f5916L, + 0x598cdbe6b98652eeL, + 0x283c1adb1c9461b3L, + 0xbaed599df3a23454L, + 0xcb5d98a056b00709L, + 0xee787a396eb5ee65L, + 0x9fc8bb04cba7dd38L, + 0x0d19f842249188dfL, + 0x7ca9397f8183bb82L, + 0x1c62589ca26ab07aL, + 0x6dd299a107788327L, + 0xff03dae7e84ed6c0L, + 0x8eb31bda4d5ce59dL, + 0x3e951921af9cc130L, + 0x4f25d81c0a8ef26dL, + 0xddf49b5ae5b8a78aL, + 0xac445a6740aa94d7L, + 0xcc8f3b8463439f2fL, + 0xbd3ffab9c651ac72L, + 0x2feeb9ff2967f995L, + 0x5e5e78c28c75cac8L, + 0x65a57ccd596c2b4dL, + 0x1415bdf0fc7e1810L, + 0x86c4feb613484df7L, + 0xf7743f8bb65a7eaaL, + 0x97bf5e6895b37552L, + 0xe60f9f5530a1460fL, + 0x74dedc13df9713e8L, + 0x056e1d2e7a8520b5L, + 0xb5481fd598450418L, + 0xc4f8dee83d573745L, + 0x56299daed26162a2L, + 0x27995c93777351ffL, + 0x47523d70549a5a07L, + 0x36e2fc4df188695aL, + 0xa433bf0b1ebe3cbdL, + 0xd5837e36bbac0fe0L, + 0xf0a69caf83a9e68cL, + 0x81165d9226bbd5d1L, + 0x13c71ed4c98d8036L, + 0x6277dfe96c9fb36bL, + 0x02bcbe0a4f76b893L, + 0x730c7f37ea648bceL, + 0xe1dd3c710552de29L, + 0x906dfd4ca040ed74L, + 0x204bffb74280c9d9L, + 0x51fb3e8ae792fa84L, + 0xc32a7dcc08a4af63L, + 0xb29abcf1adb69c3eL, + 0xd251dd128e5f97c6L, + 0xa3e11c2f2b4da49bL, + 0x31305f69c47bf17cL, + 0x40809e546169c221L, + 0x46c657766e483276L, + 0x3776964bcb5a012bL, + 0xa5a7d50d246c54ccL, + 0xd4171430817e6791L, + 0xb4dc75d3a2976c69L, + 0xc56cb4ee07855f34L, + 0x57bdf7a8e8b30ad3L, + 0x260d36954da1398eL, + 0x962b346eaf611d23L, + 0xe79bf5530a732e7eL, + 0x754ab615e5457b99L, + 0x04fa7728405748c4L, + 0x643116cb63be433cL, + 0x1581d7f6c6ac7061L, + 0x875094b0299a2586L, + 0xf6e0558d8c8816dbL, + 0xd3c5b714b48dffb7L, + 0xa2757629119fcceaL, + 0x30a4356ffea9990dL, + 0x4114f4525bbbaa50L, + 0x21df95b17852a1a8L, + 0x506f548cdd4092f5L, + 0xc2be17ca3276c712L, + 0xb30ed6f79764f44fL, + 0x0328d40c75a4d0e2L, + 0x72981531d0b6e3bfL, + 0xe04956773f80b658L, + 0x91f9974a9a928505L, + 0xf132f6a9b97b8efdL, + 0x808237941c69bda0L, + 0x125374d2f35fe847L, + 0x63e3b5ef564ddb1aL, + 0x5818b1e083543a9fL, + 0x29a870dd264609c2L, + 0xbb79339bc9705c25L, + 0xcac9f2a66c626f78L, + 0xaa0293454f8b6480L, + 0xdbb25278ea9957ddL, + 0x4963113e05af023aL, + 0x38d3d003a0bd3167L, + 0x88f5d2f8427d15caL, + 0xf94513c5e76f2697L, + 0x6b94508308597370L, + 0x1a2491bead4b402dL, + 0x7aeff05d8ea24bd5L, + 0x0b5f31602bb07888L, + 0x998e7226c4862d6fL, + 0xe83eb31b61941e32L, + 0xcd1b51825991f75eL, + 0xbcab90bffc83c403L, + 0x2e7ad3f913b591e4L, + 0x5fca12c4b6a7a2b9L, + 0x3f017327954ea941L, + 0x4eb1b21a305c9a1cL, + 0xdc60f15cdf6acffbL, + 0xadd030617a78fca6L, + 0x1df6329a98b8d80bL, + 0x6c46f3a73daaeb56L, + 0xfe97b0e1d29cbeb1L, + 0x8f2771dc778e8decL, + 0xefec103f54678614L, + 0x9e5cd102f175b549L, + 0x0c8d92441e43e0aeL, + 0x7d3d5379bb51d3f3L, + + 0x0000000000000000L, + 0xbfdb6c480f15915eL, + 0x4b6ffec346bcb1d7L, + 0xf4b4928b49a92089L, + 0x96dffd868d7963aeL, + 0x290491ce826cf2f0L, + 0xddb00345cbc5d279L, + 0x626b6f0dc4d04327L, + 0x1966dd5e42655437L, + 0xa6bdb1164d70c569L, + 0x5209239d04d9e5e0L, + 0xedd24fd50bcc74beL, + 0x8fb920d8cf1c3799L, + 0x30624c90c009a6c7L, + 0xc4d6de1b89a0864eL, + 0x7b0db25386b51710L, + 0x32cdbabc84caa86eL, + 0x8d16d6f48bdf3930L, + 0x79a2447fc27619b9L, + 0xc6792837cd6388e7L, + 0xa412473a09b3cbc0L, + 0x1bc92b7206a65a9eL, + 0xef7db9f94f0f7a17L, + 0x50a6d5b1401aeb49L, + 0x2bab67e2c6affc59L, + 0x94700baac9ba6d07L, + 0x60c4992180134d8eL, + 0xdf1ff5698f06dcd0L, + 0xbd749a644bd69ff7L, + 0x02aff62c44c30ea9L, + 0xf61b64a70d6a2e20L, + 0x49c008ef027fbf7eL, + 0x659b7579099550dcL, + 0xda4019310680c182L, + 0x2ef48bba4f29e10bL, + 0x912fe7f2403c7055L, + 0xf34488ff84ec3372L, + 0x4c9fe4b78bf9a22cL, + 0xb82b763cc25082a5L, + 0x07f01a74cd4513fbL, + 0x7cfda8274bf004ebL, + 0xc326c46f44e595b5L, + 0x379256e40d4cb53cL, + 0x88493aac02592462L, + 0xea2255a1c6896745L, + 0x55f939e9c99cf61bL, + 0xa14dab628035d692L, + 0x1e96c72a8f2047ccL, + 0x5756cfc58d5ff8b2L, + 0xe88da38d824a69ecL, + 0x1c393106cbe34965L, + 0xa3e25d4ec4f6d83bL, + 0xc189324300269b1cL, + 0x7e525e0b0f330a42L, + 0x8ae6cc80469a2acbL, + 0x353da0c8498fbb95L, + 0x4e30129bcf3aac85L, + 0xf1eb7ed3c02f3ddbL, + 0x055fec5889861d52L, + 0xba84801086938c0cL, + 0xd8efef1d4243cf2bL, + 0x673483554d565e75L, + 0x938011de04ff7efcL, + 0x2c5b7d960beaefa2L, + 0xcb36eaf2132aa1b8L, + 0x74ed86ba1c3f30e6L, + 0x805914315596106fL, + 0x3f8278795a838131L, + 0x5de917749e53c216L, + 0xe2327b3c91465348L, + 0x1686e9b7d8ef73c1L, + 0xa95d85ffd7fae29fL, + 0xd25037ac514ff58fL, + 0x6d8b5be45e5a64d1L, + 0x993fc96f17f34458L, + 0x26e4a52718e6d506L, + 0x448fca2adc369621L, + 0xfb54a662d323077fL, + 0x0fe034e99a8a27f6L, + 0xb03b58a1959fb6a8L, + 0xf9fb504e97e009d6L, + 0x46203c0698f59888L, + 0xb294ae8dd15cb801L, + 0x0d4fc2c5de49295fL, + 0x6f24adc81a996a78L, + 0xd0ffc180158cfb26L, + 0x244b530b5c25dbafL, + 0x9b903f4353304af1L, + 0xe09d8d10d5855de1L, + 0x5f46e158da90ccbfL, + 0xabf273d39339ec36L, + 0x14291f9b9c2c7d68L, + 0x7642709658fc3e4fL, + 0xc9991cde57e9af11L, + 0x3d2d8e551e408f98L, + 0x82f6e21d11551ec6L, + 0xaead9f8b1abff164L, + 0x1176f3c315aa603aL, + 0xe5c261485c0340b3L, + 0x5a190d005316d1edL, + 0x3872620d97c692caL, + 0x87a90e4598d30394L, + 0x731d9cced17a231dL, + 0xccc6f086de6fb243L, + 0xb7cb42d558daa553L, + 0x08102e9d57cf340dL, + 0xfca4bc161e661484L, + 0x437fd05e117385daL, + 0x2114bf53d5a3c6fdL, + 0x9ecfd31bdab657a3L, + 0x6a7b4190931f772aL, + 0xd5a02dd89c0ae674L, + 0x9c6025379e75590aL, + 0x23bb497f9160c854L, + 0xd70fdbf4d8c9e8ddL, + 0x68d4b7bcd7dc7983L, + 0x0abfd8b1130c3aa4L, + 0xb564b4f91c19abfaL, + 0x41d0267255b08b73L, + 0xfe0b4a3a5aa51a2dL, + 0x8506f869dc100d3dL, + 0x3add9421d3059c63L, + 0xce6906aa9aacbceaL, + 0x71b26ae295b92db4L, + 0x13d905ef51696e93L, + 0xac0269a75e7cffcdL, + 0x58b6fb2c17d5df44L, + 0xe76d976418c04e1aL, + 0xa2b4f3b77ec2d01bL, + 0x1d6f9fff71d74145L, + 0xe9db0d74387e61ccL, + 0x5600613c376bf092L, + 0x346b0e31f3bbb3b5L, + 0x8bb06279fcae22ebL, + 0x7f04f0f2b5070262L, + 0xc0df9cbaba12933cL, + 0xbbd22ee93ca7842cL, + 0x040942a133b21572L, + 0xf0bdd02a7a1b35fbL, + 0x4f66bc62750ea4a5L, + 0x2d0dd36fb1dee782L, + 0x92d6bf27becb76dcL, + 0x66622dacf7625655L, + 0xd9b941e4f877c70bL, + 0x9079490bfa087875L, + 0x2fa22543f51de92bL, + 0xdb16b7c8bcb4c9a2L, + 0x64cddb80b3a158fcL, + 0x06a6b48d77711bdbL, + 0xb97dd8c578648a85L, + 0x4dc94a4e31cdaa0cL, + 0xf21226063ed83b52L, + 0x891f9455b86d2c42L, + 0x36c4f81db778bd1cL, + 0xc2706a96fed19d95L, + 0x7dab06def1c40ccbL, + 0x1fc069d335144fecL, + 0xa01b059b3a01deb2L, + 0x54af971073a8fe3bL, + 0xeb74fb587cbd6f65L, + 0xc72f86ce775780c7L, + 0x78f4ea8678421199L, + 0x8c40780d31eb3110L, + 0x339b14453efea04eL, + 0x51f07b48fa2ee369L, + 0xee2b1700f53b7237L, + 0x1a9f858bbc9252beL, + 0xa544e9c3b387c3e0L, + 0xde495b903532d4f0L, + 0x619237d83a2745aeL, + 0x9526a553738e6527L, + 0x2afdc91b7c9bf479L, + 0x4896a616b84bb75eL, + 0xf74dca5eb75e2600L, + 0x03f958d5fef70689L, + 0xbc22349df1e297d7L, + 0xf5e23c72f39d28a9L, + 0x4a39503afc88b9f7L, + 0xbe8dc2b1b521997eL, + 0x0156aef9ba340820L, + 0x633dc1f47ee44b07L, + 0xdce6adbc71f1da59L, + 0x28523f373858fad0L, + 0x9789537f374d6b8eL, + 0xec84e12cb1f87c9eL, + 0x535f8d64beededc0L, + 0xa7eb1feff744cd49L, + 0x183073a7f8515c17L, + 0x7a5b1caa3c811f30L, + 0xc58070e233948e6eL, + 0x3134e2697a3daee7L, + 0x8eef8e2175283fb9L, + 0x698219456de871a3L, + 0xd659750d62fde0fdL, + 0x22ede7862b54c074L, + 0x9d368bce2441512aL, + 0xff5de4c3e091120dL, + 0x4086888bef848353L, + 0xb4321a00a62da3daL, + 0x0be97648a9383284L, + 0x70e4c41b2f8d2594L, + 0xcf3fa8532098b4caL, + 0x3b8b3ad869319443L, + 0x845056906624051dL, + 0xe63b399da2f4463aL, + 0x59e055d5ade1d764L, + 0xad54c75ee448f7edL, + 0x128fab16eb5d66b3L, + 0x5b4fa3f9e922d9cdL, + 0xe494cfb1e6374893L, + 0x10205d3aaf9e681aL, + 0xaffb3172a08bf944L, + 0xcd905e7f645bba63L, + 0x724b32376b4e2b3dL, + 0x86ffa0bc22e70bb4L, + 0x3924ccf42df29aeaL, + 0x42297ea7ab478dfaL, + 0xfdf212efa4521ca4L, + 0x09468064edfb3c2dL, + 0xb69dec2ce2eead73L, + 0xd4f68321263eee54L, + 0x6b2def69292b7f0aL, + 0x9f997de260825f83L, + 0x204211aa6f97ceddL, + 0x0c196c3c647d217fL, + 0xb3c200746b68b021L, + 0x477692ff22c190a8L, + 0xf8adfeb72dd401f6L, + 0x9ac691bae90442d1L, + 0x251dfdf2e611d38fL, + 0xd1a96f79afb8f306L, + 0x6e720331a0ad6258L, + 0x157fb16226187548L, + 0xaaa4dd2a290de416L, + 0x5e104fa160a4c49fL, + 0xe1cb23e96fb155c1L, + 0x83a04ce4ab6116e6L, + 0x3c7b20aca47487b8L, + 0xc8cfb227eddda731L, + 0x7714de6fe2c8366fL, + 0x3ed4d680e0b78911L, + 0x810fbac8efa2184fL, + 0x75bb2843a60b38c6L, + 0xca60440ba91ea998L, + 0xa80b2b066dceeabfL, + 0x17d0474e62db7be1L, + 0xe364d5c52b725b68L, + 0x5cbfb98d2467ca36L, + 0x27b20bdea2d2dd26L, + 0x98696796adc74c78L, + 0x6cddf51de46e6cf1L, + 0xd3069955eb7bfdafL, + 0xb16df6582fabbe88L, + 0x0eb69a1020be2fd6L, + 0xfa02089b69170f5fL, + 0x45d964d366029e01L, + + 0x0000000000000000L, + 0x3ea616bd2ae10d77L, + 0x7d4c2d7a55c21aeeL, + 0x43ea3bc77f231799L, + 0xfa985af4ab8435dcL, + 0xc43e4c49816538abL, + 0x87d4778efe462f32L, + 0xb9726133d4a72245L, + 0xc1e993ba0f9ff8d3L, + 0xff4f8507257ef5a4L, + 0xbca5bec05a5de23dL, + 0x8203a87d70bcef4aL, + 0x3b71c94ea41bcd0fL, + 0x05d7dff38efac078L, + 0x463de434f1d9d7e1L, + 0x789bf289db38da96L, + 0xb70a012747a862cdL, + 0x89ac179a6d496fbaL, + 0xca462c5d126a7823L, + 0xf4e03ae0388b7554L, + 0x4d925bd3ec2c5711L, + 0x73344d6ec6cd5a66L, + 0x30de76a9b9ee4dffL, + 0x0e786014930f4088L, + 0x76e3929d48379a1eL, + 0x4845842062d69769L, + 0x0bafbfe71df580f0L, + 0x3509a95a37148d87L, + 0x8c7bc869e3b3afc2L, + 0xb2ddded4c952a2b5L, + 0xf137e513b671b52cL, + 0xcf91f3ae9c90b85bL, + 0x5acd241dd7c756f1L, + 0x646b32a0fd265b86L, + 0x2781096782054c1fL, + 0x19271fdaa8e44168L, + 0xa0557ee97c43632dL, + 0x9ef3685456a26e5aL, + 0xdd195393298179c3L, + 0xe3bf452e036074b4L, + 0x9b24b7a7d858ae22L, + 0xa582a11af2b9a355L, + 0xe6689add8d9ab4ccL, + 0xd8ce8c60a77bb9bbL, + 0x61bced5373dc9bfeL, + 0x5f1afbee593d9689L, + 0x1cf0c029261e8110L, + 0x2256d6940cff8c67L, + 0xedc7253a906f343cL, + 0xd3613387ba8e394bL, + 0x908b0840c5ad2ed2L, + 0xae2d1efdef4c23a5L, + 0x175f7fce3beb01e0L, + 0x29f96973110a0c97L, + 0x6a1352b46e291b0eL, + 0x54b5440944c81679L, + 0x2c2eb6809ff0ccefL, + 0x1288a03db511c198L, + 0x51629bfaca32d601L, + 0x6fc48d47e0d3db76L, + 0xd6b6ec743474f933L, + 0xe810fac91e95f444L, + 0xabfac10e61b6e3ddL, + 0x955cd7b34b57eeaaL, + 0xb59a483baf8eade2L, + 0x8b3c5e86856fa095L, + 0xc8d66541fa4cb70cL, + 0xf67073fcd0adba7bL, + 0x4f0212cf040a983eL, + 0x71a404722eeb9549L, + 0x324e3fb551c882d0L, + 0x0ce829087b298fa7L, + 0x7473db81a0115531L, + 0x4ad5cd3c8af05846L, + 0x093ff6fbf5d34fdfL, + 0x3799e046df3242a8L, + 0x8eeb81750b9560edL, + 0xb04d97c821746d9aL, + 0xf3a7ac0f5e577a03L, + 0xcd01bab274b67774L, + 0x0290491ce826cf2fL, + 0x3c365fa1c2c7c258L, + 0x7fdc6466bde4d5c1L, + 0x417a72db9705d8b6L, + 0xf80813e843a2faf3L, + 0xc6ae05556943f784L, + 0x85443e921660e01dL, + 0xbbe2282f3c81ed6aL, + 0xc379daa6e7b937fcL, + 0xfddfcc1bcd583a8bL, + 0xbe35f7dcb27b2d12L, + 0x8093e161989a2065L, + 0x39e180524c3d0220L, + 0x074796ef66dc0f57L, + 0x44adad2819ff18ceL, + 0x7a0bbb95331e15b9L, + 0xef576c267849fb13L, + 0xd1f17a9b52a8f664L, + 0x921b415c2d8be1fdL, + 0xacbd57e1076aec8aL, + 0x15cf36d2d3cdcecfL, + 0x2b69206ff92cc3b8L, + 0x68831ba8860fd421L, + 0x56250d15aceed956L, + 0x2ebeff9c77d603c0L, + 0x1018e9215d370eb7L, + 0x53f2d2e62214192eL, + 0x6d54c45b08f51459L, + 0xd426a568dc52361cL, + 0xea80b3d5f6b33b6bL, + 0xa96a881289902cf2L, + 0x97cc9eafa3712185L, + 0x585d6d013fe199deL, + 0x66fb7bbc150094a9L, + 0x2511407b6a238330L, + 0x1bb756c640c28e47L, + 0xa2c537f59465ac02L, + 0x9c632148be84a175L, + 0xdf891a8fc1a7b6ecL, + 0xe12f0c32eb46bb9bL, + 0x99b4febb307e610dL, + 0xa712e8061a9f6c7aL, + 0xe4f8d3c165bc7be3L, + 0xda5ec57c4f5d7694L, + 0x632ca44f9bfa54d1L, + 0x5d8ab2f2b11b59a6L, + 0x1e608935ce384e3fL, + 0x20c69f88e4d94348L, + 0x5fedb624078ac8afL, + 0x614ba0992d6bc5d8L, + 0x22a19b5e5248d241L, + 0x1c078de378a9df36L, + 0xa575ecd0ac0efd73L, + 0x9bd3fa6d86eff004L, + 0xd839c1aaf9cce79dL, + 0xe69fd717d32deaeaL, + 0x9e04259e0815307cL, + 0xa0a2332322f43d0bL, + 0xe34808e45dd72a92L, + 0xddee1e59773627e5L, + 0x649c7f6aa39105a0L, + 0x5a3a69d7897008d7L, + 0x19d05210f6531f4eL, + 0x277644addcb21239L, + 0xe8e7b7034022aa62L, + 0xd641a1be6ac3a715L, + 0x95ab9a7915e0b08cL, + 0xab0d8cc43f01bdfbL, + 0x127fedf7eba69fbeL, + 0x2cd9fb4ac14792c9L, + 0x6f33c08dbe648550L, + 0x5195d63094858827L, + 0x290e24b94fbd52b1L, + 0x17a83204655c5fc6L, + 0x544209c31a7f485fL, + 0x6ae41f7e309e4528L, + 0xd3967e4de439676dL, + 0xed3068f0ced86a1aL, + 0xaeda5337b1fb7d83L, + 0x907c458a9b1a70f4L, + 0x05209239d04d9e5eL, + 0x3b868484faac9329L, + 0x786cbf43858f84b0L, + 0x46caa9feaf6e89c7L, + 0xffb8c8cd7bc9ab82L, + 0xc11ede705128a6f5L, + 0x82f4e5b72e0bb16cL, + 0xbc52f30a04eabc1bL, + 0xc4c90183dfd2668dL, + 0xfa6f173ef5336bfaL, + 0xb9852cf98a107c63L, + 0x87233a44a0f17114L, + 0x3e515b7774565351L, + 0x00f74dca5eb75e26L, + 0x431d760d219449bfL, + 0x7dbb60b00b7544c8L, + 0xb22a931e97e5fc93L, + 0x8c8c85a3bd04f1e4L, + 0xcf66be64c227e67dL, + 0xf1c0a8d9e8c6eb0aL, + 0x48b2c9ea3c61c94fL, + 0x7614df571680c438L, + 0x35fee49069a3d3a1L, + 0x0b58f22d4342ded6L, + 0x73c300a4987a0440L, + 0x4d651619b29b0937L, + 0x0e8f2ddecdb81eaeL, + 0x30293b63e75913d9L, + 0x895b5a5033fe319cL, + 0xb7fd4ced191f3cebL, + 0xf417772a663c2b72L, + 0xcab161974cdd2605L, + 0xea77fe1fa804654dL, + 0xd4d1e8a282e5683aL, + 0x973bd365fdc67fa3L, + 0xa99dc5d8d72772d4L, + 0x10efa4eb03805091L, + 0x2e49b25629615de6L, + 0x6da3899156424a7fL, + 0x53059f2c7ca34708L, + 0x2b9e6da5a79b9d9eL, + 0x15387b188d7a90e9L, + 0x56d240dff2598770L, + 0x68745662d8b88a07L, + 0xd10637510c1fa842L, + 0xefa021ec26fea535L, + 0xac4a1a2b59ddb2acL, + 0x92ec0c96733cbfdbL, + 0x5d7dff38efac0780L, + 0x63dbe985c54d0af7L, + 0x2031d242ba6e1d6eL, + 0x1e97c4ff908f1019L, + 0xa7e5a5cc4428325cL, + 0x9943b3716ec93f2bL, + 0xdaa988b611ea28b2L, + 0xe40f9e0b3b0b25c5L, + 0x9c946c82e033ff53L, + 0xa2327a3fcad2f224L, + 0xe1d841f8b5f1e5bdL, + 0xdf7e57459f10e8caL, + 0x660c36764bb7ca8fL, + 0x58aa20cb6156c7f8L, + 0x1b401b0c1e75d061L, + 0x25e60db13494dd16L, + 0xb0bada027fc333bcL, + 0x8e1cccbf55223ecbL, + 0xcdf6f7782a012952L, + 0xf350e1c500e02425L, + 0x4a2280f6d4470660L, + 0x7484964bfea60b17L, + 0x376ead8c81851c8eL, + 0x09c8bb31ab6411f9L, + 0x715349b8705ccb6fL, + 0x4ff55f055abdc618L, + 0x0c1f64c2259ed181L, + 0x32b9727f0f7fdcf6L, + 0x8bcb134cdbd8feb3L, + 0xb56d05f1f139f3c4L, + 0xf6873e368e1ae45dL, + 0xc821288ba4fbe92aL, + 0x07b0db25386b5171L, + 0x3916cd98128a5c06L, + 0x7afcf65f6da94b9fL, + 0x445ae0e2474846e8L, + 0xfd2881d193ef64adL, + 0xc38e976cb90e69daL, + 0x8064acabc62d7e43L, + 0xbec2ba16eccc7334L, + 0xc659489f37f4a9a2L, + 0xf8ff5e221d15a4d5L, + 0xbb1565e56236b34cL, + 0x85b3735848d7be3bL, + 0x3cc1126b9c709c7eL, + 0x026704d6b6919109L, + 0x418d3f11c9b28690L, + 0x7f2b29ace3538be7L, + + 0x0000000000000000L, + 0x169489cc969951e5L, + 0x2d2913992d32a3caL, + 0x3bbd9a55bbabf22fL, + 0x5a5227325a654794L, + 0x4cc6aefeccfc1671L, + 0x777b34ab7757e45eL, + 0x61efbd67e1ceb5bbL, + 0xb4a44e64b4ca8f28L, + 0xa230c7a82253decdL, + 0x998d5dfd99f82ce2L, + 0x8f19d4310f617d07L, + 0xeef66956eeafc8bcL, + 0xf862e09a78369959L, + 0xc3df7acfc39d6b76L, + 0xd54bf30355043a93L, + 0x5d91ba9a31028d3bL, + 0x4b053356a79bdcdeL, + 0x70b8a9031c302ef1L, + 0x662c20cf8aa97f14L, + 0x07c39da86b67caafL, + 0x11571464fdfe9b4aL, + 0x2aea8e3146556965L, + 0x3c7e07fdd0cc3880L, + 0xe935f4fe85c80213L, + 0xffa17d32135153f6L, + 0xc41ce767a8faa1d9L, + 0xd2886eab3e63f03cL, + 0xb367d3ccdfad4587L, + 0xa5f35a0049341462L, + 0x9e4ec055f29fe64dL, + 0x88da49996406b7a8L, + 0xbb23753462051a76L, + 0xadb7fcf8f49c4b93L, + 0x960a66ad4f37b9bcL, + 0x809eef61d9aee859L, + 0xe171520638605de2L, + 0xf7e5dbcaaef90c07L, + 0xcc58419f1552fe28L, + 0xdaccc85383cbafcdL, + 0x0f873b50d6cf955eL, + 0x1913b29c4056c4bbL, + 0x22ae28c9fbfd3694L, + 0x343aa1056d646771L, + 0x55d51c628caad2caL, + 0x434195ae1a33832fL, + 0x78fc0ffba1987100L, + 0x6e688637370120e5L, + 0xe6b2cfae5307974dL, + 0xf0264662c59ec6a8L, + 0xcb9bdc377e353487L, + 0xdd0f55fbe8ac6562L, + 0xbce0e89c0962d0d9L, + 0xaa7461509ffb813cL, + 0x91c9fb0524507313L, + 0x875d72c9b2c922f6L, + 0x521681cae7cd1865L, + 0x4482080671544980L, + 0x7f3f9253caffbbafL, + 0x69ab1b9f5c66ea4aL, + 0x0844a6f8bda85ff1L, + 0x1ed02f342b310e14L, + 0x256db561909afc3bL, + 0x33f93cad0603addeL, + 0x429fcc3b9c9da787L, + 0x540b45f70a04f662L, + 0x6fb6dfa2b1af044dL, + 0x7922566e273655a8L, + 0x18cdeb09c6f8e013L, + 0x0e5962c55061b1f6L, + 0x35e4f890ebca43d9L, + 0x2370715c7d53123cL, + 0xf63b825f285728afL, + 0xe0af0b93bece794aL, + 0xdb1291c605658b65L, + 0xcd86180a93fcda80L, + 0xac69a56d72326f3bL, + 0xbafd2ca1e4ab3edeL, + 0x8140b6f45f00ccf1L, + 0x97d43f38c9999d14L, + 0x1f0e76a1ad9f2abcL, + 0x099aff6d3b067b59L, + 0x3227653880ad8976L, + 0x24b3ecf41634d893L, + 0x455c5193f7fa6d28L, + 0x53c8d85f61633ccdL, + 0x6875420adac8cee2L, + 0x7ee1cbc64c519f07L, + 0xabaa38c51955a594L, + 0xbd3eb1098fccf471L, + 0x86832b5c3467065eL, + 0x9017a290a2fe57bbL, + 0xf1f81ff74330e200L, + 0xe76c963bd5a9b3e5L, + 0xdcd10c6e6e0241caL, + 0xca4585a2f89b102fL, + 0xf9bcb90ffe98bdf1L, + 0xef2830c36801ec14L, + 0xd495aa96d3aa1e3bL, + 0xc201235a45334fdeL, + 0xa3ee9e3da4fdfa65L, + 0xb57a17f13264ab80L, + 0x8ec78da489cf59afL, + 0x985304681f56084aL, + 0x4d18f76b4a5232d9L, + 0x5b8c7ea7dccb633cL, + 0x6031e4f267609113L, + 0x76a56d3ef1f9c0f6L, + 0x174ad0591037754dL, + 0x01de599586ae24a8L, + 0x3a63c3c03d05d687L, + 0x2cf74a0cab9c8762L, + 0xa42d0395cf9a30caL, + 0xb2b98a595903612fL, + 0x8904100ce2a89300L, + 0x9f9099c07431c2e5L, + 0xfe7f24a795ff775eL, + 0xe8ebad6b036626bbL, + 0xd356373eb8cdd494L, + 0xc5c2bef22e548571L, + 0x10894df17b50bfe2L, + 0x061dc43dedc9ee07L, + 0x3da05e6856621c28L, + 0x2b34d7a4c0fb4dcdL, + 0x4adb6ac32135f876L, + 0x5c4fe30fb7aca993L, + 0x67f2795a0c075bbcL, + 0x7166f0969a9e0a59L, + 0x853f9877393b4f0eL, + 0x93ab11bbafa21eebL, + 0xa8168bee1409ecc4L, + 0xbe8202228290bd21L, + 0xdf6dbf45635e089aL, + 0xc9f93689f5c7597fL, + 0xf244acdc4e6cab50L, + 0xe4d02510d8f5fab5L, + 0x319bd6138df1c026L, + 0x270f5fdf1b6891c3L, + 0x1cb2c58aa0c363ecL, + 0x0a264c46365a3209L, + 0x6bc9f121d79487b2L, + 0x7d5d78ed410dd657L, + 0x46e0e2b8faa62478L, + 0x50746b746c3f759dL, + 0xd8ae22ed0839c235L, + 0xce3aab219ea093d0L, + 0xf5873174250b61ffL, + 0xe313b8b8b392301aL, + 0x82fc05df525c85a1L, + 0x94688c13c4c5d444L, + 0xafd516467f6e266bL, + 0xb9419f8ae9f7778eL, + 0x6c0a6c89bcf34d1dL, + 0x7a9ee5452a6a1cf8L, + 0x41237f1091c1eed7L, + 0x57b7f6dc0758bf32L, + 0x36584bbbe6960a89L, + 0x20ccc277700f5b6cL, + 0x1b715822cba4a943L, + 0x0de5d1ee5d3df8a6L, + 0x3e1ced435b3e5578L, + 0x2888648fcda7049dL, + 0x1335feda760cf6b2L, + 0x05a17716e095a757L, + 0x644eca71015b12ecL, + 0x72da43bd97c24309L, + 0x4967d9e82c69b126L, + 0x5ff35024baf0e0c3L, + 0x8ab8a327eff4da50L, + 0x9c2c2aeb796d8bb5L, + 0xa791b0bec2c6799aL, + 0xb1053972545f287fL, + 0xd0ea8415b5919dc4L, + 0xc67e0dd92308cc21L, + 0xfdc3978c98a33e0eL, + 0xeb571e400e3a6febL, + 0x638d57d96a3cd843L, + 0x7519de15fca589a6L, + 0x4ea44440470e7b89L, + 0x5830cd8cd1972a6cL, + 0x39df70eb30599fd7L, + 0x2f4bf927a6c0ce32L, + 0x14f663721d6b3c1dL, + 0x0262eabe8bf26df8L, + 0xd72919bddef6576bL, + 0xc1bd9071486f068eL, + 0xfa000a24f3c4f4a1L, + 0xec9483e8655da544L, + 0x8d7b3e8f849310ffL, + 0x9befb743120a411aL, + 0xa0522d16a9a1b335L, + 0xb6c6a4da3f38e2d0L, + 0xc7a0544ca5a6e889L, + 0xd134dd80333fb96cL, + 0xea8947d588944b43L, + 0xfc1dce191e0d1aa6L, + 0x9df2737effc3af1dL, + 0x8b66fab2695afef8L, + 0xb0db60e7d2f10cd7L, + 0xa64fe92b44685d32L, + 0x73041a28116c67a1L, + 0x659093e487f53644L, + 0x5e2d09b13c5ec46bL, + 0x48b9807daac7958eL, + 0x29563d1a4b092035L, + 0x3fc2b4d6dd9071d0L, + 0x047f2e83663b83ffL, + 0x12eba74ff0a2d21aL, + 0x9a31eed694a465b2L, + 0x8ca5671a023d3457L, + 0xb718fd4fb996c678L, + 0xa18c74832f0f979dL, + 0xc063c9e4cec12226L, + 0xd6f74028585873c3L, + 0xed4ada7de3f381ecL, + 0xfbde53b1756ad009L, + 0x2e95a0b2206eea9aL, + 0x3801297eb6f7bb7fL, + 0x03bcb32b0d5c4950L, + 0x15283ae79bc518b5L, + 0x74c787807a0bad0eL, + 0x62530e4cec92fcebL, + 0x59ee941957390ec4L, + 0x4f7a1dd5c1a05f21L, + 0x7c832178c7a3f2ffL, + 0x6a17a8b4513aa31aL, + 0x51aa32e1ea915135L, + 0x473ebb2d7c0800d0L, + 0x26d1064a9dc6b56bL, + 0x30458f860b5fe48eL, + 0x0bf815d3b0f416a1L, + 0x1d6c9c1f266d4744L, + 0xc8276f1c73697dd7L, + 0xdeb3e6d0e5f02c32L, + 0xe50e7c855e5bde1dL, + 0xf39af549c8c28ff8L, + 0x9275482e290c3a43L, + 0x84e1c1e2bf956ba6L, + 0xbf5c5bb7043e9989L, + 0xa9c8d27b92a7c86cL, + 0x21129be2f6a17fc4L, + 0x3786122e60382e21L, + 0x0c3b887bdb93dc0eL, + 0x1aaf01b74d0a8debL, + 0x7b40bcd0acc43850L, + 0x6dd4351c3a5d69b5L, + 0x5669af4981f69b9aL, + 0x40fd2685176fca7fL, + 0x95b6d586426bf0ecL, + 0x83225c4ad4f2a109L, + 0xb89fc61f6f595326L, + 0xae0b4fd3f9c002c3L, + 0xcfe4f2b4180eb778L, + 0xd9707b788e97e69dL, + 0xe2cde12d353c14b2L, + 0xf45968e1a3a54557L, + + 0x0000000000000000L, + 0x0aed36d1a3bb9d7fL, + 0x15da6da347773afeL, + 0x1f375b72e4cca781L, + 0x2bb4db468eee75fcL, + 0x2159ed972d55e883L, + 0x3e6eb6e5c9994f02L, + 0x348380346a22d27dL, + 0x5769b68d1ddcebf8L, + 0x5d84805cbe677687L, + 0x42b3db2e5aabd106L, + 0x485eedfff9104c79L, + 0x7cdd6dcb93329e04L, + 0x76305b1a3089037bL, + 0x69070068d445a4faL, + 0x63ea36b977fe3985L, + 0xaed36d1a3bb9d7f0L, + 0xa43e5bcb98024a8fL, + 0xbb0900b97cceed0eL, + 0xb1e43668df757071L, + 0x8567b65cb557a20cL, + 0x8f8a808d16ec3f73L, + 0x90bddbfff22098f2L, + 0x9a50ed2e519b058dL, + 0xf9badb9726653c08L, + 0xf357ed4685dea177L, + 0xec60b634611206f6L, + 0xe68d80e5c2a99b89L, + 0xd20e00d1a88b49f4L, + 0xd8e336000b30d48bL, + 0xc7d46d72effc730aL, + 0xcd395ba34c47ee75L, + 0x697ffc672fe43c8bL, + 0x6392cab68c5fa1f4L, + 0x7ca591c468930675L, + 0x7648a715cb289b0aL, + 0x42cb2721a10a4977L, + 0x482611f002b1d408L, + 0x57114a82e67d7389L, + 0x5dfc7c5345c6eef6L, + 0x3e164aea3238d773L, + 0x34fb7c3b91834a0cL, + 0x2bcc2749754fed8dL, + 0x21211198d6f470f2L, + 0x15a291acbcd6a28fL, + 0x1f4fa77d1f6d3ff0L, + 0x0078fc0ffba19871L, + 0x0a95cade581a050eL, + 0xc7ac917d145deb7bL, + 0xcd41a7acb7e67604L, + 0xd276fcde532ad185L, + 0xd89bca0ff0914cfaL, + 0xec184a3b9ab39e87L, + 0xe6f57cea390803f8L, + 0xf9c22798ddc4a479L, + 0xf32f11497e7f3906L, + 0x90c527f009810083L, + 0x9a281121aa3a9dfcL, + 0x851f4a534ef63a7dL, + 0x8ff27c82ed4da702L, + 0xbb71fcb6876f757fL, + 0xb19cca6724d4e800L, + 0xaeab9115c0184f81L, + 0xa446a7c463a3d2feL, + 0xd2fff8ce5fc87916L, + 0xd812ce1ffc73e469L, + 0xc725956d18bf43e8L, + 0xcdc8a3bcbb04de97L, + 0xf94b2388d1260ceaL, + 0xf3a61559729d9195L, + 0xec914e2b96513614L, + 0xe67c78fa35eaab6bL, + 0x85964e43421492eeL, + 0x8f7b7892e1af0f91L, + 0x904c23e00563a810L, + 0x9aa11531a6d8356fL, + 0xae229505ccfae712L, + 0xa4cfa3d46f417a6dL, + 0xbbf8f8a68b8dddecL, + 0xb115ce7728364093L, + 0x7c2c95d46471aee6L, + 0x76c1a305c7ca3399L, + 0x69f6f87723069418L, + 0x631bcea680bd0967L, + 0x57984e92ea9fdb1aL, + 0x5d75784349244665L, + 0x42422331ade8e1e4L, + 0x48af15e00e537c9bL, + 0x2b45235979ad451eL, + 0x21a81588da16d861L, + 0x3e9f4efa3eda7fe0L, + 0x3472782b9d61e29fL, + 0x00f1f81ff74330e2L, + 0x0a1ccece54f8ad9dL, + 0x152b95bcb0340a1cL, + 0x1fc6a36d138f9763L, + 0xbb8004a9702c459dL, + 0xb16d3278d397d8e2L, + 0xae5a690a375b7f63L, + 0xa4b75fdb94e0e21cL, + 0x9034dfeffec23061L, + 0x9ad9e93e5d79ad1eL, + 0x85eeb24cb9b50a9fL, + 0x8f03849d1a0e97e0L, + 0xece9b2246df0ae65L, + 0xe60484f5ce4b331aL, + 0xf933df872a87949bL, + 0xf3dee956893c09e4L, + 0xc75d6962e31edb99L, + 0xcdb05fb340a546e6L, + 0xd28704c1a469e167L, + 0xd86a321007d27c18L, + 0x155369b34b95926dL, + 0x1fbe5f62e82e0f12L, + 0x008904100ce2a893L, + 0x0a6432c1af5935ecL, + 0x3ee7b2f5c57be791L, + 0x340a842466c07aeeL, + 0x2b3ddf56820cdd6fL, + 0x21d0e98721b74010L, + 0x423adf3e56497995L, + 0x48d7e9eff5f2e4eaL, + 0x57e0b29d113e436bL, + 0x5d0d844cb285de14L, + 0x698e0478d8a70c69L, + 0x636332a97b1c9116L, + 0x7c5469db9fd03697L, + 0x76b95f0a3c6babe8L, + 0x9126d7cfe7076147L, + 0x9bcbe11e44bcfc38L, + 0x84fcba6ca0705bb9L, + 0x8e118cbd03cbc6c6L, + 0xba920c8969e914bbL, + 0xb07f3a58ca5289c4L, + 0xaf48612a2e9e2e45L, + 0xa5a557fb8d25b33aL, + 0xc64f6142fadb8abfL, + 0xcca25793596017c0L, + 0xd3950ce1bdacb041L, + 0xd9783a301e172d3eL, + 0xedfbba047435ff43L, + 0xe7168cd5d78e623cL, + 0xf821d7a73342c5bdL, + 0xf2cce17690f958c2L, + 0x3ff5bad5dcbeb6b7L, + 0x35188c047f052bc8L, + 0x2a2fd7769bc98c49L, + 0x20c2e1a738721136L, + 0x144161935250c34bL, + 0x1eac5742f1eb5e34L, + 0x019b0c301527f9b5L, + 0x0b763ae1b69c64caL, + 0x689c0c58c1625d4fL, + 0x62713a8962d9c030L, + 0x7d4661fb861567b1L, + 0x77ab572a25aefaceL, + 0x4328d71e4f8c28b3L, + 0x49c5e1cfec37b5ccL, + 0x56f2babd08fb124dL, + 0x5c1f8c6cab408f32L, + 0xf8592ba8c8e35dccL, + 0xf2b41d796b58c0b3L, + 0xed83460b8f946732L, + 0xe76e70da2c2ffa4dL, + 0xd3edf0ee460d2830L, + 0xd900c63fe5b6b54fL, + 0xc6379d4d017a12ceL, + 0xccdaab9ca2c18fb1L, + 0xaf309d25d53fb634L, + 0xa5ddabf476842b4bL, + 0xbaeaf08692488ccaL, + 0xb007c65731f311b5L, + 0x848446635bd1c3c8L, + 0x8e6970b2f86a5eb7L, + 0x915e2bc01ca6f936L, + 0x9bb31d11bf1d6449L, + 0x568a46b2f35a8a3cL, + 0x5c67706350e11743L, + 0x43502b11b42db0c2L, + 0x49bd1dc017962dbdL, + 0x7d3e9df47db4ffc0L, + 0x77d3ab25de0f62bfL, + 0x68e4f0573ac3c53eL, + 0x6209c68699785841L, + 0x01e3f03fee8661c4L, + 0x0b0ec6ee4d3dfcbbL, + 0x14399d9ca9f15b3aL, + 0x1ed4ab4d0a4ac645L, + 0x2a572b7960681438L, + 0x20ba1da8c3d38947L, + 0x3f8d46da271f2ec6L, + 0x3560700b84a4b3b9L, + 0x43d92f01b8cf1851L, + 0x493419d01b74852eL, + 0x560342a2ffb822afL, + 0x5cee74735c03bfd0L, + 0x686df44736216dadL, + 0x6280c296959af0d2L, + 0x7db799e471565753L, + 0x775aaf35d2edca2cL, + 0x14b0998ca513f3a9L, + 0x1e5daf5d06a86ed6L, + 0x016af42fe264c957L, + 0x0b87c2fe41df5428L, + 0x3f0442ca2bfd8655L, + 0x35e9741b88461b2aL, + 0x2ade2f696c8abcabL, + 0x203319b8cf3121d4L, + 0xed0a421b8376cfa1L, + 0xe7e774ca20cd52deL, + 0xf8d02fb8c401f55fL, + 0xf23d196967ba6820L, + 0xc6be995d0d98ba5dL, + 0xcc53af8cae232722L, + 0xd364f4fe4aef80a3L, + 0xd989c22fe9541ddcL, + 0xba63f4969eaa2459L, + 0xb08ec2473d11b926L, + 0xafb99935d9dd1ea7L, + 0xa554afe47a6683d8L, + 0x91d72fd0104451a5L, + 0x9b3a1901b3ffccdaL, + 0x840d427357336b5bL, + 0x8ee074a2f488f624L, + 0x2aa6d366972b24daL, + 0x204be5b73490b9a5L, + 0x3f7cbec5d05c1e24L, + 0x3591881473e7835bL, + 0x0112082019c55126L, + 0x0bff3ef1ba7ecc59L, + 0x14c865835eb26bd8L, + 0x1e255352fd09f6a7L, + 0x7dcf65eb8af7cf22L, + 0x7722533a294c525dL, + 0x68150848cd80f5dcL, + 0x62f83e996e3b68a3L, + 0x567bbead0419badeL, + 0x5c96887ca7a227a1L, + 0x43a1d30e436e8020L, + 0x494ce5dfe0d51d5fL, + 0x8475be7cac92f32aL, + 0x8e9888ad0f296e55L, + 0x91afd3dfebe5c9d4L, + 0x9b42e50e485e54abL, + 0xafc1653a227c86d6L, + 0xa52c53eb81c71ba9L, + 0xba1b0899650bbc28L, + 0xb0f63e48c6b02157L, + 0xd31c08f1b14e18d2L, + 0xd9f13e2012f585adL, + 0xc6c66552f639222cL, + 0xcc2b53835582bf53L, + 0xf8a8d3b73fa06d2eL, + 0xf245e5669c1bf051L, + 0xed72be1478d757d0L, + 0xe79f88c5db6ccaafL, + 0x0000000000000000L, + 0xb0bc2e589204f500L, + 0x55a17ae27c9e796bL, + 0xe51d54baee9a8c6bL, + 0xab42f5c4f93cf2d6L, + 0x1bfedb9c6b3807d6L, + 0xfee38f2685a28bbdL, + 0x4e5fa17e17a67ebdL, + 0x625ccddaaaee76c7L, + 0xd2e0e38238ea83c7L, + 0x37fdb738d6700facL, + 0x874199604474faacL, + 0xc91e381e53d28411L, + 0x79a21646c1d67111L, + 0x9cbf42fc2f4cfd7aL, + 0x2c036ca4bd48087aL, + 0xc4b99bb555dced8eL, + 0x7405b5edc7d8188eL, + 0x9118e157294294e5L, + 0x21a4cf0fbb4661e5L, + 0x6ffb6e71ace01f58L, + 0xdf4740293ee4ea58L, + 0x3a5a1493d07e6633L, + 0x8ae63acb427a9333L, + 0xa6e5566fff329b49L, + 0x165978376d366e49L, + 0xf3442c8d83ace222L, + 0x43f802d511a81722L, + 0x0da7a3ab060e699fL, + 0xbd1b8df3940a9c9fL, + 0x5806d9497a9010f4L, + 0xe8baf711e894e5f4L, + 0xbdaa1139f32e4877L, + 0x0d163f61612abd77L, + 0xe80b6bdb8fb0311cL, + 0x58b745831db4c41cL, + 0x16e8e4fd0a12baa1L, + 0xa654caa598164fa1L, + 0x43499e1f768cc3caL, + 0xf3f5b047e48836caL, + 0xdff6dce359c03eb0L, + 0x6f4af2bbcbc4cbb0L, + 0x8a57a601255e47dbL, + 0x3aeb8859b75ab2dbL, + 0x74b42927a0fccc66L, + 0xc408077f32f83966L, + 0x211553c5dc62b50dL, + 0x91a97d9d4e66400dL, + 0x79138a8ca6f2a5f9L, + 0xc9afa4d434f650f9L, + 0x2cb2f06eda6cdc92L, + 0x9c0ede3648682992L, + 0xd2517f485fce572fL, + 0x62ed5110cdcaa22fL, + 0x87f005aa23502e44L, + 0x374c2bf2b154db44L, + 0x1b4f47560c1cd33eL, + 0xabf3690e9e18263eL, + 0x4eee3db47082aa55L, + 0xfe5213ece2865f55L, + 0xb00db292f52021e8L, + 0x00b19cca6724d4e8L, + 0xe5acc87089be5883L, + 0x5510e6281bbaad83L, + 0x4f8d0420becb0385L, + 0xff312a782ccff685L, + 0x1a2c7ec2c2557aeeL, + 0xaa90509a50518feeL, + 0xe4cff1e447f7f153L, + 0x5473dfbcd5f30453L, + 0xb16e8b063b698838L, + 0x01d2a55ea96d7d38L, + 0x2dd1c9fa14257542L, + 0x9d6de7a286218042L, + 0x7870b31868bb0c29L, + 0xc8cc9d40fabff929L, + 0x86933c3eed198794L, + 0x362f12667f1d7294L, + 0xd33246dc9187feffL, + 0x638e688403830bffL, + 0x8b349f95eb17ee0bL, + 0x3b88b1cd79131b0bL, + 0xde95e57797899760L, + 0x6e29cb2f058d6260L, + 0x20766a51122b1cddL, + 0x90ca4409802fe9ddL, + 0x75d710b36eb565b6L, + 0xc56b3eebfcb190b6L, + 0xe968524f41f998ccL, + 0x59d47c17d3fd6dccL, + 0xbcc928ad3d67e1a7L, + 0x0c7506f5af6314a7L, + 0x422aa78bb8c56a1aL, + 0xf29689d32ac19f1aL, + 0x178bdd69c45b1371L, + 0xa737f331565fe671L, + 0xf22715194de54bf2L, + 0x429b3b41dfe1bef2L, + 0xa7866ffb317b3299L, + 0x173a41a3a37fc799L, + 0x5965e0ddb4d9b924L, + 0xe9d9ce8526dd4c24L, + 0x0cc49a3fc847c04fL, + 0xbc78b4675a43354fL, + 0x907bd8c3e70b3d35L, + 0x20c7f69b750fc835L, + 0xc5daa2219b95445eL, + 0x75668c790991b15eL, + 0x3b392d071e37cfe3L, + 0x8b85035f8c333ae3L, + 0x6e9857e562a9b688L, + 0xde2479bdf0ad4388L, + 0x369e8eac1839a67cL, + 0x8622a0f48a3d537cL, + 0x633ff44e64a7df17L, + 0xd383da16f6a32a17L, + 0x9ddc7b68e10554aaL, + 0x2d6055307301a1aaL, + 0xc87d018a9d9b2dc1L, + 0x78c12fd20f9fd8c1L, + 0x54c24376b2d7d0bbL, + 0xe47e6d2e20d325bbL, + 0x01633994ce49a9d0L, + 0xb1df17cc5c4d5cd0L, + 0xff80b6b24beb226dL, + 0x4f3c98ead9efd76dL, + 0xaa21cc5037755b06L, + 0x1a9de208a571ae06L, + 0x9f1a08417d96070aL, + 0x2fa62619ef92f20aL, + 0xcabb72a301087e61L, + 0x7a075cfb930c8b61L, + 0x3458fd8584aaf5dcL, + 0x84e4d3dd16ae00dcL, + 0x61f98767f8348cb7L, + 0xd145a93f6a3079b7L, + 0xfd46c59bd77871cdL, + 0x4dfaebc3457c84cdL, + 0xa8e7bf79abe608a6L, + 0x185b912139e2fda6L, + 0x5604305f2e44831bL, + 0xe6b81e07bc40761bL, + 0x03a54abd52dafa70L, + 0xb31964e5c0de0f70L, + 0x5ba393f4284aea84L, + 0xeb1fbdacba4e1f84L, + 0x0e02e91654d493efL, + 0xbebec74ec6d066efL, + 0xf0e16630d1761852L, + 0x405d48684372ed52L, + 0xa5401cd2ade86139L, + 0x15fc328a3fec9439L, + 0x39ff5e2e82a49c43L, + 0x8943707610a06943L, + 0x6c5e24ccfe3ae528L, + 0xdce20a946c3e1028L, + 0x92bdabea7b986e95L, + 0x220185b2e99c9b95L, + 0xc71cd108070617feL, + 0x77a0ff509502e2feL, + 0x22b019788eb84f7dL, + 0x920c37201cbcba7dL, + 0x7711639af2263616L, + 0xc7ad4dc26022c316L, + 0x89f2ecbc7784bdabL, + 0x394ec2e4e58048abL, + 0xdc53965e0b1ac4c0L, + 0x6cefb806991e31c0L, + 0x40ecd4a2245639baL, + 0xf050fafab652ccbaL, + 0x154dae4058c840d1L, + 0xa5f18018caccb5d1L, + 0xebae2166dd6acb6cL, + 0x5b120f3e4f6e3e6cL, + 0xbe0f5b84a1f4b207L, + 0x0eb375dc33f04707L, + 0xe60982cddb64a2f3L, + 0x56b5ac95496057f3L, + 0xb3a8f82fa7fadb98L, + 0x0314d67735fe2e98L, + 0x4d4b770922585025L, + 0xfdf75951b05ca525L, + 0x18ea0deb5ec6294eL, + 0xa85623b3ccc2dc4eL, + 0x84554f17718ad434L, + 0x34e9614fe38e2134L, + 0xd1f435f50d14ad5fL, + 0x61481bad9f10585fL, + 0x2f17bad388b626e2L, + 0x9fab948b1ab2d3e2L, + 0x7ab6c031f4285f89L, + 0xca0aee69662caa89L, + 0xd0970c61c35d048fL, + 0x602b22395159f18fL, + 0x85367683bfc37de4L, + 0x358a58db2dc788e4L, + 0x7bd5f9a53a61f659L, + 0xcb69d7fda8650359L, + 0x2e74834746ff8f32L, + 0x9ec8ad1fd4fb7a32L, + 0xb2cbc1bb69b37248L, + 0x0277efe3fbb78748L, + 0xe76abb59152d0b23L, + 0x57d695018729fe23L, + 0x1989347f908f809eL, + 0xa9351a27028b759eL, + 0x4c284e9dec11f9f5L, + 0xfc9460c57e150cf5L, + 0x142e97d49681e901L, + 0xa492b98c04851c01L, + 0x418fed36ea1f906aL, + 0xf133c36e781b656aL, + 0xbf6c62106fbd1bd7L, + 0x0fd04c48fdb9eed7L, + 0xeacd18f2132362bcL, + 0x5a7136aa812797bcL, + 0x76725a0e3c6f9fc6L, + 0xc6ce7456ae6b6ac6L, + 0x23d320ec40f1e6adL, + 0x936f0eb4d2f513adL, + 0xdd30afcac5536d10L, + 0x6d8c819257579810L, + 0x8891d528b9cd147bL, + 0x382dfb702bc9e17bL, + 0x6d3d1d5830734cf8L, + 0xdd813300a277b9f8L, + 0x389c67ba4ced3593L, + 0x882049e2dee9c093L, + 0xc67fe89cc94fbe2eL, + 0x76c3c6c45b4b4b2eL, + 0x93de927eb5d1c745L, + 0x2362bc2627d53245L, + 0x0f61d0829a9d3a3fL, + 0xbfddfeda0899cf3fL, + 0x5ac0aa60e6034354L, + 0xea7c84387407b654L, + 0xa423254663a1c8e9L, + 0x149f0b1ef1a53de9L, + 0xf1825fa41f3fb182L, + 0x413e71fc8d3b4482L, + 0xa98486ed65afa176L, + 0x1938a8b5f7ab5476L, + 0xfc25fc0f1931d81dL, + 0x4c99d2578b352d1dL, + 0x02c673299c9353a0L, + 0xb27a5d710e97a6a0L, + 0x576709cbe00d2acbL, + 0xe7db27937209dfcbL, + 0xcbd84b37cf41d7b1L, + 0x7b64656f5d4522b1L, + 0x9e7931d5b3dfaedaL, + 0x2ec51f8d21db5bdaL, + 0x609abef3367d2567L, + 0xd02690aba479d067L, + 0x353bc4114ae35c0cL, + 0x8587ea49d8e7a90cL, }; + + private static final long[] M_UX2N = { + 0x0080000000000000L, + 0x0000800000000000L, + 0x0000000080000000L, + 0x9a6c9329ac4bc9b5L, + 0x10f4bb0f129310d6L, + 0x70f05dcea2ebd226L, + 0x311211205672822dL, + 0x2fc297db0f46c96eL, + 0xca4d536fabf7da84L, + 0xfb4cdc3b379ee6edL, + 0xea261148df25140aL, + 0x59ccb2c07aa6c9b4L, + 0x20b3674a839af27aL, + 0x2d8e1986da94d583L, + 0x42cdf4c20337635dL, + 0x1d78724bf0f26839L, + 0xb96c84e0afb34bd5L, + 0x5d2e1fcd2df0a3eaL, + 0xcd9506572332be42L, + 0x23bda2427f7d690fL, + 0x347a953232374f07L, + 0x1c2a807ac2a8ceeaL, + 0x9b92ad0e14fe1460L, + 0x2574114889f670b2L, + 0x4a84a6c45e3bf520L, + 0x915bbac21cd1c7ffL, + 0xb0290ec579f291f5L, + 0xcf2548505c624e6eL, + 0xb154f27bf08a8207L, + 0xce4e92344baf7d35L, + 0x51da8d7e057c5eb3L, + 0x9fb10823f5be15dfL, + 0x73b825b3ff1f71cfL, + 0x5db436c5406ebb74L, + 0xfa7ed8f3ec3f2bcaL, + 0xc4d58efdc61b9ef6L, + 0xa7e39e61e855bd45L, + 0x97ad46f9dd1bf2f1L, + 0x1a0abb01f853ee6bL, + 0x3f0827c3348f8215L, + 0x4eb68c4506134607L, + 0x4a46f6de5df34e0aL, + 0x2d855d6a1c57a8ddL, + 0x8688da58e1115812L, + 0x5232f417fc7c7300L, + 0xa4080fb2e767d8daL, + 0xd515a7e17693e562L, + 0x1181f7c862e94226L, + 0x9e23cd058204ca91L, + 0x9b8992c57a0aed82L, + 0xb2c0afb84609b6ffL, + 0x2f7160553a5ea018L, + 0x3cd378b5c99f2722L, + 0x814054ad61a3b058L, + 0xbf766189fce806d8L, + 0x85a5e898ac49f86fL, + 0x34830d11bc84f346L, + 0x9644d95b173c8c1cL, + 0x150401ac9ac759b1L, + 0xebe1f7f46fb00ebaL, + 0x8ee4ce0c2e2bd662L, + 0x4000000000000000L, + 0x2000000000000000L, + 0x0800000000000000L, }; + + /** + * Computes the CRC64 checksum for the given byte array using the Azure Storage CRC64 polynomial. + * This method processes the input data in chunks of 32 bytes for efficiency and uses lookup tables + * to update the CRC values. + * + * @param src the byte array for which the CRC64 checksum is to be computed. + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(byte[] src, long uCrc) { + return compute(src, 0, src.length, uCrc); + } + + /** + * Computes the CRC64 checksum for a region of a byte array. Avoids copying when the caller has + * an array view (e.g. {@link ByteBuffer#array()} with array offset and position) or when combined + * with {@link #compute(byte[], long)} for whole-array input. + * + * @param src the byte array. + * @param offset start index (inclusive). + * @param length number of bytes to process. + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(byte[] src, int offset, int length, long uCrc) { + int pData = offset; + long uSize = length; + long uBytes, uStop; + + uCrc = ~uCrc; // Flip all bits of uCrc + + uStop = uSize - (uSize % 32); + if (uStop >= 2 * 32) { + long uCrc0 = 0L; + long uCrc1 = 0L; + long uCrc2 = 0L; + long uCrc3 = 0L; + + int pLast = pData + (int) uStop - 32; + uSize -= uStop; + uCrc0 = uCrc; + + ByteBuffer buffer = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); + + for (; pData < pLast; pData += 32) { + long b0, b1, b2, b3; + + // Load and XOR data with CRC + b0 = buffer.getLong(pData) ^ uCrc0; + b1 = buffer.getLong(pData + 8) ^ uCrc1; + b2 = buffer.getLong(pData + 16) ^ uCrc2; + b3 = buffer.getLong(pData + 24) ^ uCrc3; + + // Unsigned updates using tables and masking + uCrc0 = M_U32[7 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 = M_U32[7 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 = M_U32[7 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 = M_U32[7 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[6 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[6 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[6 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[6 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[5 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[5 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[5 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[5 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[4 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[4 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[4 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[4 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[3 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[3 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[3 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[3 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[2 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[2 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[2 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[2 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[((int) (b0 & 0xFF))]; + uCrc1 ^= M_U32[((int) (b1 & 0xFF))]; + uCrc2 ^= M_U32[((int) (b2 & 0xFF))]; + uCrc3 ^= M_U32[((int) (b3 & 0xFF))]; + } + + // Combine CRC values + uCrc = 0; + uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc1; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc2; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc3; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + pData += 32; + } + + // Process remaining bytes + for (uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[pData]) & 0xFF)]; + } + + return ~uCrc; // Flip all bits of uCrc and return as long + } + + /** + * Concatenates two CRC64 values by combining their initial and final CRC values and sizes. + * This method ensures unsigned behavior and uses the `mulX_N` method to perform necessary + * multiplications in GF(2^64). + * + * @param uInitialCrcAB The initial CRC value for the concatenated data. + * @param uInitialCrcA The initial CRC value for the first data segment. + * @param uFinalCrcA The final CRC value for the first data segment. + * @param uSizeA The size of the first data segment. + * @param uInitialCrcB The initial CRC value for the second data segment. + * @param uFinalCrcB The final CRC value for the second data segment. + * @param uSizeB The size of the second data segment. + * @return The concatenated CRC64 value. + */ + public static long concat(long uInitialCrcAB, long uInitialCrcA, long uFinalCrcA, long uSizeA, long uInitialCrcB, + long uFinalCrcB, long uSizeB) { + long uFinalCrcAB = ~uFinalCrcA; + + // Ensure unsigned behavior when comparing uInitialCrcA and uInitialCrcAB + if ((uInitialCrcA) != (uInitialCrcAB)) { + // Apply mulX_N with proper unsigned masking + uFinalCrcAB ^= multiplyCrcByPowerOfX((uInitialCrcA ^ uInitialCrcAB), uSizeA); + } + + uFinalCrcAB ^= ~uInitialCrcB; // Ensure unsigned XOR logic + uFinalCrcAB = multiplyCrcByPowerOfX(uFinalCrcAB, uSizeB); + uFinalCrcAB ^= uFinalCrcB; + + return uFinalCrcAB; // Ensure the result is treated as unsigned + } + + /** + * Multiplies a CRC value by x^n in GF(2^64). + * This method uses a lookup table to perform the multiplication efficiently. + * + * @param a The CRC value to be multiplied. + * @param uSize The power of x by which the CRC value is to be multiplied. + * @return The resulting CRC64 value after multiplication. + */ + private static long multiplyCrcByPowerOfX(long a, long uSize) { + long i = 0; + long r = a; + + while (uSize != 0) { + if ((uSize & 1) == 1) { + r = mulPolyUnrolled(r, M_UX2N[(int) i]); // Ensure result is treated as unsigned + } + uSize >>>= 1; // Unsigned right shift + i += 1; + } + + return r; + } + + /** + * Multiplies two CRC values using the polynomial in GF(2^64). + * This method performs the multiplication using an unrolled loop for efficiency. + * + * @param a The first CRC value to be multiplied. + * @param b The second CRC value to be multiplied. + * @return The resulting CRC64 value after multiplication. + */ + private static long mulPolyUnrolled(long a, long b) { + final long p = POLY; + final long p2 = (p >>> 1) ^ (p); // Use unsigned shift + final long bw = Long.SIZE; + + final long[] vt = { 0, p2, p, p ^ p2 }; + final long[] vs = { bw - 2, bw - 1 }; + long[] vb = { (b >>> 1) ^ vt[(int) ((b & 1) << 1)], b }; // Unsigned right shift + long[] vr = { 0, 0 }; + + for (long i = 0; i < bw; i += 2) { + for (int j = 0; j < 2; ++j) { + vr[j] ^= vb[j] * ((a >>> vs[j]) & 1); // Unsigned right shift + vb[j] = (vb[j] >>> 2) ^ vt[(int) (vb[j] & 3)]; // Unsigned right shift + } + a <<= 2; // Shift remains the same as a is multiplied by 4 + } + + return vr[0] ^ vr[1]; + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java new file mode 100644 index 000000000000..8e0ed1ff6e86 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +/** + * Constants used in the structured message encoding and decoding process. + */ +public final class StructuredMessageConstants { + /** + * The default version of the structured message. + */ + public static final int DEFAULT_MESSAGE_VERSION = 1; + + /** + * The length of the header for version 1 of the structured message. + */ + public static final int V1_HEADER_LENGTH = 13; + + /** + * The length of the segment header for version 1 of the structured message. + */ + public static final int V1_SEGMENT_HEADER_LENGTH = 10; + + /** + * The length of the CRC64 checksum. + */ + public static final int CRC64_LENGTH = 8; + + /** + * The default length of segments for version 1. + */ + public static final int V1_DEFAULT_SEGMENT_CONTENT_LENGTH = 4 * 1024 * 1024; // 4 MiB + + /** + * The maximum amount of data to encode at once. + */ + public static final int STATIC_MAXIMUM_ENCODED_DATA_LENGTH = 4 * 1024 * 1024; // 4 MiB + + /** + * The maximum single part upload size to use CRC64 header. + */ + public static final int MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER = 4 * 1024 * 1024; // 4 MiB + + /** + * The structured body type header value indicating version 1 with CRC64 properties. + */ + public static final String STRUCTURED_BODY_TYPE_VALUE = "XSM/1.0; properties=crc64"; + + public static final String CONTENT_VALIDATION_MODE_KEY = "contentValidationMode"; + + public static final String USE_CRC64_CHECKSUM_HEADER_CONTEXT = "crc64ChecksumHeaderContext"; + + public static final String USE_STRUCTURED_MESSAGE_CONTEXT = "structuredMessageChecksumAlgorithm"; + + public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java new file mode 100644 index 000000000000..3a6570fe07bc --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -0,0 +1,536 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.logging.ClientLogger; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + +/** + * Streaming decoder for the storage structured message format used to validate downloaded blob/file/datalake + * content with CRC64 checksums. + * + *

This class owns the actual parsing and CRC validation. The pipeline policy hands it raw {@link ByteBuffer}s as + * they arrive on the wire (via {@link #decodeChunk(ByteBuffer)}); the decoder returns only the payload bytes that + * have already been CRC-validated and tells the policy when the entire message has been consumed + * (via {@link #isComplete()}). Any malformed input or CRC mismatch surfaces as an + * {@link IllegalArgumentException} thrown from {@code decodeChunk} so the policy can translate it into a stream + * error.

+ * + *

Wire format (V1)

+ * + *

The encoded body has the following layout (all integers little-endian):

+ * 
+ *   |-- message header (13 B) ----------------------------------------|
+ *   |  version (1)  |  total message length (8)  |  flags (2)  |  numSegments (2)  |
+ *
+ *   for each segment in 1..numSegments:
+ *     |-- segment header (10 B) -|
+ *     |  segNum (2)  |  segContentLen (8)  |
+ *     |-- segment payload (segContentLen B) --|
+ *     |-- segment CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ *   |-- message CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ * + *

Emission guarantee

+ * + * Payload bytes for a segment are never emitted to the caller until that segment's CRC64 footer + * has been validated. This matches the emission semantics used by {@code BlobDecryptionPolicy}/{@code DecryptorV2} + * (which only emits a decrypted region after its GCM tag is verified) and ensures that no unvalidated bytes are + * exposed to consumers, even if the connection is later torn down or the download is retried. + * + *

Thread-safety

+ * + *

This class is not thread-safe. A new instance is created for every HTTP response, and the + * reactive operators in the policy ({@code concatMap}) serialize access to the single instance. Retries produce new + * HTTP responses and therefore new decoder instances, so a CRC failure on one attempt cannot pollute another.

+ */ +public class StructuredMessageDecoder { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecoder.class); + + private long messageLength = -1; + private StructuredMessageFlags flags; + private int numSegments = -1; + private final long expectedEncodedMessageLength; + // Number of encoded bytes consumed so far (headers + payloads + footers). + private long messageOffset = 0; + private int currentSegmentNumber = 0; + private long currentSegmentContentLength = 0; + private long currentSegmentContentOffset = 0; + private boolean segmentHeaderRead = false; + // Running CRC64 over all payload bytes seen so far (across every segment). + private long messageCrc64 = 0; + // Running CRC64 over only the current segment's payload bytes. + private long segmentCrc64 = 0; + // Holds bytes left over from a previous decodeChunk() call when the current chunk did not contain a full + // header or footer. + private final ByteArrayOutputStream pendingBytes = new ByteArrayOutputStream(); + // Payload for the current segment, one per inbound wire buffer. Not pre-sized to the full segment length, + // avoiding unnecessary heap spikes. + // These bytes are intentionally NOT emitted until the segment's CRC footer has been validated. + private List currentSegmentPayload; + // Validated payload buffers to emit from the current decodeChunk() invocation. + private final List validatedOutput = new ArrayList<>(); + + /** + * Constructs a new StructuredMessageDecoder. + * + * @param expectedEncodedMessageLength The expected encoded structured-message length (typically HTTP + * {@code Content-Length}). + */ + public StructuredMessageDecoder(long expectedEncodedMessageLength) { + this.expectedEncodedMessageLength = expectedEncodedMessageLength; + } + + /** + * Reads the 13-byte message header (version + total length + flags + numSegments) the first time the decoder + * sees enough bytes, and validates each field. Subsequent calls are no-ops. + * + * @param buffer The buffer to read from. + * @return true if the header was successfully read (or had already been read on a previous pass); false if more + * bytes are still needed. + */ + private boolean tryReadMessageHeader(ByteBuffer buffer) { + if (messageLength != -1) { + // Header already parsed on a previous chunk; nothing to do. + return true; + } + + if (getAvailableBytes(buffer) < V1_HEADER_LENGTH) { + // Not enough bytes for the full header yet; carry over what we have. + appendToPending(buffer); + return false; + } + + ByteBuffer combined = getCombinedBuffer(buffer); + + // Byte 0: protocol version. + int messageVersion = Byte.toUnsignedInt(combined.get()); + if (messageVersion != DEFAULT_MESSAGE_VERSION) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Unsupported structured message version: " + messageVersion))); + } + + // Bytes 1-8: total encoded message length. Must be at least the header itself, and must agree with what the + // HTTP layer told us via Content-Length – any disagreement implies a truncated/extended response. + long msgLen = combined.getLong(); + if (msgLen < V1_HEADER_LENGTH) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException(enrichExceptionMessage("Message length too small: " + msgLen))); + } + if (msgLen != expectedEncodedMessageLength) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage("Structured message length " + + msgLen + " did not match content length " + expectedEncodedMessageLength))); + } + + // Bytes 9-10: flags (NONE or STORAGE_CRC64). Bytes 11-12: number of segments. + flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(combined.getShort())); + numSegments = Short.toUnsignedInt(combined.getShort()); + if (numSegments < 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Structured message must have at least one segment, got: " + numSegments))); + } + + // Commit: drop the 13 bytes we just parsed from pending/buffer and record the message length. + consumeBytes(V1_HEADER_LENGTH, buffer); + messageOffset += V1_HEADER_LENGTH; + messageLength = msgLen; + + return true; + } + + /** + * Reads the 10-byte header for the next segment (segment number + segment payload length) and resets + * per-segment state so {@link #tryReadSegmentContent(ByteBuffer)} can begin filling + * {@link #currentSegmentPayload}. + * + *

Validates that segments arrive in order and that the declared segment size leaves enough room in the + * remaining message for any subsequent segment headers, payloads, footers, and the trailing message footer – + * this catches malformed/oversized segment lengths up front instead of waiting until we run off the end of the + * stream.

+ * + * @param buffer The buffer to read from. + * @return true if the segment header was read; false if more bytes are needed. + */ + private boolean tryReadSegmentHeader(ByteBuffer buffer) { + if (getAvailableBytes(buffer) < V1_SEGMENT_HEADER_LENGTH) { + appendToPending(buffer); + return false; + } + + ByteBuffer combined = getCombinedBuffer(buffer); + + // Bytes 0-1: segment number. Bytes 2-9: declared payload length of this segment. + int segmentNum = Short.toUnsignedInt(combined.getShort()); + long segmentSize = combined.getLong(); + + // Segments must arrive strictly in order so the running CRC and "segment N follows segment N-1" assumption + // hold. Anything else implies a malformed/reordered response. + if (segmentNum != currentSegmentNumber + 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Unexpected segment number. Expected: " + (currentSegmentNumber + 1) + ", got: " + segmentNum))); + } + + // Compute an upper bound on the legal segment size: whatever is left in the message, minus the bytes that + // MUST still appear after this segment's payload (this segment's footer, the headers/payloads/footers of all + // remaining segments, and the trailing message footer). + long footerSize = flags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; + long remainingSegmentsAfterThis = (long) numSegments - segmentNum; + long reservedBytes + = footerSize + remainingSegmentsAfterThis * (V1_SEGMENT_HEADER_LENGTH + footerSize) + footerSize; + long maxSegmentSize = messageLength - messageOffset - V1_SEGMENT_HEADER_LENGTH - reservedBytes; + if (segmentSize < 0 || segmentSize > maxSegmentSize) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Invalid segment size detected: " + segmentSize + " (max=" + maxSegmentSize + ")"))); + } + + // Commit: drop the 10 header bytes and set up per-segment state so payload accumulation can start fresh. + consumeBytes(V1_SEGMENT_HEADER_LENGTH, buffer); + messageOffset += V1_SEGMENT_HEADER_LENGTH; + currentSegmentNumber = segmentNum; + currentSegmentContentLength = segmentSize; + currentSegmentContentOffset = 0; + currentSegmentPayload = new ArrayList<>(); + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Reset only the per-segment running CRC; the message-wide running CRC keeps accumulating across all + // segments so the final message footer covers the entire payload. + segmentCrc64 = 0; + } + + return true; + } + + /** + * Pulls as many payload bytes as possible (bounded by what is still owed for the current segment) from the + * pending+buffer view into {@link #currentSegmentPayload}, updating the running per-segment and per-message + * CRC64 values along the way. + * + *

Bytes accumulated here are not yet emitted to the caller. They are released only after + * {@link #tryReadSegmentFooter(ByteBuffer)} validates this segment's CRC. This is the mechanism that enforces + * "no unvalidated bytes ever leave the decoder".

+ * + * @param buffer The buffer to read from. + * @return The number of payload bytes read in this call (0 means we either had no bytes available or the + * current segment's payload was already complete). + */ + private int tryReadSegmentContent(ByteBuffer buffer) { + long remaining = currentSegmentContentLength - currentSegmentContentOffset; + if (remaining == 0) { + // Segment payload is already complete; nothing to do here. The caller will move on to read the footer. + return 0; + } + + int available = getAvailableBytes(buffer); + if (available == 0) { + return 0; + } + + // Read the minimum of "what's available right now" and "what's still owed for this segment" so we never + // accidentally consume the segment footer here. + int toRead = (int) Math.min(available, remaining); + ByteBuffer combined = getCombinedBuffer(buffer); + + // Materialize only this chunk so retained memory grows with bytes received, not the full declared segment size. + byte[] content = new byte[toRead]; + combined.get(content); + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Update both CRCs incrementally: the segment CRC will be checked at the segment footer, and the + // message CRC accumulates across every segment to be checked at the message footer. + segmentCrc64 = StorageCrc64Calculator.compute(content, 0, toRead, segmentCrc64); + messageCrc64 = StorageCrc64Calculator.compute(content, 0, toRead, messageCrc64); + } + + consumeBytes(toRead, buffer); + currentSegmentPayload.add(content); + + messageOffset += toRead; + currentSegmentContentOffset += toRead; + + return toRead; + } + + /** + * Validates the 8-byte segment CRC64 footer for the segment that has just finished accumulating. Pre-condition: + * {@code currentSegmentContentOffset == currentSegmentContentLength}. + * + *

This step is intentionally separate from reading the message footer: when the CRC matches, we want to be + * able to flush the buffered segment payload to the caller right away – even if the trailing message footer is + * not yet available in the current chunk.

+ * + * @param buffer The buffer to read from. + * @return true if the footer was successfully read (or no footer is required for this message); false if more + * bytes are still needed. + */ + private boolean tryReadSegmentFooter(ByteBuffer buffer) { + if (currentSegmentContentOffset != currentSegmentContentLength) { + // Segment payload is not complete yet; wait for more content. + return true; + } + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + return tryConsumeCrc64Footer(buffer, segmentCrc64, " in segment " + currentSegmentNumber); + } + + // No CRC was negotiated, so there is no footer to read; the caller can release the buffered payload. + return true; + } + + /** + * Validates the 8-byte message CRC64 footer that follows the last segment. + * + * @param buffer The buffer to read from. + * @return true if the footer was successfully read (or none is required); false if more bytes are still needed. + */ + private boolean tryReadMessageFooter(ByteBuffer buffer) { + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + return tryConsumeCrc64Footer(buffer, messageCrc64, " in message footer"); + } + return true; + } + + /** + * Decodes as much as possible from the given buffer and returns any fully validated + * payload bytes that are now safe to emit downstream. + * + *

The returned buffers will only ever contain bytes from segments whose CRC (when + * enabled) has already been verified. If no segments have been fully validated by + * this invocation the method returns an empty list. Callers distinguish "more bytes + * needed" from "stream complete" via {@link #isComplete()}.

+ * + * @param buffer The buffer containing encoded data. + * @return Validated payload bytes ready to emit downstream; empty when none are ready yet. + * @throws IllegalArgumentException if the input is malformed or a CRC64 check fails. + */ + public List decodeChunk(ByteBuffer buffer) { + // Decoder always reads little-endian; force the order on the caller's buffer so all our get() calls match + // the wire format regardless of how the buffer was constructed. + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Output collected during this single invocation. Each segment whose CRC validates in this call is appended + // here and ultimately returned to the policy as one or more ByteBuffers. + validatedOutput.clear(); + + // Step 1: parse the message header on the first chunk that has enough bytes for it. If this chunk doesn't, + // bail out early. + if (!tryReadMessageHeader(buffer)) { + return finishDecodeChunk(); + } + + // Step 2: walk forward through the message until we either hit the end (messageOffset == messageLength) or + // we run out of bytes for the current structural element and have to wait for the next chunk. + while (messageOffset < messageLength) { + if (!segmentHeaderRead) { + // We are between segments. If every segment has been processed, only the trailing message footer + // can still appear in the stream – read it (or wait for it) and exit. + if (currentSegmentNumber == numSegments) { + if (!tryReadMessageFooter(buffer)) { + break; + } + break; + } + // Otherwise, parse the next segment's header. May return false if it is split across chunks. + if (!tryReadSegmentHeader(buffer)) { + break; + } + segmentHeaderRead = true; + } + + // Drain as many payload bytes as are available into the per-segment buffer. + int payloadRead = tryReadSegmentContent(buffer); + + if (currentSegmentContentOffset == currentSegmentContentLength) { + // Segment payload fully buffered. Validate the CRC footer (if any). When the footer isn't fully + // available yet, break and resume on the next chunk – currentSegmentPayload keeps its contents so + // we can still emit them on the call where the footer arrives. + if (!tryReadSegmentFooter(buffer)) { + break; + } + // Segment passed validation: it is now safe to release the buffered payload to the caller. + releaseValidatedSegmentPayload(); + segmentHeaderRead = false; + // Loop continues: either consume the next segment's header or the message footer. + } else if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { + // Nothing left to read this pass and the segment is not complete – wait for the next chunk. + break; + } + } + + return finishDecodeChunk(); + } + + /** + * Hands off validated segment bytes for emission as {@link ByteBuffer} views over the accumulated payload copies + * without consolidating them into a single array. + */ + private void releaseValidatedSegmentPayload() { + List validatedCopies = currentSegmentPayload; + currentSegmentPayload = null; + for (byte[] validatedCopy : validatedCopies) { + validatedOutput.add(ByteBuffer.wrap(validatedCopy)); + } + } + + private List finishDecodeChunk() { + if (validatedOutput.isEmpty()) { + return Collections.emptyList(); + } + List result; + if (validatedOutput.size() == 1) { + result = Collections.singletonList(validatedOutput.get(0)); + } else { + result = new ArrayList<>(validatedOutput); + } + validatedOutput.clear(); + return result; + } + + /** + * @return the total number of bytes the decoder can currently see across the carry-over {@link #pendingBytes} + * plus the unread tail of the supplied buffer. Used to decide whether a structural element (header / + * footer) can be parsed in this pass or whether we must defer to the next chunk. + */ + private int getAvailableBytes(ByteBuffer buffer) { + return pendingBytes.size() + buffer.remaining(); + } + + /** + * Returns a single read-only view that logically concatenates {@link #pendingBytes} with the unread tail of + * a buffer. + * + *

The position of the supplied buffer is intentionally not advanced here – reads happen on the + * combined view, and the original buffer's position is moved later by {@link #consumeBytes(int, ByteBuffer)} + * once we know the parse succeeded.

+ * + *

When pendingBytes is empty we avoid the allocation and just return a duplicate of the buffer; + * otherwise we materialize a fresh array of size {@code pending + buffer.remaining()}.

+ */ + private ByteBuffer getCombinedBuffer(ByteBuffer buffer) { + if (pendingBytes.size() == 0) { + ByteBuffer dup = buffer.duplicate(); + dup.order(ByteOrder.LITTLE_ENDIAN); + return dup; + } + + byte[] pending = pendingBytes.toByteArray(); + ByteBuffer combined = ByteBuffer.allocate(pending.length + buffer.remaining()); + combined.order(ByteOrder.LITTLE_ENDIAN); + combined.put(pending); + combined.put(buffer.duplicate()); + combined.flip(); + return combined; + } + + /** + * Consumes the next 8 bytes as a little-endian CRC64 footer, validates it against expectedCrc64, and + * advances {@link #messageOffset}. Used for both segment and message footers. + * + *

If fewer than 8 bytes are available, the remaining buffer bytes are stashed in {@link #pendingBytes} and + * the method returns false so the caller can break out of the decode loop and wait for the next + * chunk. On a CRC mismatch, an {@link IllegalArgumentException} is thrown (the decoder is then discarded by + * the enclosing policy).

+ */ + private boolean tryConsumeCrc64Footer(ByteBuffer buffer, long expectedCrc64, String mismatchDetail) { + if (getAvailableBytes(buffer) < CRC64_LENGTH) { + // Not enough bytes yet for the footer; carry whatever we have over to the next call. + appendToPending(buffer); + return false; + } + ByteBuffer combined = getCombinedBuffer(buffer); + long reportedCrc = combined.getLong(); + if (expectedCrc64 != reportedCrc) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "CRC64 mismatch" + mismatchDetail + ". Expected: " + expectedCrc64 + ", got: " + reportedCrc))); + } + consumeBytes(CRC64_LENGTH, buffer); + messageOffset += CRC64_LENGTH; + return true; + } + + /** + * Drains {@code bytesToConsume} bytes from the logical pending+buffer stream that + * {@link #getCombinedBuffer(ByteBuffer)} produced. + * + *

Bytes are taken from {@link #pendingBytes} first, then from the live buffer. The pending stream is + * reset whenever it is fully drained, and any leftover (when {@code bytesToConsume} was less than what was in + * pending) is rewritten so the carry-over stays compact.

+ */ + private void consumeBytes(int bytesToConsume, ByteBuffer buffer) { + int pendingSize = pendingBytes.size(); + if (bytesToConsume <= pendingSize) { + // The entire consume fits in pending: rewrite whatever survives back into pending after a reset. + byte[] remaining = pendingBytes.toByteArray(); + pendingBytes.reset(); + if (bytesToConsume < pendingSize) { + pendingBytes.write(remaining, bytesToConsume, pendingSize - bytesToConsume); + } + } else { + // Pending is fully drained and the remainder comes from the live buffer; advance its position directly. + int bytesFromBuffer = bytesToConsume - pendingSize; + pendingBytes.reset(); + buffer.position(buffer.position() + bytesFromBuffer); + } + } + + /** + * Stashes everything still unread in the buffer into {@link #pendingBytes} so it can be combined with the + * next chunk on the next call to {@link #decodeChunk(ByteBuffer)}. + * + *

This is only called when the current chunk does not contain enough bytes for the next structural element, + * so the carry-over is always small (bounded by the largest header size, currently 13 bytes).

+ */ + private void appendToPending(ByteBuffer buffer) { + while (buffer.hasRemaining()) { + pendingBytes.write(buffer.get()); + } + } + + /** + * Reports whether the decoder has finished consuming the entire structured message and validated everything it + * was supposed to validate. Used by the pipeline policy to distinguish "stream ended cleanly" from "stream was + * truncated". + * + *

The check requires all of:

+ *
    + *
  • The message header has been parsed ({@code messageLength != -1}).
    • + *
  • Every byte of the declared message has been consumed.
    • + *
  • No carry-over bytes remain in pending.
    • + *
  • No segment is currently in progress (no segment header without a matching footer).
    • + *
  • The current segment's payload accumulation is itself complete.
    • + *
+ * + * @return true if all expected bytes have been decoded and validated; false otherwise. + */ + public boolean isComplete() { + return messageLength != -1 + && messageOffset == messageLength + && pendingBytes.size() == 0 + && !segmentHeaderRead + && currentSegmentContentOffset == currentSegmentContentLength; + } + + /** + * Appends the current decoder offset to an exception message so failures can be traced back to a specific + * point in the encoded stream. + * + * @param message The original exception message. + * @return The original message with {@code [decoderOffset=N]} appended. + */ + private String enrichExceptionMessage(String message) { + return String.format("%s [decoderOffset=%d]", message, messageOffset); + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java new file mode 100644 index 000000000000..35a559799e78 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.StorageImplUtils; +import reactor.core.publisher.Flux; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; + +/** + * Encoder for structured messages with support for segmenting and CRC64 checksums. + */ +public class StructuredMessageEncoder { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageEncoder.class); + private static final int CRC64_SCRATCH_BUFFER_SIZE = 64 * 1024; + + private final int messageVersion; + private final int contentLength; + private final int messageLength; + private final StructuredMessageFlags structuredMessageFlags; + private final int segmentSize; + private final int numSegments; + + private int currentContentOffset; + private int currentSegmentNumber; + private int currentSegmentOffset; + private long messageCRC64; + private final Map segmentCRC64s; + private final byte[] crc64ScratchBuffer; + + /** + * Constructs a new StructuredMessageEncoder. + * @param contentLength The length of the content to be encoded. + * @param segmentSize The size of each segment. + * @param structuredMessageFlags The structuredMessageFlags to be set. + * @throws IllegalArgumentException If the segment size is less than 1, the content length is less than 1, or the + * number of segments is greater than {@link java.lang.Short#MAX_VALUE}. + */ + public StructuredMessageEncoder(int contentLength, int segmentSize, StructuredMessageFlags structuredMessageFlags) { + if (segmentSize < 1) { + StorageImplUtils.assertInBounds("segmentSize", segmentSize, 1, Long.MAX_VALUE); + } + if (contentLength < 1) { + StorageImplUtils.assertInBounds("contentLength", contentLength, 1, Long.MAX_VALUE); + } + + this.messageVersion = DEFAULT_MESSAGE_VERSION; + this.contentLength = contentLength; + this.structuredMessageFlags = structuredMessageFlags; + this.segmentSize = segmentSize; + this.numSegments = Math.max(1, (int) Math.ceil((double) this.contentLength / this.segmentSize)); + this.messageLength = calculateMessageLength(); + this.currentContentOffset = 0; + this.currentSegmentNumber = 0; + this.currentSegmentOffset = 0; + this.messageCRC64 = 0; + this.segmentCRC64s = new HashMap<>(); + this.crc64ScratchBuffer = new byte[CRC64_SCRATCH_BUFFER_SIZE]; + + if (numSegments > Short.MAX_VALUE) { + StorageImplUtils.assertInBounds("numSegments", numSegments, 1, Short.MAX_VALUE); + } + } + + private int getMessageHeaderLength() { + return V1_HEADER_LENGTH; + } + + private int getSegmentHeaderLength() { + return V1_SEGMENT_HEADER_LENGTH; + } + + private int getSegmentFooterLength() { + return (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) ? CRC64_LENGTH : 0; + } + + private int getMessageFooterLength() { + return (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) ? CRC64_LENGTH : 0; + } + + private int getSegmentContentLength() { + // last segment size is remaining content + if (currentSegmentNumber == numSegments) { + return contentLength - ((currentSegmentNumber - 1) * segmentSize); + } else { + return segmentSize; + } + } + + private byte[] generateMessageHeader() { + // 1 byte version, 8 byte size, 2 byte structuredMessageFlags, 2 byte numSegments + ByteBuffer buffer = ByteBuffer.allocate(getMessageHeaderLength()).order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) messageVersion); + buffer.putLong(messageLength); + buffer.putShort((short) structuredMessageFlags.getValue()); + buffer.putShort((short) numSegments); + + return buffer.array(); + } + + private byte[] generateSegmentHeader() { + int segmentContentSize = Math.min(segmentSize, contentLength - currentContentOffset); + // 2 byte number, 8 byte size + ByteBuffer buffer = ByteBuffer.allocate(getSegmentHeaderLength()).order(ByteOrder.LITTLE_ENDIAN); + buffer.putShort((short) currentSegmentNumber); + buffer.putLong(segmentContentSize); + + return buffer.array(); + } + + /** + * Encodes the given buffer into a structured message format as a stream of ByteBuffers. + * + * @param unencodedBuffer The buffer to be encoded. + * @return A Flux of encoded ByteBuffers. + * @throws IllegalArgumentException If the buffer length exceeds the content length, or the content has already been + * encoded. + */ + public Flux encode(ByteBuffer unencodedBuffer) { + StorageImplUtils.assertNotNull("unencodedBuffer", unencodedBuffer); + + return Flux.defer(() -> { + if (currentContentOffset == contentLength) { + // Already encoded; return empty (e.g. extra aggregator from staging/flush, or retry re-subscription). + return Flux.empty(); + } + + if ((unencodedBuffer.remaining() + currentContentOffset) > contentLength) { + return Flux.error( + LOGGER.logExceptionAsError(new IllegalArgumentException("Buffer length exceeds content length."))); + } + + if (!unencodedBuffer.hasRemaining()) { + return Flux.empty(); + } + + // Emit buffers lazily to avoid materializing full encoded output in memory + // Could be swapped to Flux.generate if performance is impacted and we're eagerly pushing a lot of data. + return Flux.create(sink -> { + // if we are at the beginning of the message, encode message header and emit it + if (currentContentOffset == 0) { + sink.next(ByteBuffer.wrap(generateMessageHeader())); + } + + // while there are remaining bytes in the unencoded buffer, encode the segment content + while (unencodedBuffer.hasRemaining()) { + // if we are at the beginning of a segment's content, encode segment header and emit it + if (currentSegmentOffset == 0) { + incrementCurrentSegment(); + sink.next(ByteBuffer.wrap(generateSegmentHeader())); + } + + // encode the segment content and emit it + sink.next(encodeSegmentContent(unencodedBuffer)); + + // if we are at the end of a segment's content, encode segment footer + if (currentSegmentOffset == getSegmentContentLength()) { + byte[] footer = generateSegmentFooter(); + if (footer.length > 0) { + sink.next(ByteBuffer.wrap(footer)); + } + currentSegmentOffset = 0; + } + } + + // if all content has been encoded, encode message footer and emit it + if (currentContentOffset == contentLength) { + byte[] footer = generateMessageFooter(); + if (footer.length > 0) { + sink.next(ByteBuffer.wrap(footer)); + } + } + + sink.complete(); + }); + }); + } + + private byte[] generateSegmentFooter() { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + return ByteBuffer.allocate(CRC64_LENGTH) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(segmentCRC64s.get(currentSegmentNumber)) + .array(); + } + return new byte[0]; + } + + private byte[] generateMessageFooter() { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + return ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN).putLong(messageCRC64).array(); + } + return new byte[0]; + } + + private ByteBuffer encodeSegmentContent(ByteBuffer unencodedBuffer) { + // get the number of bytes to read from the unencoded buffer based on the segment content length and the current segment offset + int readSize = Math.min(unencodedBuffer.remaining(), getSegmentContentLength() - currentSegmentOffset); + + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + if (unencodedBuffer.hasArray()) { + // if the unencoded buffer has an array, compute the CRC64 checksum of the segment content + // this is more efficient than copying the array to a new byte array and computing the checksum + int pos = unencodedBuffer.arrayOffset() + unencodedBuffer.position(); + segmentCRC64s.put(currentSegmentNumber, StorageCrc64Calculator.compute(unencodedBuffer.array(), pos, + readSize, segmentCRC64s.get(currentSegmentNumber))); + messageCRC64 = StorageCrc64Calculator.compute(unencodedBuffer.array(), pos, readSize, messageCRC64); + } else { + updateCrc64sWithoutAccessibleArray(unencodedBuffer, readSize); + } + } + + currentContentOffset += readSize; + currentSegmentOffset += readSize; + + // Return a view (slice) to avoid allocating 4MB per segment; caller must consume before next segment. + ByteBuffer slice = unencodedBuffer.slice(); + slice.limit(readSize); + unencodedBuffer.position(unencodedBuffer.position() + readSize); + return slice.asReadOnlyBuffer(); + } + + private void updateCrc64sWithoutAccessibleArray(ByteBuffer unencodedBuffer, int readSize) { + ByteBuffer duplicate = unencodedBuffer.duplicate(); + duplicate.limit(duplicate.position() + readSize); + + long segmentCrc64 = segmentCRC64s.get(currentSegmentNumber); + long currentMessageCrc64 = messageCRC64; + + while (duplicate.hasRemaining()) { + int chunkSize = Math.min(duplicate.remaining(), crc64ScratchBuffer.length); + duplicate.get(crc64ScratchBuffer, 0, chunkSize); + segmentCrc64 = StorageCrc64Calculator.compute(crc64ScratchBuffer, 0, chunkSize, segmentCrc64); + currentMessageCrc64 = StorageCrc64Calculator.compute(crc64ScratchBuffer, 0, chunkSize, currentMessageCrc64); + } + + segmentCRC64s.put(currentSegmentNumber, segmentCrc64); + messageCRC64 = currentMessageCrc64; + } + + private int calculateMessageLength() { + int length = getMessageHeaderLength(); + + length += (getSegmentHeaderLength() + getSegmentFooterLength()) * numSegments; + length += contentLength; + length += getMessageFooterLength(); + return length; + } + + private void incrementCurrentSegment() { + currentSegmentNumber++; + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + segmentCRC64s.putIfAbsent(currentSegmentNumber, 0L); + } + } + + /** + * Returns the length of the message. + * + * @return The length of the message. + */ + public long getEncodedMessageLength() { + return messageLength; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java new file mode 100644 index 000000000000..a5ccb2974a6e --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +/** + * Defines values for StructuredMessageFlags. + */ +public enum StructuredMessageFlags { + /** + * No flags set. + */ + NONE(0), + + /** + * StructuredMessageFlag indicating the use of CRC64. + */ + STORAGE_CRC64(1); + + /** + * The actual serialized value for a StructuredMessageFlags instance. + */ + private final int value; + + StructuredMessageFlags(int value) { + this.value = value; + } + + /** + * Parses a serialized value to a StructuredMessageFlags instance. + * + * @param value the serialized value to parse. + * @return the parsed StructuredMessageFlags object, or null if unable to parse. + */ + public static StructuredMessageFlags fromString(String value) { + if (value == null) { + return null; + } + StructuredMessageFlags[] items = StructuredMessageFlags.values(); + for (StructuredMessageFlags item : items) { + if (item.getValue() == Integer.parseInt(value)) { + return item; + } + } + return null; + } + + /** + * Parses a serialized value to a StructuredMessageFlags instance. + * @param value the serialized value to parse. + * @return the parsed StructuredMessageFlags object. + * @throws IllegalArgumentException if unable to parse. + */ + public static StructuredMessageFlags fromValue(int value) { + for (StructuredMessageFlags flag : StructuredMessageFlags.values()) { + if (flag.getValue() == value) { + return flag; + } + } + throw new IllegalArgumentException("Invalid value for StructuredMessageFlags: " + value); + } + + /** + * Returns the value for a StructuredMessageFlags instance. + * + * @return the integer value of the StructuredMessageFlags object. + */ + public int getValue() { + return value; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java new file mode 100644 index 000000000000..74c1334e3a7a --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Package containing classes for structured message encoding and decoding. + */ +package com.azure.storage.common.implementation.contentvalidation; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java new file mode 100644 index 000000000000..7e88de8cd37e --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.FluxUtil; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * {@link HttpResponse} wrapper that exposes a decoded body stream while preserving the request and status code of + * the original response. + * + *

The policy hands this class a Flux that already represents validated, framing-stripped bytes (produced by the + * decoder pipeline). This class's only job is to make that Flux look like the body of the original + * {@link HttpResponse}. {@code Content-Length} is overridden to the decoded payload size so it matches what callers + * will actually read; all other headers are forwarded verbatim. The validation is transparent to callers.

+ */ +class DecodedResponse extends HttpResponse { + private final HttpResponse originalResponse; + private final Flux decodedBody; + private final HttpHeaders adjustedHeaders; + + /** + * Wraps {@code httpResponse} with a body backed by {@code decodedBody}. + * + *

{@code Content-Length} is overridden to {@code decodedContentLength} so callers see the size of the bytes + * they will actually read from the decoded payload, not the larger wire size of the structured message.

+ * + * @param httpResponse The original response from the storage service. + * @param decodedBody The Flux of CRC-validated, framing-stripped payload bytes produced by the decoder pipeline. + * @param decodedContentLength The size of the decoded payload that callers will consume. + */ + DecodedResponse(HttpResponse httpResponse, Flux decodedBody, long decodedContentLength) { + super(httpResponse.getRequest()); + this.originalResponse = httpResponse; + this.decodedBody = decodedBody; + HttpHeaders headers = new HttpHeaders(httpResponse.getHeaders()); + headers.set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(decodedContentLength)); + this.adjustedHeaders = headers; + } + + @Override + public int getStatusCode() { + return originalResponse.getStatusCode(); + } + + @Override + @SuppressWarnings("deprecation") + public String getHeaderValue(String name) { + return adjustedHeaders.getValue(name); + } + + @Override + public HttpHeaders getHeaders() { + return adjustedHeaders; + } + + @Override + public Flux getBody() { + return decodedBody; + } + + @Override + public Mono getBodyAsByteArray() { + return FluxUtil.collectBytesInByteBufferStream(decodedBody); + } + + @Override + public Mono getBodyAsString() { + return getBodyAsByteArray() + .map(b -> CoreUtils.bomAwareToString(b, adjustedHeaders.getValue(HttpHeaderName.CONTENT_TYPE))); + } + + @Override + public Mono getBodyAsString(Charset charset) { + return FluxUtil.collectBytesInByteBufferStream(decodedBody).map(b -> new String(b, charset)); + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecodingPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecodingPolicy.java new file mode 100644 index 000000000000..3c1ebaabf7ce --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecodingPolicy.java @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * HTTP pipeline policy that decodes the storage structured message body returned for downloads when CRC64 + * content validation is active. + * + *

The policy decides when to opt in (via the context key), tells the service to + * encode the response (via the request header), constructs the decoder and the wrapper response, and + * translates decoder-level failures (malformed framing, CRC mismatch, premature end-of-stream) into reactive + * {@link IOException} errors.

+ * + *

This policy uses {@link com.azure.core.http.HttpPipelinePosition#PER_RETRY PER_RETRY} semantics by default, so + * each retry produces a fresh response that this policy wraps with a fresh decoder. A CRC failure on one attempt + * cannot pollute another, and the storage download retry logic ({@code BlobAsyncClientBase.downloadStream...}) can + * resume by reissuing range requests; each new range response is validated end-to-end on its own.

+ * + *

Because the wrapped {@link StructuredMessageDecoder} only releases payload bytes after the corresponding + * segment's CRC has been verified, the {@link DecodedResponse}'s body Flux is guaranteed to contain only validated + * bytes – callers never see a byte that could later fail validation, even when retries are involved.

+ */ +public class StorageContentValidationDecodingPolicy implements HttpPipelinePolicy { + private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecodingPolicy.class); + + /** + * Creates a new instance of {@link StorageContentValidationDecodingPolicy}. + */ + public StorageContentValidationDecodingPolicy() { + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Check if the decoding should be applied. + if (!shouldApplyDecoding(context)) { + return next.process(); + } + + // Tell the service we want a structured-message body. + context.getHttpRequest() + .getHeaders() + .set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, + StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); + + return next.process().map(httpResponse -> { + // The HTTP Content-Length is the size of the encoded structured message body. We hand it to the + // decoder which cross-checks it against the message header. + Long contentLength = getContentLength(httpResponse.getHeaders()); + + // Only 2xx GET responses with a positive content length carry a body that we can decode. + if (!isEligibleDownload(httpResponse, contentLength)) { + return httpResponse; + } + + // Confirm the service honored our structured-body request and parse the decoded length in one step, + // failing fast with a consistent error if either header is absent or not parseable as a long. + long decodedContentLength = validateAndGetDecodedContentLength(httpResponse); + + // Fresh decoder per response so retries each get a clean state machine. + StructuredMessageDecoder decoder = new StructuredMessageDecoder(contentLength); + + Flux decodedStream = decodeStream(httpResponse.getBody(), decoder); + return new DecodedResponse(httpResponse, decodedStream, decodedContentLength); + }); + } + + /** + * @return true when the request carries the boolean opt-in flag set + * by {@code ContentValidationModeResolver.addStructuredMessageDecodingToContext}. + */ + private boolean shouldApplyDecoding(HttpPipelineCallContext context) { + return context.getData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) + .map(value -> value instanceof Boolean && (Boolean) value) + .orElse(false); + } + + /** + * Verifies the response acknowledges the structured-body request ({@code x-ms-structured-body} present) and + * parses {@code x-ms-structured-content-length} in one step. Throws with a consistent error message if either + * header is absent or the length value is not parseable as a {@code long}, matching the fail-fast behaviour + * used for other validation failures. + * + * @return the decoded payload size declared by the service. + */ + private long validateAndGetDecodedContentLength(HttpResponse httpResponse) { + String structuredBody + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String structuredContentLength + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + if (structuredBody == null || structuredContentLength == null) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Structured message was requested but the response did not acknowledge it.")); + } + try { + return Long.parseLong(structuredContentLength); + } catch (NumberFormatException e) { + throw LOGGER.logExceptionAsError(new IllegalStateException("x-ms-structured-content-length header value '" + + structuredContentLength + "' could not be parsed as a long.", e)); + } + } + + /** + * Reads {@code Content-Length} as a {@code long}, returning {@code null} when the header is missing or + * not parseable so callers can simply skip decoding for non-bodied responses. + */ + private static Long getContentLength(HttpHeaders headers) { + String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + // Header invalid; treat as not eligible. + } + } + return null; + } + + /** + * @return true for a 2xx response to a GET request, the only response shape that carries a body we + * can decode. 206 (Partial Content) on retried range downloads is included. + */ + private static boolean isDownloadResponse(HttpResponse response) { + return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; + } + + /** + * @return true when the response is one we should decode: a 2xx GET with a positive, parseable + * {@code Content-Length}. + */ + private static boolean isEligibleDownload(HttpResponse response, Long contentLength) { + return isDownloadResponse(response) && contentLength != null && contentLength > 0; + } + + /** + * Builds the body-decoding Flux: each upstream {@link ByteBuffer} is fed to the decoder in order + * ({@code concatMap} preserves order and serializes access), and a deferred stream-completion check is + * appended so a truncated body raises an error instead of completing silently. + */ + private Flux decodeStream(Flux encodedFlux, StructuredMessageDecoder decoder) { + // limitRate(1) mirrors StorageContentValidationEncodingPolicy's upload path: process one wire buffer at a time so + // the decoder can copy only the current chunk into owned storage and release the inbound buffer before the + // next segment payload bytes arrive. + return encodedFlux.limitRate(1) + .concatMap(buffer -> decodeBuffer(buffer, decoder)) + .concatWith(Mono.defer(() -> handleStreamCompletion(decoder))); + } + + /** + * Feeds a single inbound chunk to the decoder and translates its outputs into reactive emissions: + * If the decoder reports validated bytes, emit them downstream. + * If the decoder threw because the input is malformed or a CRC mismatch was detected, surface that as + * an {@link IOException}. + * If the decoder is already complete (e.g., extra trailing bytes after the message footer), drop the + * chunk silently. + */ + private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecoder decoder) { + if (decoder.isComplete()) { + // Decoding finished on a previous chunk; ignore any trailing bytes the transport might still emit. + return Flux.empty(); + } + + if (buffer == null || !buffer.hasRemaining()) { + return Flux.empty(); + } + + try { + return Flux.fromIterable(decoder.decodeChunk(buffer)); + } catch (IllegalArgumentException e) { + return Flux.error(new IOException("Failed to decode structured message: " + e.getMessage(), e)); + } catch (Exception e) { + // Anything not foreseen by the decoder, log it. + LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); + return Flux.error(new IOException("Failed to decode structured message chunk: " + e.getMessage(), e)); + } + } + + /** + * Run after the upstream Flux completes. If the decoder is not in a complete state, the response body ended + * before all expected bytes arrived – surface this as an {@link IOException} so callers don't accept a + * truncated payload. + */ + private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { + if (!decoder.isComplete()) { + return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); + } + return Mono.empty(); + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationEncodingPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationEncodingPolicy.java new file mode 100644 index 000000000000..ee6da95d3455 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationEncodingPolicy.java @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Optional; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE; + +/** + * StorageContentValidationEncodingPolicy is a policy that applies content validation to the request body. + */ +public class StorageContentValidationEncodingPolicy implements HttpPipelinePolicy { + private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationEncodingPolicy.class); + + /** + * Creates a new instance of {@link StorageContentValidationEncodingPolicy}. + */ + public StorageContentValidationEncodingPolicy() { + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Defer creating the next policy Mono until validation (and any required header mutations) has completed. + // Some downstream policies may compute auth/signatures eagerly in their `process()` method. + return applyContentValidation(context).then(Mono.defer(next::process)); + } + + /** + * Applies content validation to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when content validation has been applied to the request body. + */ + private Mono applyContentValidation(HttpPipelineCallContext context) { + Optional behaviorOptional = context.getContext().getData(CONTENT_VALIDATION_MODE_KEY); + if (!behaviorOptional.isPresent()) { + return Mono.empty(); + } + + String contentValidationBehavior = behaviorOptional.get().toString(); + if (contentValidationBehavior.isEmpty()) { + return Mono.empty(); + } + + Mono validation = Mono.empty(); + + if (contentValidationBehavior.contains(USE_CRC64_CHECKSUM_HEADER_CONTEXT)) { + validation = validation.then(applyCRC64Header(context)); + } + if (contentValidationBehavior.contains(USE_STRUCTURED_MESSAGE_CONTEXT)) { + validation = validation.then(applyStructuredMessage(context)); + } + + return validation; + } + + /** + * Applies the crc64 header to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when the crc64 header has been applied to the request body. + */ + private Mono applyCRC64Header(HttpPipelineCallContext context) { + if (context.getHttpRequest().getBody() == null) { + return Mono.empty(); + } + + // Collect request body bytes once, compute CRC64 off the reactive thread, and then restore the body + // as a replayable Flux so downstream processing / sending still sees the expected content. + Flux originalBody = context.getHttpRequest().getBody(); + return FluxUtil.collectBytesInByteBufferStream(originalBody) + .flatMap(originalBytes -> Mono.fromCallable(() -> StorageCrc64Calculator.compute(originalBytes, 0)) + // Run CRC64 on boundedElastic so synchronous work over the full body does not block the reactive + // I/O thread (e.g. Netty event loop) that drives the pipeline. + .subscribeOn(Schedulers.boundedElastic()) + .flatMap(contentCRC64 -> { + // Restore body for downstream consumers. + context.getHttpRequest().setBody(Flux.just(ByteBuffer.wrap(originalBytes))); + + // Convert the 64-bit CRC value to 8 bytes in little-endian format. + byte[] crc64Bytes = new byte[8]; + for (int i = 0; i < 8; i++) { + crc64Bytes[i] = (byte) (contentCRC64 >>> (i * 8)); + } + + // Base64 encode the binary representation. + String encodedCRC64 = Base64.getEncoder().encodeToString(crc64Bytes); + context.getHttpRequest().setHeader(CONTENT_CRC64_HEADER_NAME, encodedCRC64); + return Mono.empty(); + })); + } + + /** + * Applies the structured message to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when the structured message has been applied, or fails if the request is + * not valid for structured messaging + */ + private Mono applyStructuredMessage(HttpPipelineCallContext context) { + String contentLengthValue = context.getHttpRequest().getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH); + if (contentLengthValue == null || contentLengthValue.isEmpty()) { + return FluxUtil.monoError(LOGGER, + LOGGER.logExceptionAsError( + new IllegalArgumentException("Content-Length header is required to apply structured message " + + "and CRC64 encoding, but it was not present on the request."))); + } + + long parsedContentLength; + try { + parsedContentLength = Long.parseLong(contentLengthValue); + } catch (NumberFormatException ex) { + return FluxUtil.monoError(LOGGER, LOGGER.logExceptionAsError(new IllegalArgumentException( + "Content-Length header value '" + contentLengthValue + + "' is not a valid non-negative integer value required for structured message and CRC64 encoding.", + ex))); + } + + int unencodedContentLength = (int) parsedContentLength; + + Flux originalBody = context.getHttpRequest().getBody(); + + /* + * Replace the request body with a structured message: raw content wrapped with headers, segment + * boundaries, and CRC64 checksums so the service can validate integrity as it receives the stream. + * + * A fresh encoder is created on each subscribe (via defer) so retries re-encode correctly from the + * original replayable body. The encoded buffers are slices of the original data, produced lazily and + * consumed by the HTTP client without materialization. + * + * limitRate(1) keeps the encoder's segment boundaries aligned with buffer boundaries. + */ + Flux encodedBody = Flux.defer(() -> { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(unencodedContentLength, + V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64); + return Flux.from(originalBody).limitRate(1).concatMap(encoder::encode); + }); + + context.getHttpRequest().setBody(encodedBody); + + long encodedLength = new StructuredMessageEncoder(unencodedContentLength, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, + StructuredMessageFlags.STORAGE_CRC64).getEncodedMessageLength(); + context.getHttpRequest().setHeader(HttpHeaderName.CONTENT_LENGTH, String.valueOf(encodedLength)); + context.getHttpRequest().setHeader(STRUCTURED_BODY_TYPE_HEADER_NAME, STRUCTURED_BODY_TYPE_VALUE); + context.getHttpRequest() + .setHeader(STRUCTURED_CONTENT_LENGTH_HEADER_NAME, String.valueOf(unencodedContentLength)); + + return Mono.empty(); + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/module-info.java b/sdk/storage/azure-storage-common/src/main/java/module-info.java index f5f1cfa99b2b..412a11fe41d6 100644 --- a/sdk/storage/azure-storage-common/src/main/java/module-info.java +++ b/sdk/storage/azure-storage-common/src/main/java/module-info.java @@ -25,4 +25,7 @@ exports com.azure.storage.common.implementation.connectionstring to // FIXME this should not be a long-term solution com.azure.data.tables, com.azure.storage.blob, com.azure.storage.blob.cryptography, com.azure.storage.file.share, com.azure.storage.file.datalake, com.azure.storage.queue; + + exports com.azure.storage.common.implementation.contentvalidation to // FIXME this should not be a long-term solution + com.azure.storage.blob, com.azure.storage.file.share, com.azure.storage.file.datalake; } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java index 5e84dd31947d..ddd488ff0887 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java @@ -6,10 +6,10 @@ import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; +import com.azure.core.util.FluxUtil; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -19,14 +19,21 @@ with than was worth it. Because this type is just for BlobDownload, we don't need to accept a header type. */ public class MockDownloadHttpResponse extends HttpResponse { + private final HttpResponse originalResponse; private final int statusCode; private final HttpHeaders headers; private final Flux body; public MockDownloadHttpResponse(HttpResponse response, int statusCode, Flux body) { + this(response, statusCode, response.getHeaders(), body); + } + + public MockDownloadHttpResponse(HttpResponse response, int statusCode, HttpHeaders headers, + Flux body) { super(response.getRequest()); + this.originalResponse = response; this.statusCode = statusCode; - this.headers = response.getHeaders(); + this.headers = headers; this.body = body; } @@ -52,21 +59,26 @@ public HttpHeaders getHeaders() { @Override public Flux getBody() { - return body; + return Flux.using(() -> originalResponse, ignored -> body, HttpResponse::close); } @Override public Mono getBodyAsByteArray() { - return Mono.error(new IOException()); + return FluxUtil.collectBytesInByteBufferStream(getBody()); } @Override public Mono getBodyAsString() { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); } @Override public Mono getBodyAsString(Charset charset) { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); + } + + @Override + public void close() { + originalResponse.close(); } } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index 1f1109d8b38b..347d3ac11a59 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -8,6 +8,7 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; import reactor.core.publisher.Flux; @@ -16,50 +17,137 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class MockPartialResponsePolicy implements HttpPipelinePolicy { - static final HttpHeaderName RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); - private int tries; - private final List rangeHeaders = new ArrayList<>(); + static final HttpHeaderName X_MS_RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); + static final HttpHeaderName RANGE_HEADER = HttpHeaderName.RANGE; + private final AtomicInteger tries; + private final List rangeHeaders = Collections.synchronizedList(new ArrayList<>()); + private final int maxBytesPerResponse; + private final AtomicInteger hits = new AtomicInteger(); + private final String targetUrlPrefix; + /** + * Creates a MockPartialResponsePolicy that simulates network interruptions. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + */ public MockPartialResponsePolicy(int tries) { - this.tries = tries; + this(tries, 200, null); + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse) { + this(tries, maxBytesPerResponse, null); + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior and an optional URL filter. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + * @param targetUrlPrefix If non-null, only requests whose URL starts with this prefix will be interrupted. + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse, String targetUrlPrefix) { + this.tries = new AtomicInteger(tries); + this.maxBytesPerResponse = maxBytesPerResponse; + this.targetUrlPrefix = targetUrlPrefix; + } + + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; } @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { return next.process().flatMap(response -> { HttpHeader rangeHttpHeader = response.getRequest().getHeaders().get(RANGE_HEADER); - String rangeHeader = rangeHttpHeader == null ? null : rangeHttpHeader.getValue(); + HttpHeader xMsRangeHttpHeader = response.getRequest().getHeaders().get(X_MS_RANGE_HEADER); - if (rangeHeader != null && rangeHeader.startsWith("bytes=")) { - rangeHeaders.add(rangeHeader); + if (response.getRequest().getHttpMethod() == HttpMethod.GET) { + String recordedRange = null; + if (rangeHttpHeader != null && rangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = rangeHttpHeader.getValue(); + } else if (xMsRangeHttpHeader != null && xMsRangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = xMsRangeHttpHeader.getValue(); + } + rangeHeaders.add(recordedRange == null ? "" : recordedRange); } - if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || this.tries == 0) { + boolean urlMatches = targetUrlPrefix == null + || response.getRequest().getUrl().toString().startsWith(targetUrlPrefix); + + if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || !urlMatches) { return Mono.just(response); } else { - this.tries -= 1; - return response.getBody().collectList().flatMap(bodyBuffers -> { - ByteBuffer firstBuffer = bodyBuffers.get(0); - byte firstByte = firstBuffer.get(); - - // Simulate partial response by returning the first byte only from the requested range and timeout - return Mono.just(new MockDownloadHttpResponse(response, 206, - Flux.just(ByteBuffer.wrap(new byte[] { firstByte })) - .concatWith(Flux.error(new IOException("Simulated timeout"))) - )); - }); + int remainingTries = this.tries.getAndUpdate(value -> value > 0 ? value - 1 : value); + if (remainingTries <= 0) { + return Mono.just(response); + } + hits.incrementAndGet(); + + Flux limitedBody = limitStreamToBytes(response.getBody(), maxBytesPerResponse); + return Mono.just( + new MockDownloadHttpResponse(response, response.getStatusCode(), response.getHeaders(), + limitedBody)); } }); } + private Flux limitStreamToBytes(Flux body, int maxBytes) { + return Flux.defer(() -> { + final long[] bytesEmitted = new long[] { 0 }; + return body.concatMap(buffer -> { + if (buffer == null || !buffer.hasRemaining()) { + return Flux.just(buffer); + } + + long remaining = maxBytes - bytesEmitted[0]; + if (remaining <= 0) { + return Flux.error(new IOException("Simulated timeout")); + } + + int bufferSize = buffer.remaining(); + if (bufferSize <= remaining) { + bytesEmitted[0] += bufferSize; + if (bytesEmitted[0] >= maxBytes) { + return Flux.just(buffer).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + return Flux.just(buffer); + } else { + int bytesToEmit = (int) remaining; + ByteBuffer slice = buffer.duplicate(); + slice.limit(slice.position() + bytesToEmit); + + ByteBuffer limited = ByteBuffer.allocate(bytesToEmit); + limited.put(slice); + limited.flip(); + + bytesEmitted[0] += bytesToEmit; + return Flux.just(limited).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + }); + }); + } + public int getTriesRemaining() { - return tries; + return tries.get(); } public List getRangeHeaders() { return rangeHeaders; } + + public int getHits() { + return hits.get(); + } } diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/BufferStagingAreaTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/BufferStagingAreaTests.java index 66cac7dfeaf7..77384c6630ce 100644 --- a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/BufferStagingAreaTests.java +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/BufferStagingAreaTests.java @@ -105,8 +105,10 @@ private static Stream bufferStagingAreaSupplier() { Arguments.of(1000, 1000, 1000), // _ Arguments.of(10000, 1000, 1000), // _ Arguments.of(10000, 1, 1000), // These test variation in buffSize. - Arguments.of(100, 1, Constants.MB * 4), // _ - Arguments.of(100, Constants.MB * 4, Constants.MB * 8) // _ + // Exercise repartitioning of many variably-sized buffers by maxBuffSize. The buffer count and size + // variation are what matter here, not the absolute MB size; sizes were reduced from 4-8 MiB. + Arguments.of(100, 1, Constants.KB * 256), // _ + Arguments.of(100, Constants.KB * 256, Constants.KB * 512) // _ ); } diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java new file mode 100644 index 000000000000..14408a4163bb --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.Context; +import com.azure.core.util.ProgressListener; +import com.azure.storage.common.ContentValidationAlgorithm; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.util.stream.Stream; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ContentValidationModeResolverTests { + + private static String modeOnContext(Context context, ContentValidationAlgorithm algorithm, long contentLength, + boolean chunkedUpload) { + return ContentValidationModeResolver.addContentValidationMode(context, algorithm, contentLength, chunkedUpload) + .getData(CONTENT_VALIDATION_MODE_KEY) + .map(Object::toString) + .orElse(null); + } + + // =========================================================================================== + // addContentValidationMode (Context) — single-part + // =========================================================================================== + + static Stream singlePartDoesNotSetModeSupplier() { + return Stream.of(Arguments.of(null, 1024), Arguments.of(ContentValidationAlgorithm.NONE, 1024), + Arguments.of(null, 8 * 1024 * 1024), Arguments.of(ContentValidationAlgorithm.NONE, 8 * 1024 * 1024)); + } + + @ParameterizedTest + @MethodSource("singlePartDoesNotSetModeSupplier") + public void singlePartDoesNotSetModeForNullOrNone(ContentValidationAlgorithm algorithm, long length) { + assertEquals(null, modeOnContext(Context.NONE, algorithm, length, false)); + } + + @Test + public void singlePartSmallUploadUsesCrc64Header() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, underThreshold, false)); + } + + @Test + public void singlePartAtExact4MBBoundaryUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, + MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER, false)); + } + + @Test + public void singlePartLargeUploadUsesStructuredMessage() { + long overThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + 1; + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, overThreshold, false)); + } + + @Test + public void singlePartAutoSmallUploadUsesCrc64Header() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, underThreshold, false)); + } + + @Test + public void singlePartAutoAtExact4MBBoundaryUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, + MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER, false)); + } + + @Test + public void singlePartAutoLargeUploadUsesStructuredMessage() { + long overThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + 1; + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, overThreshold, false)); + } + + @Test + public void addContentValidationModeNullContextUsesNone() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(null, ContentValidationAlgorithm.CRC64, underThreshold, false)); + } + + // =========================================================================================== + // addContentValidationMode (Context) — chunked + // =========================================================================================== + + static Stream chunkedDoesNotSetModeSupplier() { + return Stream.of(Arguments.of((ContentValidationAlgorithm) null), + Arguments.of(ContentValidationAlgorithm.NONE)); + } + + @ParameterizedTest + @MethodSource("chunkedDoesNotSetModeSupplier") + public void chunkedDoesNotSetModeForNonCrc64Algorithms(ContentValidationAlgorithm algorithm) { + assertEquals(null, modeOnContext(Context.NONE, algorithm, 1024, true)); + } + + @Test + public void chunkedCrc64AlwaysUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, 1024, true)); + } + + @Test + public void chunkedAutoAlwaysUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, 1024, true)); + } + + // =========================================================================================== + // addContentValidationMode (Mono) + // =========================================================================================== + + @Test + public void addContentValidationModeMonoWritesReactorContextForCrc64() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + Mono source = Mono.deferContextual(ctx -> Mono.just(ctx.get(CONTENT_VALIDATION_MODE_KEY))); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.CRC64, underThreshold, false); + StepVerifier.create(augmented).expectNext(USE_CRC64_CHECKSUM_HEADER_CONTEXT).verifyComplete(); + } + + @Test + public void addContentValidationModeMonoWritesReactorContextForAuto() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + Mono source = Mono.deferContextual(ctx -> Mono.just(ctx.get(CONTENT_VALIDATION_MODE_KEY))); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.AUTO, underThreshold, false); + StepVerifier.create(augmented).expectNext(USE_CRC64_CHECKSUM_HEADER_CONTEXT).verifyComplete(); + } + + @Test + public void addContentValidationModeMonoLeavesChainUnchangedWhenNoMode() { + Mono source = Mono.deferContextual(ctx -> Mono.just("ok")); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.NONE, 1024, false); + StepVerifier.create(augmented).expectNext("ok").verifyComplete(); + } + + // =========================================================================================== + // validateTransactionalChecksumOptions (boolean computeMd5) + // =========================================================================================== + + @Test + public void validateComputeMd5PassesForCompatibleOptions() { + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(true, null)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.AUTO)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.NONE)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.CRC64)); + } + + @Test + public void validateComputeMd5ThrowsWhenAlgorithmNone() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.NONE)); + } + + @Test + public void validateComputeMd5ThrowsForCrc64() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.CRC64)); + } + + @Test + public void validateComputeMd5ThrowsForAuto() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.AUTO)); + } + + // =========================================================================================== + // validateProgressWithContentValidation + // =========================================================================================== + + @Test + public void validateProgressWithContentValidationPassesWhenNoProgress() { + assertDoesNotThrow(() -> ContentValidationModeResolver + .validateProgressWithContentValidation((ProgressListener) null, ContentValidationAlgorithm.CRC64)); + assertDoesNotThrow(() -> ContentValidationModeResolver + .validateProgressWithContentValidation((ProgressListener) null, ContentValidationAlgorithm.AUTO)); + } + + @Test + public void validateProgressWithContentValidationPassesWhenNoneOrNullAlgorithm() { + ProgressListener listener = l -> { + }; + assertDoesNotThrow(() -> ContentValidationModeResolver.validateProgressWithContentValidation(listener, null)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateProgressWithContentValidation(listener, + ContentValidationAlgorithm.NONE)); + } + + @Test + public void validateProgressWithContentValidationThrowsForCrc64() { + ProgressListener listener = l -> { + }; + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateProgressWithContentValidation(listener, ContentValidationAlgorithm.CRC64)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + @Test + public void validateProgressWithContentValidationThrowsForAuto() { + ProgressListener listener = l -> { + }; + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateProgressWithContentValidation(listener, ContentValidationAlgorithm.AUTO)); + } + + @Test + public void isCrc64OrAutoReflectsCrc64AndAutoOnly() { + assertTrue(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.CRC64)); + assertTrue(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.AUTO)); + assertFalse(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.NONE)); + assertFalse(ContentValidationModeResolver.isCrc64OrAuto(null)); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java new file mode 100644 index 000000000000..a4443ccb9aa1 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; + +import com.azure.storage.common.implementation.Constants; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Random; +import java.util.stream.IntStream; +import java.math.BigInteger; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class StorageCrc64CalculatorTests { + + @ParameterizedTest + @MethodSource("testComputeSupplier") + public void testCompute(String data, long initial, long expected) { + byte[] bytes = data.getBytes(); + long actual = StorageCrc64Calculator.compute(bytes, initial); + assertEquals(expected, actual); + } + + private static Stream testComputeSupplier() { + return Stream.of(Arguments.of("", 0, 0), Arguments.of("Hello World!", 0, 208604604655264165L), + Arguments.of("123456789!@#$%^&*()", 0, 2153758901452455624L), + Arguments.of( + "This is a test where the data is longer than 64 characters so that we can test that code path.", 0, + 2736107658526394369L)); + } + + @ParameterizedTest + @MethodSource("testComputeWithBinaryDataSupplier") + void testComputeWithBinaryData(long initial, String hexData, String expectedStr) { + byte[] hexBytes = hexStringToByteArray(hexData); + long expected = unsignedLongFromString(expectedStr); + long actual = StorageCrc64Calculator.compute(hexBytes, initial); + assertEquals(expected, actual); + } + + private static Stream testComputeWithBinaryDataSupplier() { + return Stream.of(Arguments.of(0L, "C8E11B40D793D1526018", "3386042136331673945"), + Arguments.of(208604604655264165L, "C8E11B40D793D1526018", "4570059697646401418"), + Arguments.of(2153758901452455624L, "C8E11B40D793D1526018", "13366433516720813220"), + Arguments.of(12345L, "C8E11B40D793D1526018", "5139183895903464380"), + Arguments.of(0L, "AABBCCDDEEFF", "13410969003359324163"), + Arguments.of(208604604655264165L, "AABBCCDDEEFF", "7077292804802198544"), + Arguments.of(2153758901452455624L, "AABBCCDDEEFF", "7845171950889742163"), + Arguments.of(12345L, "AABBCCDDEEFF", "1250888331951523569")); + } + + private static byte[] hexStringToByteArray(String hexData) { + int length = hexData.length(); + byte[] data = new byte[length / 2]; + for (int i = 0; i < length; i += 2) { + data[i / 2] + = (byte) ((Character.digit(hexData.charAt(i), 16) << 4) + Character.digit(hexData.charAt(i + 1), 16)); + } + return data; + } + + @ParameterizedTest + @MethodSource("testComposeSupplier") + void testCompose(int numSegments, int minBlockSize, int maxBlockSize) { + Random random = new Random(); + + List blockLengths = new ArrayList<>(); + IntStream.range(0, numSegments).forEach(i -> { + int blockSize = maxBlockSize > minBlockSize + ? random.nextInt(maxBlockSize - minBlockSize) + minBlockSize + : minBlockSize; + blockLengths.add(blockSize); + }); + + byte[] data = new byte[blockLengths.stream().mapToInt(Integer::intValue).sum()]; + random.nextBytes(data); + + long wholeCrc = StorageCrc64Calculator.compute(data, 0); + Queue blockCrcs = new LinkedList<>(); + int offset = 0; + for (int length : blockLengths) { + blockCrcs.add(StorageCrc64Calculator.compute(Arrays.copyOfRange(data, offset, offset + length), 0)); + offset += length; + } + + long composedCrc = blockCrcs.poll(); + int lengthIndex = 1; + while (!blockCrcs.isEmpty()) { + long nextBlockCrc = blockCrcs.poll(); + composedCrc = StorageCrc64Calculator.concat(0, 0, composedCrc, blockLengths.get(lengthIndex - 1), 0, + nextBlockCrc, blockLengths.get(lengthIndex)); + lengthIndex++; + } + + assertEquals(wholeCrc, composedCrc); + } + + private static Stream testComposeSupplier() { + return Stream.of(Arguments.of(2, Constants.KB, Constants.KB), Arguments.of(3, Constants.KB, Constants.KB), + Arguments.of(10, Constants.KB, Constants.KB), Arguments.of(2, Constants.KB, 4 * Constants.KB), + Arguments.of(3, Constants.KB, 4 * Constants.KB), Arguments.of(10, Constants.KB, 4 * Constants.KB), + Arguments.of(2, Constants.KB, Constants.MB), Arguments.of(3, Constants.KB, Constants.MB), + // Cover large, non-uniform block sizes to exercise concat-composition without allocating + // multi-hundred-MB buffers (previously 512 MB, which intermittently OOMed the 4 GB fork and + // triggered a multi-GB heap dump that filled the CI agent disk). + Arguments.of(2, Constants.KB, 4 * Constants.MB), Arguments.of(3, Constants.KB, 4 * Constants.MB)); + } + + @Test + void testComputeInChunks() { + int maxBlockSize = Constants.GB; + int chunkSize = 8 * Constants.KB; + String baseString = "Hello World! This is testing chunked crc."; + byte[] baseBytes = baseString.getBytes(); + int baseLength = baseBytes.length; + + long wholeCrc = 8100535992282268188L; + + long chunkedCrc = 0; + int totalLength = 0; + + while (totalLength < maxBlockSize) { + int length = Math.min(chunkSize, maxBlockSize - totalLength); + byte[] chunk = new byte[length]; + for (int i = 0; i < length; i += baseLength) { + System.arraycopy(baseBytes, 0, chunk, i, Math.min(baseLength, length - i)); + } + long chunkCrc = StorageCrc64Calculator.compute(chunk, 0); + chunkedCrc = StorageCrc64Calculator.concat(chunkedCrc, 0, chunkCrc, length, 0, chunkCrc, length); + totalLength += length; + } + + assertEquals(wholeCrc, chunkedCrc); + } + + @ParameterizedTest + @CsvSource({ + "0, 0, 0, 0, 0", + "17360427831495520774, 949533, 13068224794440996385, 99043, 2942932174470096852", + "11788770130477425887, 505156, 11825964890373840515, 543420, 11679439596881108042", + "3295333047304801182, 732633, 15304759627474960884, 315943, 31840984168952505", + "16590039424904606984, 550299, 6063316096266934453, 498277, 4430932446378441680", + "4505532069077416052, 852279, 4910763717047934640, 196297, 15119506491662968913", + "390777554329396866, 834642, 2639871931800812330, 213934, 11705441749781302341", + "3373070654000205532, 282713, 998330282635826126, 765863, 9830625244855600085", + "2124306943908111903, 737293, 11017351202683543503, 311283, 8163771928713973931", + "15994440356403990005, 85861, 13803536430055425947, 962715, 1941624554903785510", + "3705122932895036835, 444701, 17573219681510991482, 603875, 4421337306620606216" }) + void testConcat(String crc1Str, long size1, String crc2Str, long size2, String expectedStr) { + // Convert the large numbers from String to BigInteger for unsigned support + long crc1 = unsignedLongFromString(crc1Str); + long crc2 = unsignedLongFromString(crc2Str); + long expected = unsignedLongFromString(expectedStr); + + long actual = StorageCrc64Calculator.concat(0, 0, crc1, size1, 0, crc2, size2); + assertEquals(expected, actual); + } + + private long unsignedLongFromString(String value) { + // Convert a string to a signed long interpreting it as unsigned 64-bit integer + return new BigInteger(value).longValue(); + } + + @ParameterizedTest + @MethodSource("testConcatWithInitialsSupplier") + void testConcatWithInitials(String initialStr, String initial1Str, String crc1Str, String size1Str, + String initial2Str, String crc2Str, String size2Str, String expectedStr) { + // Convert large unsigned values to signed long + long initial = unsignedLongFromString(initialStr); + long initial1 = unsignedLongFromString(initial1Str); + long crc1 = unsignedLongFromString(crc1Str); + long size1 = Long.parseLong(size1Str); + long initial2 = unsignedLongFromString(initial2Str); + long crc2 = unsignedLongFromString(crc2Str); + long size2 = Long.parseLong(size2Str); + long expected = unsignedLongFromString(expectedStr); + + long actual = StorageCrc64Calculator.concat(initial, initial1, crc1, size1, initial2, crc2, size2); + assertEquals(expected, actual); + } + + private static Stream testConcatWithInitialsSupplier() { + return Stream.of(Arguments.of("0", "0", "0", "0", "0", "0", "0", "0"), + Arguments.of("556425425686929588", "346224202686926702", "16342296696377857982", "332915", + "1153230192133190692", "12153371329672466699", "715661", "1441822370130745021"), + Arguments.of("6707243468313456313", "572263087298867634", "16994544883182326144", "75745", + "9131338361339398429", "10182915179976307502", "972831", "14966971284513070994"), + Arguments.of("5753644013440131291", "5049702011265556767", "17549647897932809624", "255140", + "7204171574261853450", "1993084328138883374", "793436", "6041621697050742380"), + Arguments.of("6856094926385348025", "5380840211500611709", "9696539459657763690", "537777", + "4787042077805010903", "13660128687379374948", "510799", "17784586126519415898"), + Arguments.of("7768574238870932405", "97145001356670685", "607054043350981298", "706788", + "667444555190985522", "10677778047180339455", "341788", "5763961866513573791"), + Arguments.of("3302120679354661969", "7763531798276712053", "8827557196489825944", "490442", + "8582969104890206846", "6702182603435500761", "558134", "4787302867829109706"), + Arguments.of("7553023568245626261", "9093436341919279996", "10815569438302788871", "785480", + "8305342016037017917", "6633140726058569127", "263096", "14625483825524673467"), + Arguments.of("8894905328920043035", "9101951045389247372", "10098427678135105249", "782758", + "8101576936188464286", "8318237935995533450", "265818", "5983082645903611588"), + Arguments.of("1084935736425738155", "5378644106529179816", "13762475631325587388", "1014816", + "8473418370223760471", "10401355811619715622", "33760", "3692020299859515126"), + Arguments.of("889000539881195835", "2971048229276949174", "5346315327374690144", "307387", + "1407121768110541356", "10535852615249992663", "741189", "3634018251978804152")); + } + + @Test + void testComputeSliceMatchesFullArray() { + byte[] data = "Hello World!".getBytes(); + long expected = StorageCrc64Calculator.compute(data, 0); + long actual = StorageCrc64Calculator.compute(data, 0, data.length, 0); + assertEquals(expected, actual); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java new file mode 100644 index 000000000000..f9f460bb57fc --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -0,0 +1,631 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.FluxUtil; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import reactor.core.publisher.Flux; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for StructuredMessageDecoder. + */ +public class StructuredMessageDecoderTests { + private static final int MESSAGE_HEADER_LENGTH = StructuredMessageConstants.V1_HEADER_LENGTH; + private static final int SEGMENT_HEADER_LENGTH = StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + private static final int CRC64_LENGTH = 8; + + @Test + public void readsCompleteMessageInSingleChunk() { + byte[] originalData = getRandomData(1024); + ByteBuffer encodedData = encodeToByteBuffer(originalData, 512, StructuredMessageFlags.STORAGE_CRC64); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List decodedPayload = decoder.decodeChunk(encodedData); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + @Test + public void readsTopLevelMessageHeaderSplitAcrossChunks() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + // Split before the 13-byte message header is complete. + int messageHeaderSplitPoint = 7; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, messageHeaderSplitPoint); + ByteBuffer chunk2 + = ByteBuffer.wrap(encodedBytes, messageHeaderSplitPoint, encodedLength - messageHeaderSplitPoint); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + assertTrue(firstDecodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + + List secondDecodedPayload = decoder.decodeChunk(chunk2); + assertFalse(secondDecodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + @Test + public void readsPerSegmentHeaderSplitAcrossChunks() { + byte[] originalData = getRandomData(512); + byte[] encodedBytes = encodeToBytes(originalData, 256); + int encodedLength = encodedBytes.length; + + // Split after the full message header but before the 10-byte segment header is complete. + int segmentHeaderSplitPoint = MESSAGE_HEADER_LENGTH + 5; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, segmentHeaderSplitPoint); + ByteBuffer chunk2 + = ByteBuffer.wrap(encodedBytes, segmentHeaderSplitPoint, encodedLength - segmentHeaderSplitPoint); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + // Segment header is incomplete, so nothing is emitted yet. + assertTrue(firstDecodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + + List secondDecodedPayload = decoder.decodeChunk(chunk2); + assertFalse(secondDecodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + @Test + public void multipleChunksDecode() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + int chunkSize = 32; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + for (int offset = 0; offset < encodedLength; offset += chunkSize) { + int len = Math.min(chunkSize, encodedLength - offset); + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); + + List decodedPayload = decoder.decodeChunk(chunk); + writeDecodedPayload(output, decodedPayload); + if (decoder.isComplete()) { + break; + } + } + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, output.toByteArray()); + } + + @Test + public void decodeWithNoCrc() { + byte[] originalData = getRandomData(256); + ByteBuffer encodedData = encodeToByteBuffer(originalData, 128, StructuredMessageFlags.NONE); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List decodedPayload = decoder.decodeChunk(encodedData); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + @Test + public void handlesZeroLengthBuffer() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + List firstDecodedPayload = decoder.decodeChunk(emptyBuffer); + assertTrue(firstDecodedPayload.isEmpty()); + ByteBuffer dataBuffer = ByteBuffer.wrap(encodedBytes); + List decodedPayload = decoder.decodeChunk(dataBuffer); + assertFalse(decodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + /** + * Payload bytes for a segment must not be emitted until the segment's CRC footer has been read and + * validated. While the footer is incomplete, decodeChunk must return null. + */ + @Test + public void withholdsPayloadUntilSegmentFooterValidated() { + byte[] originalData = getRandomData(1024); + byte[] encodedBytes = encodeToBytes(originalData, 1024); + int encodedLength = encodedBytes.length; + + // The segment payload cannot be emitted until the final segment CRC byte arrives in chunk2. + int segCrcAllButLast + = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + 1024 + StructuredMessageConstants.CRC64_LENGTH - 1; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, segCrcAllButLast); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + assertTrue(firstDecodedPayload.isEmpty(), "Decoder must not emit payload before segment CRC is validated"); + assertFalse(decoder.isComplete()); + + ByteBuffer chunk2 = ByteBuffer.wrap(encodedBytes, segCrcAllButLast, encodedLength - segCrcAllButLast); + List emittedPayload = decoder.decodeChunk(chunk2); + assertFalse(emittedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, collectDecodedBytes(emittedPayload)); + } + + @Test + public void throwsOnUnsupportedStructuredMessageVersion() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Byte 0 of the message header is reserved for the version. Changing to an incompatible version (2) + encodedBytes[0] = (byte) (StructuredMessageConstants.DEFAULT_MESSAGE_VERSION + 1); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unsupported structured message version")); + } + + @Test + public void throwsOnMessageLengthMismatch() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Construct decoder with wrong expected encoded length. + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length + 1); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("did not match content length")); + } + + @Test + public void throwsOnUnexpectedSegmentNumber() { + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Corrupt first segment number from 1 to 2 (offset 13). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(MESSAGE_HEADER_LENGTH, (short) 2); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unexpected segment number")); + } + + @Test + public void throwsOnInvalidSegmentSize() { + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Corrupt first segment size to an impossible value (8 bytes at offset 15). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(MESSAGE_HEADER_LENGTH + 2, Long.MAX_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid segment size detected")); + } + + @Test + public void throwsOnSegmentCrcMismatch() { + byte[] data = getRandomData(512); + byte[] encodedBytes = encodeToBytes(data, 512); + + // msgHdr(13) + segHdr(10) + payload(512) + segCrc(8) + msgCrc(8). Flip one bit of the segment CRC. + int segmentCrcOffset = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + data.length; + encodedBytes[segmentCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in segment")); + } + + @Test + public void throwsOnSegmentCrcMismatchInLaterSegment() { + // Multi-segment message where segment 1 is intact but segment 2's CRC is corrupted; verifies + // CRC validation runs on every segment, not just the first. + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 100); + + // Per-segment block: segHdr(10) + payload(100) + segCrc(8) = 118. + // Segment 2's CRC starts at: msgHdr(13) + segBlock(118) + segHdr(10) + payload(100). + int seg2CrcOffset + = MESSAGE_HEADER_LENGTH + (SEGMENT_HEADER_LENGTH + 100 + CRC64_LENGTH) + SEGMENT_HEADER_LENGTH + 100; + encodedBytes[seg2CrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in segment 2")); + } + + @Test + public void throwsOnMessageCrcMismatch() { + byte[] data = getRandomData(512); + byte[] encodedBytes = encodeToBytes(data, 512); + + int messageCrcOffset = encodedBytes.length - CRC64_LENGTH; + byte[] corrupted = Arrays.copyOf(encodedBytes, encodedBytes.length); + corrupted[messageCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(corrupted.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(corrupted))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in message footer")); + } + + @Test + public void throwsOnUnsupportedFlags() { + // Flags value 2 is not in the StructuredMessageFlags enum (NONE=0, STORAGE_CRC64=1) and must be rejected. + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Flags live at offset 9 (2 bytes, little-endian). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(9, (short) 2); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid value for StructuredMessageFlags")); + } + + @Test + public void throwsOnZeroSegments() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // numSegments lives at offset 11 (2 bytes, little-endian). Force it to zero. + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(11, (short) 0); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("at least one segment")); + } + + @Test + public void throwsOnSkippedSegmentNumber() { + // 3 segments of 100 bytes each. Layout per segment: segHdr(10) + payload(100) + segCrc(8) = 118. + // Rewrite segment 2's number to 3 to simulate a stream that skips segment 2. + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 100); + + int seg2NumberOffset = MESSAGE_HEADER_LENGTH + (SEGMENT_HEADER_LENGTH + 100 + CRC64_LENGTH); + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(seg2NumberOffset, (short) 3); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unexpected segment number")); + } + + @Test + public void truncatedMessageHeaderLeavesDecoderIncomplete() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Feed only the first 5 bytes of the 13-byte message header. + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, 5); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedSegmentHeaderLeavesDecoderIncomplete() { + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 256); + + // Feed full message header + 5 of the 10 segment-header bytes. + int truncated = MESSAGE_HEADER_LENGTH + 5; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedSegmentFooterLeavesDecoderIncomplete() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Layout: msgHdr(13) + segHdr(10) + payload(128) + segCrc(8) + msgCrc(8). Truncate mid-segCrc. + int truncated = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + 128 + 4; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedMessageFooterLeavesDecoderIncomplete() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Feed everything except the last 4 bytes of the 8-byte message CRC footer. + int truncated = encodedBytes.length - 4; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + // Segment payload has been released, but the message footer is still incomplete. + assertFalse(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void extraBytesAfterMessageFooterAreNotConsumed() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Append garbage bytes after the message footer; the decoder must stop at the declared message length. + int extras = 16; + byte[] padded = new byte[encodedBytes.length + extras]; + System.arraycopy(encodedBytes, 0, padded, 0, encodedBytes.length); + byte[] noise = getRandomData(extras); + System.arraycopy(noise, 0, padded, encodedBytes.length, extras); + ByteBuffer buffer = ByteBuffer.wrap(padded); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(buffer); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(data, collectDecodedBytes(decodedPayload)); + // Trailing bytes must not be consumed; buffer position stops at the declared message length. + assertEquals(extras, buffer.remaining()); + } + + @Test + public void throwsOnEncodedPayloadLargerThanExpectedSize() { + // Wire payload is larger than the expectedEncodedMessageLength supplied to the decoder. Must be rejected by the msgLen check. + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length - 8); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("did not match content length")); + } + + @Test + public void throwsOnNegativeMessageLength() { + // msgLen lives at offset 1 (8 bytes, little-endian). A negative value must be rejected before + // any further bounds math runs. + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(1, Long.MIN_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Message length too small")); + } + + @Test + public void throwsOnNegativeSegmentSize() { + // Companion to throwsOnInvalidSegmentSize covering the negative-value branch of the segment-size check. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 256); + + // Segment size lives at offset MESSAGE_HEADER_LENGTH + 2 (after the 2-byte segment number). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(MESSAGE_HEADER_LENGTH + 2, Long.MIN_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid segment size detected")); + } + + @Test + public void throwsOnInjectedRandomByte() { + // Insert a single random byte at a random offset in the encoded wire bytes. The msgLen field still + // declares the original size, so any insertion must be rejected by validation. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + int insertAt = ThreadLocalRandom.current().nextInt(encodedBytes.length + 1); + byte injected = (byte) ThreadLocalRandom.current().nextInt(256); + + byte[] tampered = new byte[encodedBytes.length + 1]; + System.arraycopy(encodedBytes, 0, tampered, 0, insertAt); + tampered[insertAt] = injected; + System.arraycopy(encodedBytes, insertAt, tampered, insertAt + 1, encodedBytes.length - insertAt); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(tampered.length); + assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(tampered))); + } + + @Test + public void throwsOnRemovedRandomBytes() { + // Remove a random run of bytes from a random offset in the encoded wire. The msgLen field still + // declares the original size, so any deletion must be rejected by validation. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + int removeCount = 1 + ThreadLocalRandom.current().nextInt(8); + int removeAt = ThreadLocalRandom.current().nextInt(encodedBytes.length - removeCount); + + byte[] tampered = new byte[encodedBytes.length - removeCount]; + System.arraycopy(encodedBytes, 0, tampered, 0, removeAt); + System.arraycopy(encodedBytes, removeAt + removeCount, tampered, removeAt, + encodedBytes.length - removeAt - removeCount); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(tampered.length); + assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(tampered))); + } + + /** + * Multi-segment round-trip with CRC enabled. Exercises the per-segment CRC validation followed by the + * message-wide CRC concat fold: if the concat math is wrong the trailing message footer check would fail + * even though every individual segment CRC matched, so this test directly guards the concat optimization. + */ + @Test + public void multipleSegmentsRoundTripWithCrc() { + // 16 segments of 1 KiB each. Small enough to feed in one chunk; large enough that there is meaningful + // CRC accumulation across segments. + int segmentSize = 1024; + int numSegments = 16; + byte[] originalData = getRandomData(segmentSize * numSegments); + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentSize, StructuredMessageFlags.STORAGE_CRC64); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encoded.remaining()); + List decodedPayload = decoder.decodeChunk(encoded); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + /** + * Multi-segment round-trip with CRC where the decoder is fed many small chunks rather than the whole encoded + * blob at once. This ensures the per-segment CRC computation is correct when payload bytes for a single + * segment arrive split across many decodeChunk calls (the typical production wire pattern) and that the + * O(1) concat fold at each segment boundary still produces a matching message CRC at the end. + */ + @Test + public void multipleSegmentsRoundTripWithCrcAcrossManyChunks() { + int segmentSize = 4 * 1024; + int numSegments = 8; + byte[] originalData = getRandomData(segmentSize * numSegments); + byte[] encodedBytes = encodeToBytes(originalData, segmentSize); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + ByteArrayOutputStream collected = new ByteArrayOutputStream(); + int chunkSize = 137; // deliberately non-power-of-two and smaller than the segment so footers split + + for (int offset = 0; offset < encodedBytes.length; offset += chunkSize) { + int len = Math.min(chunkSize, encodedBytes.length - offset); + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); + writeDecodedPayload(collected, decoder.decodeChunk(chunk)); + } + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, collected.toByteArray()); + } + + /** + * Verifies that emitted payload buffers are exact-sized for the copied payload they contain and do not expose + * an oversized backing array. In this single-segment scenario the decoder emits one buffer whose backing + * array length matches the payload length, guarding against accidental reintroduction of intermediate + * buffering (for example, a growing {@code ByteArrayOutputStream} that hands off a larger-than-needed + * buffer). + */ + @Test + public void singleSegmentEmissionIsExactSized() { + int segmentSize = 2 * 1024; + byte[] originalData = getRandomData(segmentSize); + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentSize, StructuredMessageFlags.STORAGE_CRC64); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encoded.remaining()); + List decoded = decoder.decodeChunk(encoded); + + assertEquals(1, decoded.size()); + assertTrue(decoder.isComplete()); + ByteBuffer payload = decoded.get(0); + assertTrue(payload.hasArray()); + assertEquals(segmentSize, payload.remaining()); + assertEquals(segmentSize, payload.array().length); + assertArrayEquals(originalData, collectDecodedBytes(decoded)); + } + + /** + * Multi-megabyte segment size exercises per-segment length from the wire header, + * not a fixed 4 MiB assumption. + */ + @ParameterizedTest + @MethodSource("segmentPayloadSizeAndTotalPayloadSizeSupplier") + public void decodesMultiMegabyteSegments(int segmentPayloadSize, int totalPayloadSize) { + byte[] originalData = getRandomData(totalPayloadSize); + ByteBuffer encodedData = encodeToByteBuffer(originalData, segmentPayloadSize, StructuredMessageFlags.NONE); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + writeDecodedPayload(output, decoder.decodeChunk(encodedData)); + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, output.toByteArray()); + } + + private static Stream segmentPayloadSizeAndTotalPayloadSizeSupplier() { + return Stream.of(Arguments.of(10 * 1024 * 1024, 10 * 1024 * 1024 + 1), + Arguments.of(3 * 1024 * 1024, 3 * 1024 * 1024 + 1), Arguments.of(5 * 1024 * 1024 + 1, 15 * 1024 * 1024)); + } + + // For tests that pass the whole encoded message to decodeChunk. + private static ByteBuffer encodeToByteBuffer(byte[] originalData, int segmentLength, StructuredMessageFlags flags) { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, segmentLength, flags); + Flux flux = encoder.encode(ByteBuffer.wrap(originalData)); + + return ByteBuffer.wrap(Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(flux).block())); + } + + // For tests that need random access/mutation/splitting of encoded bytes. + private static byte[] encodeToBytes(byte[] originalData, int segmentLength) { + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentLength, StructuredMessageFlags.STORAGE_CRC64); + byte[] encodedBytes = new byte[encoded.remaining()]; + encoded.get(encodedBytes); + return encodedBytes; + } + + private static byte[] getRandomData(int size) { + byte[] result = new byte[size]; + ThreadLocalRandom.current().nextBytes(result); + return result; + } + + private static byte[] collectDecodedBytes(List decoded) { + if (decoded.isEmpty()) { + return null; + } + ByteArrayOutputStream output = new ByteArrayOutputStream(); + for (ByteBuffer buffer : decoded) { + if (buffer != null && buffer.hasRemaining()) { + byte[] decodedBytes = new byte[buffer.remaining()]; + buffer.get(decodedBytes); + output.write(decodedBytes, 0, decodedBytes.length); + } + } + return output.toByteArray(); + } + + private static void writeDecodedPayload(ByteArrayOutputStream output, List decoded) { + byte[] bytes = collectDecodedBytes(decoded); + if (bytes != null) { + output.write(bytes, 0, bytes.length); + } + } + +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoderTests.java new file mode 100644 index 000000000000..f04a9909b57f --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoderTests.java @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.FluxUtil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.Disabled; +import reactor.core.publisher.Flux; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Stream; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class StructuredMessageEncoderTests { + + private static byte[] getRandomData(int size) { + byte[] result = new byte[size]; + ThreadLocalRandom.current().nextBytes(result); + return result; + } + + private static void writeSegment(int number, byte[] data, long dataCrc, ByteArrayOutputStream stream) + throws IOException { + writeSegment(number, data, stream); // Call the method without CRC + ByteBuffer segFooter = ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN); + segFooter.putLong(dataCrc); + stream.write(segFooter.array()); // Write segment footer + } + + private static void writeSegment(int number, byte[] data, ByteArrayOutputStream stream) throws IOException { + ByteBuffer segHeader = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); // 2 + 8 + segHeader.putShort((short) number); + segHeader.putLong(data.length); + + stream.write(segHeader.array()); // Write segment header + stream.write(data); // Write segment content + } + + // TODO (isbr): Add tests with static inputs and expected outputs for the encoder. + // Avoid reimplementing the encoder in tests to prevent potential errors in both implementation and tests. + // Consider reusing outputs from existing scripts. This approach can also benefit future decoder tests. + + private static ByteBuffer buildStructuredMessage(ByteBuffer data, int segmentSize, + StructuredMessageFlags structuredMessageFlags) throws IOException { + int segmentCount = Math.max(1, (int) Math.ceil((double) data.capacity() / segmentSize)); + int segmentFooterLength = structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; + + int messageLength = V1_HEADER_LENGTH + ((V1_SEGMENT_HEADER_LENGTH + segmentFooterLength) * segmentCount) + + data.capacity() + (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0); + + long messageCRC = 0; + + // Message Header + ByteBuffer buffer = ByteBuffer.allocate(13).order(ByteOrder.LITTLE_ENDIAN); //1 + 8 + 2 + 2 + buffer.put((byte) 0x01); + buffer.putLong(messageLength); + buffer.putShort((short) structuredMessageFlags.getValue()); + buffer.putShort((short) segmentCount); + + ByteArrayOutputStream message = new ByteArrayOutputStream(); + message.write(buffer.array()); + + if (data.capacity() == 0) { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + writeSegment(1, data.array(), 0, message); + } else { + writeSegment(1, data.array(), message); + } + } else { + // Segments + int[] segmentSizes = new int[segmentCount]; + Arrays.fill(segmentSizes, segmentSize); + + int offset = 0; + for (int i = 1; i <= segmentCount; i++) { + int size = segmentSizes[i - 1]; + byte[] segmentData = customCopyOfRange(data, offset, size); + offset += size; + + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + long segmentCrc = StorageCrc64Calculator.compute(segmentData, 0); + writeSegment(i, segmentData, segmentCrc, message); + messageCRC = StorageCrc64Calculator.compute(segmentData, messageCRC); + } else { + writeSegment(i, segmentData, message); + } + } + } + + // Message footer + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + byte[] crcBytes + = ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN).putLong(messageCRC).array(); + message.write(crcBytes); + } + + return ByteBuffer.wrap(message.toByteArray()); + } + + public static byte[] customCopyOfRange(ByteBuffer original, int from, int size) { + int end = Math.min(from + size, original.capacity()); + return Arrays.copyOfRange(original.array(), from, end); + } + + private static Stream readAllSupplier() { + return Stream.of(Arguments.of(10, 1, StructuredMessageFlags.NONE), + Arguments.of(10, 1, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 1024, StructuredMessageFlags.NONE), + Arguments.of(1024, 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 200, StructuredMessageFlags.NONE), + Arguments.of(1024, 200, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 10, 2, StructuredMessageFlags.NONE), + Arguments.of(1024 * 10, 2, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 50, 512, StructuredMessageFlags.NONE), + Arguments.of(1024 * 50, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024, 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024, 1024, StructuredMessageFlags.STORAGE_CRC64), + // A single multi-MB case with >= 1 MiB segments is sufficient to cover large content and large + // segment sizes. Larger 8 MiB / 12 MiB variants were removed as they add no structural coverage + // (same handful of segments) while multiplying the test's heap footprint. + Arguments.of(1024 * 1024 * 4, 1024 * 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024 * 4, 1024 * 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234, 123, StructuredMessageFlags.NONE), + Arguments.of(1234, 123, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234 * 10, 12, StructuredMessageFlags.NONE), + Arguments.of(1234 * 10, 12, StructuredMessageFlags.STORAGE_CRC64), + // Non-power-of-two content and segment sizes to exercise uneven segment boundaries. + Arguments.of(1234 * 1234, 567, StructuredMessageFlags.NONE), + Arguments.of(1234 * 1234, 567, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("readAllSupplier") + public void readAll(int size, int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data = getRandomData(size); + + ByteBuffer unencodedBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(size, segmentSize, flags); + + byte[] actual + = FluxUtil.collectBytesInByteBufferStream(structuredMessageEncoder.encode(unencodedBuffer)).block(); + byte[] expected = buildStructuredMessage(unencodedBuffer, segmentSize, flags).array(); + + assertArrayEquals(expected, actual); + } + + private static Stream readMultipleSupplier() { + return Stream.of(Arguments.of(30, StructuredMessageFlags.NONE), + Arguments.of(30, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(15, StructuredMessageFlags.NONE), + Arguments.of(15, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(11, StructuredMessageFlags.NONE), + Arguments.of(11, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(8, StructuredMessageFlags.NONE), + Arguments.of(8, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("readMultipleSupplier") + public void readMultiple(int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data1 = getRandomData(10); + byte[] data2 = getRandomData(10); + byte[] data3 = getRandomData(10); + + ByteBuffer wrappedData1 = ByteBuffer.wrap(data1); + ByteBuffer wrappedData2 = ByteBuffer.wrap(data2); + ByteBuffer wrappedData3 = ByteBuffer.wrap(data3); + + ByteBuffer allWrappedData = ByteBuffer.allocate(30); + allWrappedData.put(data1); + allWrappedData.put(data2); + allWrappedData.put(data3); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(30, segmentSize, flags); + + byte[] expected = buildStructuredMessage(allWrappedData, segmentSize, flags).array(); + + Flux allActualFlux = structuredMessageEncoder.encode(wrappedData1) + .concatWith(structuredMessageEncoder.encode(wrappedData2)) + .concatWith(structuredMessageEncoder.encode(wrappedData3)); + + byte[] actual = FluxUtil.collectBytesInByteBufferStream(allActualFlux).block(); + + assertArrayEquals(expected, actual); + } + + @Test + public void emptyBuffer() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(10, 5, StructuredMessageFlags.NONE); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + byte[] result = FluxUtil.collectBytesInByteBufferStream(encoder.encode(emptyBuffer)).block(); + assertNotNull(result); + assertEquals(0, result.length); + } + + @Test + public void contentAlreadyEncoded() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(4, 2, StructuredMessageFlags.NONE); + FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 }))).block(); + // After encoding is complete, further encode calls return empty (no error) to support retries and extra buffers. + byte[] result + = FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2 }))).block(); + assertNotNull(result); + assertEquals(0, result.length); + } + + @Test + public void bufferLengthExceedsContentLength() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(4, 2, StructuredMessageFlags.NONE); + FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2, 3 }))).block(); + assertThrows(IllegalArgumentException.class, + () -> FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2 }))) + .block()); + } + + @Test + public void segmentSizeLessThanOne() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(10, 0, StructuredMessageFlags.NONE)); + } + + @Test + public void contentLengthLessThanOne() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(0, 10, StructuredMessageFlags.NONE)); + } + + @Test + public void testNumSegmentsExceedsMaxValue() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(Integer.MAX_VALUE, 1, StructuredMessageFlags.NONE)); + } + + // =========================================================================================== + // getEncodedMessageLength accuracy + // =========================================================================================== + + private static Stream encodedLengthSupplier() { + return Stream.of(Arguments.of(1, 1, StructuredMessageFlags.NONE), + Arguments.of(1, 1, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(100, 100, StructuredMessageFlags.NONE), + Arguments.of(100, 100, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(4 * 1024 * 1024, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234, 123, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("encodedLengthSupplier") + public void encodedLengthMatchesActualOutput(int size, int segmentSize, StructuredMessageFlags flags) { + byte[] data = getRandomData(size); + StructuredMessageEncoder encoder = new StructuredMessageEncoder(size, segmentSize, flags); + long predictedLength = encoder.getEncodedMessageLength(); + + byte[] actual = FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(data))).block(); + assertNotNull(actual); + assertEquals(predictedLength, actual.length, "getEncodedMessageLength() must match actual encoded output size"); + } + + // =========================================================================================== + // Direct (non-array-backed) ByteBuffer + // =========================================================================================== + + private static Stream readAllDirectSupplier() { + // Direct (non-array-backed) buffers take the scratch-buffer CRC path + // (StructuredMessageEncoder.CRC64_SCRATCH_BUFFER_SIZE == 64 KiB). These cases only need to cover + // that path: empty/uneven segment boundaries plus content larger than the scratch buffer (to force + // multiple scratch iterations). They intentionally stay small so the direct-buffer test doesn't + // re-run the large readAllSupplier cases and double their heap footprint. + return Stream.of(Arguments.of(10, 1, StructuredMessageFlags.NONE), + Arguments.of(10, 1, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 200, StructuredMessageFlags.NONE), + Arguments.of(1024, 200, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 128, 512, StructuredMessageFlags.NONE), + Arguments.of(1024 * 128, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 128, 1024 * 128, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("readAllDirectSupplier") + public void readAllDirectByteBuffer(int size, int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data = getRandomData(size); + ByteBuffer directBuffer = ByteBuffer.allocateDirect(size); + directBuffer.put(data); + directBuffer.flip(); + + ByteBuffer arrayBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder encoder = new StructuredMessageEncoder(size, segmentSize, flags); + byte[] actual = FluxUtil.collectBytesInByteBufferStream(encoder.encode(directBuffer)).block(); + byte[] expected = buildStructuredMessage(arrayBuffer, segmentSize, flags).array(); + + assertArrayEquals(expected, actual, "Direct ByteBuffer encoding must match array-backed encoding"); + } + + // =========================================================================================== + // Null buffer + // =========================================================================================== + + @Test + public void encodeNullBufferThrows() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(10, 5, StructuredMessageFlags.NONE); + assertThrows(NullPointerException.class, + () -> FluxUtil.collectBytesInByteBufferStream(encoder.encode(null)).block()); + } + + @Test + @Disabled("For local testing only") + public void bigEncode() throws IOException { + byte[] data = getRandomData(262144000); + + ByteBuffer unencodedBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(262144000, + V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64); + + byte[] actual + = FluxUtil.collectBytesInByteBufferStream(structuredMessageEncoder.encode(unencodedBuffer)).block(); + byte[] expected = buildStructuredMessage(unencodedBuffer, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, + StructuredMessageFlags.STORAGE_CRC64).array(); + assertArrayEquals(expected, actual); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java new file mode 100644 index 000000000000..61da6d3b2b4c --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class StructuredMessageFlagsTests { + @Test + public void testNoneFlag() { + StructuredMessageFlags flag = StructuredMessageFlags.NONE; + assertEquals(0, flag.getValue()); + } + + @Test + public void testStorageCrc64Flag() { + StructuredMessageFlags flag = StructuredMessageFlags.STORAGE_CRC64; + assertEquals(1, flag.getValue()); + } + + @Test + public void testFromStringValid() { + assertEquals(StructuredMessageFlags.NONE, StructuredMessageFlags.fromString("0")); + assertEquals(StructuredMessageFlags.STORAGE_CRC64, StructuredMessageFlags.fromString("1")); + } + + @Test + public void testFromStringInvalid() { + assertNull(StructuredMessageFlags.fromString("2")); + assertNull(StructuredMessageFlags.fromString(null)); + assertThrows(NumberFormatException.class, () -> { + StructuredMessageFlags.fromString("invalid"); + }); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java new file mode 100644 index 000000000000..e99c169a1763 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.util.BinaryData; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +public class DecodedResponseTests { + + private static final HttpHeaderName CUSTOM_HEADER = HttpHeaderName.fromString("x-ms-custom"); + + private static HttpRequest newRequest() { + try { + return new HttpRequest(HttpMethod.GET, new URL("http://example.com/")); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private static HttpHeaders headers(HttpHeaderName name, String value) { + return new HttpHeaders().set(name, value); + } + + private static MockHttpResponse mockResponse(int status, HttpHeaders headers, byte[] body) { + return new MockHttpResponse(newRequest(), status, headers, body); + } + + private static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + private static Flux fluxOf(byte[] data) { + return Flux.just(ByteBuffer.wrap(data)); + } + + @Test + public void preservesRequestStatusCodeAndHeaders() { + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, "100").set(CUSTOM_HEADER, "value"); + MockHttpResponse original = mockResponse(206, h, bytes("encoded")); + + DecodedResponse wrapper = new DecodedResponse(original, fluxOf(bytes("decoded")), 80L); + + assertSame(original.getRequest(), wrapper.getRequest()); + assertEquals(206, wrapper.getStatusCode()); + // Content-Length is overridden to decoded size; other headers are preserved. + assertEquals("80", wrapper.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + assertEquals("value", wrapper.getHeaders().getValue(CUSTOM_HEADER)); + } + + @Test + public void getHeaderValueByStringReturnsHeaderValue() { + HttpHeaders h = headers(CUSTOM_HEADER, "value"); + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(new byte[0]), 0L); + + assertEquals("value", wrapper.getHeaderValue(CUSTOM_HEADER.getCaseInsensitiveName())); + assertNull(wrapper.getHeaderValue("nonexistent")); + } + + @Test + public void getBodyReturnsDecodedFlux() { + byte[] decoded = bytes("decoded body"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + StepVerifier.create(wrapper.getBody().reduce(new ByteArrayOutputStream(), (sink, buf) -> { + byte[] copy = new byte[buf.remaining()]; + buf.get(copy); + sink.write(copy, 0, copy.length); + return sink; + })).expectNextMatches(sink -> Arrays.equals(decoded, sink.toByteArray())).verifyComplete(); + } + + @Test + public void getBodyAsByteArrayReturnsDecodedBytes() { + byte[] decoded = bytes("decoded body"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + StepVerifier.create(wrapper.getBodyAsByteArray()) + .expectNextMatches(b -> Arrays.equals(decoded, b)) + .verifyComplete(); + } + + @Test + public void getBodyAsStringDefaultsToUtf8WhenNoCharsetSpecified() { + // The no-arg overload routes through CoreUtils.bomAwareToString, which falls back to UTF-8 when neither a + // BOM nor a Content-Type charset parameter is present. This test pins the "no headers, no BOM" path. + String text = "héllo wörld – ✓"; + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), + fluxOf(text.getBytes(StandardCharsets.UTF_8)), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringHonorsCharsetFromContentTypeHeader() { + // Per the base HttpResponse contract, the no-arg getBodyAsString() must honor a charset declared in the + // response's Content-Type header. Without the bom-aware decoding the bytes would be (mis)interpreted as + // UTF-8 and the assertion below would fail. + String text = "ümlaut"; + byte[] iso = text.getBytes(StandardCharsets.ISO_8859_1); + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "text/plain; charset=ISO-8859-1"); + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(iso), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringDetectsUtf8BomAndStripsIt() { + // A leading UTF-8 BOM (EF BB BF) must be detected and stripped from the decoded string, matching the base + // HttpResponse contract that CoreUtils.bomAwareToString implements. + String text = "with bom"; + byte[] bom = new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF }; + byte[] payload = text.getBytes(StandardCharsets.UTF_8); + byte[] withBom = new byte[bom.length + payload.length]; + System.arraycopy(bom, 0, withBom, 0, bom.length); + System.arraycopy(payload, 0, withBom, bom.length, payload.length); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), fluxOf(withBom), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringDecodesUsingProvidedCharset() { + String text = "ümlaut"; + byte[] latin1 = text.getBytes(StandardCharsets.ISO_8859_1); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), fluxOf(latin1), 0L); + + StepVerifier.create(wrapper.getBodyAsString(StandardCharsets.ISO_8859_1)).expectNext(text).verifyComplete(); + } + + @Test + public void inheritedGetBodyAsInputStreamUsesDecodedBytes() throws IOException { + // Base getBodyAsInputStream() routes through getBodyAsByteArray(), so the override is exercised end-to-end. + byte[] decoded = bytes("decoded stream"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + try (InputStream stream = wrapper.getBodyAsInputStream().block()) { + assertNotNull(stream); + assertArrayEquals(decoded, readAll(stream)); + } + } + + @Test + public void inheritedWriteBodyToWritesDecodedBytes() throws IOException { + byte[] decoded = bytes("write me"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + ByteArrayOutputStream sink = new ByteArrayOutputStream(); + try (WritableByteChannel channel = Channels.newChannel(sink)) { + wrapper.writeBodyTo(channel); + } + + assertArrayEquals(decoded, sink.toByteArray()); + } + + @Test + public void inheritedBufferReturnsResponseBackedByDecodedBytes() { + byte[] decoded = bytes("buffered"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + HttpResponse buffered = wrapper.buffer(); + assertNotNull(buffered); + StepVerifier.create(buffered.getBodyAsByteArray()) + .expectNextMatches(b -> Arrays.equals(decoded, b)) + .verifyComplete(); + } + + @Test + public void inheritedGetBodyAsBinaryDataReturnsDecodedBytes() { + // Base HttpResponse#getBodyAsBinaryData() pulls from getBody() (our override), so the resulting BinaryData + // must contain the decoded payload, not the original wire body. A divergent Content-Length header is set + // to make the wire vs decoded distinction explicit and guard against regressions in header forwarding. + byte[] decoded = bytes("decoded payload"); + long decodedSize = decoded.length; + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(decodedSize + 32)); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, h, bytes("encoded wire body")), fluxOf(decoded), decodedSize); + + BinaryData data = wrapper.getBodyAsBinaryData(); + assertNotNull(data); + assertArrayEquals(decoded, data.toBytes()); + } + + @Test + public void contentLengthIsOverriddenToDecodedSize() { + long wireSize = 500L; + long decodedSize = 300L; + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(wireSize)) + .set(CUSTOM_HEADER, "preserve-me"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(new byte[0]), decodedSize); + + assertEquals(String.valueOf(decodedSize), wrapper.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + assertEquals("preserve-me", wrapper.getHeaders().getValue(CUSTOM_HEADER)); + // Deprecated getHeaderValue must reflect the same override. + assertEquals(String.valueOf(decodedSize), + wrapper.getHeaderValue(HttpHeaderName.CONTENT_LENGTH.getCaseInsensitiveName())); + } + + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int n; + while ((n = stream.read(buf)) != -1) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java new file mode 100644 index 000000000000..92d93d968d80 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpPipeline; +import com.azure.core.http.HttpPipelineBuilder; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.util.Context; +import com.azure.core.util.FluxUtil; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link StorageContentValidationDecodingPolicy} together with {@link StructuredMessageEncoder} / + * wire-format payloads so the reactive decode path matches what the blob download pipeline uses. + */ +public class StorageContentValidationDecoderPolicyTests { + + /** + * End-to-end through the policy: encoded body uses multi-megabyte segment payload lengths (not the default + * 4 MiB framing only); decoded flux must match the original bytes. + */ + @ParameterizedTest + @MethodSource("segmentPayloadSizeAndTotalPayloadSizeSupplier") + public void decodesDynamicallySizedSegmentStructuredMessageThroughPipeline(int segmentPayloadSize, + int totalPayloadSize) throws IOException { + byte[] originalData = new byte[totalPayloadSize]; + ThreadLocalRandom.current().nextBytes(originalData); + + byte[] encodedBytes + = encodeStructuredMessageWireBytes(originalData, segmentPayloadSize, StructuredMessageFlags.STORAGE_CRC64); + + AtomicReference requestAfterPolicies = new AtomicReference<>(); + HttpClient httpClient = request -> { + requestAfterPolicies.set(request); + HttpHeaders headers = structuredDownloadResponseHeaders(encodedBytes.length, totalPayloadSize); + return Mono.just(new MockHttpResponse(request, 200, headers, encodedBytes)); + }; + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecodingPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertTrue(response instanceof DecodedResponse); + byte[] decoded = Objects.requireNonNull(response.getBodyAsByteArray().block()); + assertArrayEquals(originalData, decoded); + } + + HttpRequest sent = requestAfterPolicies.get(); + assertNotNull(sent); + assertEquals(StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE, + sent.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME)); + } + + @Test + public void contentLengthIsOverriddenToDecodedSizeWhenDecodingApplied() throws IOException { + byte[] payload = new byte[64]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + long decodedLen = payload.length; + + HttpClient httpClient = request -> Mono.just( + new MockHttpResponse(request, 200, structuredDownloadResponseHeaders(encodedLen, decodedLen), encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecodingPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertEquals(String.valueOf(decodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + } + } + + @Test + public void contentLengthMatchesActualDecodedBodySize() throws IOException { + byte[] payload = new byte[128]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + long decodedLen = payload.length; + + HttpClient httpClient = request -> Mono.just( + new MockHttpResponse(request, 200, structuredDownloadResponseHeaders(encodedLen, decodedLen), encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecodingPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertEquals(String.valueOf(decodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + + byte[] body = Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(response.getBody()).block()); + assertEquals(decodedLen, body.length); + assertArrayEquals(payload, body); + } + } + + @Test + public void contentLengthIsUnchangedWhenDecodingNotApplied() throws IOException { + byte[] payload = new byte[64]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + + HttpHeaders responseHeaders = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(encodedLen)); + HttpClient httpClient = request -> Mono.just(new MockHttpResponse(request, 200, responseHeaders, encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies(new StorageContentValidationDecodingPolicy()) + .httpClient(httpClient) + .build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + HttpResponse response = pipeline.send(request, Context.NONE).block(); + + assertNotNull(response); + assertEquals(String.valueOf(encodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + } + + private static Stream segmentPayloadSizeAndTotalPayloadSizeSupplier() { + return Stream.of(Arguments.of(10 * 1024 * 1024, 10 * 1024 * 1024 + 1), // larger than 4 MiB + Arguments.of(3 * 1024 * 1024, 3 * 1024 * 1024 + 1), // smaller than 4 MiB, but not KB + Arguments.of(5 * 1024 * 1024 + 1, 15 * 1024 * 1024)); + } + + private static HttpHeaders structuredDownloadResponseHeaders(long contentLength, long structuredContentLength) { + HttpHeaders headers = new HttpHeaders(); + headers.set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(contentLength)); + headers.set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, + StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); + headers.set(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME, + String.valueOf(structuredContentLength)); + return headers; + } + + private static byte[] encodeStructuredMessageWireBytes(byte[] originalData, int segmentLength, + StructuredMessageFlags flags) throws IOException { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, segmentLength, flags); + Flux flux = encoder.encode(ByteBuffer.wrap(originalData)); + return Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(flux).block()); + } +} diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java index 3e3ec7595cee..adafed94a3e5 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java @@ -162,7 +162,12 @@ public enum DataLakeServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"); private final String version; @@ -184,6 +189,6 @@ public String getVersion() { * @return the latest {@link DataLakeServiceVersion} */ public static DataLakeServiceVersion getLatest() { - return V2026_06_06; + return V2026_10_06; } } diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java index fe07d636f95c..badbd66eed88 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java @@ -107,6 +107,9 @@ public static BlobServiceVersion toBlobServiceVersion(DataLakeServiceVersion ver case V2026_06_06: return BlobServiceVersion.V2026_06_06; + case V2026_10_06: + return BlobServiceVersion.V2026_10_06; + default: return null; } diff --git a/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/LargeFileTests.java b/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/LargeFileTests.java index 5e244cedd267..4d45d76dabcf 100644 --- a/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/LargeFileTests.java +++ b/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/LargeFileTests.java @@ -114,12 +114,20 @@ public void uploadLargeFile() { long tail = Constants.MB; File file = getLargeRandomFile(LARGE_BLOCK_SIZE + tail); - fc.uploadFromFile(file.toPath().toString(), new ParallelTransferOptions().setBlockSizeLong(LARGE_BLOCK_SIZE), - null, null, null, null); - - assertEquals(2, countingPolicy.count.get()); - assertTrue(countingPolicy.appendPayloadSizes.contains(LARGE_BLOCK_SIZE)); - assertTrue(countingPolicy.appendPayloadSizes.contains(tail)); + try { + fc.uploadFromFile(file.toPath().toString(), + new ParallelTransferOptions().setBlockSizeLong(LARGE_BLOCK_SIZE), null, null, null, null); + + assertEquals(2, countingPolicy.count.get()); + assertTrue(countingPolicy.appendPayloadSizes.contains(LARGE_BLOCK_SIZE)); + assertTrue(countingPolicy.appendPayloadSizes.contains(tail)); + } finally { + // Delete the ~2.5 GiB source eagerly instead of waiting for deleteOnExit() so it doesn't occupy agent + // disk for the remainder of the module's test JVM. + if (!file.delete()) { + file.deleteOnExit(); + } + } } // This test does not send large payload over the wire diff --git a/sdk/storage/azure-storage-file-share/assets.json b/sdk/storage/azure-storage-file-share/assets.json index 2096399e6a76..19ffbc8404cc 100644 --- a/sdk/storage/azure-storage-file-share/assets.json +++ b/sdk/storage/azure-storage-file-share/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "java", "TagPrefix": "java/storage/azure-storage-file-share", - "Tag": "java/storage/azure-storage-file-share_1e9eb49640" + "Tag": "java/storage/azure-storage-file-share_47ecfcd261" } diff --git a/sdk/storage/azure-storage-file-share/pom.xml b/sdk/storage/azure-storage-file-share/pom.xml index 9da516ae2d32..713045cfd849 100644 --- a/sdk/storage/azure-storage-file-share/pom.xml +++ b/sdk/storage/azure-storage-file-share/pom.xml @@ -37,6 +37,7 @@ --add-exports com.azure.core/com.azure.core.implementation.http=ALL-UNNAMED + --add-exports com.azure.storage.common/com.azure.storage.common.implementation.contentvalidation=ALL-UNNAMED --add-exports com.azure.storage.common/com.azure.storage.common.implementation.credentials=ALL-UNNAMED --add-opens com.azure.storage.common/com.azure.storage.common.implementation=ALL-UNNAMED --add-opens com.azure.storage.file.share/com.azure.storage.file.share=ALL-UNNAMED diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileAsyncClient.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileAsyncClient.java index 50bba1696fd1..c6e2bde18e29 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileAsyncClient.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileAsyncClient.java @@ -64,6 +64,7 @@ import com.azure.storage.file.share.models.ShareFilePermission; import com.azure.storage.file.share.models.ShareFileProperties; import com.azure.storage.file.share.models.ShareFileRange; +import com.azure.storage.file.share.models.ShareFileRangeItem; import com.azure.storage.file.share.models.ShareFileRangeList; import com.azure.storage.file.share.models.ShareFileSymbolicLinkInfo; import com.azure.storage.file.share.models.ShareFileUploadInfo; @@ -78,6 +79,7 @@ import com.azure.storage.file.share.options.ShareFileCreateOptions; import com.azure.storage.file.share.options.ShareFileCreateSymbolicLinkOptions; import com.azure.storage.file.share.options.ShareFileDownloadOptions; +import com.azure.storage.file.share.options.ShareFileListRangesOptions; import com.azure.storage.file.share.options.ShareFileListRangesDiffOptions; import com.azure.storage.file.share.options.ShareFileRenameOptions; import com.azure.storage.file.share.options.ShareFileSetPropertiesOptions; @@ -116,6 +118,7 @@ import static com.azure.core.util.FluxUtil.monoError; import static com.azure.core.util.FluxUtil.pagedFluxError; import static com.azure.core.util.FluxUtil.withContext; +import static com.azure.storage.file.share.implementation.util.ModelHelper.toShareFileRangeItems; /** * This class provides a client that contains all the operations for interacting with file in Azure Storage File @@ -2703,7 +2706,9 @@ private static List sliceFile(String path) { * Azure Docs.

* * @return {@link ShareFileRange ranges} in the files. + * @deprecated Use {@link #listAllRanges()} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.COLLECTION) public PagedFlux listRanges() { return listRanges(null); @@ -2729,7 +2734,9 @@ public PagedFlux listRanges() { * * @param range Optional byte range which returns file data only from the specified range. * @return {@link ShareFileRange ranges} in the files that satisfy the requirements + * @deprecated Use {@link #listAllRanges(ShareFileListRangesOptions)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.COLLECTION) public PagedFlux listRanges(ShareFileRange range) { return this.listRanges(range, null); @@ -2757,11 +2764,82 @@ public PagedFlux listRanges(ShareFileRange range) { * @param range Optional byte range which returns file data only from the specified range. * @param requestConditions {@link ShareRequestConditions} * @return {@link ShareFileRange ranges} in the files that satisfy the requirements + * @deprecated Use {@link #listAllRanges(ShareFileListRangesOptions)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.COLLECTION) public PagedFlux listRanges(ShareFileRange range, ShareRequestConditions requestConditions) { try { - return listRangesWithOptionalTimeout(range, requestConditions, null, Context.NONE); + return listRangesWithOptionalTimeout(range, requestConditions, null); + } catch (RuntimeException ex) { + return pagedFluxError(LOGGER, ex); + } + } + + /** + * Lists all ranges for the file as a paged flux. + * + *

For more information, see the + * Azure Docs.

+ * + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedFlux listAllRanges() { + return listAllRanges(new ShareFileListRangesOptions()); + } + + /** + * Lists all ranges for the file as a paged flux. + * + *

For more information, see the + * Azure Docs.

+ * + * @param options Optional parameters. + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedFlux listAllRanges(ShareFileListRangesOptions options) { + try { + ShareFileListRangesOptions finalOptions = options == null ? new ShareFileListRangesOptions() : options; + return listAllRangesInternal(finalOptions.getRange(), finalOptions.getRequestConditions(), null, null, + false, null); + } catch (RuntimeException ex) { + return pagedFluxError(LOGGER, ex); + } + } + + /** + * Lists ranges that have changed between the file and the specified snapshot as a paged flux. + * + *

For more information, see the + * Azure Docs.

+ * + * @param previousSnapshot Specifies that the response will contain only ranges that were changed between target + * file and previous snapshot. Changed ranges include both updated and cleared ranges. The target file may be a + * snapshot, as long as the snapshot specified by previousSnapshot is the older of the two. + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedFlux listAllRangesDiff(String previousSnapshot) { + return listAllRangesDiff(new ShareFileListRangesDiffOptions(previousSnapshot)); + } + + /** + * Lists ranges that have changed between the file and the specified snapshot as a paged flux. + * + *

For more information, see the + * Azure Docs.

+ * + * @param options {@link ShareFileListRangesDiffOptions} + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedFlux listAllRangesDiff(ShareFileListRangesDiffOptions options) { + try { + StorageImplUtils.assertNotNull("options", options); + return listAllRangesInternal(options.getRange(), options.getRequestConditions(), + options.getPreviousSnapshot(), options.isRenameIncluded(), true, null); } catch (RuntimeException ex) { return pagedFluxError(LOGGER, ex); } @@ -2791,7 +2869,9 @@ public PagedFlux listRanges(ShareFileRange range, ShareRequestCo * file and previous snapshot. Changed ranges include both updated and cleared ranges. The target file may be a * snapshot, as long as the snapshot specified by previousSnapshot is the older of the two. * @return {@link ShareFileRange ranges} in the files that satisfy the requirements + * @deprecated Use {@link #listAllRangesDiff(String)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public Mono listRangesDiff(String previousSnapshot) { try { @@ -2826,23 +2906,26 @@ public Mono listRangesDiff(String previousSnapshot) { * * @param options {@link ShareFileListRangesDiffOptions}. * @return {@link ShareFileRange ranges} in the files that satisfy the requirements + * @deprecated Use {@link #listAllRangesDiff(ShareFileListRangesDiffOptions)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public Mono> listRangesDiffWithResponse(ShareFileListRangesDiffOptions options) { try { StorageImplUtils.assertNotNull("options", options); return listRangesWithResponse(options.getRange(), options.getRequestConditions(), - options.getPreviousSnapshot(), options.isRenameIncluded(), Context.NONE); + options.getPreviousSnapshot(), options.isRenameIncluded(), null, null, Context.NONE); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } } PagedFlux listRangesWithOptionalTimeout(ShareFileRange range, - ShareRequestConditions requestConditions, Duration timeout, Context context) { + ShareRequestConditions requestConditions, Duration timeout) { Function>> retriever = marker -> StorageImplUtils - .applyOptionalTimeout(this.listRangesWithResponse(range, requestConditions, null, null, context), timeout) + .applyOptionalTimeout( + this.listRangesWithResponse(range, requestConditions, null, null, null, null, Context.NONE), timeout) .map(response -> new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), response.getValue() @@ -2856,8 +2939,25 @@ PagedFlux listRangesWithOptionalTimeout(ShareFileRange range, return new PagedFlux<>(() -> retriever.apply(null), retriever); } + PagedFlux listAllRangesInternal(ShareFileRange range, ShareRequestConditions requestConditions, + String previousSnapshot, Boolean supportRename, boolean includeClearRanges, Duration timeout) { + BiFunction>> nextPageRetriever + = (marker, pageSize) -> StorageImplUtils + .applyOptionalTimeout(this.listRangesWithResponse(range, requestConditions, previousSnapshot, + supportRename, marker, pageSize, Context.NONE), timeout) + .map(response -> new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), + response.getHeaders(), toShareFileRangeItems(response.getValue(), includeClearRanges), + response.getValue().getNextMarker(), response.getHeaders())); + + Function>> firstPageRetriever + = pageSize -> nextPageRetriever.apply(null, pageSize); + + return new PagedFlux<>(firstPageRetriever, nextPageRetriever); + } + Mono> listRangesWithResponse(ShareFileRange range, - ShareRequestConditions requestConditions, String previousSnapshot, Boolean supportRename, Context context) { + ShareRequestConditions requestConditions, String previousSnapshot, Boolean supportRename, String marker, + Integer maxResultsPerPage, Context context) { ShareRequestConditions finalRequestConditions = requestConditions == null ? new ShareRequestConditions() : requestConditions; @@ -2866,7 +2966,7 @@ Mono> listRangesWithResponse(ShareFileRange range, return this.azureFileStorageClient.getFiles() .getRangeListWithResponseAsync(shareName, filePath, snapshot, previousSnapshot, null, rangeString, - finalRequestConditions.getLeaseId(), supportRename, context) + finalRequestConditions.getLeaseId(), supportRename, marker, maxResultsPerPage, context) .map(response -> new SimpleResponse<>(response, response.getValue())); } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileClient.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileClient.java index 0d0339d1c579..81febcdf9130 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileClient.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareFileClient.java @@ -70,6 +70,7 @@ import com.azure.storage.file.share.models.ShareFilePermission; import com.azure.storage.file.share.models.ShareFileProperties; import com.azure.storage.file.share.models.ShareFileRange; +import com.azure.storage.file.share.models.ShareFileRangeItem; import com.azure.storage.file.share.models.ShareFileRangeList; import com.azure.storage.file.share.models.ShareFileSymbolicLinkInfo; import com.azure.storage.file.share.models.ShareFileUploadInfo; @@ -84,6 +85,7 @@ import com.azure.storage.file.share.options.ShareFileCreateOptions; import com.azure.storage.file.share.options.ShareFileCreateSymbolicLinkOptions; import com.azure.storage.file.share.options.ShareFileDownloadOptions; +import com.azure.storage.file.share.options.ShareFileListRangesOptions; import com.azure.storage.file.share.options.ShareFileListRangesDiffOptions; import com.azure.storage.file.share.options.ShareFileRenameOptions; import com.azure.storage.file.share.options.ShareFileSeekableByteChannelReadOptions; @@ -111,6 +113,7 @@ import java.util.stream.Collectors; import static com.azure.storage.common.implementation.StorageImplUtils.sendRequest; +import static com.azure.storage.file.share.implementation.util.ModelHelper.toShareFileRangeItems; /** * This class provides a client that contains all the operations for interacting files under Azure Storage File Service. @@ -2469,7 +2472,9 @@ public void uploadFromFile(String uploadFilePath, ShareRequestConditions request * Azure Docs.

* * @return {@link ShareFileRange ranges} in the files. + * @deprecated Use {@link #listAllRanges()} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.COLLECTION) public PagedIterable listRanges() { return listRanges((ShareFileRange) null, null, null); @@ -2500,7 +2505,9 @@ public PagedIterable listRanges() { * @param context Additional context that is passed through the Http pipeline during the service call. * @return {@link ShareFileRange ranges} in the files that satisfy the requirements * @throws RuntimeException if the operation doesn't complete before the timeout concludes. + * @deprecated Use {@link #listAllRanges(ShareFileListRangesOptions, Duration, Context)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.COLLECTION) public PagedIterable listRanges(ShareFileRange range, Duration timeout, Context context) { return this.listRanges(range, null, timeout, context); @@ -2533,7 +2540,9 @@ public PagedIterable listRanges(ShareFileRange range, Duration t * @param context Additional context that is passed through the Http pipeline during the service call. * @return {@link ShareFileRange ranges} in the files that satisfy the requirements * @throws RuntimeException if the operation doesn't complete before the timeout concludes. + * @deprecated Use {@link #listAllRanges(ShareFileListRangesOptions, Duration, Context)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.COLLECTION) public PagedIterable listRanges(ShareFileRange range, ShareRequestConditions requestConditions, Duration timeout, Context context) { @@ -2541,12 +2550,11 @@ public PagedIterable listRanges(ShareFileRange range, ShareReque ShareRequestConditions finalRequestConditions = requestConditions == null ? new ShareRequestConditions() : requestConditions; String rangeString = range == null ? null : range.toString(); - try { Callable> operation = () -> this.azureFileStorageClient.getFiles() .getRangeListWithResponse(shareName, filePath, snapshot, null, null, rangeString, - finalRequestConditions.getLeaseId(), null, finalContext); + finalRequestConditions.getLeaseId(), null, null, null, finalContext); ResponseBase response = sendRequest(operation, timeout, ShareStorageException.class); @@ -2569,6 +2577,96 @@ public PagedIterable listRanges(ShareFileRange range, ShareReque } } + /** + * Lists all ranges for the file as a paged iterable. + * + *

For more information, see the + * Azure Docs.

+ * + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedIterable listAllRanges() { + return listAllRanges(new ShareFileListRangesOptions(), null, Context.NONE); + } + + /** + * Lists all ranges for the file as a paged iterable. + * + *

For more information, see the + * Azure Docs.

+ * + * @param options Optional parameters. + * @param timeout An optional timeout applied to the operation. If a response is not returned before the timeout + * concludes a {@link RuntimeException} will be thrown. + * @param context Additional context that is passed through the Http pipeline during the service call. + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedIterable listAllRanges(ShareFileListRangesOptions options, Duration timeout, + Context context) { + StorageImplUtils.assertNotNull("options", options); + return listAllRangesInternal(options.getRange(), options.getRequestConditions(), null, null, false, timeout, + context); + } + + /** + * Lists ranges that have changed between the file and the specified snapshot as a paged iterable. + * + *

For more information, see the + * Azure Docs.

+ * + * @param previousSnapshot Specifies that the response will contain only ranges that were changed between target + * file and previous snapshot. Changed ranges include both updated and cleared ranges. The target file may be a + * snapshot, as long as the snapshot specified by previousSnapshot is the older of the two. + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedIterable listAllRangesDiff(String previousSnapshot) { + return listAllRangesDiff(new ShareFileListRangesDiffOptions(previousSnapshot), null, Context.NONE); + } + + /** + * Lists ranges that have changed between the file and the specified snapshot as a paged iterable. + * + *

For more information, see the + * Azure Docs.

+ * + * @param options {@link ShareFileListRangesDiffOptions} + * @param timeout An optional timeout applied to the operation. If a response is not returned before the timeout + * concludes a {@link RuntimeException} will be thrown. + * @param context Additional context that is passed through the Http pipeline during the service call. + * @return {@link ShareFileRangeItem ranges} in the file. + */ + @ServiceMethod(returns = ReturnType.COLLECTION) + public PagedIterable listAllRangesDiff(ShareFileListRangesDiffOptions options, Duration timeout, + Context context) { + StorageImplUtils.assertNotNull("options", options); + return listAllRangesInternal(options.getRange(), options.getRequestConditions(), options.getPreviousSnapshot(), + options.isRenameIncluded(), true, timeout, context); + } + + private PagedIterable listAllRangesInternal(ShareFileRange range, + ShareRequestConditions requestConditions, String previousSnapshot, Boolean supportRename, + boolean includeClearRanges, Duration timeout, Context context) { + Context finalContext = context == null ? Context.NONE : context; + ShareRequestConditions finalRequestConditions + = requestConditions == null ? new ShareRequestConditions() : requestConditions; + + BiFunction> nextPageRetriever = (marker, pageSize) -> { + ResponseBase response = listRangesWithResponse(range, + finalRequestConditions, previousSnapshot, supportRename, marker, pageSize, timeout, finalContext); + + return new PagedResponseBase<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), + toShareFileRangeItems(response.getValue(), includeClearRanges), response.getValue().getNextMarker(), + response.getDeserializedHeaders()); + }; + Function> firstPageRetriever + = pageSize -> nextPageRetriever.apply(null, pageSize); + + return new PagedIterable<>(firstPageRetriever, nextPageRetriever); + } + /** * List of valid ranges for a file. * @@ -2594,7 +2692,9 @@ public PagedIterable listRanges(ShareFileRange range, ShareReque * snapshot, as long as the snapshot specified by previousSnapshot is the older of the two. * @return {@link ShareFileRange ranges} in the files that satisfy the requirements * @throws RuntimeException if the operation doesn't complete before the timeout concludes. + * @deprecated Use {@link #listAllRangesDiff(String)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public ShareFileRangeList listRangesDiff(String previousSnapshot) { return this.listRangesDiffWithResponse(new ShareFileListRangesDiffOptions(previousSnapshot), null, Context.NONE) @@ -2629,7 +2729,9 @@ public ShareFileRangeList listRangesDiff(String previousSnapshot) { * @param context Additional context that is passed through the Http pipeline during the service call. * @return {@link ShareFileRange ranges} in the files that satisfy the requirements * @throws RuntimeException if the operation doesn't complete before the timeout concludes. + * @deprecated Use {@link #listAllRangesDiff(ShareFileListRangesDiffOptions, Duration, Context)} instead. */ + @Deprecated @ServiceMethod(returns = ReturnType.SINGLE) public Response listRangesDiffWithResponse(ShareFileListRangesDiffOptions options, Duration timeout, Context context) { @@ -2639,7 +2741,7 @@ public Response listRangesDiffWithResponse(ShareFileListRang String rangeString = options.getRange() == null ? null : options.getRange().toString(); Callable> operation = () -> this.azureFileStorageClient.getFiles() .getRangeListNoCustomHeadersWithResponse(shareName, filePath, snapshot, options.getPreviousSnapshot(), null, - rangeString, requestConditions.getLeaseId(), options.isRenameIncluded(), finalContext); + rangeString, requestConditions.getLeaseId(), options.isRenameIncluded(), null, null, finalContext); return sendRequest(operation, timeout, ShareStorageException.class); } @@ -3306,4 +3408,19 @@ public String generateUserDelegationSas(ShareServiceSasSignatureValues shareServ return new ShareSasImplUtil(shareServiceSasSignatureValues, getShareName(), getFilePath()) .generateUserDelegationSas(userDelegationKey, accountName, stringToSignHandler, context); } + + ResponseBase listRangesWithResponse(ShareFileRange range, + ShareRequestConditions requestConditions, String previousSnapshot, Boolean supportRename, String marker, + Integer maxResultsPerPage, Duration timeout, Context context) { + ShareRequestConditions finalRequestConditions + = requestConditions == null ? new ShareRequestConditions() : requestConditions; + String rangeString = range == null ? null : range.toString(); + + Callable> operation + = () -> this.azureFileStorageClient.getFiles() + .getRangeListWithResponse(shareName, filePath, snapshot, previousSnapshot, null, rangeString, + finalRequestConditions.getLeaseId(), supportRename, marker, maxResultsPerPage, context); + + return sendRequest(operation, timeout, ShareStorageException.class); + } } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java index 20d1e6de7cea..edf9c237d29c 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java @@ -162,7 +162,12 @@ public enum ShareServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"); private final String version; @@ -184,6 +189,6 @@ public String getVersion() { * @return the latest {@link ShareServiceVersion} */ public static ShareServiceVersion getLatest() { - return V2026_06_06; + return V2026_10_06; } } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/FilesImpl.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/FilesImpl.java index e47e219591fd..e2391f3e7923 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/FilesImpl.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/FilesImpl.java @@ -963,8 +963,8 @@ Mono> getRangeList(@H @HeaderParam("x-ms-lease-id") String leaseId, @HeaderParam("x-ms-allow-trailing-dot") Boolean allowTrailingDot, @HeaderParam("x-ms-file-request-intent") ShareTokenIntent fileRequestIntent, - @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @HeaderParam("Accept") String accept, - Context context); + @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @QueryParam("marker") String marker, + @QueryParam("maxresults") Integer maxresults, @HeaderParam("Accept") String accept, Context context); @Get("/{shareName}/{fileName}") @ExpectedResponses({ 200 }) @@ -977,8 +977,8 @@ Mono> getRangeListNoCustomHeaders(@HostParam("url") @HeaderParam("x-ms-lease-id") String leaseId, @HeaderParam("x-ms-allow-trailing-dot") Boolean allowTrailingDot, @HeaderParam("x-ms-file-request-intent") ShareTokenIntent fileRequestIntent, - @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @HeaderParam("Accept") String accept, - Context context); + @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @QueryParam("marker") String marker, + @QueryParam("maxresults") Integer maxresults, @HeaderParam("Accept") String accept, Context context); @Get("/{shareName}/{fileName}") @ExpectedResponses({ 200 }) @@ -991,8 +991,8 @@ ResponseBase getRangeListSync(@Hos @HeaderParam("x-ms-lease-id") String leaseId, @HeaderParam("x-ms-allow-trailing-dot") Boolean allowTrailingDot, @HeaderParam("x-ms-file-request-intent") ShareTokenIntent fileRequestIntent, - @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @HeaderParam("Accept") String accept, - Context context); + @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @QueryParam("marker") String marker, + @QueryParam("maxresults") Integer maxresults, @HeaderParam("Accept") String accept, Context context); @Get("/{shareName}/{fileName}") @ExpectedResponses({ 200 }) @@ -1005,8 +1005,8 @@ Response getRangeListNoCustomHeadersSync(@HostParam("url") S @HeaderParam("x-ms-lease-id") String leaseId, @HeaderParam("x-ms-allow-trailing-dot") Boolean allowTrailingDot, @HeaderParam("x-ms-file-request-intent") ShareTokenIntent fileRequestIntent, - @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @HeaderParam("Accept") String accept, - Context context); + @HeaderParam("x-ms-file-support-rename") Boolean supportRename, @QueryParam("marker") String marker, + @QueryParam("maxresults") Integer maxresults, @HeaderParam("Accept") String accept, Context context); @Put("/{shareName}/{fileName}") @ExpectedResponses({ 202 }) @@ -6557,6 +6557,12 @@ public Response uploadRangeFromURLNoCustomHeadersWithResponse(String share * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. @@ -6565,10 +6571,10 @@ public Response uploadRangeFromURLNoCustomHeadersWithResponse(String share @ServiceMethod(returns = ReturnType.SINGLE) public Mono> getRangeListWithResponseAsync( String shareName, String fileName, String sharesnapshot, String prevsharesnapshot, Integer timeout, - String range, String leaseId, Boolean supportRename) { + String range, String leaseId, Boolean supportRename, String marker, Integer maxresults) { return FluxUtil .withContext(context -> getRangeListWithResponseAsync(shareName, fileName, sharesnapshot, prevsharesnapshot, - timeout, range, leaseId, supportRename, context)) + timeout, range, leaseId, supportRename, marker, maxresults, context)) .onErrorMap(ShareStorageExceptionInternal.class, ModelHelper::mapToShareStorageException); } @@ -6591,6 +6597,12 @@ public Mono> getRange * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. @@ -6600,13 +6612,13 @@ public Mono> getRange @ServiceMethod(returns = ReturnType.SINGLE) public Mono> getRangeListWithResponseAsync( String shareName, String fileName, String sharesnapshot, String prevsharesnapshot, Integer timeout, - String range, String leaseId, Boolean supportRename, Context context) { + String range, String leaseId, Boolean supportRename, String marker, Integer maxresults, Context context) { final String comp = "rangelist"; final String accept = "application/xml"; return service .getRangeList(this.client.getUrl(), shareName, fileName, comp, sharesnapshot, prevsharesnapshot, timeout, this.client.getVersion(), range, leaseId, this.client.isAllowTrailingDot(), - this.client.getFileRequestIntent(), supportRename, accept, context) + this.client.getFileRequestIntent(), supportRename, marker, maxresults, accept, context) .onErrorMap(ShareStorageExceptionInternal.class, ModelHelper::mapToShareStorageException); } @@ -6629,6 +6641,12 @@ public Mono> getRange * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. @@ -6636,9 +6654,10 @@ public Mono> getRange */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono getRangeListAsync(String shareName, String fileName, String sharesnapshot, - String prevsharesnapshot, Integer timeout, String range, String leaseId, Boolean supportRename) { + String prevsharesnapshot, Integer timeout, String range, String leaseId, Boolean supportRename, String marker, + Integer maxresults) { return getRangeListWithResponseAsync(shareName, fileName, sharesnapshot, prevsharesnapshot, timeout, range, - leaseId, supportRename) + leaseId, supportRename, marker, maxresults) .onErrorMap(ShareStorageExceptionInternal.class, ModelHelper::mapToShareStorageException) .flatMap(res -> Mono.justOrEmpty(res.getValue())); } @@ -6662,6 +6681,12 @@ public Mono getRangeListAsync(String shareName, String fileN * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. @@ -6670,10 +6695,10 @@ public Mono getRangeListAsync(String shareName, String fileN */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono getRangeListAsync(String shareName, String fileName, String sharesnapshot, - String prevsharesnapshot, Integer timeout, String range, String leaseId, Boolean supportRename, - Context context) { + String prevsharesnapshot, Integer timeout, String range, String leaseId, Boolean supportRename, String marker, + Integer maxresults, Context context) { return getRangeListWithResponseAsync(shareName, fileName, sharesnapshot, prevsharesnapshot, timeout, range, - leaseId, supportRename, context) + leaseId, supportRename, marker, maxresults, context) .onErrorMap(ShareStorageExceptionInternal.class, ModelHelper::mapToShareStorageException) .flatMap(res -> Mono.justOrEmpty(res.getValue())); } @@ -6697,6 +6722,12 @@ public Mono getRangeListAsync(String shareName, String fileN * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. @@ -6705,10 +6736,10 @@ public Mono getRangeListAsync(String shareName, String fileN @ServiceMethod(returns = ReturnType.SINGLE) public Mono> getRangeListNoCustomHeadersWithResponseAsync(String shareName, String fileName, String sharesnapshot, String prevsharesnapshot, Integer timeout, String range, String leaseId, - Boolean supportRename) { + Boolean supportRename, String marker, Integer maxresults) { return FluxUtil .withContext(context -> getRangeListNoCustomHeadersWithResponseAsync(shareName, fileName, sharesnapshot, - prevsharesnapshot, timeout, range, leaseId, supportRename, context)) + prevsharesnapshot, timeout, range, leaseId, supportRename, marker, maxresults, context)) .onErrorMap(ShareStorageExceptionInternal.class, ModelHelper::mapToShareStorageException); } @@ -6731,6 +6762,12 @@ public Mono> getRangeListNoCustomHeadersWithRespons * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. @@ -6740,13 +6777,13 @@ public Mono> getRangeListNoCustomHeadersWithRespons @ServiceMethod(returns = ReturnType.SINGLE) public Mono> getRangeListNoCustomHeadersWithResponseAsync(String shareName, String fileName, String sharesnapshot, String prevsharesnapshot, Integer timeout, String range, String leaseId, - Boolean supportRename, Context context) { + Boolean supportRename, String marker, Integer maxresults, Context context) { final String comp = "rangelist"; final String accept = "application/xml"; return service .getRangeListNoCustomHeaders(this.client.getUrl(), shareName, fileName, comp, sharesnapshot, prevsharesnapshot, timeout, this.client.getVersion(), range, leaseId, this.client.isAllowTrailingDot(), - this.client.getFileRequestIntent(), supportRename, accept, context) + this.client.getFileRequestIntent(), supportRename, marker, maxresults, accept, context) .onErrorMap(ShareStorageExceptionInternal.class, ModelHelper::mapToShareStorageException); } @@ -6769,6 +6806,12 @@ public Mono> getRangeListNoCustomHeadersWithRespons * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. @@ -6778,13 +6821,13 @@ public Mono> getRangeListNoCustomHeadersWithRespons @ServiceMethod(returns = ReturnType.SINGLE) public ResponseBase getRangeListWithResponse(String shareName, String fileName, String sharesnapshot, String prevsharesnapshot, Integer timeout, String range, String leaseId, - Boolean supportRename, Context context) { + Boolean supportRename, String marker, Integer maxresults, Context context) { try { final String comp = "rangelist"; final String accept = "application/xml"; return service.getRangeListSync(this.client.getUrl(), shareName, fileName, comp, sharesnapshot, prevsharesnapshot, timeout, this.client.getVersion(), range, leaseId, this.client.isAllowTrailingDot(), - this.client.getFileRequestIntent(), supportRename, accept, context); + this.client.getFileRequestIntent(), supportRename, marker, maxresults, accept, context); } catch (ShareStorageExceptionInternal internalException) { throw ModelHelper.mapToShareStorageException(internalException); } @@ -6809,6 +6852,12 @@ public ResponseBase getRangeListWi * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. * @throws RuntimeException all other wrapped checked exceptions if the request fails to be sent. @@ -6816,10 +6865,11 @@ public ResponseBase getRangeListWi */ @ServiceMethod(returns = ReturnType.SINGLE) public ShareFileRangeList getRangeList(String shareName, String fileName, String sharesnapshot, - String prevsharesnapshot, Integer timeout, String range, String leaseId, Boolean supportRename) { + String prevsharesnapshot, Integer timeout, String range, String leaseId, Boolean supportRename, String marker, + Integer maxresults) { try { return getRangeListWithResponse(shareName, fileName, sharesnapshot, prevsharesnapshot, timeout, range, - leaseId, supportRename, Context.NONE).getValue(); + leaseId, supportRename, marker, maxresults, Context.NONE).getValue(); } catch (ShareStorageExceptionInternal internalException) { throw ModelHelper.mapToShareStorageException(internalException); } @@ -6844,6 +6894,12 @@ public ShareFileRangeList getRangeList(String shareName, String fileName, String * file) and the previous snapshot should be listed. If the value is true, the valid changed ranges for the file * will be returned. If the value is false, the operation will result in a failure with 409 (Conflict) response. The * default value is false. + * @param marker A string value that identifies the portion of the list to be returned with the next list operation. + * The operation returns a marker value within the response body if the list returned was not complete. The marker + * value may then be used in a subsequent call to request the next set of list items. The marker value is opaque to + * the client. + * @param maxresults Specifies the maximum number of entries to return. If the request does not specify maxresults, + * or specifies a value greater than 5,000, the server will return up to 5,000 items. * @param context The context to associate with this operation. * @throws IllegalArgumentException thrown if parameters fail the validation. * @throws ShareStorageExceptionInternal thrown if the request is rejected by server. @@ -6853,13 +6909,14 @@ public ShareFileRangeList getRangeList(String shareName, String fileName, String @ServiceMethod(returns = ReturnType.SINGLE) public Response getRangeListNoCustomHeadersWithResponse(String shareName, String fileName, String sharesnapshot, String prevsharesnapshot, Integer timeout, String range, String leaseId, - Boolean supportRename, Context context) { + Boolean supportRename, String marker, Integer maxresults, Context context) { try { final String comp = "rangelist"; final String accept = "application/xml"; return service.getRangeListNoCustomHeadersSync(this.client.getUrl(), shareName, fileName, comp, sharesnapshot, prevsharesnapshot, timeout, this.client.getVersion(), range, leaseId, - this.client.isAllowTrailingDot(), this.client.getFileRequestIntent(), supportRename, accept, context); + this.client.isAllowTrailingDot(), this.client.getFileRequestIntent(), supportRename, marker, maxresults, + accept, context); } catch (ShareStorageExceptionInternal internalException) { throw ModelHelper.mapToShareStorageException(internalException); } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/util/ModelHelper.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/util/ModelHelper.java index 8652bbdb4dcd..a6830e7cb854 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/util/ModelHelper.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/implementation/util/ModelHelper.java @@ -51,9 +51,11 @@ import com.azure.storage.file.share.implementation.models.SharesCreateSnapshotHeaders; import com.azure.storage.file.share.implementation.models.SharesGetPropertiesHeaders; import com.azure.storage.file.share.implementation.models.StringEncoded; +import com.azure.storage.file.share.models.ClearRange; import com.azure.storage.file.share.models.CopyStatusType; import com.azure.storage.file.share.models.CopyableFileSmbPropertiesList; import com.azure.storage.file.share.models.FilePosixProperties; +import com.azure.storage.file.share.models.FileRange; import com.azure.storage.file.share.models.HandleItem; import com.azure.storage.file.share.models.LeaseDurationType; import com.azure.storage.file.share.models.LeaseStateType; @@ -69,6 +71,9 @@ import com.azure.storage.file.share.models.ShareFileItemProperties; import com.azure.storage.file.share.models.ShareFileMetadataInfo; import com.azure.storage.file.share.models.ShareFileProperties; +import com.azure.storage.file.share.models.ShareFileRange; +import com.azure.storage.file.share.models.ShareFileRangeItem; +import com.azure.storage.file.share.models.ShareFileRangeList; import com.azure.storage.file.share.models.ShareFileSymbolicLinkInfo; import com.azure.storage.file.share.models.ShareFileUploadInfo; import com.azure.storage.file.share.models.ShareFileUploadRangeFromUrlInfo; @@ -731,4 +736,28 @@ public static ShareStorageException mapToShareStorageException(ShareStorageExcep return new ShareStorageException(StorageImplUtils.convertStorageExceptionMessage(internal.getMessage(), internal.getResponse(), code, headerName), internal.getResponse(), internal.getValue()); } + + /** + * Converts a {@link ShareFileRangeList} into a list of {@link ShareFileRangeItem}s, optionally including cleared + * ranges (only present on diff responses). + * + * @param rangeList The range list returned by the service. + * @param includeClearRanges Whether to include cleared ranges. Should be {@code true} only for diff responses. + * @return The combined list of {@link ShareFileRangeItem}s. + */ + public static List toShareFileRangeItems(ShareFileRangeList rangeList, + boolean includeClearRanges) { + List ranges = new ArrayList<>(); + if (rangeList.getRanges() != null) { + for (FileRange r : rangeList.getRanges()) { + ranges.add(new ShareFileRangeItem(new ShareFileRange(r.getStart(), r.getEnd()), false)); + } + } + if (includeClearRanges && rangeList.getClearRanges() != null) { + for (ClearRange r : rangeList.getClearRanges()) { + ranges.add(new ShareFileRangeItem(new ShareFileRange(r.getStart(), r.getEnd()), true)); + } + } + return ranges; + } } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/NfsFileType.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/NfsFileType.java index c252495b5def..42d241442282 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/NfsFileType.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/NfsFileType.java @@ -32,7 +32,7 @@ public final class NfsFileType extends ExpandableStringEnum { /** * Creates a new instance of NfsFileType value. - * + * * @deprecated Use the {@link #fromString(String)} factory method. */ @Generated @@ -42,7 +42,7 @@ public NfsFileType() { /** * Creates or finds a NfsFileType from its string representation. - * + * * @param name a name to look for. * @return the corresponding NfsFileType. */ @@ -53,7 +53,7 @@ public static NfsFileType fromString(String name) { /** * Gets known NfsFileType values. - * + * * @return known NfsFileType values. */ @Generated diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/ShareFileRangeItem.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/ShareFileRangeItem.java new file mode 100644 index 000000000000..636cc532b664 --- /dev/null +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/ShareFileRangeItem.java @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.file.share.models; + +import com.azure.core.annotation.Immutable; + +/** + * Represents a range returned by a paged file range-list operation. + */ +@Immutable +public final class ShareFileRangeItem { + private final ShareFileRange range; + private final boolean isClear; + + /** + * Creates a new {@link ShareFileRangeItem}. + * + * @param range The file range. + * @param isClear Whether the range is cleared. + */ + public ShareFileRangeItem(ShareFileRange range, boolean isClear) { + this.range = range; + this.isClear = isClear; + } + + /** + * Gets the file range. + * + * @return The file range. + */ + public ShareFileRange getRange() { + return range; + } + + /** + * Gets whether the range is cleared. + * + * @return Whether the range is cleared. + */ + public boolean isClear() { + return isClear; + } +} diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/ShareFileRangeList.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/ShareFileRangeList.java index 9f670646d07c..17df7e78798a 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/ShareFileRangeList.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/models/ShareFileRangeList.java @@ -32,6 +32,12 @@ public final class ShareFileRangeList implements XmlSerializable clearRanges = new ArrayList<>(); + /* + * The NextMarker property. + */ + @Generated + private String nextMarker; + /** * Creates an instance of ShareFileRangeList class. */ @@ -83,6 +89,28 @@ public ShareFileRangeList setClearRanges(List clearRanges) { return this; } + /** + * Get the nextMarker property: The NextMarker property. + * + * @return the nextMarker value. + */ + @Generated + public String getNextMarker() { + return this.nextMarker; + } + + /** + * Set the nextMarker property: The NextMarker property. + * + * @param nextMarker the nextMarker value to set. + * @return the ShareFileRangeList object itself. + */ + @Generated + public ShareFileRangeList setNextMarker(String nextMarker) { + this.nextMarker = nextMarker; + return this; + } + @Generated @Override public XmlWriter toXml(XmlWriter xmlWriter) throws XMLStreamException { @@ -104,6 +132,7 @@ public XmlWriter toXml(XmlWriter xmlWriter, String rootElementName) throws XMLSt xmlWriter.writeXml(element, "ClearRange"); } } + xmlWriter.writeStringElement("NextMarker", this.nextMarker); return xmlWriter.writeEndElement(); } @@ -142,6 +171,8 @@ public static ShareFileRangeList fromXml(XmlReader xmlReader, String rootElement deserializedShareFileRangeList.ranges.add(FileRange.fromXml(reader, "Range")); } else if ("ClearRange".equals(elementName.getLocalPart())) { deserializedShareFileRangeList.clearRanges.add(ClearRange.fromXml(reader, "ClearRange")); + } else if ("NextMarker".equals(elementName.getLocalPart())) { + deserializedShareFileRangeList.nextMarker = reader.getStringElement(); } else { reader.skipElement(); } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/options/ShareFileListRangesOptions.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/options/ShareFileListRangesOptions.java new file mode 100644 index 000000000000..3969057bb6fa --- /dev/null +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/options/ShareFileListRangesOptions.java @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.file.share.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.file.share.models.ShareFileRange; +import com.azure.storage.file.share.models.ShareRequestConditions; + +/** + * Extended options that may be passed when listing ranges for a file. + */ +@Fluent +public class ShareFileListRangesOptions { + private ShareFileRange range; + private ShareRequestConditions requestConditions; + + /** + * Creates a new instance of {@link ShareFileListRangesOptions}. + */ + public ShareFileListRangesOptions() { + } + + /** + * Gets the range of bytes over which to list ranges, inclusively. + * + * @return The range of bytes over which to list ranges, inclusively. + */ + public ShareFileRange getRange() { + return range; + } + + /** + * Sets the range of bytes over which to list ranges, inclusively. + * + * @param range The range of bytes over which to list ranges, inclusively. + * @return The updated options. + */ + public ShareFileListRangesOptions setRange(ShareFileRange range) { + this.range = range; + return this; + } + + /** + * Gets the {@link ShareRequestConditions}. + * + * @return {@link ShareRequestConditions} + */ + public ShareRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link ShareRequestConditions}. + * + * @param requestConditions {@link ShareRequestConditions} for the file. + * @return The updated options. + */ + public ShareFileListRangesOptions setRequestConditions(ShareRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } +} diff --git a/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileApiTests.java b/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileApiTests.java index 4558fba36bfe..f2d1e78db613 100644 --- a/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileApiTests.java +++ b/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileApiTests.java @@ -6,6 +6,7 @@ import com.azure.core.exception.UnexpectedLengthException; import com.azure.core.http.HttpHeader; import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.rest.PagedResponse; import com.azure.core.http.rest.Response; import com.azure.core.test.TestMode; import com.azure.core.util.BinaryData; @@ -55,6 +56,7 @@ import com.azure.storage.file.share.models.ShareFilePermission; import com.azure.storage.file.share.models.ShareFileProperties; import com.azure.storage.file.share.models.ShareFileRange; +import com.azure.storage.file.share.models.ShareFileRangeItem; import com.azure.storage.file.share.models.ShareFileRangeList; import com.azure.storage.file.share.models.ShareFileSymbolicLinkInfo; import com.azure.storage.file.share.models.ShareFileUploadInfo; @@ -73,6 +75,7 @@ import com.azure.storage.file.share.options.ShareFileCreateSymbolicLinkOptions; import com.azure.storage.file.share.options.ShareFileDownloadOptions; import com.azure.storage.file.share.options.ShareFileListRangesDiffOptions; +import com.azure.storage.file.share.options.ShareFileListRangesOptions; import com.azure.storage.file.share.options.ShareFileRenameOptions; import com.azure.storage.file.share.options.ShareFileSetPropertiesOptions; import com.azure.storage.file.share.options.ShareFileUploadRangeFromUrlOptions; @@ -83,11 +86,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -100,6 +105,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.LocalDateTime; @@ -676,7 +682,7 @@ public void downloadToFileWithPartialDownloads() throws Exception { } outFile.deleteOnExit(); - MockPartialResponsePolicy policy = new MockPartialResponsePolicy(5); + MockPartialResponsePolicy policy = new MockPartialResponsePolicy(5, 1); // Create a ShareFileClient for download using the custom pipeline ShareFileClient downloadClient @@ -724,7 +730,7 @@ public void downloadToFileRetryExhausted() throws Exception { } outFile.deleteOnExit(); - MockPartialResponsePolicy policy = new MockPartialResponsePolicy(6); + MockPartialResponsePolicy policy = new MockPartialResponsePolicy(6, 1); // Create a ShareFileClient for download using the custom pipeline ShareFileClient downloadClient @@ -2235,6 +2241,95 @@ public void listRanges() throws IOException { FileShareTestHelper.deleteFileIfExists(testFolder.getPath(), fileName); } + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRanges(@TempDir Path tempDir) throws IOException { + int uploadFileSize = Constants.MB; + primaryFileClient.create(uploadFileSize); + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(Constants.KB)); + ShareFileRange offsetRange = FileShareTestHelper.rangeFromOffsetAndLength(Constants.KB, Constants.KB); + ShareFileRange startRange = FileShareTestHelper.rangeFromLength(uploadFileSize); + + uploadRangeFromFile(uploadFilePath, offsetRange); + + ShareFileListRangesOptions options = new ShareFileListRangesOptions().setRange(startRange); + List ranges = new ArrayList<>(); + primaryFileClient.listAllRanges(options, null, null).forEach(ranges::add); + + assertEquals(1, ranges.size()); + assertFalse(ranges.get(0).isClear()); + FileShareTestHelper.assertFileRangeEquals(offsetRange, ranges.get(0).getRange()); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesAccessConditions(@TempDir Path tempDir) throws IOException { + int uploadFileSize = Constants.MB; + primaryFileClient.create(uploadFileSize); + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(Constants.KB)); + ShareFileRange offsetRange = FileShareTestHelper.rangeFromOffsetAndLength(Constants.KB, Constants.KB); + ShareFileRange startRange = FileShareTestHelper.rangeFromLength(uploadFileSize); + String leaseId = setupFileLeaseCondition(primaryFileClient, RECEIVED_LEASE_ID); + ShareRequestConditions conditions = new ShareRequestConditions().setLeaseId(leaseId); + + uploadRangeFromFile(uploadFilePath, offsetRange, conditions); + + ShareFileListRangesOptions options + = new ShareFileListRangesOptions().setRange(startRange).setRequestConditions(conditions); + List> pages = new ArrayList<>(); + primaryFileClient.listAllRanges(options, null, null).iterableByPage().forEach(pages::add); + + assertFalse(pages.isEmpty()); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesAccessConditionsFails() { + int uploadFileSize = Constants.MB; + primaryFileClient.create(uploadFileSize); + ShareFileRange startRange = FileShareTestHelper.rangeFromLength(uploadFileSize); + ShareRequestConditions conditions = new ShareRequestConditions().setLeaseId(testResourceNamer.randomUuid()); + ShareFileListRangesOptions options + = new ShareFileListRangesOptions().setRange(startRange).setRequestConditions(conditions); + + // file created, but no upload + + ShareStorageException e = assertThrows(ShareStorageException.class, + () -> primaryFileClient.listAllRanges(options, null, null).forEach(ignored -> { + })); + assertEquals("LeaseNotPresentWithFileOperation", e.getErrorCode().getValue()); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesPagedByPage(@TempDir Path tempDir) throws IOException { + int uploadFileSize = Constants.MB; + int rangeLength = Constants.KB; + primaryFileClient.create(uploadFileSize); + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(rangeLength)); + ShareFileRange firstRange = FileShareTestHelper.rangeFromOffsetAndLength(0, rangeLength); + ShareFileRange secondRange = FileShareTestHelper.rangeFromOffsetAndLength(2L * rangeLength, rangeLength); + ShareFileRange thirdRange = FileShareTestHelper.rangeFromOffsetAndLength(4L * rangeLength, rangeLength); + + uploadRangeFromFile(uploadFilePath, firstRange); + uploadRangeFromFile(uploadFilePath, secondRange); + uploadRangeFromFile(uploadFilePath, thirdRange); + + List> pages = new ArrayList<>(); + primaryFileClient.listAllRanges().iterableByPage(1).forEach(pages::add); + + assertEquals(3, pages.size()); + assertEquals(1, pages.get(0).getValue().size()); + FileShareTestHelper.assertFileRangeItemEquals(firstRange, pages.get(0).getValue().get(0)); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(0).getContinuationToken())); + assertEquals(1, pages.get(1).getValue().size()); + FileShareTestHelper.assertFileRangeItemEquals(secondRange, pages.get(1).getValue().get(0)); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(1).getContinuationToken())); + assertEquals(1, pages.get(2).getValue().size()); + FileShareTestHelper.assertFileRangeItemEquals(thirdRange, pages.get(2).getValue().get(0)); + assertNull(pages.get(2).getContinuationToken()); + } + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2022-11-02") @Test public void listRangesTrailingDot() throws IOException { @@ -2371,6 +2466,129 @@ public void listRangesDiff(List rangesToUpdate, List range } } + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiff(@TempDir Path tempDir) throws IOException { + int rangeLength = Constants.KB; + primaryFileClient.create(Constants.MB); + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(rangeLength)); + + ShareFileRange initialPopulatedRange = FileShareTestHelper.rangeFromOffsetAndLength(0, rangeLength); + uploadRangeFromFile(uploadFilePath, initialPopulatedRange); + + ShareSnapshotInfo previousSnapshot = shareClient.createSnapshot(); + + ShareFileRange populatedRange = FileShareTestHelper.rangeFromOffsetAndLength(3L * Constants.KB, rangeLength); + ClearRange clearedSubrange + = FileShareTestHelper.clearRangeFromOffsetAndLength(initialPopulatedRange.getStart(), 512); + uploadRangeFromFile(uploadFilePath, populatedRange); + primaryFileClient.clearRangeWithResponse(512, clearedSubrange.getStart(), null, null); + + ShareSnapshotInfo snapshot = shareClient.createSnapshot(); + ShareFileClient snapshotFileClient + = fileBuilderHelper(shareName, filePath).snapshot(snapshot.getSnapshot()).buildFileClient(); + List rangeItems = new ArrayList<>(); + snapshotFileClient + .listAllRangesDiff(new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot()), null, null) + .forEach(rangeItems::add); + + List populated + = rangeItems.stream().filter(item -> !item.isClear()).collect(Collectors.toList()); + List cleared + = rangeItems.stream().filter(ShareFileRangeItem::isClear).collect(Collectors.toList()); + assertEquals(1, populated.size()); + FileShareTestHelper.assertFileRangeItemEquals(populatedRange, populated.get(0)); + assertEquals(1, cleared.size()); + FileShareTestHelper.assertClearRangeItemEquals(clearedSubrange, cleared.get(0)); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiffPagedByPage(@TempDir Path tempDir) throws IOException { + int rangeLength = Constants.KB; + primaryFileClient.create(Constants.MB); + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(rangeLength)); + + ShareFileRange initialPopulatedRange = FileShareTestHelper.rangeFromOffsetAndLength(0, rangeLength); + uploadRangeFromFile(uploadFilePath, initialPopulatedRange); + + ShareSnapshotInfo previousSnapshot = shareClient.createSnapshot(); + + ShareFileRange firstPopulatedRange + = FileShareTestHelper.rangeFromOffsetAndLength(3L * Constants.KB, rangeLength); + ShareFileRange secondPopulatedRange + = FileShareTestHelper.rangeFromOffsetAndLength(5L * Constants.KB, rangeLength); + ClearRange clearedSubrange + = FileShareTestHelper.clearRangeFromOffsetAndLength(initialPopulatedRange.getStart(), 512); + uploadRangeFromFile(uploadFilePath, firstPopulatedRange); + uploadRangeFromFile(uploadFilePath, secondPopulatedRange); + primaryFileClient.clearRangeWithResponse(512, clearedSubrange.getStart(), null, null); + + ShareSnapshotInfo snapshot = shareClient.createSnapshot(); + ShareFileClient snapshotFileClient + = fileBuilderHelper(shareName, filePath).snapshot(snapshot.getSnapshot()).buildFileClient(); + ShareFileListRangesDiffOptions options = new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot()); + + List> pages = new ArrayList<>(); + snapshotFileClient.listAllRangesDiff(options, null, null).iterableByPage(1).forEach(pages::add); + + assertEquals(3, pages.size()); + pages.forEach(page -> assertEquals(1, page.getValue().size())); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(0).getContinuationToken())); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(1).getContinuationToken())); + assertNull(pages.get(2).getContinuationToken()); + + List rangeItems + = pages.stream().flatMap(page -> page.getValue().stream()).collect(Collectors.toList()); + List populated + = rangeItems.stream().filter(item -> !item.isClear()).collect(Collectors.toList()); + List cleared + = rangeItems.stream().filter(ShareFileRangeItem::isClear).collect(Collectors.toList()); + assertEquals(2, populated.size()); + FileShareTestHelper.assertFileRangeItemEquals(firstPopulatedRange, populated.get(0)); + FileShareTestHelper.assertFileRangeItemEquals(secondPopulatedRange, populated.get(1)); + assertEquals(1, cleared.size()); + FileShareTestHelper.assertClearRangeItemEquals(clearedSubrange, cleared.get(0)); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiffAccessConditions() { + int uploadFileSize = Constants.MB; + primaryFileClient.create(uploadFileSize); + ShareFileRange uploadRange = FileShareTestHelper.rangeFromOffsetAndLength(0L, uploadFileSize); + + ShareSnapshotInfo previousSnapshot = shareClient.createSnapshot(); + String leaseId = setupFileLeaseCondition(primaryFileClient, RECEIVED_LEASE_ID); + ShareFileListRangesDiffOptions options + = new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot()).setRange(uploadRange) + .setRequestConditions(new ShareRequestConditions().setLeaseId(leaseId)); + + List> pages = new ArrayList<>(); + primaryFileClient.listAllRangesDiff(options, null, null).iterableByPage().forEach(pages::add); + + assertFalse(pages.isEmpty()); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiffAccessConditionsFail() { + int uploadFileSize = Constants.MB; + primaryFileClient.create(uploadFileSize); + ShareFileRange uploadRange = FileShareTestHelper.rangeFromOffsetAndLength(0L, uploadFileSize); + + ShareSnapshotInfo previousSnapshot = shareClient.createSnapshot(); + String leaseId = setupFileLeaseCondition(primaryFileClient, GARBAGE_LEASE_ID); + ShareFileListRangesDiffOptions options + = new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot()).setRange(uploadRange) + .setRequestConditions(new ShareRequestConditions().setLeaseId(leaseId)); + + ShareStorageException e = assertThrows(ShareStorageException.class, + () -> primaryFileClient.listAllRangesDiff(options, null, null).forEach(ignored -> { + })); + assertEquals("LeaseIdMismatchWithFileOperation", e.getErrorCode().getValue()); + } + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2021-04-10") @Test public void listRangesDiffOAuth() { @@ -3509,4 +3727,21 @@ public void createFileWithLargeBinaryDataPartiallyEmptyMD5() throws NoSuchAlgori assertArrayEquals(MessageDigest.getInstance("MD5").digest(randomByteArray), decodedContentMd5); } + + private void uploadRangeFromFile(Path uploadFilePath, ShareFileRange range) throws IOException { + uploadRangeFromFile(uploadFilePath, range, null); + } + + private void uploadRangeFromFile(Path uploadFilePath, ShareFileRange range, ShareRequestConditions conditions) + throws IOException { + try (InputStream uploadStream = new BufferedInputStream(Files.newInputStream(uploadFilePath))) { + long length = range.getEnd() - range.getStart() + 1; + ShareFileUploadRangeOptions uploadRangeOptions + = new ShareFileUploadRangeOptions(uploadStream, length).setOffset(range.getStart()); + if (conditions != null) { + uploadRangeOptions.setRequestConditions(conditions); + } + primaryFileClient.uploadRangeWithResponse(uploadRangeOptions, null, Context.NONE); + } + } } diff --git a/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileAsyncApiTests.java b/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileAsyncApiTests.java index 1a90a9f411bf..99d9120e0dd9 100644 --- a/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileAsyncApiTests.java +++ b/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileAsyncApiTests.java @@ -41,6 +41,7 @@ import com.azure.storage.file.share.models.ShareFilePermission; import com.azure.storage.file.share.models.ShareFileProperties; import com.azure.storage.file.share.models.ShareFileRange; +import com.azure.storage.file.share.models.ShareFileRangeItem; import com.azure.storage.file.share.models.ShareFileSymbolicLinkInfo; import com.azure.storage.file.share.models.ShareFileUploadInfo; import com.azure.storage.file.share.models.ShareFileUploadRangeOptions; @@ -55,6 +56,7 @@ import com.azure.storage.file.share.options.ShareFileCreateOptions; import com.azure.storage.file.share.options.ShareFileCreateSymbolicLinkOptions; import com.azure.storage.file.share.options.ShareFileListRangesDiffOptions; +import com.azure.storage.file.share.options.ShareFileListRangesOptions; import com.azure.storage.file.share.options.ShareFileRenameOptions; import com.azure.storage.file.share.options.ShareFileSetPropertiesOptions; import com.azure.storage.file.share.sas.ShareFileSasPermission; @@ -64,6 +66,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -83,6 +86,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Duration; @@ -98,6 +102,7 @@ import java.util.Map; import java.util.Objects; import java.util.Scanner; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.azure.storage.common.implementation.Constants.HeaderConstants.ERROR_CODE_HEADER_NAME; @@ -646,7 +651,7 @@ public void downloadToFileWithPartialDownloads() throws Exception { } outFile.deleteOnExit(); - MockPartialResponsePolicy policy = new MockPartialResponsePolicy(5); + MockPartialResponsePolicy policy = new MockPartialResponsePolicy(5, 1); // Create a ShareFileAsyncClient for download using the custom pipeline ShareFileAsyncClient downloadClient = getFileAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), @@ -696,7 +701,7 @@ public void downloadToFileRetryExhausted() throws Exception { } outFile.deleteOnExit(); - MockPartialResponsePolicy policy = new MockPartialResponsePolicy(6); + MockPartialResponsePolicy policy = new MockPartialResponsePolicy(6, 1); // Create a ShareFileAsyncClient for download using the custom pipeline ShareFileAsyncClient downloadClient = getFileAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), @@ -1599,6 +1604,92 @@ public void listRanges() throws IOException { FileShareTestHelper.deleteFileIfExists(testFolder.getPath(), fileName); } + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRanges(@TempDir Path tempDir) throws IOException { + int uploadFileSize = Constants.MB; + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(Constants.KB)); + ShareFileRange offsetRange = FileShareTestHelper.rangeFromOffsetAndLength(Constants.KB, Constants.KB); + ShareFileRange startRange = FileShareTestHelper.rangeFromLength(uploadFileSize); + ShareFileListRangesOptions options = new ShareFileListRangesOptions().setRange(startRange); + + StepVerifier.create(primaryFileAsyncClient.create(uploadFileSize) + .then(uploadRangeFromFile(uploadFilePath, offsetRange)) + .thenMany(primaryFileAsyncClient.listAllRanges(options)) + .collectList()).assertNext(ranges -> { + assertEquals(1, ranges.size()); + assertFalse(ranges.get(0).isClear()); + FileShareTestHelper.assertFileRangeEquals(offsetRange, ranges.get(0).getRange()); + }).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesAccessConditions(@TempDir Path tempDir) throws IOException { + int uploadFileSize = Constants.MB; + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(Constants.KB)); + ShareFileRange offsetRange = FileShareTestHelper.rangeFromOffsetAndLength(Constants.KB, Constants.KB); + ShareFileRange startRange = FileShareTestHelper.rangeFromLength(uploadFileSize); + + StepVerifier.create(primaryFileAsyncClient.create(uploadFileSize) + .then(setupFileLeaseCondition(primaryFileAsyncClient, RECEIVED_LEASE_ID)) + .flatMapMany(leaseId -> { + ShareRequestConditions conditions = new ShareRequestConditions().setLeaseId(leaseId); + ShareFileListRangesOptions options + = new ShareFileListRangesOptions().setRange(startRange).setRequestConditions(conditions); + return uploadRangeFromFile(uploadFilePath, offsetRange, conditions) + .thenMany(primaryFileAsyncClient.listAllRanges(options).byPage()); + }) + .collectList()).assertNext(pages -> assertFalse(pages.isEmpty())).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesAccessConditionsFails() { + int uploadFileSize = Constants.MB; + ShareFileRange startRange = FileShareTestHelper.rangeFromLength(uploadFileSize); + ShareRequestConditions conditions = new ShareRequestConditions().setLeaseId(testResourceNamer.randomUuid()); + ShareFileListRangesOptions options + = new ShareFileListRangesOptions().setRange(startRange).setRequestConditions(conditions); + + StepVerifier + .create( + primaryFileAsyncClient.create(uploadFileSize).thenMany(primaryFileAsyncClient.listAllRanges(options))) + .verifyErrorSatisfies(r -> { + ShareStorageException e = assertInstanceOf(ShareStorageException.class, r); + assertEquals("LeaseNotPresentWithFileOperation", e.getErrorCode().getValue()); + }); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesPagedByPage(@TempDir Path tempDir) throws IOException { + int uploadFileSize = Constants.MB; + int rangeLength = Constants.KB; + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(rangeLength)); + ShareFileRange firstRange = FileShareTestHelper.rangeFromOffsetAndLength(0, rangeLength); + ShareFileRange secondRange = FileShareTestHelper.rangeFromOffsetAndLength(2L * rangeLength, rangeLength); + ShareFileRange thirdRange = FileShareTestHelper.rangeFromOffsetAndLength(4L * rangeLength, rangeLength); + + StepVerifier.create(primaryFileAsyncClient.create(uploadFileSize) + .then(uploadRangeFromFile(uploadFilePath, firstRange)) + .then(uploadRangeFromFile(uploadFilePath, secondRange)) + .then(uploadRangeFromFile(uploadFilePath, thirdRange)) + .thenMany(primaryFileAsyncClient.listAllRanges().byPage(1)) + .collectList()).assertNext(pages -> { + assertEquals(3, pages.size()); + assertEquals(1, pages.get(0).getValue().size()); + FileShareTestHelper.assertFileRangeItemEquals(firstRange, pages.get(0).getValue().get(0)); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(0).getContinuationToken())); + assertEquals(1, pages.get(1).getValue().size()); + FileShareTestHelper.assertFileRangeItemEquals(secondRange, pages.get(1).getValue().get(0)); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(1).getContinuationToken())); + assertEquals(1, pages.get(2).getValue().size()); + FileShareTestHelper.assertFileRangeItemEquals(thirdRange, pages.get(2).getValue().get(0)); + assertNull(pages.get(2).getContinuationToken()); + }).verifyComplete(); + } + @Test public void listRangesWithRange() throws IOException { Mono createFileMono = primaryFileAsyncClient.createWithResponse(1024, null, null, null, null).then(); @@ -1692,6 +1783,133 @@ public void listRangesDiff(List rangesToUpdate, List range }).verifyComplete(); } + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiffPopulatedAndCleared(@TempDir Path tempDir) throws IOException { + int rangeLength = Constants.KB; + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(rangeLength)); + ShareFileRange initialPopulatedRange = FileShareTestHelper.rangeFromOffsetAndLength(0, rangeLength); + ShareFileRange populatedRange = FileShareTestHelper.rangeFromOffsetAndLength(3L * Constants.KB, rangeLength); + ClearRange clearedSubrange + = FileShareTestHelper.clearRangeFromOffsetAndLength(initialPopulatedRange.getStart(), 512); + + StepVerifier.create(primaryFileAsyncClient.create(Constants.MB) + .then(uploadRangeFromFile(uploadFilePath, initialPopulatedRange)) + .then(shareAsyncClient.createSnapshot()) + .flatMap(previousSnapshot -> uploadRangeFromFile(uploadFilePath, populatedRange) + .then(primaryFileAsyncClient.clearRangeWithResponse(512, clearedSubrange.getStart())) + .then(shareAsyncClient.createSnapshot()) + .flatMap(snapshot -> { + ShareFileAsyncClient snapshotFileClient + = fileBuilderHelper(shareName, filePath).snapshot(snapshot.getSnapshot()) + .buildFileAsyncClient(); + return snapshotFileClient + .listAllRangesDiff(new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot())) + .collectList(); + }))) + .assertNext(rangeItems -> { + List populated + = rangeItems.stream().filter(item -> !item.isClear()).collect(Collectors.toList()); + List cleared + = rangeItems.stream().filter(ShareFileRangeItem::isClear).collect(Collectors.toList()); + assertEquals(1, populated.size()); + FileShareTestHelper.assertFileRangeItemEquals(populatedRange, populated.get(0)); + assertEquals(1, cleared.size()); + FileShareTestHelper.assertClearRangeItemEquals(clearedSubrange, cleared.get(0)); + }) + .verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiffPagedByPage(@TempDir Path tempDir) throws IOException { + int rangeLength = Constants.KB; + Path uploadFilePath = Files.write(tempDir.resolve(generatePathName()), getRandomByteArray(rangeLength)); + ShareFileRange initialPopulatedRange = FileShareTestHelper.rangeFromOffsetAndLength(0, rangeLength); + ShareFileRange firstPopulatedRange + = FileShareTestHelper.rangeFromOffsetAndLength(3L * Constants.KB, rangeLength); + ShareFileRange secondPopulatedRange + = FileShareTestHelper.rangeFromOffsetAndLength(5L * Constants.KB, rangeLength); + ClearRange clearedSubrange + = FileShareTestHelper.clearRangeFromOffsetAndLength(initialPopulatedRange.getStart(), 512); + + StepVerifier.create(primaryFileAsyncClient.create(Constants.MB) + .then(uploadRangeFromFile(uploadFilePath, initialPopulatedRange)) + .then(shareAsyncClient.createSnapshot()) + .flatMap(previousSnapshot -> uploadRangeFromFile(uploadFilePath, firstPopulatedRange) + .then(uploadRangeFromFile(uploadFilePath, secondPopulatedRange)) + .then(primaryFileAsyncClient.clearRangeWithResponse(512, clearedSubrange.getStart())) + .then(shareAsyncClient.createSnapshot()) + .flatMap(snapshot -> { + ShareFileAsyncClient snapshotFileClient + = fileBuilderHelper(shareName, filePath).snapshot(snapshot.getSnapshot()) + .buildFileAsyncClient(); + ShareFileListRangesDiffOptions options + = new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot()); + + return snapshotFileClient.listAllRangesDiff(options).byPage(1).collectList(); + }))) + .assertNext(pages -> { + assertEquals(3, pages.size()); + pages.forEach(page -> assertEquals(1, page.getValue().size())); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(0).getContinuationToken())); + assertFalse(CoreUtils.isNullOrEmpty(pages.get(1).getContinuationToken())); + assertNull(pages.get(2).getContinuationToken()); + + List rangeItems + = pages.stream().flatMap(page -> page.getValue().stream()).collect(Collectors.toList()); + List populated + = rangeItems.stream().filter(item -> !item.isClear()).collect(Collectors.toList()); + List cleared + = rangeItems.stream().filter(ShareFileRangeItem::isClear).collect(Collectors.toList()); + assertEquals(2, populated.size()); + FileShareTestHelper.assertFileRangeItemEquals(firstPopulatedRange, populated.get(0)); + FileShareTestHelper.assertFileRangeItemEquals(secondPopulatedRange, populated.get(1)); + assertEquals(1, cleared.size()); + FileShareTestHelper.assertClearRangeItemEquals(clearedSubrange, cleared.get(0)); + }) + .verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiffAccessConditions() { + int uploadFileSize = Constants.MB; + ShareFileRange uploadRange = FileShareTestHelper.rangeFromOffsetAndLength(0L, uploadFileSize); + + StepVerifier.create(primaryFileAsyncClient.create(uploadFileSize) + .then(shareAsyncClient.createSnapshot()) + .flatMapMany(previousSnapshot -> setupFileLeaseCondition(primaryFileAsyncClient, RECEIVED_LEASE_ID) + .flatMapMany(leaseId -> { + ShareFileListRangesDiffOptions options + = new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot()).setRange(uploadRange) + .setRequestConditions(new ShareRequestConditions().setLeaseId(leaseId)); + return primaryFileAsyncClient.listAllRangesDiff(options).byPage(); + })) + .collectList()).assertNext(pages -> assertFalse(pages.isEmpty())).verifyComplete(); + } + + @Test + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2026-10-06") + public void listAllRangesDiffAccessConditionsFail() { + int uploadFileSize = Constants.MB; + ShareFileRange uploadRange = FileShareTestHelper.rangeFromOffsetAndLength(0L, uploadFileSize); + + StepVerifier.create(primaryFileAsyncClient.create(uploadFileSize) + .then(shareAsyncClient.createSnapshot()) + .flatMapMany(previousSnapshot -> setupFileLeaseCondition(primaryFileAsyncClient, GARBAGE_LEASE_ID) + .flatMapMany(leaseId -> { + ShareFileListRangesDiffOptions options + = new ShareFileListRangesDiffOptions(previousSnapshot.getSnapshot()).setRange(uploadRange) + .setRequestConditions(new ShareRequestConditions().setLeaseId(leaseId)); + return primaryFileAsyncClient.listAllRangesDiff(options); + }))) + .verifyErrorSatisfies(r -> { + ShareStorageException e = assertInstanceOf(ShareStorageException.class, r); + assertEquals("LeaseIdMismatchWithFileOperation", e.getErrorCode().getValue()); + }); + } + @RequiredServiceVersion(clazz = ShareServiceVersion.class, min = "2024-05-04") @ParameterizedTest @MethodSource("listRangesDiffWithRenameSupplier") @@ -2365,4 +2583,23 @@ public void createFileWithLargeBinaryDataPartiallyEmptyMD5() throws NoSuchAlgori assertArrayEquals(expectedMd5, decodedContentMd5); }).verifyComplete(); } + + private Mono> uploadRangeFromFile(Path uploadFilePath, ShareFileRange range) { + return uploadRangeFromFile(uploadFilePath, range, null); + } + + private Mono> uploadRangeFromFile(Path uploadFilePath, ShareFileRange range, + ShareRequestConditions conditions) { + return Mono.fromCallable(() -> Files.readAllBytes(uploadFilePath)).flatMap(uploadContent -> { + long length = range.getEnd() - range.getStart() + 1; + ShareFileUploadRangeOptions uploadRangeOptions + = new ShareFileUploadRangeOptions(Flux.defer(() -> Flux.just(ByteBuffer.wrap(uploadContent))), length) + .setOffset(range.getStart()); + if (conditions != null) { + uploadRangeOptions.setRequestConditions(conditions); + } + + return primaryFileAsyncClient.uploadRangeWithResponse(uploadRangeOptions); + }); + } } diff --git a/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileShareTestHelper.java b/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileShareTestHelper.java index 21563e7f65da..bd329ede2ed5 100644 --- a/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileShareTestHelper.java +++ b/sdk/storage/azure-storage-file-share/src/test/java/com/azure/storage/file/share/FileShareTestHelper.java @@ -14,6 +14,8 @@ import com.azure.storage.file.share.models.ShareCorsRule; import com.azure.storage.file.share.models.ShareErrorCode; import com.azure.storage.file.share.models.ShareFileHttpHeaders; +import com.azure.storage.file.share.models.ShareFileRange; +import com.azure.storage.file.share.models.ShareFileRangeItem; import com.azure.storage.file.share.models.ShareItem; import com.azure.storage.file.share.models.ShareMetrics; import com.azure.storage.file.share.models.ShareRetentionPolicy; @@ -43,8 +45,10 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class FileShareTestHelper { private static final ClientLogger LOGGER = new ClientLogger(FileShareTestHelper.class); @@ -102,6 +106,34 @@ protected static List createClearRanges(long... offsets) { return clearRanges; } + protected static ShareFileRange rangeFromOffsetAndLength(long offset, long length) { + return new ShareFileRange(offset, offset + length - 1); + } + + protected static ShareFileRange rangeFromLength(long length) { + return rangeFromOffsetAndLength(0, length); + } + + protected static ClearRange clearRangeFromOffsetAndLength(long offset, long length) { + return new ClearRange().setStart(offset).setEnd(offset + length - 1); + } + + protected static void assertFileRangeEquals(ShareFileRange expected, ShareFileRange actual) { + assertEquals(expected.getStart(), actual.getStart()); + assertEquals(expected.getEnd(), actual.getEnd()); + } + + protected static void assertFileRangeItemEquals(ShareFileRange expected, ShareFileRangeItem actual) { + assertFalse(actual.isClear()); + assertFileRangeEquals(expected, actual.getRange()); + } + + protected static void assertClearRangeItemEquals(ClearRange expected, ShareFileRangeItem actual) { + assertTrue(actual.isClear()); + assertEquals(expected.getStart(), actual.getRange().getStart()); + assertEquals(expected.getEnd(), actual.getRange().getEnd()); + } + protected static InputStream getInputStream(byte[] data) { return new ByteArrayInputStream(data); } diff --git a/sdk/storage/azure-storage-file-share/swagger/README.md b/sdk/storage/azure-storage-file-share/swagger/README.md index 41515fb4e938..40fab42ccbdf 100644 --- a/sdk/storage/azure-storage-file-share/swagger/README.md +++ b/sdk/storage/azure-storage-file-share/swagger/README.md @@ -16,7 +16,7 @@ autorest ### Code generation settings ``` yaml use: '@autorest/java@4.1.63' -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/15d7f54a5389d5906ffb4e56bb2f38fe5525c0d3/specification/storage/data-plane/Microsoft.FileStorage/stable/2026-06-06/file.json +input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/99ba8f1136fe3b3ba173b613c3291386fb0687d1/specification/storage/data-plane/Microsoft.FileStorage/stable/2026-12-06/file.json java: true output-folder: ../ namespace: com.azure.storage.file.share @@ -401,6 +401,15 @@ directive: delete $["x-ms-pageable"]; ``` +### Delete File_GetRangeList x-ms-pageable as autorest does not currently support multiple return types for pageable +``` yaml +directive: +- from: swagger-document + where: $["x-ms-paths"]["/{shareName}/{fileName}?comp=rangelist"].get + transform: > + delete $["x-ms-pageable"]; +``` + ### Change PutRange response file-last-write-time to ISO8601 ``` yaml directive: diff --git a/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java b/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java index efd0cded7d73..78c60cac21cd 100644 --- a/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java +++ b/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java @@ -162,7 +162,12 @@ public enum QueueServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"); private final String version; @@ -184,6 +189,6 @@ public String getVersion() { * @return the latest {@link QueueServiceVersion} */ public static QueueServiceVersion getLatest() { - return V2026_06_06; + return V2026_10_06; } }