From 9da6d27f4b68f0575275cf76bbd105aabd849746 Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sun, 14 Feb 2021 16:39:08 +0000 Subject: [PATCH 1/3] add an extra unit test for doing an async read on a ZipFile InputStream --- .../Zip/ZipEncryptionHandling.cs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs index 34dde202b..15247f1d3 100644 --- a/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs +++ b/test/ICSharpCode.SharpZipLib.Tests/Zip/ZipEncryptionHandling.cs @@ -4,6 +4,7 @@ using System.IO; using System.Text; using ICSharpCode.SharpZipLib.Tests.TestSupport; +using System.Threading.Tasks; namespace ICSharpCode.SharpZipLib.Tests.Zip { @@ -180,6 +181,53 @@ public void ZipFileStoreAes() } } + /// + /// As , but with Async reads + /// + [Test] + [Category("Encryption")] + [Category("Zip")] + public async Task ZipFileStoreAesAsync() + { + string password = "password"; + + using (var memoryStream = new MemoryStream()) + { + // Try to create a zip stream + WriteEncryptedZipToStream(memoryStream, password, 256, CompressionMethod.Stored); + + // reset + memoryStream.Seek(0, SeekOrigin.Begin); + + // try to read it + var zipFile = new ZipFile(memoryStream, leaveOpen: true) + { + Password = password + }; + + foreach (ZipEntry entry in zipFile) + { + if (!entry.IsFile) continue; + + // Should be stored rather than deflated + Assert.That(entry.CompressionMethod, Is.EqualTo(CompressionMethod.Stored), "Entry should be stored"); + + using (var zis = zipFile.GetInputStream(entry)) + { + var buffer = new byte[entry.Size]; + + using (var inputStream = zipFile.GetInputStream(entry)) + { + await zis.ReadAsync(buffer, 0, buffer.Length); + } + + var content = Encoding.UTF8.GetString(buffer); + Assert.That(content, Is.EqualTo(DummyDataString), "Decompressed content does not match input data"); + } + } + } + } + /// /// Test using AES encryption on a file whose contents are Stored rather than deflated /// From 08f81860821d6bf47591a5111c972ede8bd23faf Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sun, 14 Feb 2021 16:39:47 +0000 Subject: [PATCH 2/3] Add an async version of ReadRequestedBytes --- .../Core/StreamUtils.cs | 60 +++++++++++++++---- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Core/StreamUtils.cs b/src/ICSharpCode.SharpZipLib/Core/StreamUtils.cs index 6d0d9b304..f4feeb2b3 100644 --- a/src/ICSharpCode.SharpZipLib/Core/StreamUtils.cs +++ b/src/ICSharpCode.SharpZipLib/Core/StreamUtils.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Threading.Tasks; namespace ICSharpCode.SharpZipLib.Core { @@ -64,16 +65,8 @@ static public void ReadFully(Stream stream, byte[] buffer, int offset, int count } } - /// - /// Read as much data as possible from a ", up to the requested number of bytes - /// - /// The stream to read data from. - /// The buffer to store data in. - /// The offset at which to begin storing data. - /// The number of bytes of data to store. - /// Required parameter is null - /// and or are invalid. - static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, int count) + // A helper function to share between the async and sync versions of ReadRequestedBytes + private static void ValidateArgumentsForRead(Stream stream, byte[] buffer, int offset, int count) { if (stream == null) { @@ -95,7 +88,23 @@ static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, i { throw new ArgumentOutOfRangeException(nameof(count)); } + } + /// + /// Read as much data as possible from a ", up to the requested number of bytes + /// + /// The stream to read data from. + /// The buffer to store data in. + /// The offset at which to begin storing data. + /// The number of bytes of data to store. + /// Required parameter is null + /// and or are invalid. + static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, int count) + { + // Common validation function + ValidateArgumentsForRead(stream, buffer, offset, count); + + // read the data using Read int totalReadCount = 0; while (count > 0) { @@ -112,6 +121,37 @@ static public int ReadRequestedBytes(Stream stream, byte[] buffer, int offset, i return totalReadCount; } + /// + /// Read as much data as possible from a ", up to the requested number of bytes + /// + /// The stream to read data from. + /// The buffer to store data in. + /// The offset at which to begin storing data. + /// The number of bytes of data to store. + /// Required parameter is null + /// and or are invalid. + static public async Task ReadRequestedBytesAsync(Stream stream, byte[] buffer, int offset, int count) + { + // Common validation function + ValidateArgumentsForRead(stream, buffer, offset, count); + + // read the data using ReadAsync + int totalReadCount = 0; + while (count > 0) + { + int readCount = await stream.ReadAsync(buffer, offset, count); + if (readCount <= 0) + { + break; + } + offset += readCount; + count -= readCount; + totalReadCount += readCount; + } + + return totalReadCount; + } + /// /// Copy the contents of one to another. /// From 24234846de1470b4eb625029b193a2cc4166918f Mon Sep 17 00:00:00 2001 From: Richard Webb Date: Sun, 14 Feb 2021 16:40:51 +0000 Subject: [PATCH 3/3] start adding an implementation of ReadAsync to ZipAESStream --- .../Encryption/ZipAESStream.cs | 185 ++++++++++++------ 1 file changed, 130 insertions(+), 55 deletions(-) diff --git a/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs b/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs index 4f649e8a9..f06f35a75 100644 --- a/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs +++ b/src/ICSharpCode.SharpZipLib/Encryption/ZipAESStream.cs @@ -1,6 +1,8 @@ using System; using System.IO; using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; using ICSharpCode.SharpZipLib.Core; namespace ICSharpCode.SharpZipLib.Encryption @@ -70,18 +72,11 @@ public override int Read(byte[] buffer, int offset, int count) return 0; // If we have buffered data, read that first - int nBytes = 0; - if (HasBufferedData) - { - nBytes = ReadBufferedData(buffer, offset, count); + int nBytes = ReadBufferedData(buffer, ref offset, ref count); - // Read all requested data from the buffer - if (nBytes == count) - return nBytes; - - offset += nBytes; - count -= nBytes; - } + // Read all requested data from the buffer + if (nBytes == count) + return nBytes; // Read more data from the input, if available if (_slideBuffer != null) @@ -90,6 +85,27 @@ public override int Read(byte[] buffer, int offset, int count) return nBytes; } + /// + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + // Nothing to do + if (count == 0) + return 0; + + // If we have buffered data, read that first + int nBytes = ReadBufferedData(buffer, ref offset, ref count); + + // Read all requested data from the buffer + if (nBytes == count) + return nBytes; + + // Read more data from the input, if available + if (_slideBuffer != null) + nBytes += await ReadAndTransformAsync(buffer, offset, count); + + return nBytes; + } + // Read data from the underlying stream and decrypt it private int ReadAndTransform(byte[] buffer, int offset, int count) { @@ -105,68 +121,127 @@ private int ReadAndTransform(byte[] buffer, int offset, int count) // Maintain a read-ahead equal to the length of (crypto block + Auth Code). // When that runs out we can detect these final sections. int lengthToRead = BLOCK_AND_AUTH - byteCount; - if (_slideBuffer.Length - _slideBufFreePos < lengthToRead) + UpdateSlideBufferIfNeeded(lengthToRead); + + int obtained = StreamUtils.ReadRequestedBytes(_stream, _slideBuffer, _slideBufFreePos, lengthToRead); + _slideBufFreePos += obtained; + + // Transform data from the slide buffer + if (TransformFromSlideBuffer(buffer, ref offset, bytesLeftToRead, ref nBytes)) { - // Shift the data to the beginning of the buffer - int iTo = 0; - for (int iFrom = _slideBufStartPos; iFrom < _slideBufFreePos; iFrom++, iTo++) - { - _slideBuffer[iTo] = _slideBuffer[iFrom]; - } - _slideBufFreePos -= _slideBufStartPos; // Note the -= - _slideBufStartPos = 0; + // Reached the auth code + break; } - int obtained = StreamUtils.ReadRequestedBytes(_stream, _slideBuffer, _slideBufFreePos, lengthToRead); + } + return nBytes; + } + + // Read data from the underlying stream asynchronously and decrypt it + private async Task ReadAndTransformAsync(byte[] buffer, int offset, int count) + { + int nBytes = 0; + while (nBytes < count) + { + int bytesLeftToRead = count - nBytes; + + // Calculate buffer quantities vs read-ahead size, and check for sufficient free space + int byteCount = _slideBufFreePos - _slideBufStartPos; + + // Need to handle final block and Auth Code specially, but don't know total data length. + // Maintain a read-ahead equal to the length of (crypto block + Auth Code). + // When that runs out we can detect these final sections. + int lengthToRead = BLOCK_AND_AUTH - byteCount; + UpdateSlideBufferIfNeeded(lengthToRead); + + int obtained = await StreamUtils.ReadRequestedBytesAsync(_stream, _slideBuffer, _slideBufFreePos, lengthToRead); _slideBufFreePos += obtained; - // Recalculate how much data we now have - byteCount = _slideBufFreePos - _slideBufStartPos; - if (byteCount >= BLOCK_AND_AUTH) + // Transform data from the slide buffer + if (TransformFromSlideBuffer(buffer, ref offset, bytesLeftToRead, ref nBytes)) { - var read = TransformAndBufferBlock(buffer, offset, bytesLeftToRead, CRYPTO_BLOCK_SIZE); - nBytes += read; - offset += read; + // Reached the auth code + break; } - else + } + return nBytes; + } + + // Helper function to update the slide buffer, if we need to + private void UpdateSlideBufferIfNeeded(int lengthToRead) + { + if (_slideBuffer.Length - _slideBufFreePos < lengthToRead) + { + // Shift the data to the beginning of the buffer + int iTo = 0; + for (int iFrom = _slideBufStartPos; iFrom < _slideBufFreePos; iFrom++, iTo++) { - // Last round. - if (byteCount > AUTH_CODE_LENGTH) - { - // At least one byte of data plus auth code - int finalBlock = byteCount - AUTH_CODE_LENGTH; - nBytes += TransformAndBufferBlock(buffer, offset, bytesLeftToRead, finalBlock); - } - else if (byteCount < AUTH_CODE_LENGTH) - throw new Exception("Internal error missed auth code"); // Coding bug - // Final block done. Check Auth code. - byte[] calcAuthCode = _transform.GetAuthCode(); - for (int i = 0; i < AUTH_CODE_LENGTH; i++) + _slideBuffer[iTo] = _slideBuffer[iFrom]; + } + _slideBufFreePos -= _slideBufStartPos; // Note the -= + _slideBufStartPos = 0; + } + } + + // A helper to do the non-async crypto transform, using data from the in-memory slide buffer + // Returns true if the auth code has been reached, false if not. + private bool TransformFromSlideBuffer(byte[] buffer, ref int offset, int bytesLeftToRead, ref int nBytes) + { + // Recalculate how much data we now have + int byteCount = _slideBufFreePos - _slideBufStartPos; + if (byteCount >= BLOCK_AND_AUTH) + { + var read = TransformAndBufferBlock(buffer, offset, bytesLeftToRead, CRYPTO_BLOCK_SIZE); + nBytes += read; + offset += read; + + return false; + } + else + { + // Last round. + if (byteCount > AUTH_CODE_LENGTH) + { + // At least one byte of data plus auth code + int finalBlock = byteCount - AUTH_CODE_LENGTH; + nBytes += TransformAndBufferBlock(buffer, offset, bytesLeftToRead, finalBlock); + } + else if (byteCount < AUTH_CODE_LENGTH) + throw new Exception("Internal error missed auth code"); // Coding bug + // Final block done. Check Auth code. + byte[] calcAuthCode = _transform.GetAuthCode(); + for (int i = 0; i < AUTH_CODE_LENGTH; i++) + { + if (calcAuthCode[i] != _slideBuffer[_slideBufStartPos + i]) { - if (calcAuthCode[i] != _slideBuffer[_slideBufStartPos + i]) - { - throw new Exception("AES Authentication Code does not match. This is a super-CRC check on the data in the file after compression and encryption. \r\n" - + "The file may be damaged."); - } + throw new Exception("AES Authentication Code does not match. This is a super-CRC check on the data in the file after compression and encryption. \r\n" + + "The file may be damaged."); } + } - // don't need this any more, so use it as a 'complete' flag - _slideBuffer = null; + // don't need this any more, so use it as a 'complete' flag + _slideBuffer = null; - break; // Reached the auth code - } + return true; // Reached the auth code } - return nBytes; } // read some buffered data - private int ReadBufferedData(byte[] buffer, int offset, int count) + private int ReadBufferedData(byte[] buffer, ref int offset, ref int count) { - int copyCount = Math.Min(count, _transformBufferFreePos - _transformBufferStartPos); + if (HasBufferedData) + { + int copyCount = Math.Min(count, _transformBufferFreePos - _transformBufferStartPos); + + Array.Copy(_transformBuffer, _transformBufferStartPos, buffer, offset, copyCount); + _transformBufferStartPos += copyCount; - Array.Copy(_transformBuffer, _transformBufferStartPos, buffer, offset, copyCount); - _transformBufferStartPos += copyCount; + offset += copyCount; + count -= copyCount; + + return copyCount; + } - return copyCount; + return 0; } // Perform the crypto transform, and buffer the data if less than one block has been requested.