Merge pull request #140 from bezzad/feature/inmemory_buffering_limitation

bezzad · web-flow · commit 7726685a2255 · 2023-06-03T16:04:14.000+03:30
Feature/inmemory buffering limitation
diff --git a/README.md b/README.md
@@ -52,6 +52,7 @@ Downloader is compatible with .NET Standard 2.0 and above, running on Windows, L
 - Live streaming support, suitable for playing music at the same time as downloading.
 - Ability to download just a certain range of bytes of a large file.
 - Code is tiny, fast and does not depend on external libraries.
+- Control the amount of system memroy (RAM) that the Downloader consumes during downloading.
 
 ---
 
@@ -93,6 +94,8 @@ var downloadOpt = new DownloadConfiguration()
     MaximumBytesPerSecond = 1024*1024*2, 
     // the maximum number of times to fail
     MaxTryAgainOnFailover = 5,    
+    // release memory buffer after each 50 MB
+    MaximumMemoryBufferBytes = 1024 * 1024 * 50, 
     // download parts of file as parallel or not. Default value is false
     ParallelDownload = true,
     // number of parallel downloads. The default value is the same as the chunk count
diff --git a/src/Downloader.Test/Downloader.Test.csproj b/src/Downloader.Test/Downloader.Test.csproj
@@ -24,19 +24,19 @@
     <PackageReference Include="AssertMessage.Fody" Version="2.1.0">
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Fody" Version="6.6.4">
+    <PackageReference Include="Fody" Version="6.7.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
-    <PackageReference Include="Moq" Version="4.18.2" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
-    <PackageReference Include="coverlet.collector" Version="3.2.0">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="Moq" Version="4.18.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.0.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.0.3" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs b/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs
@@ -124,7 +124,7 @@ public void TestPackageSituationAfterDispose()
             Package.TotalFileSize = sampleDataLength * 64;
             Options.ChunkCount = 1;
             new ChunkHub(Options).SetFileChunks(Package);
-            Package.BuildStorage(false);
+            Package.BuildStorage(false, 1024 * 1024);
             Package.Storage.WriteAsync(0, sampleData, sampleDataLength);
             Package.Storage.Flush();
 
@@ -145,7 +145,7 @@ public async Task TestPackageChunksDataAfterDispose()
             var dummyData = DummyData.GenerateOrderedBytes(chunkSize);
             Options.ChunkCount = 64;
             Package.TotalFileSize = chunkSize * 64;
-            Package.BuildStorage(false);
+            Package.BuildStorage(false, 1024 * 1024);
             new ChunkHub(Options).SetFileChunks(Package);
             for (int i = 0; i < Package.Chunks.Length; i++)
             {
diff --git a/src/Downloader.Test/UnitTests/DownloadPackageTest.cs b/src/Downloader.Test/UnitTests/DownloadPackageTest.cs
@@ -19,7 +19,7 @@ public virtual void Initial()
         {
             Config = new DownloadConfiguration() { ChunkCount = 8 };
             Data = DummyData.GenerateOrderedBytes(DummyFileHelper.FileSize16Kb);
-            Package.BuildStorage(false);
+            Package.BuildStorage(false, 1024 * 1024);
             new ChunkHub(Config).SetFileChunks(Package);
             Package.Storage.WriteAsync(0, Data, DummyFileHelper.FileSize16Kb);
             Package.Storage.Flush();
diff --git a/src/Downloader.Test/UnitTests/DownloadPackageTestOnFile.cs b/src/Downloader.Test/UnitTests/DownloadPackageTestOnFile.cs
@@ -53,7 +53,7 @@ private void BuildStorageTest(bool reserveSpace)
             };
 
             // act
-            Package.BuildStorage(reserveSpace);
+            Package.BuildStorage(reserveSpace, 1024*1024);
 
             // assert
             Assert.IsInstanceOfType(Package.Storage.OpenRead(), typeof(FileStream));
diff --git a/src/Downloader.Test/UnitTests/RequestTest.cs b/src/Downloader.Test/UnitTests/RequestTest.cs
@@ -389,7 +389,7 @@ public void GetRedirectUrlByLocationTest()
 
             // assert
             Assert.AreNotEqual(url, redirectUrl);
-            Assert.AreNotEqual(request.Address, redirectUrl);
+            Assert.AreNotEqual(request.Address.ToString(), redirectUrl);
             Assert.AreEqual(redirectUrl, actualRedirectUrl.AbsoluteUri);
         }
 
@@ -408,7 +408,7 @@ public void GetRedirectUrlWithoutLocationTest()
 
             // assert
             Assert.AreNotEqual(url, redirectUrl);
-            Assert.AreNotEqual(request.Address, redirectUrl);
+            Assert.AreNotEqual(request.Address.ToString(), redirectUrl);
             Assert.AreEqual(redirectUrl, actualRedirectUrl.AbsoluteUri);
         }
 
diff --git a/src/Downloader/ConcurrentStream.cs b/src/Downloader/ConcurrentStream.cs
@@ -8,11 +8,11 @@ namespace Downloader
 {
     public class ConcurrentStream : IDisposable
     {
-        private readonly SemaphoreSlim _queueCheckerSemaphore = new SemaphoreSlim(0);
+        private readonly SemaphoreSlim _queueConsumerLocker = new SemaphoreSlim(0);
         private readonly ManualResetEventSlim _completionEvent = new ManualResetEventSlim(true);
+        private readonly ManualResetEventSlim _stopWriteNewPacketEvent = new ManualResetEventSlim(true);
         private readonly ConcurrentBag<Packet> _inputBag = new ConcurrentBag<Packet>();
-        private int? _resourceReleaseThreshold;
-        private long _packetCounter = 0;
+        private long _maxMemoryBufferBytes = 0;
         private bool _disposed;
         private Stream _stream;
         private string _path;
@@ -29,7 +29,7 @@ public string Path
                 }
             }
         }
-        
+
         public byte[] Data
         {
             get
@@ -49,24 +49,35 @@ public byte[] Data
 
         public long Length => _stream?.Length ?? 0;
 
-        public ConcurrentStream(Stream stream)
+        public long MaxMemoryBufferBytes
+        {
+            get => _maxMemoryBufferBytes;
+            set
+            {
+                _maxMemoryBufferBytes = (value <= 0) ? long.MaxValue : value;
+            }
+        }
+
+        public ConcurrentStream(Stream stream, long maxMemoryBufferBytes = 0)
         {
             _stream = stream;
+            MaxMemoryBufferBytes = maxMemoryBufferBytes;
             Initial();
         }
 
-        public ConcurrentStream(string filename, long initSize)
+        public ConcurrentStream(string filename, long initSize, long maxMemoryBufferBytes = 0)
         {
             _path = filename;
             _stream = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read);
+            MaxMemoryBufferBytes = maxMemoryBufferBytes;
 
             if (initSize > 0)
                 _stream.SetLength(initSize);
 
             Initial();
         }
 
-        public ConcurrentStream()
+        public ConcurrentStream() // parameterless constructor for deserialization
         {
             _stream = new MemoryStream();
             Initial();
@@ -88,44 +99,46 @@ public Stream OpenRead()
 
         public void WriteAsync(long position, byte[] bytes, int length)
         {
+            _stopWriteNewPacketEvent.Wait();
             _inputBag.Add(new Packet(position, bytes, length));
             _completionEvent.Reset();
-            _queueCheckerSemaphore.Release();
+            _queueConsumerLocker.Release();
+            ReleaseQueue(length);
         }
 
         private async Task Watcher()
         {
             while (!_disposed)
             {
-                await _queueCheckerSemaphore.WaitAsync().ConfigureAwait(false);
+                await _queueConsumerLocker.WaitAsync().ConfigureAwait(false);
                 if (_inputBag.TryTake(out var packet))
                 {
                     await WritePacket(packet).ConfigureAwait(false);
-                    ReleasePackets(packet.Data.Length);
                     packet.Dispose();
                 }
             }
         }
+
         private async Task WritePacket(Packet packet)
         {
             if (_stream.CanSeek)
             {
                 _stream.Position = packet.Position;
                 await _stream.WriteAsync(packet.Data, 0, packet.Length).ConfigureAwait(false);
-                _packetCounter++;
             }
 
             if (_inputBag.IsEmpty)
                 _completionEvent.Set();
         }
 
-        private void ReleasePackets(int packetSize)
+        private void ReleaseQueue(int packetSize)
         {
-            _resourceReleaseThreshold ??= 1024 * 1024 * 50 / packetSize; // 50MB / a packet size
-
             // Clean up RAM every _resourceReleaseThreshold packet
-            if (_packetCounter % _resourceReleaseThreshold == 0)
-                GC.Collect();
+            if (MaxMemoryBufferBytes < packetSize * _inputBag.Count)
+            {
+                _stopWriteNewPacketEvent.Set();
+                Flush();
+            }
         }
 
         public void Flush()
@@ -141,7 +154,7 @@ public void Dispose()
             {
                 Flush();
                 _disposed = true;
-                _queueCheckerSemaphore.Dispose();
+                _queueConsumerLocker.Dispose();
                 _stream.Dispose();
             }
         }
diff --git a/src/Downloader/DownloadConfiguration.cs b/src/Downloader/DownloadConfiguration.cs
@@ -9,8 +9,9 @@ public class DownloadConfiguration : ICloneable, INotifyPropertyChanged
         private int _bufferBlockSize;
         private int _chunkCount;
         private long _maximumBytesPerSecond;
+        private int _maximumTryAgainOnFailover;
+        private long _maximumMemoryBufferBytes;
         private bool _checkDiskSizeBeforeDownload;
-        private int _maxTryAgainOnFailover;
         private bool _parallelDownload;
         private int _parallelCount;
         private int _timeout;
@@ -26,7 +27,7 @@ public class DownloadConfiguration : ICloneable, INotifyPropertyChanged
         public DownloadConfiguration()
         {
             RequestConfiguration = new RequestConfiguration(); // default requests configuration
-            _maxTryAgainOnFailover = int.MaxValue; // the maximum number of times to fail.
+            _maximumTryAgainOnFailover = int.MaxValue; // the maximum number of times to fail.
             _parallelDownload = false; // download parts of file as parallel or not
             _parallelCount = 0; // number of parallel downloads
             _chunkCount = 1; // file parts to download
@@ -117,10 +118,10 @@ public long MaximumBytesPerSecond
         /// </summary>
         public int MaxTryAgainOnFailover
         {
-            get => _maxTryAgainOnFailover;
+            get => _maximumTryAgainOnFailover;
             set
             {
-                _maxTryAgainOnFailover = value;
+                _maximumTryAgainOnFailover = value;
                 OnPropertyChanged();
             }
         }
@@ -249,6 +250,29 @@ public bool ReserveStorageSpaceBeforeStartingDownload
             }
         }
 
+        /// <summary>
+        /// Gets or sets the maximum amount of memory, in bytes, that the Downloader library is allowed
+        /// to allocate for buffering downloaded content. Once this limit is reached, the library will
+        /// stop downloading and start writing the buffered data to a file stream before continuing.
+        /// The default value for is 0, which indicates unlimited buffering.
+        /// </summary>
+        /// <example>
+        /// The following example sets the maximum memory buffer to 50 MB, causing the library to release
+        /// the memory buffer after each 50 MB of downloaded content:
+        /// <code>
+        /// MaximumMemoryBufferBytes = 1024 * 1024 * 50
+        /// </code>
+        /// </example>
+        public long MaximumMemoryBufferBytes
+        {
+            get => _maximumMemoryBufferBytes;
+            set
+            {
+                _maximumMemoryBufferBytes = value;
+                OnPropertyChanged();
+            }
+        }
+
         public object Clone()
         {
             return MemberwiseClone();
diff --git a/src/Downloader/DownloadPackage.cs b/src/Downloader/DownloadPackage.cs
@@ -50,12 +50,12 @@ public void Validate()
             }
         }
 
-        public void BuildStorage(bool reserveFileSize)
+        public void BuildStorage(bool reserveFileSize, long maxMemoryBufferBytes = 0)
         {
             if (string.IsNullOrWhiteSpace(FileName))
-                Storage = new ConcurrentStream();
+                Storage = new ConcurrentStream() { MaxMemoryBufferBytes = maxMemoryBufferBytes };
             else
-                Storage = new ConcurrentStream(FileName, reserveFileSize ? TotalFileSize : 0);
+                Storage = new ConcurrentStream(FileName, reserveFileSize ? TotalFileSize : 0, maxMemoryBufferBytes);
         }
     }
 }
diff --git a/src/Downloader/DownloadService.cs b/src/Downloader/DownloadService.cs
@@ -178,7 +178,7 @@ private async Task<Stream> StartDownload()
                 await _singleInstanceSemaphore.WaitAsync();
                 Package.TotalFileSize = await _requestInstance.GetFileSize().ConfigureAwait(false);
                 Package.IsSupportDownloadInRange = await _requestInstance.IsSupportDownloadInRange().ConfigureAwait(false);
-                Package.BuildStorage(Options.ReserveStorageSpaceBeforeStartingDownload);
+                Package.BuildStorage(Options.ReserveStorageSpaceBeforeStartingDownload, Options.MaximumMemoryBufferBytes);
                 ValidateBeforeChunking();
                 _chunkHub.SetFileChunks(Package);
 
diff --git a/src/Downloader/Downloader.csproj b/src/Downloader/Downloader.csproj
@@ -10,7 +10,7 @@
     <PackageProjectUrl>https://github.com/bezzad/Downloader</PackageProjectUrl>
     <RepositoryUrl>https://github.com/bezzad/Downloader</RepositoryUrl>
     <PackageTags>download-manager, downloader, download, idm, internet, streaming, download-file, stream-downloader, multipart-download</PackageTags>
-    <PackageReleaseNotes>Added task async method for the download cancel operation. #133</PackageReleaseNotes>
+    <PackageReleaseNotes>Added task async method for the download cancel operation. #132</PackageReleaseNotes>
     <SignAssembly>true</SignAssembly>
     <AssemblyOriginatorKeyFile>Downloader.snk</AssemblyOriginatorKeyFile>
     <Copyright>Copyright (C) 2019-2022 Behzad Khosravifar</Copyright>
diff --git a/src/Samples/Downloader.Sample/DownloadList.json b/src/Samples/Downloader.Sample/DownloadList.json
@@ -8,8 +8,8 @@
   //  "Url": "https://c2rsetup.officeapps.live.com/c2r/downloadVS.aspx?sku=community&channel=Release&version=VS2022&source=VSLandingPage&includeRecommended=true&cid=2030:9bf2104738684908988ca7dcd5dafed1"
   //},
   {
-    "FileName": "D:\\TestDownload\\LocalFile100MB_Raw.dat",
-    "Url": "http://localhost:3333/dummyfile/file/size/104857600"
+    "FileName": "D:\\TestDownload\\LocalFile1GB_Raw.dat",
+    "Url": "http://localhost:3333/dummyfile/file/size/1073741824"
   },
   {
     "FolderPath": "D:\\TestDownload",
diff --git a/src/Samples/Downloader.Sample/Downloader.Sample.csproj b/src/Samples/Downloader.Sample/Downloader.Sample.csproj
@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="ShellProgressBar" Version="5.2.0" />
   </ItemGroup>
 
diff --git a/src/Samples/Downloader.Sample/Helper.cs b/src/Samples/Downloader.Sample/Helper.cs
@@ -1,9 +1,12 @@
 ﻿using System;
+using System.Diagnostics;
 
 namespace Downloader.Sample
 {
     public static class Helper
     {
+        private static Process CurrentProcess = Process.GetCurrentProcess();
+
         public static string CalcMemoryMensurableUnit(this long bytes)
         {
             return CalcMemoryMensurableUnit((double)bytes);
@@ -51,12 +54,14 @@ public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e, bool
             string bytesReceived = e.ReceivedBytesSize.CalcMemoryMensurableUnit();
             string totalBytesToReceive = e.TotalBytesToReceive.CalcMemoryMensurableUnit();
             string progressPercentage = $"{e.ProgressPercentage:F3}".Replace("/", ".");
+            string usedMemory = CurrentProcess.WorkingSet64.CalcMemoryMensurableUnit();
 
             Console.Title = $"{progressPercentage}%  -  " +
                             $"{speed}/s (avg: {avgSpeed}/s)  -  " +
                             $"{estimateTime} {timeLeftUnit} left   -  " +
                             $"Active Chunks: {e.ActiveChunks}   -   " +
                             $"[{bytesReceived} of {totalBytesToReceive}]   " +
+                            $"[{usedMemory} memory buffer]   " +
                             (isPaused ? " - Paused" : "");
         }
     }
diff --git a/src/Samples/Downloader.Sample/Program.cs b/src/Samples/Downloader.Sample/Program.cs
@@ -117,6 +117,7 @@ private static DownloadConfiguration GetDownloadConfiguration()
                 ChunkCount = 8,             // file parts to download, default value is 1
                 MaximumBytesPerSecond = 1024 * 1024 * 10, // download speed limited to 10MB/s, default values is zero or unlimited
                 MaxTryAgainOnFailover = 5,  // the maximum number of times to fail
+                MaximumMemoryBufferBytes = 1024 * 1024 * 50, // release memory buffer after each 50 MB
                 ParallelDownload = true,    // download parts of file as parallel or not. Default value is false
                 ParallelCount = 4,          // number of parallel downloads. The default value is the same as the chunk count
                 Timeout = 3000,             // timeout (millisecond) per stream block reader, default value is 1000

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ public virtual void Initial()`
`19`	`19`	`{`
`20`	`20`	`Config = new DownloadConfiguration() { ChunkCount = 8 };`
`21`	`21`	`Data = DummyData.GenerateOrderedBytes(DummyFileHelper.FileSize16Kb);`
`22`		`- Package.BuildStorage(false);`
	`22`	`+ Package.BuildStorage(false, 1024 * 1024);`
`23`	`23`	`new ChunkHub(Config).SetFileChunks(Package);`
`24`	`24`	`Package.Storage.WriteAsync(0, Data, DummyFileHelper.FileSize16Kb);`
`25`	`25`	`Package.Storage.Flush();`
Original file line number	Diff line number	Diff line change
`@@ -389,7 +389,7 @@ public void GetRedirectUrlByLocationTest()`
`389`	`389`
`390`	`390`	`// assert`
`391`	`391`	`Assert.AreNotEqual(url, redirectUrl);`
`392`		`- Assert.AreNotEqual(request.Address, redirectUrl);`
	`392`	`+ Assert.AreNotEqual(request.Address.ToString(), redirectUrl);`
`393`	`393`	`Assert.AreEqual(redirectUrl, actualRedirectUrl.AbsoluteUri);`
`394`	`394`	`}`
`395`	`395`
`@@ -408,7 +408,7 @@ public void GetRedirectUrlWithoutLocationTest()`
`408`	`408`
`409`	`409`	`// assert`
`410`	`410`	`Assert.AreNotEqual(url, redirectUrl);`
`411`		`- Assert.AreNotEqual(request.Address, redirectUrl);`
	`411`	`+ Assert.AreNotEqual(request.Address.ToString(), redirectUrl);`
`412`	`412`	`Assert.AreEqual(redirectUrl, actualRedirectUrl.AbsoluteUri);`
`413`	`413`	`}`
`414`	`414`
Original file line number	Diff line number	Diff line change
`@@ -8,11 +8,11 @@ namespace Downloader`
`8`	`8`	`{`
`9`	`9`	`public class ConcurrentStream : IDisposable`
`10`	`10`	`{`
`11`		`- private readonly SemaphoreSlim _queueCheckerSemaphore = new SemaphoreSlim(0);`
	`11`	`+ private readonly SemaphoreSlim _queueConsumerLocker = new SemaphoreSlim(0);`
`12`	`12`	`private readonly ManualResetEventSlim _completionEvent = new ManualResetEventSlim(true);`
	`13`	`+ private readonly ManualResetEventSlim _stopWriteNewPacketEvent = new ManualResetEventSlim(true);`
`13`	`14`	`private readonly ConcurrentBag<Packet> _inputBag = new ConcurrentBag<Packet>();`
`14`		`- private int? _resourceReleaseThreshold;`
`15`		`- private long _packetCounter = 0;`
	`15`	`+ private long _maxMemoryBufferBytes = 0;`
`16`	`16`	`private bool _disposed;`
`17`	`17`	`private Stream _stream;`
`18`	`18`	`private string _path;`
`@@ -29,7 +29,7 @@ public string Path`
`29`	`29`	`}`
`30`	`30`	`}`
`31`	`31`	`}`
`32`		`-`
	`32`	`+`
`33`	`33`	`public byte[] Data`
`34`	`34`	`{`
`35`	`35`	`get`
`@@ -49,24 +49,35 @@ public byte[] Data`
`49`	`49`
`50`	`50`	`public long Length => _stream?.Length ?? 0;`
`51`	`51`
`52`		`- public ConcurrentStream(Stream stream)`
	`52`	`+ public long MaxMemoryBufferBytes`
	`53`	`+ {`
	`54`	`+ get => _maxMemoryBufferBytes;`
	`55`	`+ set`
	`56`	`+ {`
	`57`	`+ _maxMemoryBufferBytes = (value <= 0) ? long.MaxValue : value;`
	`58`	`+ }`
	`59`	`+ }`
	`60`	`+`
	`61`	`+ public ConcurrentStream(Stream stream, long maxMemoryBufferBytes = 0)`
`53`	`62`	`{`
`54`	`63`	`_stream = stream;`
	`64`	`+ MaxMemoryBufferBytes = maxMemoryBufferBytes;`
`55`	`65`	`Initial();`
`56`	`66`	`}`
`57`	`67`
`58`		`- public ConcurrentStream(string filename, long initSize)`
	`68`	`+ public ConcurrentStream(string filename, long initSize, long maxMemoryBufferBytes = 0)`
`59`	`69`	`{`
`60`	`70`	`_path = filename;`
`61`	`71`	`_stream = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read);`
	`72`	`+ MaxMemoryBufferBytes = maxMemoryBufferBytes;`
`62`	`73`
`63`	`74`	`if (initSize > 0)`
`64`	`75`	`_stream.SetLength(initSize);`
`65`	`76`
`66`	`77`	`Initial();`
`67`	`78`	`}`
`68`	`79`
`69`		`- public ConcurrentStream()`
	`80`	`+ public ConcurrentStream() // parameterless constructor for deserialization`
`70`	`81`	`{`
`71`	`82`	`_stream = new MemoryStream();`
`72`	`83`	`Initial();`
`@@ -88,44 +99,46 @@ public Stream OpenRead()`
`88`	`99`
`89`	`100`	`public void WriteAsync(long position, byte[] bytes, int length)`
`90`	`101`	`{`
	`102`	`+ _stopWriteNewPacketEvent.Wait();`
`91`	`103`	`_inputBag.Add(new Packet(position, bytes, length));`
`92`	`104`	`_completionEvent.Reset();`
`93`		`- _queueCheckerSemaphore.Release();`
	`105`	`+ _queueConsumerLocker.Release();`
	`106`	`+ ReleaseQueue(length);`
`94`	`107`	`}`
`95`	`108`
`96`	`109`	`private async Task Watcher()`
`97`	`110`	`{`
`98`	`111`	`while (!_disposed)`
`99`	`112`	`{`
`100`		`- await _queueCheckerSemaphore.WaitAsync().ConfigureAwait(false);`
	`113`	`+ await _queueConsumerLocker.WaitAsync().ConfigureAwait(false);`
`101`	`114`	`if (_inputBag.TryTake(out var packet))`
`102`	`115`	`{`
`103`	`116`	`await WritePacket(packet).ConfigureAwait(false);`
`104`		`- ReleasePackets(packet.Data.Length);`
`105`	`117`	`packet.Dispose();`
`106`	`118`	`}`
`107`	`119`	`}`
`108`	`120`	`}`
	`121`	`+`
`109`	`122`	`private async Task WritePacket(Packet packet)`
`110`	`123`	`{`
`111`	`124`	`if (_stream.CanSeek)`
`112`	`125`	`{`
`113`	`126`	`_stream.Position = packet.Position;`
`114`	`127`	`await _stream.WriteAsync(packet.Data, 0, packet.Length).ConfigureAwait(false);`
`115`		`- _packetCounter++;`
`116`	`128`	`}`
`117`	`129`
`118`	`130`	`if (_inputBag.IsEmpty)`
`119`	`131`	`_completionEvent.Set();`
`120`	`132`	`}`
`121`	`133`
`122`		`- private void ReleasePackets(int packetSize)`
	`134`	`+ private void ReleaseQueue(int packetSize)`
`123`	`135`	`{`
`124`		`- _resourceReleaseThreshold ??= 1024 * 1024 * 50 / packetSize; // 50MB / a packet size`
`125`		`-`
`126`	`136`	`// Clean up RAM every _resourceReleaseThreshold packet`
`127`		`- if (_packetCounter % _resourceReleaseThreshold == 0)`
`128`		`- GC.Collect();`
	`137`	`+ if (MaxMemoryBufferBytes < packetSize * _inputBag.Count)`
	`138`	`+ {`
	`139`	`+ _stopWriteNewPacketEvent.Set();`
	`140`	`+ Flush();`
	`141`	`+ }`
`129`	`142`	`}`
`130`	`143`
`131`	`144`	`public void Flush()`
`@@ -141,7 +154,7 @@ public void Dispose()`
`141`	`154`	`{`
`142`	`155`	`Flush();`
`143`	`156`	`_disposed = true;`
`144`		`- _queueCheckerSemaphore.Dispose();`
	`157`	`+ _queueConsumerLocker.Dispose();`
`145`	`158`	`_stream.Dispose();`
`146`	`159`	`}`
`147`	`160`	`}`
Original file line number	Diff line number	Diff line change
`@@ -50,12 +50,12 @@ public void Validate()`
`50`	`50`	`}`
`51`	`51`	`}`
`52`	`52`
`53`		`- public void BuildStorage(bool reserveFileSize)`
	`53`	`+ public void BuildStorage(bool reserveFileSize, long maxMemoryBufferBytes = 0)`
`54`	`54`	`{`
`55`	`55`	`if (string.IsNullOrWhiteSpace(FileName))`
`56`		`- Storage = new ConcurrentStream();`
	`56`	`+ Storage = new ConcurrentStream() { MaxMemoryBufferBytes = maxMemoryBufferBytes };`
`57`	`57`	`else`
`58`		`- Storage = new ConcurrentStream(FileName, reserveFileSize ? TotalFileSize : 0);`
	`58`	`+ Storage = new ConcurrentStream(FileName, reserveFileSize ? TotalFileSize : 0, maxMemoryBufferBytes);`
`59`	`59`	`}`
`60`	`60`	`}`
`61`	`61`	`}`
Original file line number	Diff line number	Diff line change
`@@ -8,8 +8,8 @@`
`8`	`8`	`// "Url": "https://c2rsetup.officeapps.live.com/c2r/downloadVS.aspx?sku=community&channel=Release&version=VS2022&source=VSLandingPage&includeRecommended=true&cid=2030:9bf2104738684908988ca7dcd5dafed1"`
`9`	`9`	`//},`
`10`	`10`	`{`
`11`		`- "FileName": "D:\\TestDownload\\LocalFile100MB_Raw.dat",`
`12`		`- "Url": "http://localhost:3333/dummyfile/file/size/104857600"`
	`11`	`+ "FileName": "D:\\TestDownload\\LocalFile1GB_Raw.dat",`
	`12`	`+ "Url": "http://localhost:3333/dummyfile/file/size/1073741824"`
`13`	`13`	`},`
`14`	`14`	`{`
`15`	`15`	`"FolderPath": "D:\\TestDownload",`
Original file line number	Diff line number	Diff line change
`@@ -1,9 +1,12 @@`
`1`	`1`	`using System;`
	`2`	`+using System.Diagnostics;`
`2`	`3`
`3`	`4`	`namespace Downloader.Sample`
`4`	`5`	`{`
`5`	`6`	`public static class Helper`
`6`	`7`	`{`
	`8`	`+ private static Process CurrentProcess = Process.GetCurrentProcess();`
	`9`	`+`
`7`	`10`	`public static string CalcMemoryMensurableUnit(this long bytes)`
`8`	`11`	`{`
`9`	`12`	`return CalcMemoryMensurableUnit((double)bytes);`
`@@ -51,12 +54,14 @@ public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e, bool`
`51`	`54`	`string bytesReceived = e.ReceivedBytesSize.CalcMemoryMensurableUnit();`
`52`	`55`	`string totalBytesToReceive = e.TotalBytesToReceive.CalcMemoryMensurableUnit();`
`53`	`56`	`string progressPercentage = $"{e.ProgressPercentage:F3}".Replace("/", ".");`
	`57`	`+ string usedMemory = CurrentProcess.WorkingSet64.CalcMemoryMensurableUnit();`
`54`	`58`
`55`	`59`	`Console.Title = $"{progressPercentage}% - " +`
`56`	`60`	`$"{speed}/s (avg: {avgSpeed}/s) - " +`
`57`	`61`	`$"{estimateTime} {timeLeftUnit} left - " +`
`58`	`62`	`$"Active Chunks: {e.ActiveChunks} - " +`
`59`	`63`	`$"[{bytesReceived} of {totalBytesToReceive}] " +`
	`64`	`+ $"[{usedMemory} memory buffer] " +`
`60`	`65`	`(isPaused ? " - Paused" : "");`
`61`	`66`	`}`
`62`	`67`	`}`