uploads end in-order

Author: Martijn Hoekstra

Hotsapi.Uploader.Common.Test/ManagerTests.cs

@@ -1,6 +1,7 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 
 namespace Hotsapi.Uploader.Common.Test
@@ -14,68 +15,27 @@ public static Task ShortRandomDelay()
             var delay = r.Next(100, 200);
             return Task.Delay(delay);
         }
-        private static IEnumerable<ReplayFile> ThreeInOrder
+        private static IEnumerable<ReplayFile> FilesInOrder
         {
             get {
-                var one = new ReplayFile("one") {
-                    Created = new DateTime(2020, 1, 1, 0, 0, 1)
-                };
-                var two = new ReplayFile("two") {
-                    Created = new DateTime(2020, 1, 1, 0, 0, 10)
-                };
-                var three = new ReplayFile("three") {
-                    Created = new DateTime(2020, 1, 1, 0, 0, 20)
-                };
-                var initialFiles = new List<ReplayFile>() { one, two, three };
-                return initialFiles;
+                var next = new DateTime(2020, 1, 1, 0, 0, 0);
+                var increment = new TimeSpan(0, 0, 1);
+                var rand = new Random();
+                var nums = Enumerable.Range(1, 24).OrderBy(rf => rand.NextDouble());
+                
+                return nums.Select(i => {
+                    next += increment;
+                    return new ReplayFile($"upload_{i}") {
+                        Created = next
+                    };
+                });
+
             }
         }
-
-        [TestMethod]
-        [Ignore("Known intermittant failure: multiple uploads are started in parallel and don't always start in order")]
-        public async Task InitialFilesStartInOrder()
-        {
-            var initialFiles = ThreeInOrder;
-
-            var manager = new Manager(new MockStorage(initialFiles));
-            var uploadTester = new MockUploader();
-            
-            var promise = new TaskCompletionSource<int>();
-            Task done = promise.Task;
-
-            var uploadsSeen = 0;
-            var l = new object();
-            ReplayFile lastUploadStarted = null;
-            uploadTester.SetUploadCallback(async rf => {
-                if (lastUploadStarted != null) {
-                    try {
-                        Assert.IsTrue(rf.Created >= lastUploadStarted.Created, $"upload started out of order, {lastUploadStarted} started after {rf}");
-                    } catch (Exception e) {
-                        promise.TrySetException(e);
-                    }
-                }
-                lastUploadStarted = rf;
-                await ShortRandomDelay();
-                var isDone = false;
-                lock (l) {
-                    uploadsSeen++;
-                    isDone = uploadsSeen >= 3;
-                }
-                if (isDone) {
-                    promise.TrySetResult(uploadsSeen);
-                }
-            });
-
-            manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
-            await done;
-        }
-
-
 
         [TestMethod]
-        [Ignore("Known intermittant failure: multiple uploads are started in parallel and don't always end in order")]
         public async Task InitialFilesEndInorder() {
-            var initialFiles = ThreeInOrder;
+            var initialFiles = FilesInOrder;
 
             var manager = new Manager(new MockStorage(initialFiles));
             var uploadTester = new MockUploader();
@@ -85,11 +45,11 @@ public async Task InitialFilesEndInorder() {
             var uploadsSeen = 0;
             var l = new object();
             ReplayFile lastUploadFinished = null;
-            uploadTester.SetUploadCallback(async rf => {
-                await ShortRandomDelay();
+            uploadTester.UploadFinished = async rf => {
                 if (lastUploadFinished != null) {
                     try {
-                        Assert.IsTrue(rf.Created >= lastUploadFinished.Created, $"upload completed out of order, {lastUploadFinished} completed after {rf}");
+                        var isInOrder = rf.Created >= lastUploadFinished.Created;
+                        Assert.IsTrue(isInOrder, $"upload completed out of order, {rf} completed after {lastUploadFinished}");
                     }
                     catch (Exception e) {
                         promise.TrySetException(e);
@@ -104,7 +64,7 @@ public async Task InitialFilesEndInorder() {
                 if (isDone) {
                     promise.TrySetResult(uploadsSeen);
                 }
-            });
+            };
 
             manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
             await done;
@@ -113,29 +73,27 @@ public async Task InitialFilesEndInorder() {
         [TestMethod]
         public async Task AllInitialFilesProcessed()
         {
-            var initialFiles = ThreeInOrder;
+            var initialFiles = FilesInOrder;
 
             var manager = new Manager(new MockStorage(initialFiles));
             var uploadTester = new MockUploader();
             var done = new TaskCompletionSource<int>();
 
             var uploadsSeen = 0;
             object l = new object();
-            uploadTester.SetUploadCallback(async rf => {
+            uploadTester.UploadFinished = async rf => {
                 await ShortRandomDelay();
                 lock (l) {
                     uploadsSeen++;
                     if (uploadsSeen >= 3) {
                         done.SetResult(uploadsSeen);
                     }
                 }
-            });
+            };
 
             manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
             var num = await done.Task;
-            //var finished = await Task.WhenAny(Task.Delay(4000), done.Task);
-            //await finished;
-            Assert.AreEqual(3, uploadsSeen);
+            Assert.AreEqual(3, num);
         }
     }
 }

Hotsapi.Uploader.Common.Test/MockAnalizer.cs

@@ -9,7 +9,9 @@ private class MockAnalizer : IAnalyzer
             public int MinimumBuild { get; set; }
             public Replay Analyze(ReplayFile file) {
                 file.UploadStatus = UploadStatus.Preprocessed;
-                return new Replay();
+                return new Replay() {
+                    Timestamp = file.Created
+                };
             }
             public string GetFingerprint(Replay replay) => "dummy fingerprint";
         }

Hotsapi.Uploader.Common.Test/MockUploader.cs

@@ -10,33 +10,27 @@ private class MockUploader : IUploader
         {
             public bool UploadToHotslogs { get; set; }
 
-            private Func<ReplayFile, Task> UploadCallback = _ => Task.CompletedTask;
-            public void SetUploadCallback(Func<ReplayFile, Task> onUpload)
-            {
-                var old = UploadCallback;
-
-                UploadCallback = async (ReplayFile file) => {
-                    await old(file);
-                    await onUpload(file);
-                };
-            }
+            public Func<ReplayFile, Task> UploadStarted { get; set; } = _ => Task.CompletedTask;
+            public Func<ReplayFile, Task> UploadFinished { get; set; } = _ => Task.CompletedTask;
 
             public async Task CheckDuplicate(IEnumerable<ReplayFile> replays)
             {
                 foreach (var replay in replays) {
                     replay.UploadStatus = UploadStatus.ReadyForUpload;
                 }
-                await ShortRandomDelay(); //todo: put this elsewhere
+                await ShortRandomDelay();
             }
             public Task<int> GetMinimumBuild() => Task.FromResult(1);
-            public Task Upload(ReplayFile file)
+            public async Task Upload(ReplayFile file, Task mayComplete)
             {
-                UploadCallback(file);
-                return Task.CompletedTask;
+                await UploadStarted(file);
+                await Upload(file.Filename, mayComplete);
+                await UploadFinished(file);
             }
-            public async Task<UploadStatus> Upload(string file)
+            public async Task<UploadStatus> Upload(string file, Task mayComplete)
             {
-                await Task.Delay(100);
+                await ShortRandomDelay();
+                await mayComplete;
                 return UploadStatus.Success;
             }
         }

Hotsapi.Uploader.Common/IUploader.cs

@@ -8,7 +8,7 @@ public interface IUploader
         bool UploadToHotslogs { get; set; }
         Task CheckDuplicate(IEnumerable<ReplayFile> replays);
         Task<int> GetMinimumBuild();
-        Task Upload(ReplayFile file);
-        Task<UploadStatus> Upload(string file);
+        Task Upload(ReplayFile file, Task mayComplete);
+        Task<UploadStatus> Upload(string file, Task mayComplete);
     }
 }