Skip to content

Commit d20f99c

Browse files
author
Martijn Hoekstra
committed
ensure MaxUploads is respected
1 parent 9ab3b36 commit d20f99c

File tree

5 files changed

+82
-74
lines changed

5 files changed

+82
-74
lines changed

‎Hotsapi.Uploader.Common.Test/ManagerTests.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System;
33
using System.Collections.Generic;
44
using System.Linq;
5+
using System.Threading;
56
using System.Threading.Tasks;
67

78
namespace Hotsapi.Uploader.Common.Test
@@ -95,5 +96,38 @@ public async Task AllInitialFilesProcessed()
9596
var num = await done.Task;
9697
Assert.AreEqual(3, num);
9798
}
99+
100+
[TestMethod]
101+
public async Task UploadIsRateLimited()
102+
{
103+
var initialFiles = FilesInOrder;
104+
105+
var manager = new Manager(new MockStorage(initialFiles));
106+
var uploadTester = new MockUploader();
107+
var simulaneousUploads = 0;
108+
var processedUploads = 0;
109+
var totalFiles = initialFiles.Count();
110+
var isDone = new TaskCompletionSource<int>();
111+
112+
uploadTester.UploadStarted = rf => {
113+
var inFlight = Interlocked.Increment(ref simulaneousUploads);
114+
try {
115+
Assert.IsTrue(inFlight <= Manager.MaxUploads, "may not have more uploads in flight than Manager.MaxUploads");
116+
} catch (Exception e) {
117+
isDone.TrySetException(e);
118+
}
119+
return Task.CompletedTask;
120+
};
121+
uploadTester.UploadFinished = rf => {
122+
Interlocked.Decrement(ref simulaneousUploads);
123+
if(Interlocked.Increment(ref processedUploads) >= totalFiles) {
124+
isDone.SetResult(processedUploads);
125+
}
126+
return Task.CompletedTask;
127+
};
128+
129+
manager.Start(new NoNewFilesMonitor(), new MockAnalizer(), uploadTester);
130+
await isDone.Task;
131+
}
98132
}
99133
}

‎Hotsapi.Uploader.Common/Analyzer.cs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,15 +51,16 @@ public Replay Analyze(ReplayFile file)
5151
return UploadStatus.PtrRegion;
5252

5353
case DataParser.ReplayParseResult.PreAlphaWipe:
54-
return UploadStatus.TooOld;
54+
return UploadStatus.TooOld;
5555
case DataParser.ReplayParseResult.Incomplete:
56+
case DataParser.ReplayParseResult.UnexpectedResult:
5657
return UploadStatus.Incomplete;
5758
}
5859

59-
return parseResult != DataParser.ReplayParseResult.Success ? null
60-
: replay.GameMode == GameMode.Custom ? (UploadStatus?)UploadStatus.CustomGame
61-
: replay.ReplayBuild < MinimumBuild ? (UploadStatus?)UploadStatus.TooOld
62-
: (UploadStatus?)UploadStatus.Preprocessed;
60+
return parseResult != DataParser.ReplayParseResult.Success ? null
61+
: replay.GameMode == GameMode.Custom ? (UploadStatus?)UploadStatus.CustomGame
62+
: replay.ReplayBuild < MinimumBuild ? (UploadStatus?)UploadStatus.TooOld
63+
: (UploadStatus?)UploadStatus.Preprocessed;
6364
}
6465

6566
/// <summary>

‎Hotsapi.Uploader.Common/Manager.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,12 @@ private async Task ParseLoop()
150150
//don't submit files for fingerprinting if we have a younger file in-flight
151151
_ = WorkerPool.RunBackground(async () => {
152152
var replay = _analyzer.Analyze(file);
153+
if(replay == null) {
154+
file.UploadStatus = UploadStatus.UploadError;
155+
}
153156
var doEnqueue = Task.CompletedTask;
154157
lock (l) {
155158
_ = inFlight.Remove(file);
156-
157159
if (replay != null && file.UploadStatus == UploadStatus.Preprocessed) {
158160
submissionBatch.Add((replay, file));
159161
var youngestSubmit = submissionBatch.Select(rp => rp.Item2.Created).Min();
@@ -190,7 +192,7 @@ private async Task FingerprintLoop() {
190192
}
191193
private async Task UploadLoop() {
192194
//Make sure that the next upload doesn't *end* before the previous ended
193-
//but it's OK for multiple uploads to run concurrently
195+
//but it's OK for up to MaxUploads uploads to run concurrently
194196
var previousDone = Task.CompletedTask;
195197
var l = new object();
196198
using (var rateLimitUploading = new SemaphoreSlim(MaxUploads)){
@@ -199,7 +201,7 @@ private async Task UploadLoop() {
199201
foreach (var (replay, replayfile) in parsed) {
200202
if (replayfile.UploadStatus == UploadStatus.ReadyForUpload) {
201203
//don't await the upload task, but bound it by the upload ratelimiter
202-
_ = rateLimitUploading.Locked(async () => {
204+
_ = rateLimitUploading.LockedTask(async () => {
203205
Task thisDone;
204206
lock (l) {
205207
thisDone = DoFileUpload(replayfile, replay, previousDone);

‎Hotsapi.Uploader.Common/Uploader.cs‎

Lines changed: 36 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
using Newtonsoft.Json.Linq;
1+
using Newtonsoft.Json;
2+
using Newtonsoft.Json.Linq;
23
using NLog;
34
using System;
45
using System.Collections.Generic;
5-
using System.IO;
66
using System.Linq;
77
using System.Net;
88
using System.Net.Http;
@@ -14,14 +14,24 @@ public class Uploader : IUploader
1414
{
1515
private static readonly Logger _log = LogManager.GetCurrentClassLogger();
1616
#if DEBUG
17-
private const string ApiEndpoint = "http://hotsapi.local/api/v1";
17+
private const string ApiEndpoint = "https://hotsapi.net/api/v1";
1818
#else
19-
const string ApiEndpoint = "https://hotsapi.net/api/v1";
19+
private const string ApiEndpoint = "https://hotsapi.net/api/v1";
2020
#endif
2121

2222
private string UploadUrl => $"{ApiEndpoint}/upload?uploadToHotslogs={UploadToHotslogs}";
2323
private string BulkFingerprintUrl => $"{ApiEndpoint}/replays/fingerprints?uploadToHotslogs={UploadToHotslogs}";
2424
private string FingerprintOneUrl(string fingerprint) => $"{ApiEndpoint}/replays/fingerprints/v3/{fingerprint}?uploadToHotslogs={UploadToHotslogs}";
25+
private HttpClient _httpClient;
26+
private HttpClient HttpClient
27+
{
28+
get {
29+
if (_httpClient == null) {
30+
_httpClient = new HttpClient();
31+
}
32+
return _httpClient;
33+
}
34+
}
2535

2636

2737
public bool UploadToHotslogs { get; set; }
@@ -41,62 +51,15 @@ public Uploader()
4151
public async Task Upload(ReplayFile file, Task mayComplete)
4252
{
4353
var doDuplicateCheck = file.UploadStatus != UploadStatus.ReadyForUpload;
44-
file.UploadStatus = UploadStatus.Uploading;
4554
if (file.Fingerprint != null && doDuplicateCheck && await CheckDuplicate(file.Fingerprint)) {
4655
_log.Debug($"File {file} marked as duplicate");
4756
file.UploadStatus = UploadStatus.Duplicate;
4857
} else {
58+
file.UploadStatus = UploadStatus.Uploading;
4959
file.UploadStatus = await Upload(file.Filename, mayComplete);
5060
}
5161
}
5262

53-
private class ReplayUpload : HttpContent
54-
{
55-
private const int defaultBuffersize = 1024;
56-
private readonly string filename;
57-
private readonly int buffersize;
58-
private readonly Task mayComplete;
59-
60-
public ReplayUpload(string filename, int buffersize, Task mayComplete)
61-
{
62-
this.filename = filename;
63-
this.buffersize = buffersize;
64-
this.mayComplete = mayComplete;
65-
}
66-
67-
public ReplayUpload(string filename, int buffersize) : this(filename, buffersize, Task.CompletedTask) { }
68-
public ReplayUpload(string filename, Task canComplete) : this(filename, defaultBuffersize, canComplete) { }
69-
public ReplayUpload(string filename) : this(filename, defaultBuffersize, Task.CompletedTask) { }
70-
71-
72-
protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context)
73-
{
74-
using (var input = File.OpenRead(filename)) {
75-
var buffer = new byte[buffersize];
76-
var i = 0;
77-
var done = false;
78-
while (!done) {
79-
var availableSpace = buffer.Length - i;
80-
var bytesRead = await input.ReadAsync(buffer, i, availableSpace);
81-
if (availableSpace > bytesRead) {
82-
done = true;
83-
await mayComplete;
84-
}
85-
await stream.WriteAsync(buffer, i, bytesRead);
86-
i = 0;
87-
}
88-
await stream.FlushAsync();
89-
stream.Close();
90-
input.Close();
91-
}
92-
}
93-
protected override bool TryComputeLength(out long length)
94-
{
95-
length = -1;
96-
return false;
97-
}
98-
}
99-
10063
/// <summary>
10164
/// Upload replay
10265
/// </summary>
@@ -105,23 +68,29 @@ protected override bool TryComputeLength(out long length)
10568
public async Task<UploadStatus> Upload(string file, Task mayComplete)
10669
{
10770
try {
108-
string response;
109-
using (var client = new HttpClient()) {
110-
var upload = new ReplayUpload(file, mayComplete);
111-
var responseMessage = await client.PostAsync(UploadUrl, upload);
112-
response = await responseMessage.Content.ReadAsStringAsync();
113-
}
114-
dynamic json = JObject.Parse(response);
115-
if ((bool)json.success) {
116-
if (Enum.TryParse<UploadStatus>((string)json.status, out var status)) {
117-
_log.Debug($"Uploaded file '{file}': {status}");
118-
return status;
71+
var upload = new ReplayUpload(file, mayComplete);
72+
var multipart = new MultipartFormDataContent {
73+
{ upload, "file", file }
74+
};
75+
var responseMessage = await HttpClient.PostAsync(UploadUrl, multipart);
76+
var response = await responseMessage.Content.ReadAsStringAsync();
77+
try {
78+
dynamic json = JObject.Parse(response);
79+
if ((bool)json.success) {
80+
if (Enum.TryParse<UploadStatus>((string)json.status, out var status)) {
81+
_log.Debug($"Uploaded file '{file}': {status}");
82+
return status;
83+
} else {
84+
_log.Error($"Unknown upload status '{file}': {json.status}");
85+
return UploadStatus.UploadError;
86+
}
11987
} else {
120-
_log.Error($"Unknown upload status '{file}': {json.status}");
88+
_log.Warn($"Error uploading file '{file}': {response}");
12189
return UploadStatus.UploadError;
12290
}
123-
} else {
124-
_log.Warn($"Error uploading file '{file}': {response}");
91+
}
92+
catch(JsonReaderException jre) {
93+
_log.Warn($"Error processing upload response for file '{file}': {jre.Message}");
12594
return UploadStatus.UploadError;
12695
}
12796
}
@@ -237,4 +206,5 @@ private static async Task<bool> CheckApiThrottling(WebResponse response)
237206
}
238207
}
239208
}
209+
240210
}

‎Hotsapi.Uploader.Windows/UIHelpers/UploadColorConverter.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ protected override Brush Convert(UploadStatus value)
1515

1616
case UploadStatus.Preprocessing:
1717
case UploadStatus.Preprocessed:
18+
case UploadStatus.ReadyForUpload:
1819
case UploadStatus.Uploading:
1920
return GetBrush("StatusUploadInProgressBrush");
2021

0 commit comments

Comments
 (0)