Worker + Discord Webhook refactor (#575)

- Refactor workers to use DataContext
This makes Workers compatible with `IDataConvertable`. Should help us
expand Workers in the future as a side bonus. Admittedly, this causes a
bit of jank particularly in tests and having to store the match service,
but I think it should be okay.
- Convert to API data types in DiscordIntegrationWorker
This allows us to get the same results as the API (e.g. applying PSP
path, cropped asset URLs, etc.) for free, avoiding code duplication.
Closes #574.

Refresh.GameServer/RefreshGameServer.cs

@@ -36,6 +36,7 @@ public class RefreshGameServer : RefreshServer
 
     protected readonly GameDatabaseProvider _databaseProvider;
     protected readonly IDataStore _dataStore;
+    protected MatchService _matchService = null!;
 
     protected GameServerConfig? _config;
     protected IntegrationConfig? _integrationConfig;
@@ -63,7 +64,6 @@ public RefreshGameServer(
             GameDatabaseProvider provider = databaseProvider.Invoke();
 
             this.WorkerManager?.Stop();
-            this.WorkerManager = new WorkerManager(this.Logger, this._dataStore, provider);
 
             authProvider ??= new GameAuthenticationProvider(this._config!);
 
@@ -114,7 +114,7 @@ protected override void SetupServices()
         this.Server.AddService<TimeProviderService>(this.GetTimeProvider());
         this.Server.AddRateLimitService(new RateLimitSettings(60, 400, 30, "global"));
         this.Server.AddService<CategoryService>();
-        this.Server.AddService<MatchService>();
+        this.Server.AddService(this._matchService = new MatchService(this.Server.Logger));
         this.Server.AddService<ImportService>();
         this.Server.AddService<DocumentationService>();
         this.Server.AddService<GuidCheckerService>();
@@ -147,7 +147,7 @@ protected override void SetupServices()
 
     protected virtual void SetupWorkers()
     {
-        if (this.WorkerManager == null) return;
+        this.WorkerManager = new WorkerManager(this.Logger, this._dataStore, this._databaseProvider, this._matchService);
 
         this.WorkerManager.AddWorker<PunishmentExpiryWorker>();
         this.WorkerManager.AddWorker<ExpiredObjectWorker>();

Refresh.GameServer/Types/Data/DataContextService.cs

@@ -15,8 +15,8 @@ public class DataContextService : Service
     private readonly StorageService _storageService;
     private readonly MatchService _matchService;
     private readonly AuthenticationService _authService;
-    
-    internal DataContextService(StorageService storage, MatchService match, AuthenticationService auth, Logger logger) : base(logger)
+
+    public DataContextService(StorageService storage, MatchService match, AuthenticationService auth, Logger logger) : base(logger)
     {
         this._storageService = storage;
         this._matchService = match;

Refresh.GameServer/Workers/CoolLevelsWorker.cs

@@ -10,6 +10,7 @@
 using Bunkum.Core.Storage;
 using NotEnoughLogs;
 using Refresh.GameServer.Database;
+using Refresh.GameServer.Types.Data;
 using Refresh.GameServer.Types.Levels;
 using Refresh.GameServer.Types.Reviews;
 using Refresh.GameServer.Types.Roles;
@@ -19,10 +20,10 @@ namespace Refresh.GameServer.Workers;
 public class CoolLevelsWorker : IWorker
 {
     public int WorkInterval => 600_000; // Every 10 minutes
-    public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext database)
+    public void DoWork(DataContext context)
     {
         const int pageSize = 1000;
-        DatabaseList<GameLevel> levels = database.GetUserLevelsChunk(0, pageSize);
+        DatabaseList<GameLevel> levels = context.Database.GetUserLevelsChunk(0, pageSize);
 
         // Don't do anything if there are no levels to process.
         if (levels.TotalItems <= 0) return;
@@ -43,34 +44,34 @@ public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext data
 
             foreach (GameLevel level in levels.Items)
             {
-                Log(logger, LogLevel.Trace, "Calculating score for '{0}' ({1})", level.Title, level.LevelId);
-                float decayMultiplier = CalculateLevelDecayMultiplier(logger, now, level);
+                Log(context.Logger, LogLevel.Trace, "Calculating score for '{0}' ({1})", level.Title, level.LevelId);
+                float decayMultiplier = CalculateLevelDecayMultiplier(context.Logger, now, level);
 
                 // Calculate positive & negative score separately so we don't run into issues with
                 // the multiplier having an opposite effect with the negative score as time passes
-                float positiveScore = CalculatePositiveScore(logger, level, database);
-                float negativeScore = CalculateNegativeScore(logger, level, database);
+                float positiveScore = CalculatePositiveScore(level, context);
+                float negativeScore = CalculateNegativeScore(level, context);
 
                 // Increase to tweak how little negative score gets affected by decay
                 const int negativeScoreDecayMultiplier = 2;
 
                 // Weigh everything with the multiplier and set a final score
                 float finalScore = (positiveScore * decayMultiplier) - (negativeScore * Math.Min(1.0f, decayMultiplier * negativeScoreDecayMultiplier));
 
-                Log(logger, LogLevel.Debug, "Score for '{0}' ({1}) is {2}", level.Title, level.LevelId, finalScore);
+                Log(context.Logger, LogLevel.Debug, "Score for '{0}' ({1}) is {2}", level.Title, level.LevelId, finalScore);
                 scoresToSet.Add(level, finalScore);
                 remaining--;
             }
 
             // Commit scores to database. This method lets us use a dictionary so we can batch everything in one write
-            database.SetLevelScores(scoresToSet);
+            context.Database.SetLevelScores(scoresToSet);
 
             // Load the next page
-            levels = database.GetUserLevelsChunk(levels.Items.Count(), pageSize);
+            levels = context.Database.GetUserLevelsChunk(levels.Items.Count(), pageSize);
         }
 
         stopwatch.Stop();
-        logger.LogInfo(RefreshContext.CoolLevels,  "Calculated scores for {0} levels in {1}ms", levels.TotalItems, stopwatch.ElapsedMilliseconds);
+        context.Logger.LogInfo(RefreshContext.CoolLevels,  "Calculated scores for {0} levels in {1}ms", levels.TotalItems, stopwatch.ElapsedMilliseconds);
     }
 
     [Conditional("COOL_DEBUG")]
@@ -97,7 +98,7 @@ private static float CalculateLevelDecayMultiplier(Logger logger, long now, Game
         return multiplier;
     }
 
-    private static float CalculatePositiveScore(Logger logger, GameLevel level, GameDatabaseContext database)
+    private static float CalculatePositiveScore(GameLevel level, DataContext context)
     {
         // Start levels off with a few points to prevent one dislike from bombing the level
         // Don't apply this bonus to reuploads to discourage a flood of 15CR levels.
@@ -111,13 +112,13 @@ private static float CalculatePositiveScore(Logger logger, GameLevel level, Game
         if (level.TeamPicked)
             score += 50;
 
-        int positiveRatings = database.GetTotalRatingsForLevel(level, RatingType.Yay);
-        int negativeRatings = database.GetTotalRatingsForLevel(level, RatingType.Boo);
-        int uniquePlays = database.GetUniquePlaysForLevel(level);
+        int positiveRatings = context.Database.GetTotalRatingsForLevel(level, RatingType.Yay);
+        int negativeRatings = context.Database.GetTotalRatingsForLevel(level, RatingType.Boo);
+        int uniquePlays = context.Database.GetUniquePlaysForLevel(level);
 
         score += positiveRatings * positiveRatingPoints;
         score += uniquePlays * uniquePlayPoints;
-        score += database.GetFavouriteCountForLevel(level) * heartPoints;
+        score += context.Database.GetFavouriteCountForLevel(level) * heartPoints;
 
         // Reward for a good ratio between plays and yays
         float ratingRatio = (positiveRatings - negativeRatings) / (float)uniquePlays;
@@ -129,11 +130,11 @@ private static float CalculatePositiveScore(Logger logger, GameLevel level, Game
         if (level.Publisher?.Role == GameUserRole.Trusted)
             score += trustedAuthorPoints;
 
-        Log(logger, LogLevel.Trace, "positiveScore is {0}", score);
+        Log(context.Logger, LogLevel.Trace, "positiveScore is {0}", score);
         return score;
     }
 
-    private static float CalculateNegativeScore(Logger logger, GameLevel level, GameDatabaseContext database)
+    private static float CalculateNegativeScore(GameLevel level, DataContext context)
     {
         float penalty = 0;
         const float negativeRatingPenalty = 5;
@@ -144,7 +145,7 @@ private static float CalculateNegativeScore(Logger logger, GameLevel level, Game
         // The percentage of how much penalty should be applied at the end of the calculation.
         const float penaltyMultiplier = 0.75f;
 
-        penalty += database.GetTotalRatingsForLevel(level, RatingType.Boo) * negativeRatingPenalty;
+        penalty += context.Database.GetTotalRatingsForLevel(level, RatingType.Boo) * negativeRatingPenalty;
 
         if (level.Publisher == null)
             penalty += noAuthorPenalty;
@@ -153,7 +154,7 @@ private static float CalculateNegativeScore(Logger logger, GameLevel level, Game
         else if (level.Publisher?.Role == GameUserRole.Banned)
             penalty += bannedAuthorPenalty;
 
-        Log(logger, LogLevel.Trace, "negativeScore is {0}", penalty);
+        Log(context.Logger, LogLevel.Trace, "negativeScore is {0}", penalty);
         return penalty * penaltyMultiplier;
     }
 }

Refresh.GameServer/Workers/DiscordIntegrationWorker.cs

@@ -1,15 +1,12 @@
-using Bunkum.Core.Storage;
 using Discord;
 using Discord.Webhook;
-using NotEnoughLogs;
-using Refresh.GameServer.Authentication;
 using Refresh.GameServer.Configuration;
 using Refresh.GameServer.Database;
+using Refresh.GameServer.Endpoints.ApiV3.DataTypes.Response.Levels;
+using Refresh.GameServer.Endpoints.ApiV3.DataTypes.Response.Users;
+using Refresh.GameServer.Endpoints.ApiV3.DataTypes.Response.Users.Photos;
 using Refresh.GameServer.Types.Activity;
-using Refresh.GameServer.Types.Levels;
-using Refresh.GameServer.Types.Photos;
-using Refresh.GameServer.Types.UserData;
-using Refresh.GameServer.Types.UserData.Leaderboard;
+using Refresh.GameServer.Types.Data;
 
 namespace Refresh.GameServer.Workers;
 
@@ -44,14 +41,22 @@ private string GetAssetUrl(string hash)
         return $"{this._externalUrl}/api/v3/assets/{hash}/image";
     }
 
-    private Embed? GenerateEmbedFromEvent(Event @event, GameDatabaseContext database)
+    private Embed? GenerateEmbedFromEvent(Event @event, DataContext context)
     {
         EmbedBuilder embed = new();
 
-        GameLevel? level = @event.StoredDataType == EventDataType.Level ? database.GetLevelById(@event.StoredSequentialId!.Value) : null;
-        GameUser? user = @event.StoredDataType == EventDataType.User ? database.GetUserByObjectId(@event.StoredObjectId) : null;
-        GameSubmittedScore? score = @event.StoredDataType == EventDataType.Score ? database.GetScoreByObjectId(@event.StoredObjectId) : null;
-        GamePhoto? photo = @event.StoredDataType == EventDataType.Photo ? database.GetPhotoFromEvent(@event) : null;
+        ApiGameLevelResponse? level = @event.StoredDataType == EventDataType.Level ? 
+            ApiGameLevelResponse.FromOld(context.Database.GetLevelById(@event.StoredSequentialId!.Value), context)
+            : null;
+        ApiGameUserResponse? user = @event.StoredDataType == EventDataType.User ? 
+            ApiGameUserResponse.FromOld(context.Database.GetUserByObjectId(@event.StoredObjectId), context)
+            : null;
+        ApiGameScoreResponse? score = @event.StoredDataType == EventDataType.Score ? 
+            ApiGameScoreResponse.FromOld(context.Database.GetScoreByObjectId(@event.StoredObjectId), context)
+            : null;
+        ApiGamePhotoResponse? photo = @event.StoredDataType == EventDataType.Photo ? 
+            ApiGamePhotoResponse.FromOld(context.Database.GetPhotoFromEvent(@event), context)
+            : null;
 
         if (photo != null)
             level = photo.Level;
@@ -85,30 +90,26 @@ private string GetAssetUrl(string hash)
         embed.WithDescription($"[{@event.User.Username}]({this._externalUrl}/u/{@event.User.UserId}) {description}");
 
         if (photo != null)
-        {
-            embed.WithImageUrl(this.GetAssetUrl(photo.LargeAsset.IsPSP ? $"psp/{photo.LargeAsset.AssetHash}" : photo.LargeAsset.AssetHash));
-        } else if (level != null)
-        {
-            embed.WithThumbnailUrl(this.GetAssetUrl(level.GameVersion == TokenGame.LittleBigPlanetPSP ? $"psp/{level.IconHash}" : level.IconHash));
-        } else if (user != null)
-        {
+            embed.WithImageUrl(this.GetAssetUrl(photo.LargeHash));
+        else if (level != null) 
+            embed.WithThumbnailUrl(this.GetAssetUrl(level.IconHash));
+        else if (user != null)
             embed.WithThumbnailUrl(this.GetAssetUrl(user.IconHash));
-        }
 
         embed.WithTimestamp(DateTimeOffset.FromUnixTimeMilliseconds(@event.Timestamp));
         embed.WithAuthor(@event.User.Username, this.GetAssetUrl(@event.User.IconHash), $"{this._externalUrl}/u/{@event.UserId}");
 
         return embed.Build();
     }
 
-    public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext database)
+    public void DoWork(DataContext context)
     {
         if (this._firstCycle)
         {
             this.DoFirstCycle();
         }
 
-        DatabaseList<Event> activity = database.GetGlobalRecentActivity(new ActivityQueryParameters
+        DatabaseList<Event> activity = context.Database.GetGlobalRecentActivity(new ActivityQueryParameters
         {
             Timestamp = Now,
             EndTimestamp = this._lastTimestamp,
@@ -123,7 +124,7 @@ public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext data
 
         IEnumerable<Embed> embeds = activity.Items
             .Reverse() // events are descending
-            .Select(e => this.GenerateEmbedFromEvent(e, database))
+            .Select(e => this.GenerateEmbedFromEvent(e, context))
             .Where(e => e != null)
             .ToList()!;
 
@@ -132,6 +133,6 @@ public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext data
         ulong id = this._client.SendMessageAsync(embeds: embeds, 
             username: this._config.DiscordNickname, avatarUrl: this._config.DiscordAvatarUrl).Result;
 
-        logger.LogInfo(RefreshContext.Worker, $"Posted webhook containing {activity.Items.Count()} events with id {id}");
+        context.Logger.LogInfo(RefreshContext.Worker, $"Posted webhook containing {activity.Items.Count()} events with id {id}");
     }
 }

Refresh.GameServer/Workers/ExpiredObjectWorker.cs

@@ -2,37 +2,38 @@
 using NotEnoughLogs;
 using Refresh.GameServer.Authentication;
 using Refresh.GameServer.Database;
+using Refresh.GameServer.Types.Data;
 using Refresh.GameServer.Types.UserData;
 
 namespace Refresh.GameServer.Workers;
 
 public class ExpiredObjectWorker : IWorker
 {
     public int WorkInterval => 60_000; // 1 minute
-    public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext database)
+    public void DoWork(DataContext context)
     {
-        foreach (QueuedRegistration registration in database.GetAllQueuedRegistrations().Items)
+        foreach (QueuedRegistration registration in context.Database.GetAllQueuedRegistrations().Items)
         {
-            if (!database.IsRegistrationExpired(registration)) continue;
+            if (!context.Database.IsRegistrationExpired(registration)) continue;
 
-            logger.LogInfo(RefreshContext.Worker, $"Removed {registration.Username}'s queued registration since it has expired");
-            database.RemoveRegistrationFromQueue(registration);
+            context.Logger.LogInfo(RefreshContext.Worker, $"Removed {registration.Username}'s queued registration since it has expired");
+            context.Database.RemoveRegistrationFromQueue(registration);
         }
 
-        foreach (EmailVerificationCode code in database.GetAllVerificationCodes().Items)
+        foreach (EmailVerificationCode code in context.Database.GetAllVerificationCodes().Items)
         {
-            if (!database.IsVerificationCodeExpired(code)) continue;
+            if (!context.Database.IsVerificationCodeExpired(code)) continue;
 
-            logger.LogInfo(RefreshContext.Worker, $"Removed {code.User}'s verification code since it has expired");
-            database.RemoveEmailVerificationCode(code);
+            context.Logger.LogInfo(RefreshContext.Worker, $"Removed {code.User}'s verification code since it has expired");
+            context.Database.RemoveEmailVerificationCode(code);
         }
 
-        foreach (Token token in database.GetAllTokens().Items)
+        foreach (Token token in context.Database.GetAllTokens().Items)
         {
-            if (!database.IsTokenExpired(token)) continue;
+            if (!context.Database.IsTokenExpired(token)) continue;
 
-            logger.LogInfo(RefreshContext.Worker, $"Removed {token.User}'s {token.TokenType} token since it has expired {DateTimeOffset.Now - token.ExpiresAt} ago");
-            database.RevokeToken(token);
+            context.Logger.LogInfo(RefreshContext.Worker, $"Removed {token.User}'s {token.TokenType} token since it has expired {DateTimeOffset.Now - token.ExpiresAt} ago");
+            context.Database.RevokeToken(token);
         }
     }
 }

Refresh.GameServer/Workers/IWorker.cs

@@ -1,6 +1,5 @@
-using Bunkum.Core.Storage;
 using NotEnoughLogs;
-using Refresh.GameServer.Database;
+using Refresh.GameServer.Types.Data;
 
 namespace Refresh.GameServer.Workers;
 
@@ -10,13 +9,13 @@ public interface IWorker
     /// How often to perform work, in milliseconds
     /// </summary>
     public int WorkInterval { get; }
-    
+
     /// <summary>
     /// Instructs the worker to do work.
     /// </summary>
-    /// <param name="logger">A Refresh logger, able to be operated by the worker.</param>
+    /// <param name="context"></param>
     /// <param name="dataStore">The server's data store, for workers to use.</param>
     /// <param name="database">A database context, for workers to use.</param>
     /// <returns>True if the worker did work, false if it did not.</returns>
-    public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext database);
+    public void DoWork(DataContext context);
 }

Refresh.GameServer/Workers/PunishmentExpiryWorker.cs

@@ -1,6 +1,7 @@
 using Bunkum.Core.Storage;
 using NotEnoughLogs;
 using Refresh.GameServer.Database;
+using Refresh.GameServer.Types.Data;
 using Refresh.GameServer.Types.Roles;
 using Refresh.GameServer.Types.UserData;
 
@@ -13,25 +14,25 @@ public class PunishmentExpiryWorker : IWorker
 {
     public int WorkInterval => 60_000; // 1 minute
 
-    public void DoWork(Logger logger, IDataStore dataStore, GameDatabaseContext database)
+    public void DoWork(DataContext context)
     {
-        DatabaseList<GameUser> bannedUsers = database.GetAllUsersWithRole(GameUserRole.Banned);
-        DatabaseList<GameUser> restrictedUsers = database.GetAllUsersWithRole(GameUserRole.Restricted);
+        DatabaseList<GameUser> bannedUsers = context.Database.GetAllUsersWithRole(GameUserRole.Banned);
+        DatabaseList<GameUser> restrictedUsers = context.Database.GetAllUsersWithRole(GameUserRole.Restricted);
 
         foreach (GameUser user in bannedUsers.Items)
         {
-            if (database.IsUserBanned(user)) continue;
+            if (context.Database.IsUserBanned(user)) continue;
 
-            logger.LogInfo(RefreshContext.Worker, $"Unbanned {user.Username} since their punishment has expired");
-            database.SetUserRole(user, GameUserRole.User);
+            context.Logger.LogInfo(RefreshContext.Worker, $"Unbanned {user.Username} since their punishment has expired");
+            context.Database.SetUserRole(user, GameUserRole.User);
         }
 
         foreach (GameUser user in restrictedUsers.Items)
         {
-            if (database.IsUserRestricted(user)) continue;
+            if (context.Database.IsUserRestricted(user)) continue;
 
-            logger.LogInfo(RefreshContext.Worker, $"Unrestricted {user.Username} since their punishment has expired");
-            database.SetUserRole(user, GameUserRole.User);
+            context.Logger.LogInfo(RefreshContext.Worker, $"Unrestricted {user.Username} since their punishment has expired");
+            context.Database.SetUserRole(user, GameUserRole.User);
         }
     }
 }