Metadata Downloading (#3525)

API.Tests/AbstractDbTest.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Data.Common;
 using System.IO.Abstractions.TestingHelpers;
 using System.Linq;
@@ -20,14 +21,15 @@
 
 namespace API.Tests;
 
-public abstract class AbstractDbTest
+public abstract class AbstractDbTest : IDisposable
 {
     protected readonly DbConnection _connection;
     protected readonly DataContext _context;
     protected readonly IUnitOfWork _unitOfWork;
 
 
     protected const string CacheDirectory = "C:/kavita/config/cache/";
+    protected const string CacheLongDirectory = "C:/kavita/config/cache-long/";
     protected const string CoverImageDirectory = "C:/kavita/config/covers/";
     protected const string BackupDirectory = "C:/kavita/config/backups/";
     protected const string LogDirectory = "C:/kavita/config/logs/";
@@ -38,21 +40,22 @@ public abstract class AbstractDbTest
 
     protected AbstractDbTest()
     {
-        var contextOptions = new DbContextOptionsBuilder()
+        var contextOptions = new DbContextOptionsBuilder<DataContext>()
             .UseSqlite(CreateInMemoryDatabase())
             .Options;
+
         _connection = RelationalOptionsExtension.Extract(contextOptions).Connection;
 
         _context = new DataContext(contextOptions);
+
+        _context.Database.EnsureCreated(); // Ensure DB schema is created
+
         Task.Run(SeedDb).GetAwaiter().GetResult();
 
         var config = new MapperConfiguration(cfg => cfg.AddProfile<AutoMapperProfiles>());
         var mapper = config.CreateMapper();
 
-        // Set up Hangfire to use in-memory storage for testing
         GlobalConfiguration.Configuration.UseInMemoryStorage();
-
-
         _unitOfWork = new UnitOfWork(_context, mapper, null);
     }
 
@@ -66,29 +69,43 @@ private static DbConnection CreateInMemoryDatabase()
 
     private async Task<bool> SeedDb()
     {
-        await _context.Database.MigrateAsync();
-        var filesystem = CreateFileSystem();
+        try
+        {
+            await _context.Database.EnsureCreatedAsync();
+            var filesystem = CreateFileSystem();
+
+            await Seed.SeedSettings(_context, new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem));
 
-        await Seed.SeedSettings(_context, new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem));
+            var setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.CacheDirectory).SingleAsync();
+            setting.Value = CacheDirectory;
 
-        var setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.CacheDirectory).SingleAsync();
-        setting.Value = CacheDirectory;
+            setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BackupDirectory).SingleAsync();
+            setting.Value = BackupDirectory;
 
-        setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BackupDirectory).SingleAsync();
-        setting.Value = BackupDirectory;
+            setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BookmarkDirectory).SingleAsync();
+            setting.Value = BookmarkDirectory;
 
-        setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BookmarkDirectory).SingleAsync();
-        setting.Value = BookmarkDirectory;
+            setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.TotalLogs).SingleAsync();
+            setting.Value = "10";
 
-        setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.TotalLogs).SingleAsync();
-        setting.Value = "10";
+            _context.ServerSetting.Update(setting);
 
-        _context.ServerSetting.Update(setting);
 
-        _context.Library.Add(new LibraryBuilder("Manga")
-            .WithFolderPath(new FolderPathBuilder("C:/data/").Build())
-            .Build());
-        return await _context.SaveChangesAsync() > 0;
+            _context.Library.Add(new LibraryBuilder("Manga")
+                .WithFolderPath(new FolderPathBuilder(DataDirectory).Build())
+                .Build());
+
+            await _context.SaveChangesAsync();
+
+            await Seed.SeedMetadataSettings(_context);
+
+            return true;
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"[SeedDb] Error: {ex.Message}");
+            return false;
+        }
     }
 
     protected abstract Task ResetDb();
@@ -99,6 +116,7 @@ protected static MockFileSystem CreateFileSystem()
         fileSystem.Directory.SetCurrentDirectory("C:/kavita/");
         fileSystem.AddDirectory("C:/kavita/config/");
         fileSystem.AddDirectory(CacheDirectory);
+        fileSystem.AddDirectory(CacheLongDirectory);
         fileSystem.AddDirectory(CoverImageDirectory);
         fileSystem.AddDirectory(BackupDirectory);
         fileSystem.AddDirectory(BookmarkDirectory);
@@ -109,4 +127,10 @@ protected static MockFileSystem CreateFileSystem()
 
         return fileSystem;
     }
+
+    public void Dispose()
+    {
+        _context.Dispose();
+        _connection.Dispose();
+    }
 }

API.Tests/Extensions/SeriesFilterTests.cs

@@ -932,7 +932,8 @@ private async Task<AppUser> SetupHasRating()
 
         var seriesService = new SeriesService(_unitOfWork, Substitute.For<IEventHub>(),
             Substitute.For<ITaskScheduler>(), Substitute.For<ILogger<SeriesService>>(),
-            Substitute.For<IScrobblingService>(), Substitute.For<ILocalizationService>());
+            Substitute.For<IScrobblingService>(), Substitute.For<ILocalizationService>()
+            , Substitute.For<IImageService>());
 
         // Select 0 Rating
         var zeroRating = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(2);

API.Tests/Services/SeriesServiceTests.cs

@@ -56,7 +56,7 @@ public SeriesServiceTests()
 
         _seriesService = new SeriesService(_unitOfWork, Substitute.For<IEventHub>(),
             Substitute.For<ITaskScheduler>(), Substitute.For<ILogger<SeriesService>>(),
-            Substitute.For<IScrobblingService>(), locService);
+            Substitute.For<IScrobblingService>(), locService, Substitute.For<IImageService>());
     }
     #region Setup
 

API/API.csproj

@@ -197,6 +197,7 @@
     <Content Include="EmailTemplates\**">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
+    <Folder Include="Data\ManualMigrations\v0.8.3\" />
     <Folder Include="Extensions\KavitaPlus\" />
     <None Include="I18N\**" />
   </ItemGroup>

API/Controllers/LibraryController.cs

@@ -81,7 +81,8 @@ public async Task<ActionResult> AddLibrary(UpdateLibraryDto dto)
             .WithIncludeInDashboard(dto.IncludeInDashboard)
             .WithManageCollections(dto.ManageCollections)
             .WithManageReadingLists(dto.ManageReadingLists)
-            .WIthAllowScrobbling(dto.AllowScrobbling)
+            .WithAllowScrobbling(dto.AllowScrobbling)
+            .WithAllowMetadataMatching(dto.AllowMetadataMatching)
             .Build();
 
         library.LibraryFileTypes = dto.FileGroupTypes

API/Controllers/SeriesController.cs

@@ -631,13 +631,13 @@ public async Task<ActionResult<IList<ExternalSeriesMatchDto>>> MatchSeries(Match
     /// <summary>
     /// This will perform the fix match
     /// </summary>
-    /// <param name="dto"></param>
+    /// <param name="aniListId"></param>
     /// <param name="seriesId"></param>
     /// <returns></returns>
     [HttpPost("update-match")]
-    public async Task<ActionResult> UpdateSeriesMatch(ExternalSeriesDetailDto dto, [FromQuery] int seriesId)
+    public async Task<ActionResult> UpdateSeriesMatch([FromQuery] int seriesId, [FromQuery] int aniListId)
     {
-        await _externalMetadataService.FixSeriesMatch(seriesId, dto);
+        await _externalMetadataService.FixSeriesMatch(seriesId, aniListId);
 
         return Ok();
     }

API/Controllers/SettingsController.cs

@@ -5,6 +5,7 @@
 using System.Threading.Tasks;
 using API.Data;
 using API.DTOs.Email;
+using API.DTOs.KavitaPlus.Metadata;
 using API.DTOs.Settings;
 using API.Entities;
 using API.Entities.Enums;
@@ -534,4 +535,73 @@ public async Task<ActionResult<EmailTestResultDto>> TestEmailServiceUrl()
         if (string.IsNullOrEmpty(user?.Email)) return BadRequest("Your account has no email on record. Cannot email.");
         return Ok(await _emailService.SendTestEmail(user!.Email));
     }
+
+    /// <summary>
+    /// Get the metadata settings for Kavita+ users.
+    /// </summary>
+    /// <returns></returns>
+    [Authorize(Policy = "RequireAdminRole")]
+    [HttpGet("metadata-settings")]
+    public async Task<ActionResult<MetadataSettingsDto>> GetMetadataSettings()
+    {
+        return Ok(await _unitOfWork.SettingsRepository.GetMetadataSettingDto());
+
+    }
+
+    /// <summary>
+    /// Update the metadata settings for Kavita+ users
+    /// </summary>
+    /// <param name="dto"></param>
+    /// <returns></returns>
+    [Authorize(Policy = "RequireAdminRole")]
+    [HttpPost("metadata-settings")]
+    public async Task<ActionResult<MetadataSettingsDto>> UpdateMetadataSettings(MetadataSettingsDto dto)
+    {
+        var existingMetadataSetting = await _unitOfWork.SettingsRepository.GetMetadataSettings();
+        existingMetadataSetting.Enabled = dto.Enabled;
+        existingMetadataSetting.EnableSummary = dto.EnableSummary;
+        existingMetadataSetting.EnablePublicationStatus = dto.EnablePublicationStatus;
+        existingMetadataSetting.EnableRelationships = dto.EnableRelationships;
+        existingMetadataSetting.EnablePeople = dto.EnablePeople;
+        existingMetadataSetting.EnableStartDate = dto.EnableStartDate;
+        existingMetadataSetting.EnableGenres = dto.EnableGenres;
+        existingMetadataSetting.EnableTags = dto.EnableTags;
+        existingMetadataSetting.PersonRoles = dto.PersonRoles;
+        existingMetadataSetting.FirstLastPeopleNaming = dto.FirstLastPeopleNaming;
+
+        existingMetadataSetting.AgeRatingMappings = dto.AgeRatingMappings ?? [];
+
+        existingMetadataSetting.Blacklist = dto.Blacklist.DistinctBy(d => d.ToNormalized()).ToList() ?? [];
+        existingMetadataSetting.Whitelist = dto.Whitelist.DistinctBy(d => d.ToNormalized()).ToList() ?? [];
+
+        // Handle Field Mappings
+        if (dto.FieldMappings != null)
+        {
+            // Clear existing mappings
+            existingMetadataSetting.FieldMappings ??= [];
+            _unitOfWork.SettingsRepository.RemoveRange(existingMetadataSetting.FieldMappings);
+
+            existingMetadataSetting.FieldMappings.Clear();
+
+
+            // Add new mappings
+            foreach (var mappingDto in dto.FieldMappings)
+            {
+                existingMetadataSetting.FieldMappings.Add(new MetadataFieldMapping
+                {
+                    SourceType = mappingDto.SourceType,
+                    DestinationType = mappingDto.DestinationType,
+                    SourceValue = mappingDto.SourceValue,
+                    DestinationValue = mappingDto.DestinationValue,
+                    ExcludeFromSource = mappingDto.ExcludeFromSource
+                });
+            }
+        }
+
+        // Save changes
+        await _unitOfWork.CommitAsync();
+
+        // Return updated settings
+        return Ok(await _unitOfWork.SettingsRepository.GetMetadataSettingDto());
+    }
 }

API/Controllers/UploadController.cs

@@ -8,6 +8,7 @@
 using API.Entities.Enums;
 using API.Extensions;
 using API.Services;
+using API.Services.Tasks.Metadata;
 using API.SignalR;
 using Flurl.Http;
 using Microsoft.AspNetCore.Authorization;
@@ -31,11 +32,12 @@ public class UploadController : BaseApiController
     private readonly IEventHub _eventHub;
     private readonly IReadingListService _readingListService;
     private readonly ILocalizationService _localizationService;
+    private readonly ICoverDbService _coverDbService;
 
     /// <inheritdoc />
     public UploadController(IUnitOfWork unitOfWork, IImageService imageService, ILogger<UploadController> logger,
         ITaskScheduler taskScheduler, IDirectoryService directoryService, IEventHub eventHub, IReadingListService readingListService,
-        ILocalizationService localizationService)
+        ILocalizationService localizationService, ICoverDbService coverDbService)
     {
         _unitOfWork = unitOfWork;
         _imageService = imageService;
@@ -45,6 +47,7 @@ public UploadController(IUnitOfWork unitOfWork, IImageService imageService, ILog
         _eventHub = eventHub;
         _readingListService = readingListService;
         _localizationService = localizationService;
+        _coverDbService = coverDbService;
     }
 
     /// <summary>
@@ -495,34 +498,8 @@ public async Task<ActionResult> UploadPersonCoverImageFromUrl(UploadFileDto uplo
             var person = await _unitOfWork.PersonRepository.GetPersonById(uploadFileDto.Id);
             if (person == null) return BadRequest(await _localizationService.Translate(User.GetUserId(), "person-doesnt-exist"));
 
-            if (!string.IsNullOrEmpty(uploadFileDto.Url))
-            {
-                var filePath = await CreateThumbnail(uploadFileDto, $"{ImageService.GetPersonFormat(uploadFileDto.Id)}");
-
-                if (!string.IsNullOrEmpty(filePath))
-                {
-                    person.CoverImage = filePath;
-                    person.CoverImageLocked = true;
-                    _imageService.UpdateColorScape(person);
-                    _unitOfWork.PersonRepository.Update(person);
-                }
-            }
-            else
-            {
-                person.CoverImage = string.Empty;
-                person.CoverImageLocked = false;
-                _imageService.UpdateColorScape(person);
-                _unitOfWork.PersonRepository.Update(person);
-            }
-
-            if (_unitOfWork.HasChanges())
-            {
-                await _unitOfWork.CommitAsync();
-                await _eventHub.SendMessageAsync(MessageFactory.CoverUpdate,
-                    MessageFactory.CoverUpdateEvent(person.Id, MessageFactoryEntityTypes.Person), false);
-                return Ok();
-            }
-
+            await _coverDbService.SetPersonCoverImage(person, uploadFileDto.Url, true);
+            return Ok();
         }
         catch (Exception e)
         {

API/Controllers/UsersController.cs

@@ -135,6 +135,14 @@ public async Task<ActionResult<UserPreferencesDto>> UpdatePreferences(UserPrefer
         existingPreferences.PdfScrollMode = preferencesDto.PdfScrollMode;
         existingPreferences.PdfSpreadMode = preferencesDto.PdfSpreadMode;
 
+        if (await _licenseService.HasActiveLicense())
+        {
+            existingPreferences.AniListScrobblingEnabled = preferencesDto.AniListScrobblingEnabled;
+            existingPreferences.WantToReadSync = preferencesDto.WantToReadSync;
+        }
+
+
+
         if (preferencesDto.Theme != null && existingPreferences.Theme.Id != preferencesDto.Theme?.Id)
         {
             var theme = await _unitOfWork.SiteThemeRepository.GetTheme(preferencesDto.Theme!.Id);
@@ -147,6 +155,7 @@ public async Task<ActionResult<UserPreferencesDto>> UpdatePreferences(UserPrefer
             existingPreferences.Locale = preferencesDto.Locale;
         }
 
+
         _unitOfWork.UserRepository.Update(existingPreferences);
 
         if (!await _unitOfWork.CommitAsync()) return BadRequest(await _localizationService.Translate(User.GetUserId(), "generic-user-pref"));

API/DTOs/KavitaPlus/ExternalMetadata/SeriesDetailPlusApiDto.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using API.DTOs.Recommendation;
 using API.DTOs.Scrobbling;
 using API.DTOs.SeriesDetail;
 
@@ -9,6 +10,7 @@ internal class SeriesDetailPlusApiDto
     public IEnumerable<MediaRecommendationDto> Recommendations { get; set; }
     public IEnumerable<UserReviewDto> Reviews { get; set; }
     public IEnumerable<RatingDto> Ratings { get; set; }
+    public ExternalSeriesDetailDto? Series { get; set; }
     public int? AniListId { get; set; }
     public long? MalId { get; set; }
 }