Stats & More Polish on Metadata Matching (#3538)

API/Controllers/SettingsController.cs

@@ -572,8 +572,8 @@ public async Task<ActionResult<MetadataSettingsDto>> UpdateMetadataSettings(Meta
 
         existingMetadataSetting.AgeRatingMappings = dto.AgeRatingMappings ?? [];
 
-        existingMetadataSetting.Blacklist = dto.Blacklist.DistinctBy(d => d.ToNormalized()).ToList() ?? [];
-        existingMetadataSetting.Whitelist = dto.Whitelist.DistinctBy(d => d.ToNormalized()).ToList() ?? [];
+        existingMetadataSetting.Blacklist = dto.Blacklist.Where(s => !string.IsNullOrWhiteSpace(s)).DistinctBy(d => d.ToNormalized()).ToList() ?? [];
+        existingMetadataSetting.Whitelist = dto.Whitelist.Where(s => !string.IsNullOrWhiteSpace(s)).DistinctBy(d => d.ToNormalized()).ToList() ?? [];
         existingMetadataSetting.Overrides = dto.Overrides.ToList() ?? [];
 
         // Handle Field Mappings

API/DTOs/Stats/V3/ServerInfoV3Dto.cs

@@ -55,6 +55,11 @@ public class ServerInfoV3Dto
     /// </summary>
     /// <remarks>This pings a health check and does not capture any IP Information</remarks>
     public long TimeToPingKavitaStatsApi { get; set; }
+    /// <summary>
+    /// If using the downloading metadata feature
+    /// </summary>
+    /// <remarks>Kavita+ Only</remarks>
+    public bool MatchedMetadataEnabled { get; set; }
 
 
 

API/Data/Repositories/ExternalSeriesMetadataRepository.cs

@@ -36,7 +36,7 @@ public interface IExternalSeriesMetadataRepository
     Task<SeriesDetailPlusDto?> GetSeriesDetailPlusDto(int seriesId);
     Task LinkRecommendationsToSeries(Series series);
     Task<bool> IsBlacklistedSeries(int seriesId);
-    Task<IList<int>> GetAllSeriesIdsWithoutMetadata(int limit);
+    Task<IList<int>> GetSeriesThatNeedExternalMetadata(int limit, bool includeStaleData = false);
     Task<IList<ManageMatchSeriesDto>> GetAllSeries(ManageMatchFilterDto filter);
 }
 
@@ -209,11 +209,13 @@ public Task<bool> IsBlacklistedSeries(int seriesId)
     }
 
 
-    public async Task<IList<int>> GetAllSeriesIdsWithoutMetadata(int limit)
+    public async Task<IList<int>> GetSeriesThatNeedExternalMetadata(int limit, bool includeStaleData = false)
     {
         return await _context.Series
             .Where(s => !ExternalMetadataService.NonEligibleLibraryTypes.Contains(s.Library.Type))
-            .Where(s => s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc < DateTime.UtcNow)
+            .WhereIf(includeStaleData, s => s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc < DateTime.UtcNow)
+            .Where(s => s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc == DateTime.MinValue)
+            .Where(s => s.Library.AllowMetadataMatching)
             .OrderByDescending(s => s.Library.Type)
             .ThenBy(s => s.NormalizedName)
             .Select(s => s.Id)

API/Extensions/FlurlExtensions.cs

@@ -18,4 +18,16 @@ public static IFlurlRequest WithKavitaPlusHeaders(this string request, string li
             .WithHeader("Content-Type", "application/json")
             .WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs));
     }
+
+    public static IFlurlRequest WithBasicHeaders(this string request, string apiKey)
+    {
+        return request
+            .WithHeader("Accept", "application/json")
+            .WithHeader("User-Agent", "Kavita")
+            .WithHeader("x-api-key", apiKey)
+            .WithHeader("x-installId", HashUtil.ServerToken())
+            .WithHeader("x-kavita-version", BuildInfo.Version)
+            .WithHeader("Content-Type", "application/json")
+            .WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs));
+    }
 }

API/Extensions/QueryExtensions/QueryableExtensions.cs

@@ -294,7 +294,7 @@ public static IQueryable<Series> FilterMatchState(this IQueryable<Series> query,
             MatchStateOption.NotMatched => query.
                 Include(s => s.ExternalSeriesMetadata)
                 .Where(s => (s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc == DateTime.MinValue) && !s.IsBlacklisted && !s.DontMatch),
-            MatchStateOption.Error => query.Where(s => s.IsBlacklisted),
+            MatchStateOption.Error => query.Where(s => s.IsBlacklisted && !s.DontMatch),
             MatchStateOption.DontMatch => query.Where(s => s.DontMatch),
             _ => query
         };

API/Services/Plus/ExternalMetadataService.cs

@@ -44,8 +44,8 @@ public interface IExternalMetadataService
     /// </summary>
     /// <param name="seriesId"></param>
     /// <param name="libraryType"></param>
-    /// <returns></returns>
-    Task FetchSeriesMetadata(int seriesId, LibraryType libraryType);
+    /// <returns>If the fetch was made</returns>
+    Task<bool> FetchSeriesMetadata(int seriesId, LibraryType libraryType);
 
     Task<IList<MalStackDto>> GetStacksForUser(int userId);
     Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesDto dto);
@@ -73,6 +73,7 @@ public class ExternalMetadataService : IExternalMetadataService
     };
     // Allow 50 requests per 24 hours
     private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(24), false);
+    static bool IsRomanCharacters(string input) => Regex.IsMatch(input, @"^[\p{IsBasicLatin}\p{IsLatin-1Supplement}]+$");
 
     public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper,
         ILicenseService licenseService, IScrobblingService scrobblingService, IEventHub eventHub, ICoverDbService coverDbService)
@@ -109,7 +110,7 @@ public static bool IsPlusEligible(LibraryType type)
     public async Task FetchExternalDataTask()
     {
         // Find all Series that are eligible and limit
-        var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetAllSeriesIdsWithoutMetadata(25);
+        var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetSeriesThatNeedExternalMetadata(25);
         if (ids.Count == 0) return;
 
         _logger.LogInformation("[Kavita+ Data Refresh] Started Refreshing {Count} series data from Kavita+", ids.Count);
@@ -118,9 +119,9 @@ public async Task FetchExternalDataTask()
         foreach (var seriesId in ids)
         {
             var libraryType = libTypes[seriesId];
-            await FetchSeriesMetadata(seriesId, libraryType);
+            var success = await FetchSeriesMetadata(seriesId, libraryType);
+            if (success) count++;
             await Task.Delay(1500);
-            count++;
         }
         _logger.LogInformation("[Kavita+ Data Refresh] Finished Refreshing {Count} series data from Kavita+", count);
     }
@@ -131,25 +132,25 @@ public async Task FetchExternalDataTask()
     /// </summary>
     /// <param name="seriesId"></param>
     /// <param name="libraryType"></param>
-    public async Task FetchSeriesMetadata(int seriesId, LibraryType libraryType)
+    public async Task<bool> FetchSeriesMetadata(int seriesId, LibraryType libraryType)
     {
-        if (!IsPlusEligible(libraryType)) return;
-        if (!await _licenseService.HasActiveLicense()) return;
+        if (!IsPlusEligible(libraryType)) return false;
+        if (!await _licenseService.HasActiveLicense()) return false;
 
         // Generate key based on seriesId and libraryType or any unique identifier for the request
         // Check if the request is allowed based on the rate limit
         if (!RateLimiter.TryAcquire(string.Empty))
         {
             // Request not allowed due to rate limit
             _logger.LogDebug("Rate Limit hit for Kavita+ prefetch");
-            return;
+            return false;
         }
 
         _logger.LogDebug("Prefetching Kavita+ data for Series {SeriesId}", seriesId);
 
         // Prefetch SeriesDetail data
         await GetSeriesDetailPlus(seriesId, libraryType);
-
+        return true;
     }
 
     public async Task<IList<MalStackDto>> GetStacksForUser(int userId)
@@ -512,31 +513,43 @@ private async Task<bool> WriteExternalMetadataToSeries(ExternalSeriesDetailDto e
 
         var madeModification = false;
 
-        if (settings.EnableLocalizedName && (!series.LocalizedNameLocked || settings.HasOverride(MetadataSettingField.LocalizedName)))
+        if (settings.EnableLocalizedName && (settings.HasOverride(MetadataSettingField.LocalizedName)
+                                             || !series.LocalizedNameLocked && !string.IsNullOrWhiteSpace(series.LocalizedName)))
         {
             // We need to make the best appropriate guess
             if (externalMetadata.Name == series.Name)
             {
                 // Choose closest (usually last) synonym
-                series.LocalizedName = externalMetadata.Synonyms.Last();
+                var validSynonyms = externalMetadata.Synonyms
+                    .Where(IsRomanCharacters)
+                    .Where(s => s.ToNormalized() != series.Name.ToNormalized())
+                    .ToList();
+                if (validSynonyms.Count != 0)
+                {
+                    series.LocalizedName = validSynonyms[^1];
+                    series.LocalizedNameLocked = true;
+                }
             }
-            else
+            else if (IsRomanCharacters(externalMetadata.Name))
             {
                 series.LocalizedName = externalMetadata.Name;
+                series.LocalizedNameLocked = true;
             }
 
+
             madeModification = true;
         }
 
-        if (settings.EnableSummary && (!series.Metadata.SummaryLocked ||
-                                       settings.HasOverride(MetadataSettingField.Summary)))
+        if (settings.EnableSummary && (settings.HasOverride(MetadataSettingField.Summary) ||
+                                       (!series.Metadata.SummaryLocked && !string.IsNullOrWhiteSpace(series.Metadata.Summary))))
         {
             series.Metadata.Summary = CleanSummary(externalMetadata.Summary);
             madeModification = true;
         }
 
-        if (settings.EnableStartDate && externalMetadata.StartDate.HasValue && (!series.Metadata.ReleaseYearLocked ||
-                settings.HasOverride(MetadataSettingField.StartDate)))
+        if (settings.EnableStartDate && externalMetadata.StartDate.HasValue && (settings.HasOverride(MetadataSettingField.StartDate) ||
+                                                                               (!series.Metadata.ReleaseYearLocked &&
+                                                                                   series.Metadata.ReleaseYear == 0)))
         {
             series.Metadata.ReleaseYear = externalMetadata.StartDate.Value.Year;
             madeModification = true;

API/Services/SeriesService.cs

@@ -382,10 +382,12 @@ public static async Task HandlePeopleUpdateAsync(SeriesMetadata metadata, IColle
             // Check if the person exists in the dictionary
             if (existingPeopleDictionary.TryGetValue(normalizedPersonName, out var p))
             {
+                // TODO: Should I add more controls here to map back?
                 if (personDto.AniListId > 0 && p.AniListId <= 0 && p.AniListId != personDto.AniListId)
                 {
                     p.AniListId = personDto.AniListId;
                 }
+                p.Description = string.IsNullOrEmpty(p.Description) ? personDto.Description : p.Description;
                 continue; // If we ever want to update metadata for existing people, we'd do it here
             }
 

API/Services/Tasks/StatsService.cs

@@ -13,14 +13,17 @@
 using API.DTOs.Stats.V3;
 using API.Entities;
 using API.Entities.Enums;
+using API.Extensions;
 using API.Services.Plus;
 using API.Services.Tasks.Scanner.Parser;
 using Flurl.Http;
+using Kavita.Common;
 using Kavita.Common.EnvironmentInfo;
 using Kavita.Common.Helpers;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 
 namespace API.Services.Tasks;
@@ -45,12 +48,12 @@ public class StatsService : IStatsService
     private readonly UserManager<AppUser> _userManager;
     private readonly IEmailService _emailService;
     private readonly ICacheService _cacheService;
-    private const string ApiUrl = "https://stats.kavitareader.com";
+    private readonly string _apiUrl = "";
     private const string ApiKey = "MsnvA2DfQqxSK5jh"; // It's not important this is public, just a way to keep bots from hitting the API willy-nilly
 
     public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork, DataContext context,
         ILicenseService licenseService, UserManager<AppUser> userManager, IEmailService emailService,
-        ICacheService cacheService)
+        ICacheService cacheService, IHostEnvironment environment)
     {
         _logger = logger;
         _unitOfWork = unitOfWork;
@@ -60,7 +63,9 @@ public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork, DataCo
         _emailService = emailService;
         _cacheService = cacheService;
 
-        FlurlConfiguration.ConfigureClientForUrl(ApiUrl);
+        FlurlConfiguration.ConfigureClientForUrl(Configuration.StatsApiUrl);
+
+        _apiUrl = environment.IsDevelopment() ? "http://localhost:5001" : Configuration.StatsApiUrl;
     }
 
     /// <summary>
@@ -98,13 +103,8 @@ private async Task SendDataToStatsServer(ServerInfoV3Dto data)
 
         try
         {
-            var response = await (ApiUrl + "/api/v3/stats")
-                .WithHeader("Accept", "application/json")
-                .WithHeader("User-Agent", "Kavita")
-                .WithHeader("x-api-key", ApiKey)
-                .WithHeader("x-kavita-version", BuildInfo.Version)
-                .WithHeader("Content-Type", "application/json")
-                .WithTimeout(TimeSpan.FromSeconds(30))
+            var response = await (_apiUrl + "/api/v3/stats")
+                .WithBasicHeaders(ApiKey)
                 .PostJsonAsync(data);
 
             if (response.StatusCode != StatusCodes.Status200OK)
@@ -151,12 +151,8 @@ public async Task SendCancellation()
 
         try
         {
-            var response = await (ApiUrl + "/api/v2/stats/opt-out?installId=" + installId)
-                .WithHeader("Accept", "application/json")
-                .WithHeader("User-Agent", "Kavita")
-                .WithHeader("x-api-key", ApiKey)
-                .WithHeader("x-kavita-version", BuildInfo.Version)
-                .WithHeader("Content-Type", "application/json")
+            var response = await (_apiUrl + "/api/v2/stats/opt-out?installId=" + installId)
+                .WithBasicHeaders(ApiKey)
                 .WithTimeout(TimeSpan.FromSeconds(30))
                 .PostAsync();
 
@@ -180,12 +176,8 @@ private static async Task<long> PingStatsApi()
         try
         {
             var sw = Stopwatch.StartNew();
-            var response = await (ApiUrl + "/api/health/")
-                .WithHeader("Accept", "application/json")
-                .WithHeader("User-Agent", "Kavita")
-                .WithHeader("x-api-key", ApiKey)
-                .WithHeader("x-kavita-version", BuildInfo.Version)
-                .WithHeader("Content-Type", "application/json")
+            var response = await (Configuration.StatsApiUrl + "/api/health/")
+                .WithBasicHeaders(ApiKey)
                 .WithTimeout(TimeSpan.FromSeconds(30))
                 .GetAsync();
 
@@ -244,6 +236,7 @@ private async Task<int> MaxChaptersInASeries()
     private async Task<ServerInfoV3Dto> GetStatV3Payload()
     {
         var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
+        var mediaSettings = await _unitOfWork.SettingsRepository.GetMetadataSettings();
         var dto = new ServerInfoV3Dto()
         {
             InstallId = serverSettings.InstallId,
@@ -256,6 +249,7 @@ private async Task<ServerInfoV3Dto> GetStatV3Payload()
             DotnetVersion = Environment.Version.ToString(),
             OpdsEnabled = serverSettings.EnableOpds,
             EncodeMediaAs = serverSettings.EncodeMediaAs,
+            MatchedMetadataEnabled = mediaSettings.Enabled
         };
 
         dto.OsLocale = CultureInfo.CurrentCulture.EnglishName;

Kavita.Common/Configuration.cs

@@ -17,6 +17,7 @@ public static class Configuration
     private static readonly string AppSettingsFilename = Path.Join("config", GetAppSettingFilename());
 
     public static string KavitaPlusApiUrl = "https://plus.kavitareader.com";
+    public static string StatsApiUrl = "https://stats.kavitareader.com";
 
     public static int Port
     {

README.md

@@ -32,6 +32,7 @@ your reading collection with your friends and family!
 - Rich web readers supporting webtoon, continuous reading mode (continue without leaving the reader), virtual pages (epub), etc
 - Ability to customize your dashboard and side nav with smart filters, custom order and visibility toggles
 - Full Localization Support
+- Ability to download metadata (available via [Kavita+](https://wiki.kavitareader.com/kavita+))
 
 
 ## Support

UI/Web/src/app/_single-module/match-series-modal/match-series-modal.component.ts

@@ -70,8 +70,10 @@ export class MatchSeriesModalComponent implements OnInit {
     const model: any = this.formGroup.value;
     model.seriesId = this.series.id;
 
+    const dontMatchChanged = this.series.dontMatch !== model.dontMatch;
+
     // We need to update the dontMatch status
-    if (model.dontMatch) {
+    if (dontMatchChanged) {
       this.seriesService.updateDontMatch(this.series.id, model.dontMatch).subscribe(_ => {
         this.modalService.close(true);
       });

UI/Web/src/app/_single-module/user-scrobble-history/user-scrobble-history.component.html

@@ -37,7 +37,7 @@
     [limit]="pageInfo.size"
   >
 
-    <ngx-datatable-column name="lastModifiedUtc" [sortable]="true" [draggable]="false" [resizeable]="false">
+    <ngx-datatable-column prop="lastModifiedUtc" [sortable]="true" [draggable]="false" [resizeable]="false">
       <ng-template let-column="column" ngx-datatable-header-template>
         {{t('last-modified-header')}}
       </ng-template>
@@ -46,7 +46,7 @@
       </ng-template>
     </ngx-datatable-column>
 
-    <ngx-datatable-column name="scrobbleEventType" [sortable]="true" [draggable]="false" [resizeable]="false">
+    <ngx-datatable-column prop="scrobbleEventType" [sortable]="true" [draggable]="false" [resizeable]="false">
       <ng-template let-column="column" ngx-datatable-header-template>
         {{t('type-header')}}
       </ng-template>
@@ -55,7 +55,7 @@
       </ng-template>
     </ngx-datatable-column>
 
-    <ngx-datatable-column name="seriesName" [sortable]="true" [draggable]="false" [resizeable]="false">
+    <ngx-datatable-column prop="seriesName" [sortable]="true" [draggable]="false" [resizeable]="false">
       <ng-template let-column="column" ngx-datatable-header-template>
         {{t('series-header')}}
       </ng-template>
@@ -64,7 +64,7 @@
       </ng-template>
     </ngx-datatable-column>
 
-    <ngx-datatable-column name="data" [sortable]="false" [draggable]="false" [resizeable]="false">
+    <ngx-datatable-column prop="data" [sortable]="false" [draggable]="false" [resizeable]="false">
       <ng-template let-column="column" ngx-datatable-header-template>
         {{t('data-header')}}
       </ng-template>
@@ -98,7 +98,7 @@
       </ng-template>
     </ngx-datatable-column>
 
-    <ngx-datatable-column name="isPorcessed" [sortable]="true" [draggable]="false" [resizeable]="false">
+    <ngx-datatable-column prop="isPorcessed" [sortable]="true" [draggable]="false" [resizeable]="false">
       <ng-template let-column="column" ngx-datatable-header-template>
         {{t('is-processed-header')}}
       </ng-template>