From 94cf665be791ac8796fc6ffafe9ddf0a2b2deb2f Mon Sep 17 00:00:00 2001 From: Mbucari <37587114+Mbucari@users.noreply.github.com> Date: Sun, 11 Jan 2026 15:40:11 -0700 Subject: [PATCH] Fix books not being marked absent on large imports --- Source/DtoImporterService/BookImporter.cs | 9 + .../DtoImporterService/LibraryBookImporter.cs | 247 +++++++++--------- Source/LibationUiBase/GridView/EntryStatus.cs | 4 +- 3 files changed, 139 insertions(+), 121 deletions(-) diff --git a/Source/DtoImporterService/BookImporter.cs b/Source/DtoImporterService/BookImporter.cs index 1d85fbfb..c39622f3 100644 --- a/Source/DtoImporterService/BookImporter.cs +++ b/Source/DtoImporterService/BookImporter.cs @@ -18,6 +18,14 @@ namespace DtoImporterService private SeriesImporter seriesImporter { get; } private CategoryImporter categoryImporter { get; } + /// + /// Indicates whether loaded every Book from the during import. + /// If true, the DbContext was queried for all Books, rather than just those being imported. + /// If means that all objects in the DbContext will have their property populated. + /// If false, only those Books being imported were loaded, and some objects will have a null property for books not included in the import set. + /// + internal bool LoadedEntireLibrary {get; private set; } + public BookImporter(LibationContext context) : base(context) { contributorImporter = new ContributorImporter(DbContext); @@ -56,6 +64,7 @@ namespace DtoImporterService .ToArray() .Where(b => productIds.Contains(b.AudibleProductId)) .ToDictionarySafe(b => b.AudibleProductId); + LoadedEntireLibrary = true; } else { diff --git a/Source/DtoImporterService/LibraryBookImporter.cs b/Source/DtoImporterService/LibraryBookImporter.cs index 34ef85f9..b3426e46 100644 --- a/Source/DtoImporterService/LibraryBookImporter.cs +++ b/Source/DtoImporterService/LibraryBookImporter.cs @@ -6,126 +6,135 @@ using DataLayer; using Dinah.Core; using Dinah.Core.Collections.Generic; -namespace DtoImporterService +namespace DtoImporterService; + +public class LibraryBookImporter : ItemsImporterBase { - public class LibraryBookImporter : ItemsImporterBase + protected override IValidator Validator => new LibraryValidator(); + + private BookImporter bookImporter { get; } + + public LibraryBookImporter(LibationContext context) : base(context) { - protected override IValidator Validator => new LibraryValidator(); - - private BookImporter bookImporter { get; } - - public LibraryBookImporter(LibationContext context) : base(context) - { - bookImporter = new BookImporter(DbContext); - } - - protected override int DoImport(IEnumerable importItems) - { - bookImporter.Import(importItems); - - var qtyNew = upsertLibraryBooks(importItems); - return qtyNew; - } - - private int upsertLibraryBooks(IEnumerable importItems) - { - // technically, we should be able to have duplicate books from separate accounts. - // this would violate the current pk and would be difficult to deal with elsewhere: - // - what to show in the grid - // - which to consider liberated - // - // sqlite cannot alter pk. the work around is an extensive headache - // - update: now possible in .net5/efcore5 - // - // currently, inserting LibraryBook will throw error if the same book is in multiple accounts for the same region. - // - // CURRENT SOLUTION: don't re-insert - - - //When Books are upserted during the BookImporter run, they are linked to their LibraryBook in the DbContext - //instance. If a LibraryBook has a null book here, that means it's Book was not imported during by BookImporter. - //There should never be duplicates, but this is defensive. - var existingEntries = DbContext.LibraryBooks.AsEnumerable().Where(l => l.Book is not null).ToDictionarySafe(l => l.Book.AudibleProductId); - - //If importItems are contains duplicates by asin, keep the Item that's "available" - var uniqueImportItems = ToDictionarySafe(importItems, dto => dto.DtoItem.ProductId, tieBreak); - - int qtyNew = 0; - - foreach (var item in uniqueImportItems.Values) - { - if (existingEntries.TryGetValue(item.DtoItem.ProductId, out LibraryBook existing)) - { - if (existing.Account != item.AccountId) - { - //Book is absent from the existing LibraryBook's account. Use the alternate account. - existing.SetAccount(item.AccountId); - } - - existing.AbsentFromLastScan = isUnavailable(item); - } - else - { - existing = new LibraryBook( - bookImporter.Cache[item.DtoItem.ProductId], - item.DtoItem.DateAdded, - item.AccountId) - { - AbsentFromLastScan = isUnavailable(item) - }; - - try - { - DbContext.LibraryBooks.Add(existing); - qtyNew++; - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Error adding library book. {@DebugInfo}", new { existing.Book, existing.Account }); - } - } - - existing.SetIncludedUntil(item.DtoItem.GetExpirationDate()); - existing.SetIsAudiblePlus(item.DtoItem.IsAyce is true); - } - - var scannedAccounts = importItems.Select(i => i.AccountId).Distinct().ToList(); - - //If an existing Book wasn't found in the import, the owning LibraryBook's Book will be null. - //Only change AbsentFromLastScan for LibraryBooks of accounts that were scanned. - foreach (var nullBook in DbContext.LibraryBooks.AsEnumerable().Where(lb => lb.Book is null && lb.Account.In(scannedAccounts))) - nullBook.AbsentFromLastScan = true; - - return qtyNew; - } - - private static Dictionary ToDictionarySafe(IEnumerable source, Func keySelector, Func tieBreaker) - { - var dictionary = new Dictionary(); - - foreach (TSource newItem in source) - { - TKey key = keySelector(newItem); - - dictionary[key] - = dictionary.TryGetValue(key, out TSource existingItem) - ? tieBreaker(existingItem, newItem) - : newItem; - } - return dictionary; - } - - private static ImportItem tieBreak(ImportItem item1, ImportItem item2) - => isUnavailable(item1) && !isUnavailable(item2) ? item2 : item1; - - private static bool isUnavailable(ImportItem item) - => isFutureRelease(item) || isPlusTitleUnavailable(item); - - private static bool isFutureRelease(ImportItem item) - => item.DtoItem.IssueDate is DateTimeOffset dt && dt > DateTimeOffset.UtcNow; - - private static bool isPlusTitleUnavailable(ImportItem item) - => item.DtoItem.ContentType is null - || (item.DtoItem.IsAyce is true && item.DtoItem.Plans?.Any(p => p.IsAyce) is not true); + bookImporter = new BookImporter(DbContext); } + + protected override int DoImport(IEnumerable importItems) + { + bookImporter.Import(importItems); + + var qtyNew = upsertLibraryBooks(importItems); + return qtyNew; + } + + private int upsertLibraryBooks(IEnumerable importItems) + { + // technically, we should be able to have duplicate books from separate accounts. + // this would violate the current pk and would be difficult to deal with elsewhere: + // - what to show in the grid + // - which to consider liberated + // + // sqlite cannot alter pk. the work around is an extensive headache + // - update: now possible in .net5/efcore5 + // + // currently, inserting LibraryBook will throw error if the same book is in multiple accounts for the same region. + // + // CURRENT SOLUTION: don't re-insert + + + //When Books are upserted during the BookImporter run, they are linked to their LibraryBook in the DbContext + //instance. If a LibraryBook has a null book here, that means it's Book was not imported during by BookImporter. + //There should never be duplicates, but this is defensive. + var existingEntries = DbContext.LibraryBooks.AsEnumerable().Where(l => l.Book is not null).ToDictionarySafe(l => l.Book.AudibleProductId); + + //If importItems are contains duplicates by asin, keep the Item that's "available" + var uniqueImportItems = ToDictionarySafe(importItems, dto => dto.DtoItem.ProductId, tieBreak); + + int qtyNew = 0; + foreach (var item in uniqueImportItems.Values) + { + if (existingEntries.TryGetValue(item.DtoItem.ProductId, out LibraryBook existing)) + { + if (existing.Account != item.AccountId) + { + //Book is absent from the existing LibraryBook's account. Use the alternate account. + existing.SetAccount(item.AccountId); + } + + existing.AbsentFromLastScan = isUnavailable(item); + } + else + { + existing = new LibraryBook( + bookImporter.Cache[item.DtoItem.ProductId], + item.DtoItem.DateAdded, + item.AccountId) + { + AbsentFromLastScan = isUnavailable(item) + }; + + try + { + DbContext.LibraryBooks.Add(existing); + qtyNew++; + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "Error adding library book. {@DebugInfo}", new { existing.Book, existing.Account }); + } + } + + existing.SetIncludedUntil(item.DtoItem.GetExpirationDate()); + existing.SetIsAudiblePlus(item.DtoItem.IsAyce is true); + } + + var scannedAccounts = importItems.Select(i => i.AccountId).Distinct().ToHashSet(); + var allInScannedAccounts = DbContext.LibraryBooks.Where(lb => scannedAccounts.Contains(lb.Account)).ToArray(); + + if (bookImporter.LoadedEntireLibrary) + { + //If the entire library was loaded, we can be sure that all existing LibraryBooks have their Book property populated. + //Find LibraryBooks which have a Book but weren't found in the import, and mark them as absent. + foreach (var absentBook in allInScannedAccounts.Where(lb => !uniqueImportItems.ContainsKey(lb.Book.AudibleProductId))) + absentBook.AbsentFromLastScan = true; + } + else + { + //If an existing Book wasn't found in the import, the owning LibraryBook's Book will be null. + //Only change AbsentFromLastScan for LibraryBooks of accounts that were scanned. + foreach (var nullBook in allInScannedAccounts.Where(lb => lb.Book is null && lb.Account.In(scannedAccounts))) + nullBook.AbsentFromLastScan = true; + } + + return qtyNew; + } + + private static Dictionary ToDictionarySafe(IEnumerable source, Func keySelector, Func tieBreaker) + { + var dictionary = new Dictionary(); + + foreach (TSource newItem in source) + { + TKey key = keySelector(newItem); + + dictionary[key] + = dictionary.TryGetValue(key, out TSource existingItem) + ? tieBreaker(existingItem, newItem) + : newItem; + } + return dictionary; + } + + private static ImportItem tieBreak(ImportItem item1, ImportItem item2) + => isUnavailable(item1) && !isUnavailable(item2) ? item2 : item1; + + private static bool isUnavailable(ImportItem item) + => isFutureRelease(item) || isPlusTitleUnavailable(item); + + private static bool isFutureRelease(ImportItem item) + => item.DtoItem.IssueDate is DateTimeOffset dt && dt > DateTimeOffset.UtcNow; + + private static bool isPlusTitleUnavailable(ImportItem item) + => item.DtoItem.ContentType is null + || (item.DtoItem.IsAyce is true && item.DtoItem.Plans?.Any(p => p.IsAyce) is not true); } \ No newline at end of file diff --git a/Source/LibationUiBase/GridView/EntryStatus.cs b/Source/LibationUiBase/GridView/EntryStatus.cs index e10687ad..72d0898e 100644 --- a/Source/LibationUiBase/GridView/EntryStatus.cs +++ b/Source/LibationUiBase/GridView/EntryStatus.cs @@ -47,8 +47,8 @@ namespace LibationUiBase.GridView public bool IsBook => !IsSeries && !IsEpisode; public bool IsUnavailable => !IsSeries - & isAbsent - & ( + && isAbsent + && ( BookStatus is not LiberatedStatus.Liberated || PdfStatus is not null and not LiberatedStatus.Liberated );