update dependencies

Update Libation to work with new AAXClean.Codecs

AaxDecrypter/AaxDecrypter.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AAXClean" Version="0.1.10" />
+    <PackageReference Include="AAXClean" Version="0.3.2" />
+    <PackageReference Include="AAXClean.Codecs" Version="0.1.10" />
   </ItemGroup>
 
   <ItemGroup>

AaxDecrypter/AaxcDownloadConvertBase.cs

@@ -6,6 +6,8 @@ namespace AaxDecrypter
 {
 	public abstract class AaxcDownloadConvertBase : AudiobookDownloadBase
 	{
+		public event EventHandler<AppleTags> RetrievedMetadata;
+
 		protected OutputFormat OutputFormat { get; }
 
 		protected AaxFile AaxFile;
@@ -20,8 +22,8 @@ namespace AaxDecrypter
 		public override void SetCoverArt(byte[] coverArt)
 		{
 			base.SetCoverArt(coverArt);
-			if (coverArt is not null)
-				AaxFile?.AppleTags.SetCoverArt(coverArt);
+			if (coverArt is not null && AaxFile?.AppleTags is not null)
+				AaxFile.AppleTags.Cover = coverArt;
 		}
 
 		protected bool Step_GetMetadata()
@@ -33,6 +35,8 @@ namespace AaxDecrypter
 			OnRetrievedNarrators(AaxFile.AppleTags.Narrator ?? "[unknown]");
 			OnRetrievedCoverArt(AaxFile.AppleTags.Cover);
 
+			RetrievedMetadata?.Invoke(this, AaxFile.AppleTags);
+
 			return !IsCanceled;
 		}
 

AaxDecrypter/AaxcDownloadMultiConverter.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using AAXClean;
+using AAXClean.Codecs;
 using Dinah.Core.StepRunner;
 using FileManager;
 
@@ -117,7 +118,7 @@ That naming may not be desirable for everyone, but it's an easy change to instea
             AaxFile.ConvertToMultiMp3(splitChapters, newSplitCallback =>
             {
                 createOutputFileStream(++chapterCount, splitChapters, newSplitCallback);
-                newSplitCallback.LameConfig.ID3.Track = chapterCount.ToString();
+                ((NAudio.Lame.LameConfig)newSplitCallback.UserState).ID3.Track = chapterCount.ToString();
             });
         }
 

AaxDecrypter/AaxcDownloadSingleConverter.cs

@@ -1,6 +1,7 @@
 using System;
 using System.IO;
 using AAXClean;
+using AAXClean.Codecs;
 using Dinah.Core.StepRunner;
 using FileManager;
 

AaxDecrypter/AudiobookDownloadBase.cs

@@ -67,16 +67,18 @@ namespace AaxDecrypter
 				Serilog.Log.Logger.Error("Conversion failed");
 
 			return IsSuccess;
-		}		
+		}
 
-		protected void OnRetrievedTitle(string title) 
+		protected void OnRetrievedTitle(string title)
 			=> RetrievedTitle?.Invoke(this, title);
-		protected void OnRetrievedAuthors(string authors) 
+		protected void OnRetrievedAuthors(string authors)
 			=> RetrievedAuthors?.Invoke(this, authors);
-		protected void OnRetrievedNarrators(string narrators) 
+		protected void OnRetrievedNarrators(string narrators)
 			=> RetrievedNarrators?.Invoke(this, narrators);
-		protected void OnRetrievedCoverArt(byte[] coverArt) 
+
+		protected void OnRetrievedCoverArt(byte[] coverArt)
 			=> RetrievedCoverArt?.Invoke(this, coverArt);
+
 		protected void OnDecryptProgressUpdate(DownloadProgress downloadProgress) 
 			=> DecryptProgressUpdate?.Invoke(this, downloadProgress);
 		protected void OnDecryptTimeRemaining(TimeSpan timeRemaining)

AppScaffolding/AppScaffolding.csproj

@@ -2,8 +2,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
-    <Version>6.7.3.1</Version>
+    <TargetFramework>net6.0-windows</TargetFramework>
+    <Version>7.0.0.1</Version>
   </PropertyGroup>
 
   <ItemGroup>
@@ -11,7 +11,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Octokit" Version="0.50.0" />
+    <PackageReference Include="Octokit" Version="0.51.0" />
   </ItemGroup>
 
   <ItemGroup>

AppScaffolding/LibationScaffolding.cs

@@ -230,10 +230,10 @@ namespace AppScaffolding
 				config.InProgress,
 
 				AudibleFileStorage.DownloadsInProgressDirectory,
-				DownloadsInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
+				DownloadsInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
 
 				AudibleFileStorage.DecryptInProgressDirectory,
-				DecryptInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
+				DecryptInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
 			});
 		}
 

ApplicationServices/ApplicationServices.csproj

@@ -1,12 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="CsvHelper" Version="27.2.1" />
-    <PackageReference Include="NPOI" Version="2.5.5" />
+    <PackageReference Include="NPOI" Version="2.5.6" />
   </ItemGroup>
 
   <ItemGroup>

ApplicationServices/LibraryCommands.cs

@@ -89,6 +89,9 @@ namespace ApplicationServices
 				var totalCount = importItems.Count;
 				Log.Logger.Information($"GetAllLibraryItems: Total count {totalCount}");
 
+				if (totalCount == 0)
+					return default;
+
 				Log.Logger.Information("Begin long-running import");
 				logTime($"pre {nameof(importIntoDbAsync)}");
 				var newCount = await importIntoDbAsync(importItems);
@@ -164,25 +167,46 @@ namespace ApplicationServices
 		}
 
 		private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)
+        {
+            logTime("importIntoDbAsync -- pre db");
+            using var context = DbContexts.GetContext();
+            var libraryBookImporter = new LibraryBookImporter(context);
+            var newCount = await Task.Run(() => libraryBookImporter.Import(importItems));
+            logTime("importIntoDbAsync -- post Import()");
+            int qtyChanges = saveChanges(context);
+            logTime("importIntoDbAsync -- post SaveChanges");
+
+            if (qtyChanges > 0)
+                await Task.Run(() => finalizeLibrarySizeChange());
+            logTime("importIntoDbAsync -- post finalizeLibrarySizeChange");
+
+            return newCount;
+        }
+
+        private static int saveChanges(LibationContext context)
 		{
-			logTime("importIntoDbAsync -- pre db");
-			using var context = DbContexts.GetContext();
-			var libraryBookImporter = new LibraryBookImporter(context);
-			var newCount = await Task.Run(() => libraryBookImporter.Import(importItems));
-			logTime("importIntoDbAsync -- post Import()");
-			var qtyChanges = context.SaveChanges();
-			logTime("importIntoDbAsync -- post SaveChanges");
+			try
+			{
+				return context.SaveChanges();
+			}
+			catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
+			{
+				// DbUpdateException exceptions can wreck serilog. Condense it until we can find a better solution. I suspect the culpret is the "WithExceptionDetails" serilog extension
 
-			if (qtyChanges > 0)
-				await Task.Run(() => finalizeLibrarySizeChange());
-			logTime("importIntoDbAsync -- post finalizeLibrarySizeChange");
+				static string format(Exception ex) => $"\r\nMessage: {ex.Message}\r\nStack Trace:\r\n{ex.StackTrace}";
 
-			return newCount;
-		}
-		#endregion
+				var msg = "Microsoft.EntityFrameworkCore.DbUpdateException";
+				if (ex.InnerException is null)
+					throw new Exception($"{msg}{format(ex)}");
+				throw new Exception(
+					$"{msg}{format(ex)}",
+					new Exception($"Inner Exception{format(ex.InnerException)}"));
+			}
+        }
+        #endregion
 
-		#region remove books
-		public static Task<List<LibraryBook>> RemoveBooksAsync(List<string> idsToRemove) => Task.Run(() => removeBooks(idsToRemove));
+        #region remove books
+        public static Task<List<LibraryBook>> RemoveBooksAsync(List<string> idsToRemove) => Task.Run(() => removeBooks(idsToRemove));
 		private static List<LibraryBook> removeBooks(List<string> idsToRemove)
 		{
 			using var context = DbContexts.GetContext();

AudibleUtilities/AudibleUtilities.csproj

@@ -1,11 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AudibleApi" Version="2.7.2.1" />
+    <PackageReference Include="AudibleApi" Version="2.7.3.1" />
   </ItemGroup>
 
   <ItemGroup>

DataLayer/Configurations/BookContributorConfig.cs

@@ -9,8 +9,8 @@ namespace DataLayer.Configurations
         {
             entity.HasKey(bc => new { bc.BookId, bc.ContributorId, bc.Role });
 
-            entity.HasIndex(b => b.BookId);
-            entity.HasIndex(b => b.ContributorId);
+            entity.HasIndex(bc => bc.BookId);
+            entity.HasIndex(bc => bc.ContributorId);
 
             entity
                 .HasOne(bc => bc.Book)

DataLayer/Configurations/LibraryBookConfig.cs

@@ -21,12 +21,12 @@ namespace DataLayer.Configurations
             // - update LibraryBook import code
             // - would likely challenge assumptions throughout Libation which have been true up until now
 
-            entity.HasKey(b => b.BookId);
+            entity.HasKey(lb => lb.BookId);
 
             entity
-                .HasOne(le => le.Book)
+                .HasOne(lb => lb.Book)
                 .WithOne()
-                .HasForeignKey<LibraryBook>(le => le.BookId);
+                .HasForeignKey<LibraryBook>(lb => lb.BookId);
         }
     }
 }

DataLayer/Configurations/SeriesBookConfig.cs

@@ -7,10 +7,10 @@ namespace DataLayer.Configurations
     {
         public void Configure(EntityTypeBuilder<SeriesBook> entity)
         {
-            entity.HasKey(bc => new { bc.SeriesId, bc.BookId });
+            entity.HasKey(sb => new { sb.SeriesId, sb.BookId });
 
-            entity.HasIndex(b => b.SeriesId);
-            entity.HasIndex(b => b.BookId);
+            entity.HasIndex(sb => sb.SeriesId);
+            entity.HasIndex(sb => sb.BookId);
 
             entity
                 .HasOne(sb => sb.Series)

DataLayer/Configurations/SeriesConfig.cs

@@ -7,8 +7,8 @@ namespace DataLayer.Configurations
     {
         public void Configure(EntityTypeBuilder<Series> entity)
         {
-            entity.HasKey(b => b.SeriesId);
-            entity.HasIndex(b => b.AudibleSeriesId);
+            entity.HasKey(s => s.SeriesId);
+            entity.HasIndex(s => s.AudibleSeriesId);
 
             entity
                 .Metadata

DataLayer/DataLayer.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -13,12 +13,12 @@
   
   <ItemGroup>
     <PackageReference Include="Dinah.EntityFrameworkCore" Version="4.0.0.3" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.3">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.4">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.3" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.3">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.4" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.4">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

DataLayer/EfClasses/Category.cs

@@ -18,7 +18,7 @@ namespace DataLayer
     public class Category
     {
         // Empty is a special case. use private ctor w/o validation
-        public static Category GetEmpty() => new Category { CategoryId = -1, AudibleCategoryId = "", Name = "" };
+        public static Category GetEmpty() => new() { CategoryId = -1, AudibleCategoryId = "", Name = "" };
 
         internal int CategoryId { get; private set; }
         public string AudibleCategoryId { get; private set; }

DataLayer/EfClasses/Contributor.cs

@@ -7,7 +7,7 @@ namespace DataLayer
     public class Contributor
 	{
 		// Empty is a special case. use private ctor w/o validation
-		public static Contributor GetEmpty() => new Contributor { ContributorId = -1, Name = "" };
+		public static Contributor GetEmpty() => new() { ContributorId = -1, Name = "" };
 
 		// contributors search links are just name with url-encoding. space can be + or %20
 		//   author search link:   /search?searchAuthor=Robert+Bevan
@@ -31,16 +31,12 @@ namespace DataLayer
         public string AudibleContributorId { get; private set; }
 
         private Contributor() { }
-        public Contributor(string name)
+        public Contributor(string name, string audibleContributorId = null)
         {
-            ArgumentValidator.EnsureNotNullOrWhiteSpace(name, nameof(name));
+            Name = ArgumentValidator.EnsureNotNullOrWhiteSpace(name, nameof(name));
 
             _booksLink = new HashSet<BookContributor>();
 
-            Name = name;
-        }
-		public Contributor(string name, string audibleContributorId) : this(name)
-		{
 			// don't overwrite with null or whitespace but not an error
 			if (!string.IsNullOrWhiteSpace(audibleContributorId))
 				AudibleContributorId = audibleContributorId;

DataLayer/QueryObjects/BookQueries.cs

@@ -19,7 +19,8 @@ namespace DataLayer
         public static Book GetBook(this IQueryable<Book> books, string productId)
             => books
                 .GetBooks()
-                .SingleOrDefault(b => b.AudibleProductId == productId);
+                // 'Single' is more accurate but 'First' is faster and less error prone
+                .FirstOrDefault(b => b.AudibleProductId == productId);
 
         /// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary>
         public static IQueryable<Book> GetBooks(this IQueryable<Book> books, Expression<Func<Book, bool>> predicate)

DtoImporterService/BookImporter.cs

@@ -4,62 +4,53 @@ using System.Linq;
 using AudibleApi.Common;
 using AudibleUtilities;
 using DataLayer;
+using Dinah.Core.Collections.Generic;
 
 namespace DtoImporterService
 {
 	public class BookImporter : ItemsImporterBase
 	{
-		public BookImporter(LibationContext context) : base(context) { }
+		protected override IValidator Validator => new BookValidator();
 
-		public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new BookValidator().Validate(importItems.Select(i => i.DtoItem));
+		public Dictionary<string, Book> Cache { get; private set; } = new();
+
+		private ContributorImporter contributorImporter { get; }
+		private SeriesImporter seriesImporter { get; }
+		private CategoryImporter categoryImporter { get; }
+
+		public BookImporter(LibationContext context) : base(context)
+		{
+			contributorImporter = new ContributorImporter(DbContext);
+			seriesImporter = new SeriesImporter(DbContext);
+			categoryImporter = new CategoryImporter(DbContext);
+		}
 
 		protected override int DoImport(IEnumerable<ImportItem> importItems)
 		{
 			// pre-req.s
-			new ContributorImporter(DbContext).Import(importItems);
-			new SeriesImporter(DbContext).Import(importItems);
-			new CategoryImporter(DbContext).Import(importItems);
-			
-			// get distinct
-			var productIds = importItems.Select(i => i.DtoItem.ProductId).Distinct().ToList();
-			
-			// load db existing => .Local
-			loadLocal_books(productIds);
-			
+			contributorImporter.Import(importItems);
+			seriesImporter.Import(importItems);
+			categoryImporter.Import(importItems);
+
+			// load db existing => hash table
+			loadLocal_books(importItems);
+
 			// upsert
 			var qtyNew = upsertBooks(importItems);
 			return qtyNew;
 		}
 
-		private void loadLocal_books(List<string> productIds)
+		private void loadLocal_books(IEnumerable<ImportItem> importItems)
 		{
-			// if this context has already loaded books, don't need to reload them. vestige from when context was long-lived. in practice, we now typically use a fresh context. this is quick though so no harm in leaving it.
-			var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId).ToList();
-			var remainingProductIds = productIds
-				.Except(localProductIds)
+			// get distinct
+			var productIds = importItems
+				.Select(i => i.DtoItem.ProductId)
+				.Distinct()
 				.ToList();
 
-			#region // explanation of DbContext.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
-			/*
-			articles suggest loading to Local with
-			    context.Books.Load();
-			we want Books and associated fields
-			    context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
-			this is emulating Load() but with also getting associated fields
-
-			from: Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions
-			// Summary:
-			//     Enumerates the query. When using Entity Framework, this causes the results of
-			//     the query to be loaded into the associated context. This is equivalent to calling
-			//     ToList and then throwing away the list (without the overhead of actually creating
-			//     the list).
-			public static void Load<TSource>([NotNullAttribute] this IQueryable<TSource> source); 
-			*/
-			#endregion
-
-			// GetBooks() eager loads Series, category, et al
-			if (remainingProductIds.Any())
-				DbContext.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
+			Cache = DbContext.Books
+				.GetBooks(b => productIds.Contains(b.AudibleProductId))
+				.ToDictionarySafe(b => b.AudibleProductId);
 		}
 
 		private int upsertBooks(IEnumerable<ImportItem> importItems)
@@ -68,8 +59,7 @@ namespace DtoImporterService
 
 			foreach (var item in importItems)
 			{
-				var book = DbContext.Books.Local.FirstOrDefault(p => p.AudibleProductId == item.DtoItem.ProductId);
-				if (book is null)
+				if (!Cache.TryGetValue(item.DtoItem.ProductId, out var book))
 				{
 					book = createNewBook(item);
 					qtyNew++;
@@ -94,8 +84,7 @@ namespace DtoImporterService
 			// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
 			var authors = item
 				.Authors
-				// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
-				.Select(a => DbContext.Contributors.Local.FirstOrDefault(c => a.Name == c.Name))
+				.Select(a => contributorImporter.Cache[a.Name])
 				.ToList();
 
 			var narrators
@@ -105,8 +94,7 @@ namespace DtoImporterService
 				// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
 				: item
 					.Narrators
-					// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
-					.Select(n => DbContext.Contributors.Local.FirstOrDefault(c => n.Name == c.Name))
+					.Select(n => contributorImporter.Cache[n.Name])
 					.ToList();
 
 			// categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd
@@ -120,8 +108,7 @@ namespace DtoImporterService
 				// 2+
 				: item.Categories[1].CategoryId;
 
-			// This should properly be SingleOrDefault() not FirstOrDefault(), but FirstOrDefault is defensive
-			var category = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == lastCategory);
+			var category = categoryImporter.Cache[lastCategory];
 
 			Book book;
 			try
@@ -137,6 +124,7 @@ namespace DtoImporterService
 					category,
 					importItem.LocaleName)
 					).Entity;
+				Cache.Add(book.AudibleProductId, book);
 			}
 			catch (Exception ex)
 			{
@@ -157,8 +145,7 @@ namespace DtoImporterService
 			var publisherName = item.Publisher;
 			if (!string.IsNullOrWhiteSpace(publisherName))
 			{
-				// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
-				var publisher = DbContext.Contributors.Local.FirstOrDefault(c => publisherName == c.Name);
+				var publisher = contributorImporter.Cache[publisherName];
 				book.ReplacePublisher(publisher);
 			}
 
@@ -189,7 +176,7 @@ namespace DtoImporterService
 			{
 				foreach (var seriesEntry in item.Series)
 				{
-					var series = DbContext.Series.Local.FirstOrDefault(s => seriesEntry.SeriesId == s.AudibleSeriesId);
+					var series = seriesImporter.Cache[seriesEntry.SeriesId];
 					book.UpsertSeries(series, seriesEntry.Sequence);
 				}
 			}

DtoImporterService/CategoryImporter.cs

@@ -4,14 +4,17 @@ using System.Linq;
 using AudibleApi.Common;
 using AudibleUtilities;
 using DataLayer;
+using Dinah.Core.Collections.Generic;
 
 namespace DtoImporterService
 {
 	public class CategoryImporter : ItemsImporterBase
 	{
-		public CategoryImporter(LibationContext context) : base(context) { }
+		protected override IValidator Validator => new CategoryValidator();
 
-		public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new CategoryValidator().Validate(importItems.Select(i => i.DtoItem));
+		public Dictionary<string, Category> Cache { get; private set; } = new();
+
+		public CategoryImporter(LibationContext context) : base(context) { }
 
 		protected override int DoImport(IEnumerable<ImportItem> importItems)
 		{
@@ -19,7 +22,9 @@ namespace DtoImporterService
 			var categoryIds = importItems
 				.Select(i => i.DtoItem)
 				.GetCategoriesDistinct()
-				.Select(c => c.CategoryId).ToList();
+				.Select(c => c.CategoryId)
+				.Distinct()
+				.ToList();
 
 			// load db existing => .Local
 			loadLocal_categories(categoryIds);
@@ -35,17 +40,13 @@ namespace DtoImporterService
 
 		private void loadLocal_categories(List<string> categoryIds)
 		{
-			var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId).ToList();
-			var remainingCategoryIds = categoryIds
-				.Distinct()
-				.Except(localIds)
-				.ToList();
+			// must include default/empty/missing
+			categoryIds.Add(Category.GetEmpty().AudibleCategoryId);
 
 			// load existing => local
-			// remember to include default/empty/missing
-			var emptyName = Contributor.GetEmpty().Name;
-			if (remainingCategoryIds.Any())
-				DbContext.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId) || c.Name == emptyName).ToList();
+			Cache = DbContext.Categories
+				.Where(c => categoryIds.Contains(c.AudibleCategoryId))
+				.ToDictionarySafe(c => c.AudibleCategoryId);
 		}
 
 		// only use after loading contributors => local
@@ -66,22 +67,11 @@ namespace DtoImporterService
 
 					Category parentCategory = null;
 					if (i == 1)
-						// should be "Single()" but user is getting a strange error
-						parentCategory = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == pair[0].CategoryId);
+						Cache.TryGetValue(pair[0].CategoryId, out parentCategory);
 
-					// should be "SingleOrDefault()" but user is getting a strange error
-					var category = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == id);
-					if (category is null)
+					if (!Cache.TryGetValue(id, out var category))
 					{
-						try
-						{
-							category = DbContext.Categories.Add(new Category(new AudibleCategoryId(id), name)).Entity;
-						}
-						catch (Exception ex)
-						{
-							Serilog.Log.Logger.Error(ex, "Error adding category. {@DebugInfo}", new { id, name });
-							throw;
-						}
+						category = addCategory(id, name);
 						qtyNew++;
 					}
 
@@ -91,5 +81,24 @@ namespace DtoImporterService
 
 			return qtyNew;
 		}
+
+		private Category addCategory(string id, string name)
+		{
+			try
+			{
+				var category = new Category(new AudibleCategoryId(id), name);
+
+				var entityEntry = DbContext.Categories.Add(category);
+				var entity = entityEntry.Entity;
+
+				Cache.Add(entity.AudibleCategoryId, entity);
+				return entity;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Error adding category. {@DebugInfo}", new { id, name });
+				throw;
+			}
+		}
 	}
 }

DtoImporterService/ContributorImporter.cs

@@ -4,14 +4,17 @@ using System.Linq;
 using AudibleApi.Common;
 using AudibleUtilities;
 using DataLayer;
+using Dinah.Core.Collections.Generic;
 
 namespace DtoImporterService
 {
 	public class ContributorImporter : ItemsImporterBase
 	{
-		public ContributorImporter(LibationContext context) : base(context) { }
+		protected override IValidator Validator => new ContributorValidator();
 
-		public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new ContributorValidator().Validate(importItems.Select(i => i.DtoItem));
+		public Dictionary<string, Contributor> Cache { get; private set; } = new();
+
+		public ContributorImporter(LibationContext context) : base(context) { }
 
 		protected override int DoImport(IEnumerable<ImportItem> importItems)
 		{
@@ -50,78 +53,61 @@ namespace DtoImporterService
 			// must include default/empty/missing
 			contributorNames.Add(Contributor.GetEmpty().Name);
 
-			//// BAD: very inefficient
-			// var x = context.Contributors.Local.Where(c => !contribNames.Contains(c.Name));
-
-			// GOOD: Except() is efficient. Due to hashing, it's close to O(n)
-			var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList();
-			var remainingContribNames = contributorNames
-				.Distinct()
-				.Except(localNames)
-				.ToList();
-
 			// load existing => local
-			if (remainingContribNames.Any())
-				DbContext.Contributors.Where(c => remainingContribNames.Contains(c.Name)).ToList();
+			Cache = DbContext.Contributors
+				.Where(c => contributorNames.Contains(c.Name))
+				.ToDictionarySafe(c => c.Name);
 		}
 
-		// only use after loading contributors => local
 		private int upsertPeople(List<Person> people)
 		{
-			var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList();
-			var newPeople = people
-				.Select(p => p.Name)
-				.Distinct()
-				.Except(localNames)
-				.ToList();
+			var hash = people
+				// new people only
+				.Where(p => !Cache.ContainsKey(p.Name))
+				// remove duplicates by Name. first in wins
+				.ToDictionarySafe(p => p.Name);
 
-			var groupby = people.GroupBy(
-				p => p.Name,
-				p => p,
-				(key, g) => new { Name = key, People = g.ToList() }
-				);
-			foreach (var name in newPeople)
+			foreach (var kvp in hash)
 			{
-				// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
-				var p = groupby.FirstOrDefault(g => g.Name == name).People.First();
-
-				try
-				{
-					DbContext.Contributors.Add(new Contributor(p.Name, p.Asin));
-				}
-				catch (Exception ex)
-				{
-					Serilog.Log.Logger.Error(ex, "Error adding person. {@DebugInfo}", new { p?.Name, p?.Asin });
-					throw;
-				}
+				var person = kvp.Value;
+				addContributor(person.Name, person.Asin);
 			}
 
-			return newPeople.Count;
+			return hash.Count;
 		}
 
 		// only use after loading contributors => local
 		private int upsertPublishers(List<string> publishers)
 		{
-			var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList();
-			var newPublishers = publishers
-				.Distinct()
-				.Except(localNames)
-				.ToList();
+			var hash = publishers
+				// new publishers only
+				.Where(p => !Cache.ContainsKey(p))
+				// remove duplicates
+				.ToHashSet();
 
-			foreach (var pub in newPublishers)
-			{
-				try
-				{
-					DbContext.Contributors.Add(new Contributor(pub));
-				}
-				catch (Exception ex)
-				{
-					Serilog.Log.Logger.Error(ex, "Error adding publisher. {@DebugInfo}", new { pub });
-					throw;
-				}
-			}
+			foreach (var pub in hash)
+				addContributor(pub);
 
-			return newPublishers.Count;
+			return hash.Count;
 		}
-	}
+
+        private Contributor addContributor(string name, string id = null)
+		{
+			try
+			{
+				var newContrib = new Contributor(name);
+
+				var entityEntry = DbContext.Contributors.Add(newContrib);
+				var entity = entityEntry.Entity;
+
+				Cache.Add(entity.Name, entity);
+				return entity;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Error adding contributor. {@DebugInfo}", new { name, id });
+				throw;
+			}
+        }
+    }
 }

DtoImporterService/DtoImporterService.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

DtoImporterService/ImporterBase.cs

@@ -53,5 +53,9 @@ namespace DtoImporterService
 	public abstract class ItemsImporterBase : ImporterBase<IEnumerable<ImportItem>>
 	{
 		protected ItemsImporterBase(LibationContext context) : base(context) { }
+
+		protected abstract IValidator Validator { get; }
+        public sealed override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems)
+			=> Validator.Validate(importItems.Select(i => i.DtoItem));
 	}
 }

DtoImporterService/LibraryBookImporter.cs

@@ -3,18 +3,24 @@ using System.Collections.Generic;
 using System.Linq;
 using AudibleUtilities;
 using DataLayer;
+using Dinah.Core.Collections.Generic;
 
 namespace DtoImporterService
 {
 	public class LibraryBookImporter : ItemsImporterBase
 	{
-		public LibraryBookImporter(LibationContext context) : base(context) { }
+		protected override IValidator Validator => new LibraryValidator();
 
-		public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new LibraryValidator().Validate(importItems.Select(i => i.DtoItem));
+		private BookImporter bookImporter { get; }
+
+		public LibraryBookImporter(LibationContext context) : base(context)
+		{
+			bookImporter = new BookImporter(DbContext);
+		}
 
 		protected override int DoImport(IEnumerable<ImportItem> importItems)
 		{
-			new BookImporter(DbContext).Import(importItems);
+			bookImporter.Import(importItems);
 
 			var qtyNew = upsertLibraryBooks(importItems);
 			return qtyNew;
@@ -36,25 +42,18 @@ namespace DtoImporterService
 
 			var currentLibraryProductIds = DbContext.LibraryBooks.Select(l => l.Book.AudibleProductId).ToList();
 			var newItems = importItems
-				.Where(dto => !currentLibraryProductIds
-				.Contains(dto.DtoItem.ProductId))
+				.Where(dto => !currentLibraryProductIds.Contains(dto.DtoItem.ProductId))
 				.ToList();
 
 			// if 2 accounts try to import the same book in the same transaction: error since we're only tracking and pulling by asin.
 			// just use the first
-			var groupby = newItems.GroupBy(
-				i => i.DtoItem.ProductId,
-				i => i,
-				(key, g) => new { ProductId = key, ImportItems = g.ToList() }
-				)
-				.ToList();
-			foreach (var gb in groupby)
-			{
-				var newItem = gb.ImportItems.First();
+			var hash = newItems.ToDictionarySafe(dto => dto.DtoItem.ProductId);
+			foreach (var kvp in hash)
+            {
+				var newItem = kvp.Value;
 
 				var libraryBook = new LibraryBook(
-					// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
-					DbContext.Books.Local.FirstOrDefault(b => b.AudibleProductId == newItem.DtoItem.ProductId),
+					bookImporter.Cache[newItem.DtoItem.ProductId],
 					newItem.DtoItem.DateAdded,
 					newItem.AccountId);
 				try
@@ -67,7 +66,7 @@ namespace DtoImporterService
 				}
 			}
 
-			var qtyNew = groupby.Count;
+			var qtyNew = hash.Count;
 			return qtyNew;
 		}
 	}

DtoImporterService/PerfLogger.cs

@@ -8,8 +8,8 @@ namespace DtoImporterService
 	public record timeLogEntry(string msg, long totalElapsed, long delta);
 	public static class PerfLogger
 	{
-		private static Stopwatch sw = new Stopwatch();
-		private static List<timeLogEntry> __log { get; } = new List<timeLogEntry> { new("begin", 0, 0) };
+		private static Stopwatch sw { get; } = new();
+		private static List<timeLogEntry> __log { get; } = new() { new("begin", 0, 0) };
 
 		public static void logTime(string s)
 		{

DtoImporterService/SeriesImporter.cs

@@ -4,14 +4,17 @@ using System.Linq;
 using AudibleApi.Common;
 using AudibleUtilities;
 using DataLayer;
+using Dinah.Core.Collections.Generic;
 
 namespace DtoImporterService
 {
 	public class SeriesImporter : ItemsImporterBase
 	{
-		public SeriesImporter(LibationContext context) : base(context) { }
+		protected override IValidator Validator => new SeriesValidator();
 
-		public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new SeriesValidator().Validate(importItems.Select(i => i.DtoItem));
+		public Dictionary<string, DataLayer.Series> Cache { get; private set; } = new();
+
+		public SeriesImporter(LibationContext context) : base(context) { }
 
 		protected override int DoImport(IEnumerable<ImportItem> importItems)
 		{
@@ -31,15 +34,12 @@ namespace DtoImporterService
 
 		private void loadLocal_series(List<AudibleApi.Common.Series> series)
 		{
-			var seriesIds = series.Select(s => s.SeriesId).ToList();
-			var localIds = DbContext.Series.Local.Select(s => s.AudibleSeriesId).ToList();
-			var remainingSeriesIds = seriesIds
-				.Distinct()
-				.Except(localIds)
-				.ToList();
+			var seriesIds = series.Select(s => s.SeriesId).Distinct().ToList();
 
-			if (remainingSeriesIds.Any())
-				DbContext.Series.Where(s => remainingSeriesIds.Contains(s.AudibleSeriesId)).ToList();
+			if (seriesIds.Any())
+				Cache = DbContext.Series
+					.Where(s => seriesIds.Contains(s.AudibleSeriesId))
+					.ToDictionarySafe(s => s.AudibleSeriesId);
 		}
 
 		private int upsertSeries(List<AudibleApi.Common.Series> requestedSeries)
@@ -48,18 +48,10 @@ namespace DtoImporterService
 
 			foreach (var s in requestedSeries)
 			{
-				var series = DbContext.Series.Local.FirstOrDefault(c => c.AudibleSeriesId == s.SeriesId);
-				if (series is null)
+				// AudibleApi.Common.Series.SeriesId == DataLayer.AudibleSeriesId
+				if (!Cache.TryGetValue(s.SeriesId, out var series))
 				{
-					try
-					{
-						series = DbContext.Series.Add(new DataLayer.Series(new AudibleSeriesId(s.SeriesId))).Entity;
-					}
-					catch (Exception ex)
-					{
-						Serilog.Log.Logger.Error(ex, "Error adding series. {@DebugInfo}", new { s?.SeriesId });
-						throw;
-					}
+					series = addSeries(s.SeriesId);
 					qtyNew++;
 				}
 				series.UpdateName(s.SeriesName);
@@ -67,5 +59,24 @@ namespace DtoImporterService
 
 			return qtyNew;
 		}
+
+		private DataLayer.Series addSeries(string seriesId)
+		{
+			try
+			{
+				var series = new DataLayer.Series(new AudibleSeriesId(seriesId));
+
+				var entityEntry = DbContext.Series.Add(series);
+				var entity = entityEntry.Entity;
+
+				Cache.Add(entity.AudibleSeriesId, entity);
+				return entity;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Error adding series. {@DebugInfo}", new { seriesId });
+				throw;
+			}
+		}
 	}
 }

DtoImporterService/_importer notes.txt

@@ -0,0 +1,37 @@
+* Local (eg DbContext.Books.Local): indexes/hashes PK and nothing else. Local.Find(PK) is fast. All other searches (eg FirstOrDefault) have awful performance. It deceptively *feels* like we get this partially for free since added/modified entries live here.
+* live db: for all importers, fields used for lookup are indexed
+
+Using BookImporter as an example: since AudibleProductId is indexed, hitting the live db is much faster than using Local. Fastest is putting all in a local hash table
+
+Note: GetBook/GetBooks eager loads Series, category, et al
+
+for 1,200 iterations
+* load to LocalView
+  DbContext.Books.Local.FirstOrDefault(p => p.AudibleProductId == item.DtoItem.ProductId)
+  27,125 ms
+* read from live db
+  DbContext.Books.GetBook(item.DtoItem.ProductId)
+  12,224 ms
+* load to hash table: Dictionary<string, Book>
+  dictionary[item.DtoItem.ProductId];
+  1 ms (yes: ONE)
+
+With hashtable, somehow memory usage was not significantly affected
+
+-----------------------------------
+
+why were we using Local to begin with?
+
+articles suggest loading to Local with
+    context.Books.Load();
+this loads this table but not associated fields
+we want Books and associated fields
+    context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
+this is emulating Load() but with also getting associated fields
+			from: Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions
+			// Summary:
+			//     Enumerates the query. When using Entity Framework, this causes the results of
+			//     the query to be loaded into the associated context. This is equivalent to calling
+			//     ToList and then throwing away the list (without the overhead of actually creating
+			//     the list).
+			public static void Load<TSource>([NotNullAttribute] this IQueryable<TSource> source);

FileLiberator/AudioDecodable.cs

@@ -11,30 +11,33 @@ namespace FileLiberator
         public event EventHandler<byte[]> CoverImageDiscovered;
         public abstract void Cancel();
 
-        protected void OnRequestCoverArt(Action<byte[]> setCoverArtDel)
-		{
-            Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(RequestCoverArt) });
-            RequestCoverArt?.Invoke(this, setCoverArtDel);
-        }
-
-        protected void OnTitleDiscovered(string title)
+        protected void OnTitleDiscovered(string title) => OnTitleDiscovered(null, title);
+        protected void OnTitleDiscovered(object _, string title)
 		{
             Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(TitleDiscovered), Title = title });
             TitleDiscovered?.Invoke(this, title);
 		}
 
-        protected void OnAuthorsDiscovered(string authors)
+        protected void OnAuthorsDiscovered(string authors) => OnAuthorsDiscovered(null, authors);
+        protected void OnAuthorsDiscovered(object _, string authors)
 		{
             Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(AuthorsDiscovered), Authors = authors });
             AuthorsDiscovered?.Invoke(this, authors);
 		}
 
-        protected void OnNarratorsDiscovered(string narrators)
+        protected void OnNarratorsDiscovered(string narrators) => OnNarratorsDiscovered(null, narrators);
+        protected void OnNarratorsDiscovered(object _, string narrators)
 		{
             Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(NarratorsDiscovered), Narrators = narrators });
             NarratorsDiscovered?.Invoke(this, narrators);
 		}
 
+        protected void OnRequestCoverArt(Action<byte[]> setCoverArtDel)
+		{
+            Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(RequestCoverArt) });
+            RequestCoverArt?.Invoke(this, setCoverArtDel);
+        }
+
         protected void OnCoverImageDiscovered(byte[] coverImage)
 		{
             Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(CoverImageDiscovered), CoverImageBytes = coverImage?.Length });

FileLiberator/ConvertToMp3.cs

@@ -2,6 +2,7 @@
 using System.IO;
 using System.Threading.Tasks;
 using AAXClean;
+using AAXClean.Codecs;
 using DataLayer;
 using Dinah.Core;
 using Dinah.Core.ErrorHandling;
@@ -72,8 +73,8 @@ namespace FileLiberator
         private void M4bBook_ConversionProgressUpdate(object sender, ConversionProgressEventArgs e)
         {
             var duration = m4bBook.Duration;
-            double remainingSecsToProcess = (duration - e.ProcessPosition).TotalSeconds;
-            double estTimeRemaining = remainingSecsToProcess / e.ProcessSpeed;
+            var remainingSecsToProcess = (duration - e.ProcessPosition).TotalSeconds;
+            var estTimeRemaining = remainingSecsToProcess / e.ProcessSpeed;
 
             if (double.IsNormal(estTimeRemaining))
                 OnStreamingTimeRemaining(TimeSpan.FromSeconds(estTimeRemaining));

FileLiberator/DownloadDecryptBook.cs

@@ -119,18 +119,28 @@ namespace FileLiberator
 
                 var cacheDir = AudibleFileStorage.DownloadsInProgressDirectory;
 
-                abDownloader
-                    = contentLic.DrmType != AudibleApi.Common.DrmType.Adrm ? new UnencryptedAudiobookDownloader(outFileName, cacheDir, audiobookDlLic)
-                    : Configuration.Instance.SplitFilesByChapter ? new AaxcDownloadMultiConverter(
-                        outFileName, cacheDir, audiobookDlLic, outputFormat,
-                        AudibleFileStorage.Audio.CreateMultipartRenamerFunc(libraryBook)
-                        )
-                    : new AaxcDownloadSingleConverter(outFileName, cacheDir, audiobookDlLic, outputFormat);
-                abDownloader.DecryptProgressUpdate += (_, progress) => OnStreamingProgressChanged(progress);
-                abDownloader.DecryptTimeRemaining += (_, remaining) => OnStreamingTimeRemaining(remaining);
-                abDownloader.RetrievedTitle += (_, title) => OnTitleDiscovered(title);
-                abDownloader.RetrievedAuthors += (_, authors) => OnAuthorsDiscovered(authors);
-                abDownloader.RetrievedNarrators += (_, narrators) => OnNarratorsDiscovered(narrators);
+                if (contentLic.DrmType != AudibleApi.Common.DrmType.Adrm)
+                    abDownloader = new UnencryptedAudiobookDownloader(outFileName, cacheDir, audiobookDlLic);
+                else
+                {
+                    AaxcDownloadConvertBase converter
+                        = Configuration.Instance.SplitFilesByChapter ? new AaxcDownloadMultiConverter(
+                            outFileName, cacheDir, audiobookDlLic, outputFormat,
+                            AudibleFileStorage.Audio.CreateMultipartRenamerFunc(libraryBook))
+                        : new AaxcDownloadSingleConverter(outFileName, cacheDir, audiobookDlLic, outputFormat);
+
+                    if (Configuration.Instance.AllowLibationFixup)
+                        converter.RetrievedMetadata += (_, tags) => tags.Generes = string.Join(", ", libraryBook.Book.CategoriesNames);
+
+                    abDownloader = converter;
+                }
+
+                abDownloader.DecryptProgressUpdate += OnStreamingProgressChanged;
+                abDownloader.DecryptTimeRemaining += OnStreamingTimeRemaining;
+
+                abDownloader.RetrievedTitle += OnTitleDiscovered;
+                abDownloader.RetrievedAuthors += OnAuthorsDiscovered;
+                abDownloader.RetrievedNarrators += OnNarratorsDiscovered;
                 abDownloader.RetrievedCoverArt += AaxcDownloader_RetrievedCoverArt;
                 abDownloader.FileCreated += (_, path) => OnFileCreated(libraryBook, path);
 
@@ -166,7 +176,7 @@ namespace FileLiberator
                 throw new Exception(errorString("Locale"));
         }
 
-        private void AaxcDownloader_RetrievedCoverArt(object sender, byte[] e)
+        private void AaxcDownloader_RetrievedCoverArt(object _, byte[] e)
         {
             if (e is not null)
                 OnCoverImageDiscovered(e);

FileLiberator/DownloadFile.cs

@@ -12,8 +12,7 @@ namespace FileLiberator
 		{
 			var client = new HttpClient();
 
-			var progress = new Progress<DownloadProgress>();
-			progress.ProgressChanged += (_, e) => OnStreamingProgressChanged(e);
+			var progress = new Progress<DownloadProgress>(OnStreamingProgressChanged);
 
 			OnStreamingBegin(proposedDownloadFilePath);
 

FileLiberator/DownloadPdf.cs

@@ -62,8 +62,7 @@ namespace FileLiberator
 				var api = await libraryBook.GetApiAsync();
 				var downloadUrl = await api.GetPdfDownloadLinkAsync(libraryBook.Book.AudibleProductId);
 
-				var progress = new Progress<DownloadProgress>();
-				progress.ProgressChanged += (_, e) => OnStreamingProgressChanged(e);
+				var progress = new Progress<DownloadProgress>(OnStreamingProgressChanged);
 
 				var client = new HttpClient();
 

FileLiberator/FileLiberator.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

FileLiberator/Streamable.cs

@@ -18,12 +18,14 @@ namespace FileLiberator
             StreamingBegin?.Invoke(this, filePath);
         }
 
-        protected void OnStreamingProgressChanged(DownloadProgress progress)
+        protected void OnStreamingProgressChanged(DownloadProgress progress) => OnStreamingProgressChanged(null, progress);
+        protected void OnStreamingProgressChanged(object _, DownloadProgress progress)
         {
             StreamingProgressChanged?.Invoke(this, progress);
         }
 
-        protected void OnStreamingTimeRemaining(TimeSpan timeRemaining)
+        protected void OnStreamingTimeRemaining(TimeSpan timeRemaining) => OnStreamingTimeRemaining(null, timeRemaining);
+        protected void OnStreamingTimeRemaining(object _, TimeSpan timeRemaining)
         {
             StreamingTimeRemaining?.Invoke(this, timeRemaining);
         }

FileManager/BackgroundFileSystem.cs

@@ -43,7 +43,7 @@ namespace FileManager
             lock (fsCacheLocker)
             {
                 fsCache.Clear();
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
             }
         }
 
@@ -52,7 +52,7 @@ namespace FileManager
             Stop();
 
             lock (fsCacheLocker)
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
 
             directoryChangesEvents = new BlockingCollection<FileSystemEventArgs>();
 			fileSystemWatcher = new FileSystemWatcher(RootDirectory)
@@ -135,7 +135,7 @@ namespace FileManager
         private void AddPath(string path)
         {
             if (File.GetAttributes(path).HasFlag(FileAttributes.Directory))
-                AddUniqueFiles(Directory.EnumerateFiles(path, SearchPattern, SearchOption));
+                AddUniqueFiles(FileUtility.SaferEnumerateFiles(path, SearchPattern, SearchOption));
             else
                 AddUniqueFile(path);
         }

FileManager/FileManager.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core" Version="4.0.4.1" />
+    <PackageReference Include="Dinah.Core" Version="4.0.6.1" />
     <PackageReference Include="Polly" Version="7.2.3" />
   </ItemGroup>
 

FileManager/FileUtility.cs

@@ -189,15 +189,19 @@ namespace FileManager
             {
                 try
                 {
-                    if (File.Exists(source))
+                    if (!File.Exists(source))
                     {
-                        File.Delete(source);
-                        Serilog.Log.Logger.Information("File successfully deleted", new { source });
+                        Serilog.Log.Logger.Debug("No file to delete: {@DebugText}", new { source });
+                        return;
                     }
+
+                    Serilog.Log.Logger.Debug("Attempt to delete file: {@DebugText}", new { source });
+                    File.Delete(source);
+                    Serilog.Log.Logger.Information("File successfully deleted: {@DebugText}", new { source });
                 }
                 catch (Exception e)
                 {
-                    Serilog.Log.Logger.Error(e, "Failed to delete file", new { source });
+                    Serilog.Log.Logger.Error(e, "Failed to delete file: {@DebugText}", new { source });
                     throw;
                 }
             });
@@ -208,19 +212,61 @@ namespace FileManager
             {
                 try
                 {
-                    if (File.Exists(source))
+                    if (!File.Exists(source))
                     {
-                        SaferDelete(destination);
-                        Directory.CreateDirectory(Path.GetDirectoryName(destination));
-                        File.Move(source, destination);
-                        Serilog.Log.Logger.Information("File successfully moved", new { source, destination });
+                        Serilog.Log.Logger.Debug("No file to move: {@DebugText}", new { source });
+                        return;
                     }
+
+                    SaferDelete(destination);
+
+                    var dir = Path.GetDirectoryName(destination);
+                    Serilog.Log.Logger.Debug("Attempt to create directory: {@DebugText}", new { dir });
+                    Directory.CreateDirectory(dir);
+
+                    Serilog.Log.Logger.Debug("Attempt to move file: {@DebugText}", new { source, destination });
+                    File.Move(source, destination);
+                    Serilog.Log.Logger.Information("File successfully moved: {@DebugText}", new { source, destination });
                 }
                 catch (Exception e)
                 {
-                    Serilog.Log.Logger.Error(e, "Failed to move file", new { source, destination });
+                    Serilog.Log.Logger.Error(e, "Failed to move file: {@DebugText}", new { source, destination });
                     throw;
                 }
             });
+
+        /// <summary>
+        /// A safer way to get all the files in a directory and sub directory without crashing on UnauthorizedException or PathTooLongException
+        /// </summary>
+        /// <param name="rootPath">Starting directory</param>
+        /// <param name="patternMatch">Filename pattern match</param>
+        /// <param name="searchOption">Search subdirectories or only top level directory for files</param>
+        /// <returns>List of files</returns>
+        public static IEnumerable<string> SaferEnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly)
+        {
+            var foundFiles = Enumerable.Empty<string>();
+
+            if (searchOption == SearchOption.AllDirectories)
+            {
+                try
+                {
+                    IEnumerable<string> subDirs = Directory.EnumerateDirectories(path);
+                    // Add files in subdirectories recursively to the list
+                    foreach (string dir in subDirs)
+                        foundFiles = foundFiles.Concat(SaferEnumerateFiles(dir, searchPattern, searchOption));
+                }
+                catch (UnauthorizedAccessException) { }
+                catch (PathTooLongException) { }
+            }
+
+            try
+            {
+                // Add files from the current directory
+                foundFiles = foundFiles.Concat(Directory.EnumerateFiles(path, searchPattern));
+            }
+            catch (UnauthorizedAccessException) { }
+
+            return foundFiles;
+        }
     }
 }

LibationCli/LibationCli.csproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
 
     <PublishTrimmed>true</PublishTrimmed>
     <PublishReadyToRun>true</PublishReadyToRun>

LibationCli/Options/ScanOptions.cs

@@ -35,7 +35,8 @@ namespace LibationCli
 			var (TotalBooksProcessed, NewBooksAdded) = await LibraryCommands.ImportAccountAsync((a) => ApiExtended.CreateAsync(a), _accounts);
 
 			Console.WriteLine("Scan complete.");
-			Console.WriteLine($"Total processed: {TotalBooksProcessed}\r\nNew: {NewBooksAdded}");
+			Console.WriteLine($"Total processed: {TotalBooksProcessed}");
+			Console.WriteLine($"New: {NewBooksAdded}");
 		}
 
 		private Account[] getAccounts()

LibationFileManager/AudibleFileStorage.cs

@@ -73,8 +73,8 @@ namespace LibationFileManager
         protected override string GetFilePathCustom(string productId)
         {
             var regex = GetBookSearchRegex(productId);
-            return Directory
-                .EnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
+            return FileUtility
+                .SaferEnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
                 .FirstOrDefault(s => regex.IsMatch(s));
         }
 

LibationFileManager/LibationFileManager.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

LibationFileManager/TagsPersistence.cs

@@ -54,11 +54,17 @@ namespace LibationFileManager
 
 		private static void ensureCache()
 		{
-			if (cache is null)
-				lock (locker)
-					cache = !File.Exists(TagsFile)
-						? new Dictionary<string, string>()
-						: JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
+			if (cache is not null)
+				return;
+
+			lock (locker)
+			{
+				if (File.Exists(TagsFile))
+					cache = JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
+
+				// if file doesn't exist. or if file is corrupt and deserialize returns null
+				cache ??= new Dictionary<string, string>();
+			}
 		}
 	}
 }

LibationSearchEngine/LibationSearchEngine.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

LibationWinForms/BookLiberation/AudioDecodeForm.cs

@@ -14,11 +14,6 @@ namespace LibationWinForms.BookLiberation
 
 		private Func<byte[]> GetCoverArtDelegate;
 
-		// book info
-		private string title;
-		private string authorNames;
-		private string narratorNames;
-
 		#region Processable event handler overrides
 		public override void Processable_Begin(object sender, LibraryBook libraryBook)
 		{
@@ -31,8 +26,8 @@ namespace LibationWinForms.BookLiberation
 
 			//Set default values from library
 			AudioDecodable_TitleDiscovered(sender, libraryBook.Book.Title);
-			AudioDecodable_AuthorsDiscovered(sender, string.Join(", ", libraryBook.Book.Authors));
-			AudioDecodable_NarratorsDiscovered(sender, string.Join(", ", libraryBook.Book.NarratorNames));
+			AudioDecodable_AuthorsDiscovered(sender, libraryBook.Book.AuthorNames);
+			AudioDecodable_NarratorsDiscovered(sender, libraryBook.Book.NarratorNames);
 			AudioDecodable_CoverImageDiscovered(sender,
 					PictureStorage.GetPicture(
 						new PictureDefinition(
@@ -60,14 +55,23 @@ namespace LibationWinForms.BookLiberation
 			updateRemainingTime((int)timeRemaining.TotalSeconds);
 		}
 
+		private void updateRemainingTime(int remaining)
+			=> remainingTimeLbl.UIThreadAsync(() => remainingTimeLbl.Text = $"ETA:\r\n{formatTime(remaining)}");
+
+		private string formatTime(int seconds)
+		{
+			var timeSpan = new TimeSpan(0, 0, seconds);
+			return
+				timeSpan.TotalHours >= 1 ? $"{timeSpan:%h}h {timeSpan:mm}m {timeSpan:ss}s"
+				: timeSpan.TotalMinutes >= 1 ? $"{timeSpan:%m}m {timeSpan:ss}s"
+				: $"{seconds} sec";
+		}
 		#endregion
 
 		#region AudioDecodable event handlers
-		public override void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate)
-		{
-			base.AudioDecodable_RequestCoverArt(sender, setCoverArtDelegate);
-			setCoverArtDelegate(GetCoverArtDelegate?.Invoke());
-		}
+		private string title;
+		private string authorNames;
+		private string narratorNames;
 
 		public override void AudioDecodable_TitleDiscovered(object sender, string title)
 		{
@@ -91,27 +95,20 @@ namespace LibationWinForms.BookLiberation
 			updateBookInfo();
 		}
 
+		private void updateBookInfo()
+			=> bookInfoLbl.UIThreadAsync(() => bookInfoLbl.Text = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}");
+
+		public override void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate)
+		{
+			base.AudioDecodable_RequestCoverArt(sender, setCoverArtDelegate);
+			setCoverArtDelegate(GetCoverArtDelegate?.Invoke());
+		}
+
 		public override void AudioDecodable_CoverImageDiscovered(object sender, byte[] coverArt)
 		{
 			base.AudioDecodable_CoverImageDiscovered(sender, coverArt);
 			pictureBox1.UIThreadAsync(() => pictureBox1.Image = Dinah.Core.Drawing.ImageReader.ToImage(coverArt));
 		}
 		#endregion
-
-		// thread-safe UI updates
-		private void updateBookInfo()
-			=> bookInfoLbl.UIThreadAsync(() => bookInfoLbl.Text = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}");
-
-		private void updateRemainingTime(int remaining)
-			=> remainingTimeLbl.UIThreadAsync(() => remainingTimeLbl.Text = $"ETA:\r\n{formatTime(remaining)}");
-
-		private string formatTime(int seconds)
-		{
-			var timeSpan = new TimeSpan(0, 0, seconds);
-			return
-				timeSpan.TotalHours >= 1 ? $"{timeSpan:%h}h {timeSpan:mm}m {timeSpan:ss}s"
-				: timeSpan.TotalMinutes >= 1 ? $"{timeSpan:%m}m {timeSpan:ss}s"
-				: $"{seconds} sec";
-		}
 	}
 }

LibationWinForms/BookLiberation/BaseForms/LiberationBaseForm.cs

@@ -152,11 +152,12 @@ namespace LibationWinForms.BookLiberation.BaseForms
 		#endregion
 
 		#region AudioDecodable event handlers
-		public virtual void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate) { }
 		public virtual void AudioDecodable_TitleDiscovered(object sender, string title) { }
 		public virtual void AudioDecodable_AuthorsDiscovered(object sender, string authors) { }
 		public virtual void AudioDecodable_NarratorsDiscovered(object sender, string narrators) { }
+
 		public virtual void AudioDecodable_CoverImageDiscovered(object sender, byte[] coverArt) { }
+		public virtual void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate) { }
 		#endregion
 	}
 }

LibationWinForms/Dialogs/IndexLibraryDialog.cs

@@ -31,7 +31,7 @@ namespace LibationWinForms.Dialogs
 
 				try
 				{
-					(TotalBooksProcessed, NewBooksAdded) = await LibraryCommands.ImportAccountAsync((account) => ApiExtended.CreateAsync(account, new WinformLoginChoiceEager(account)), _accounts);
+					(TotalBooksProcessed, NewBooksAdded) = await LibraryCommands.ImportAccountAsync(account => ApiExtended.CreateAsync(account, new WinformLoginChoiceEager(account)), _accounts);
 				}
 				catch (Exception ex)
 				{

LibationWinForms/Form1.cs

@@ -61,7 +61,7 @@ namespace LibationWinForms
 			// suppressed filter while init'ing UI
 			var prev_isProcessingGridSelect = isProcessingGridSelect;
 			isProcessingGridSelect = true;
-			this.UIThreadSync(() => setGrid());
+			this.UIThreadSync(setGrid);
 			isProcessingGridSelect = prev_isProcessingGridSelect;
 
 			// UI init complete. now we can apply filter

LibationWinForms/LibationWinForms.csproj

@@ -28,7 +28,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.4.1" />
+    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.6.1" />
   </ItemGroup>
 
   <ItemGroup>

LibationWinForms/ProductsGrid.cs

@@ -107,6 +107,11 @@ namespace LibationWinForms
 
 		#endregion
 
+		private SortableBindingList<GridEntry> bindingList;
+
+		/// <summary>Insert ad hoc library books to top of grid</summary>
+		public void AddToTop(DataLayer.LibraryBook libraryBook) => bindingList.Insert(0, new GridEntry(libraryBook));
+
 		#region UI display functions
 
 		private bool hasBeenDisplayed = false;
@@ -145,7 +150,8 @@ namespace LibationWinForms
 				.ToList();
 
 			// BIND
-			gridEntryBindingSource.DataSource = new SortableBindingList<GridEntry>(orderedGridEntries);
+			bindingList = new SortableBindingList<GridEntry>(orderedGridEntries);
+			gridEntryBindingSource.DataSource = bindingList;
 
 			// FILTER
 			Filter();

LibationWinForms/Program.cs

@@ -228,7 +228,6 @@ namespace LibationWinForms
 				var libhackFiles = Directory.EnumerateDirectories(config.Books, "*.libhack", SearchOption.AllDirectories);
 
 				using var context = ApplicationServices.DbContexts.GetContext();
-				context.Books.Load();
 
 				var jArr = JArray.Parse(File.ReadAllText(filePaths));
 
@@ -248,7 +247,7 @@ namespace LibationWinForms
 					if (fileType == FileType.Unknown || fileType == FileType.AAXC)
 						continue;
 
-					var book = context.Books.Local.FirstOrDefault(b => b.AudibleProductId == asin);
+					var book = context.Books.FirstOrDefault(b => b.AudibleProductId == asin);
 					if (book is null)
 						continue;
 

README.md

@@ -92,6 +92,26 @@ Or if you have multiple accounts, you'll get to choose whether to scan all accou
 
 ![Import which accounts](images/v40_import.png)
 
+If this is a new installation, or you're scanning an account you haven't scanned before, you'll be prompted to enter your password for the Audible account.
+
+![Login password](images/alt-login1.png)
+
+Enter the password and click Submit. Audible will prompt you with a CAPTCHA image. 
+
+![Login captcha](images/alt-login2.png)
+
+Enter the CAPTCHA answer characters and click Submit. If all has gone well, Libation will start scanning the account. 
+
+In rare instances, the Captcha image/response will fail in an endless loop. If this happens, delete the problem account, and then click Save. Re-add the account and click Save again. Now try to scan the account again. This time, instead of typing your password, click the link that says "Or click here". This will open the Audible External Login dialog shown below.
+
+![Login alternative setup](images/alt-login3.png)
+
+You can either copy the URL shown and paste it into your browser or launch the browser directly by clicking Launch in Browser. Audible will display its standard login page. Login, including answering the CAPTCHA on the next page. In some cases, you might have to approve the login from the email account associated with that login, but once the login is successful, you'll see an error message.
+
+![Login alternative login result](images/alt-login4.png)
+
+This actually means you've successfully logged in. Copy the entire URL shown in your browser and return to Libation. Paste that URL into the text box at the bottom of the Audible External Login window and click Submit.
+
 You'll see this window while it's scanning:
 
 ![Import step 2](images/Import2.png)

_Tests/AudibleUtilities.Tests/AudibleUtilities.Tests.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="FluentAssertions" Version="6.6.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="Moq" Version="4.17.2" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
     <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

_Tests/FileLiberator.Tests/FileLiberator.Tests.csproj

@@ -1,16 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="FluentAssertions" Version="6.6.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
     <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

_Tests/FileManager.Tests/FileManager.Tests.csproj

@@ -7,10 +7,10 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="FluentAssertions" Version="6.6.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
     <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

_Tests/LibationFileManager.Tests/LibationFileManager.Tests.csproj

@@ -1,16 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="FluentAssertions" Version="6.6.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
     <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj

@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net6.0-windows</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="FluentAssertions" Version="6.6.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="Moq" Version="4.17.2" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
     <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>