Move source code into "Source" folder

Source/DtoImporterService/BookImporter.cs

@@ -0,0 +1,185 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApi.Common;
+using AudibleUtilities;
+using DataLayer;
+using Dinah.Core.Collections.Generic;
+
+namespace DtoImporterService
+{
+	public class BookImporter : ItemsImporterBase
+	{
+		protected override IValidator Validator => new BookValidator();
+
+		public Dictionary<string, Book> Cache { get; private set; } = new();
+
+		private ContributorImporter contributorImporter { get; }
+		private SeriesImporter seriesImporter { get; }
+		private CategoryImporter categoryImporter { get; }
+
+		public BookImporter(LibationContext context) : base(context)
+		{
+			contributorImporter = new ContributorImporter(DbContext);
+			seriesImporter = new SeriesImporter(DbContext);
+			categoryImporter = new CategoryImporter(DbContext);
+		}
+
+		protected override int DoImport(IEnumerable<ImportItem> importItems)
+		{
+			// pre-req.s
+			contributorImporter.Import(importItems);
+			seriesImporter.Import(importItems);
+			categoryImporter.Import(importItems);
+
+			// load db existing => hash table
+			loadLocal_books(importItems);
+
+			// upsert
+			var qtyNew = upsertBooks(importItems);
+			return qtyNew;
+		}
+
+		private void loadLocal_books(IEnumerable<ImportItem> importItems)
+		{
+			// get distinct
+			var productIds = importItems
+				.Select(i => i.DtoItem.ProductId)
+				.Distinct()
+				.ToList();
+
+			Cache = DbContext.Books
+				.GetBooks(b => productIds.Contains(b.AudibleProductId))
+				.ToDictionarySafe(b => b.AudibleProductId);
+		}
+
+		private int upsertBooks(IEnumerable<ImportItem> importItems)
+		{
+			var qtyNew = 0;
+
+			foreach (var item in importItems)
+			{
+				if (!Cache.TryGetValue(item.DtoItem.ProductId, out var book))
+				{
+					book = createNewBook(item);
+					qtyNew++;
+				}
+
+				updateBook(item, book);
+			}
+
+			return qtyNew;
+		}
+
+		private Book createNewBook(ImportItem importItem)
+		{
+			var item = importItem.DtoItem;
+
+			var contentType = item.IsEpisodes ? DataLayer.ContentType.Episode : DataLayer.ContentType.Product;
+
+			// absence of authors is very rare, but possible
+			if (!item.Authors?.Any() ?? true)
+				item.Authors = new[] { new Person { Name = "", Asin = null } };
+
+			// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
+			var authors = item
+				.Authors
+				.Select(a => contributorImporter.Cache[a.Name])
+				.ToList();
+
+			var narrators
+				= item.Narrators is null || !item.Narrators.Any()
+				// if no narrators listed, author is the narrator
+				? authors
+				// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
+				: item
+					.Narrators
+					.Select(n => contributorImporter.Cache[n.Name])
+					.ToList();
+
+			// categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd
+			// absence of categories is also possible
+
+			// CATEGORY HACK: only use the 1st 2 categories
+			// (real impl: var lastCategory = item.Categories.LastOrDefault()?.CategoryId ?? "";)
+			var lastCategory
+				= item.Categories.Length == 0 ? ""
+				: item.Categories.Length == 1 ? item.Categories[0].CategoryId
+				// 2+
+				: item.Categories[1].CategoryId;
+
+			var category = categoryImporter.Cache[lastCategory];
+
+			Book book;
+			try
+			{
+				book = DbContext.Books.Add(new Book(
+					new AudibleProductId(item.ProductId),
+					item.TitleWithSubtitle,
+					item.Description,
+					item.LengthInMinutes,
+					contentType,
+					authors,
+					narrators,
+					category,
+					importItem.LocaleName)
+					).Entity;
+				Cache.Add(book.AudibleProductId, book);
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Error adding book. {@DebugInfo}", new {
+					item.ProductId,
+					item.TitleWithSubtitle,
+					item.Description,
+					item.LengthInMinutes,
+					contentType,
+					QtyAuthors = authors?.Count,
+					QtyNarrators = narrators?.Count,
+					Category = category?.Name,
+					importItem.LocaleName
+				});
+				throw;
+			}
+
+			var publisherName = item.Publisher;
+			if (!string.IsNullOrWhiteSpace(publisherName))
+			{
+				var publisher = contributorImporter.Cache[publisherName];
+				book.ReplacePublisher(publisher);
+			}
+
+			book.UpdateBookDetails(item.IsAbridged, item.DatePublished);
+
+			if (item.PdfUrl is not null)
+				book.AddSupplementDownloadUrl(item.PdfUrl.ToString());
+
+			return book;
+		}
+
+		private void updateBook(ImportItem importItem, Book book)
+		{
+			var item = importItem.DtoItem;
+
+			// set/update book-specific info which may have changed
+			if (item.PictureId is not null)
+				book.PictureId = item.PictureId;
+
+			book.UpdateProductRating(item.Product_OverallStars, item.Product_PerformanceStars, item.Product_StoryStars);
+
+			// important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import
+			book.UserDefinedItem.UpdateRating(item.MyUserRating_Overall, item.MyUserRating_Performance, item.MyUserRating_Story);
+
+			// update series even for existing books. these are occasionally updated
+			// these will upsert over library-scraped series, but will not leave orphans
+			if (item.Series is not null)
+			{
+				foreach (var seriesEntry in item.Series)
+				{
+					var series = seriesImporter.Cache[seriesEntry.SeriesId];
+					book.UpsertSeries(series, seriesEntry.Sequence);
+				}
+			}
+		}
+	}
+}

Source/DtoImporterService/CategoryImporter.cs

@@ -0,0 +1,104 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApi.Common;
+using AudibleUtilities;
+using DataLayer;
+using Dinah.Core.Collections.Generic;
+
+namespace DtoImporterService
+{
+	public class CategoryImporter : ItemsImporterBase
+	{
+		protected override IValidator Validator => new CategoryValidator();
+
+		public Dictionary<string, Category> Cache { get; private set; } = new();
+
+		public CategoryImporter(LibationContext context) : base(context) { }
+
+		protected override int DoImport(IEnumerable<ImportItem> importItems)
+		{
+			// get distinct
+			var categoryIds = importItems
+				.Select(i => i.DtoItem)
+				.GetCategoriesDistinct()
+				.Select(c => c.CategoryId)
+				.Distinct()
+				.ToList();
+
+			// load db existing => .Local
+			loadLocal_categories(categoryIds);
+
+			// upsert
+			var categoryPairs = importItems
+				.Select(i => i.DtoItem)
+				.GetCategoryPairsDistinct()
+				.ToList();
+			var qtyNew = upsertCategories(categoryPairs);
+			return qtyNew;
+		}
+
+		private void loadLocal_categories(List<string> categoryIds)
+		{
+			// must include default/empty/missing
+			categoryIds.Add(Category.GetEmpty().AudibleCategoryId);
+
+			// load existing => local
+			Cache = DbContext.Categories
+				.Where(c => categoryIds.Contains(c.AudibleCategoryId))
+				.ToDictionarySafe(c => c.AudibleCategoryId);
+		}
+
+		// only use after loading contributors => local
+		private int upsertCategories(List<Ladder[]> categoryPairs)
+		{
+			var qtyNew = 0;
+
+			foreach (var pair in categoryPairs)
+			{
+				for (var i = 0; i < pair.Length; i++)
+				{
+					// CATEGORY HACK: not yet supported: depth beyond 0 and 1
+					if (i > 1)
+						break;
+
+					var id = pair[i].CategoryId;
+					var name = pair[i].CategoryName;
+
+					Category parentCategory = null;
+					if (i == 1)
+						Cache.TryGetValue(pair[0].CategoryId, out parentCategory);
+
+					if (!Cache.TryGetValue(id, out var category))
+					{
+						category = addCategory(id, name);
+						qtyNew++;
+					}
+
+					category.UpdateParentCategory(parentCategory);
+				}
+			}
+
+			return qtyNew;
+		}
+
+		private Category addCategory(string id, string name)
+		{
+			try
+			{
+				var category = new Category(new AudibleCategoryId(id), name);
+
+				var entityEntry = DbContext.Categories.Add(category);
+				var entity = entityEntry.Entity;
+
+				Cache.Add(entity.AudibleCategoryId, entity);
+				return entity;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Error adding category. {@DebugInfo}", new { id, name });
+				throw;
+			}
+		}
+	}
+}

Source/DtoImporterService/ContributorImporter.cs

@@ -0,0 +1,113 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApi.Common;
+using AudibleUtilities;
+using DataLayer;
+using Dinah.Core.Collections.Generic;
+
+namespace DtoImporterService
+{
+	public class ContributorImporter : ItemsImporterBase
+	{
+		protected override IValidator Validator => new ContributorValidator();
+
+		public Dictionary<string, Contributor> Cache { get; private set; } = new();
+
+		public ContributorImporter(LibationContext context) : base(context) { }
+
+		protected override int DoImport(IEnumerable<ImportItem> importItems)
+		{
+			// get distinct
+			var authors = importItems
+				.Select(i => i.DtoItem)
+				.GetAuthorsDistinct()
+				.ToList();
+			var narrators = importItems
+				.Select(i => i.DtoItem)
+				.GetNarratorsDistinct()
+				.ToList();
+			var publishers = importItems
+				.Select(i => i.DtoItem)
+				.GetPublishersDistinct()
+				.ToList();
+
+			// load db existing => .Local
+			var allNames = publishers
+				.Union(authors.Select(n => n.Name))
+				.Union(narrators.Select(n => n.Name))
+				.Where(name => !string.IsNullOrWhiteSpace(name))
+				.ToList();
+			loadLocal_contributors(allNames);
+
+			// upsert
+			var qtyNew = 0;
+			qtyNew += upsertPeople(authors);
+			qtyNew += upsertPeople(narrators);
+			qtyNew += upsertPublishers(publishers);
+			return qtyNew;
+		}
+
+		private void loadLocal_contributors(List<string> contributorNames)
+		{
+			// must include default/empty/missing
+			contributorNames.Add(Contributor.GetEmpty().Name);
+
+			// load existing => local
+			Cache = DbContext.Contributors
+				.Where(c => contributorNames.Contains(c.Name))
+				.ToDictionarySafe(c => c.Name);
+		}
+
+		private int upsertPeople(List<Person> people)
+		{
+			var hash = people
+				// new people only
+				.Where(p => !Cache.ContainsKey(p.Name))
+				// remove duplicates by Name. first in wins
+				.ToDictionarySafe(p => p.Name);
+
+			foreach (var kvp in hash)
+			{
+				var person = kvp.Value;
+				addContributor(person.Name, person.Asin);
+			}
+
+			return hash.Count;
+		}
+
+		// only use after loading contributors => local
+		private int upsertPublishers(List<string> publishers)
+		{
+			var hash = publishers
+				// new publishers only
+				.Where(p => !Cache.ContainsKey(p))
+				// remove duplicates
+				.ToHashSet();
+
+			foreach (var pub in hash)
+				addContributor(pub);
+
+			return hash.Count;
+		}
+
+        private Contributor addContributor(string name, string id = null)
+		{
+			try
+			{
+				var newContrib = new Contributor(name);
+
+				var entityEntry = DbContext.Contributors.Add(newContrib);
+				var entity = entityEntry.Entity;
+
+				Cache.Add(entity.Name, entity);
+				return entity;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Error adding contributor. {@DebugInfo}", new { name, id });
+				throw;
+			}
+        }
+    }
+}

Source/DtoImporterService/DtoImporterService.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0-windows</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\AudibleUtilities\AudibleUtilities.csproj" />
+    <ProjectReference Include="..\DataLayer\DataLayer.csproj" />
+  </ItemGroup>
+
+</Project>

Source/DtoImporterService/ImportItem.cs

@@ -0,0 +1,12 @@
+using System;
+using AudibleApi.Common;
+
+namespace DtoImporterService
+{
+	public class ImportItem
+	{
+		public Item DtoItem { get; set; }
+		public string AccountId { get; set; }
+		public string LocaleName { get; set; }
+	}
+}

Source/DtoImporterService/ImporterBase.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleUtilities;
+using DataLayer;
+using Dinah.Core;
+
+namespace DtoImporterService
+{
+	public abstract class ImporterBase<T>
+	{
+		protected LibationContext DbContext { get; }
+
+		protected ImporterBase(LibationContext context)
+		{
+			ArgumentValidator.EnsureNotNull(context, nameof(context));
+			DbContext = context;
+		}
+
+		/// <summary>LONG RUNNING. call with await Task.Run</summary>
+		public int Import(T param) => Run(DoImport, param);
+
+		public TResult Run<TResult>(Func<T, TResult> func, T param)
+		{
+			try
+			{
+				var exceptions = Validate(param);
+				if (exceptions is not null && exceptions.Any())
+					throw new AggregateException($"Importer validation failed", exceptions);
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Import error: validation");
+				throw;
+			}
+
+			try
+			{
+				var result = func(param);
+				return result;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Import error: post-validation importing");
+				throw;
+			}
+		}
+
+		protected abstract int DoImport(T elements);
+		public abstract IEnumerable<Exception> Validate(T param);
+	}
+
+	public abstract class ItemsImporterBase : ImporterBase<IEnumerable<ImportItem>>
+	{
+		protected ItemsImporterBase(LibationContext context) : base(context) { }
+
+		protected abstract IValidator Validator { get; }
+        public sealed override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems)
+			=> Validator.Validate(importItems.Select(i => i.DtoItem));
+	}
+}

Source/DtoImporterService/LibraryBookImporter.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleUtilities;
+using DataLayer;
+using Dinah.Core.Collections.Generic;
+
+namespace DtoImporterService
+{
+	public class LibraryBookImporter : ItemsImporterBase
+	{
+		protected override IValidator Validator => new LibraryValidator();
+
+		private BookImporter bookImporter { get; }
+
+		public LibraryBookImporter(LibationContext context) : base(context)
+		{
+			bookImporter = new BookImporter(DbContext);
+		}
+
+		protected override int DoImport(IEnumerable<ImportItem> importItems)
+		{
+			bookImporter.Import(importItems);
+
+			var qtyNew = upsertLibraryBooks(importItems);
+			return qtyNew;
+		}
+
+		private int upsertLibraryBooks(IEnumerable<ImportItem> importItems)
+		{
+			// technically, we should be able to have duplicate books from separate accounts.
+			// this would violate the current pk and would be difficult to deal with elsewhere:
+			// - what to show in the grid
+			// - which to consider liberated
+			//
+			// sqlite cannot alter pk. the work around is an extensive headache
+			// - update: now possible in .net5/efcore5
+			//
+			// currently, inserting LibraryBook will throw error if the same book is in multiple accounts for the same region.
+			//
+			// CURRENT SOLUTION: don't re-insert
+
+			var currentLibraryProductIds = DbContext.LibraryBooks.Select(l => l.Book.AudibleProductId).ToList();
+			var newItems = importItems
+				.Where(dto => !currentLibraryProductIds.Contains(dto.DtoItem.ProductId))
+				.ToList();
+
+			// if 2 accounts try to import the same book in the same transaction: error since we're only tracking and pulling by asin.
+			// just use the first
+			var hash = newItems.ToDictionarySafe(dto => dto.DtoItem.ProductId);
+			foreach (var kvp in hash)
+            {
+				var newItem = kvp.Value;
+
+				var libraryBook = new LibraryBook(
+					bookImporter.Cache[newItem.DtoItem.ProductId],
+					newItem.DtoItem.DateAdded,
+					newItem.AccountId);
+				try
+				{
+					DbContext.LibraryBooks.Add(libraryBook);
+				}
+				catch (Exception ex)
+				{
+					Serilog.Log.Logger.Error(ex, "Error adding library book. {@DebugInfo}", new { libraryBook.Book, libraryBook.Account });
+				}
+			}
+
+			var qtyNew = hash.Count;
+			return qtyNew;
+		}
+	}
+}

Source/DtoImporterService/PerfLogger.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+
+namespace DtoImporterService
+{
+	public record timeLogEntry(string msg, long totalElapsed, long delta);
+	public static class PerfLogger
+	{
+		private static Stopwatch sw { get; } = new();
+		private static List<timeLogEntry> __log { get; } = new() { new("begin", 0, 0) };
+
+		public static void logTime(string s)
+		{
+			var totalElapsed = sw.ElapsedMilliseconds;
+
+			var prev = __log.Last().totalElapsed;
+			var delta = totalElapsed - prev;
+
+			__log.Add(new(s, totalElapsed, delta));
+		}
+		public static void logRestart()
+		{
+			__log.Clear();
+			__log.Add(new("begin", 0, 0));
+			sw.Restart();
+		}
+		public static void stop() => sw.Stop();
+		public static string logOutput =>
+			$"{nameof(timeLogEntry.msg)}\t{nameof(timeLogEntry.totalElapsed)}\t{nameof(timeLogEntry.delta)}\r\n"
+			+ __log.Select(t => $"{t.msg}\t{t.totalElapsed}\t{t.delta}").Aggregate((a, b) => $"{a}\r\n{b}");
+	}
+}

Source/DtoImporterService/SeriesImporter.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApi.Common;
+using AudibleUtilities;
+using DataLayer;
+using Dinah.Core.Collections.Generic;
+
+namespace DtoImporterService
+{
+	public class SeriesImporter : ItemsImporterBase
+	{
+		protected override IValidator Validator => new SeriesValidator();
+
+		public Dictionary<string, DataLayer.Series> Cache { get; private set; } = new();
+
+		public SeriesImporter(LibationContext context) : base(context) { }
+
+		protected override int DoImport(IEnumerable<ImportItem> importItems)
+		{
+			// get distinct
+			var series = importItems
+				.Select(i => i.DtoItem)
+				.GetSeriesDistinct()
+				.ToList();
+
+			// load db existing => .Local
+			loadLocal_series(series);
+
+			// upsert
+			var qtyNew = upsertSeries(series);
+			return qtyNew;
+		}
+
+		private void loadLocal_series(List<AudibleApi.Common.Series> series)
+		{
+			var seriesIds = series.Select(s => s.SeriesId).Distinct().ToList();
+
+			if (seriesIds.Any())
+				Cache = DbContext.Series
+					.Where(s => seriesIds.Contains(s.AudibleSeriesId))
+					.ToDictionarySafe(s => s.AudibleSeriesId);
+		}
+
+		private int upsertSeries(List<AudibleApi.Common.Series> requestedSeries)
+		{
+			var qtyNew = 0;
+
+			foreach (var s in requestedSeries)
+			{
+				// AudibleApi.Common.Series.SeriesId == DataLayer.AudibleSeriesId
+				if (!Cache.TryGetValue(s.SeriesId, out var series))
+				{
+					series = addSeries(s.SeriesId);
+					qtyNew++;
+				}
+				series.UpdateName(s.SeriesName);
+			}
+
+			return qtyNew;
+		}
+
+		private DataLayer.Series addSeries(string seriesId)
+		{
+			try
+			{
+				var series = new DataLayer.Series(new AudibleSeriesId(seriesId));
+
+				var entityEntry = DbContext.Series.Add(series);
+				var entity = entityEntry.Entity;
+
+				Cache.Add(entity.AudibleSeriesId, entity);
+				return entity;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "Error adding series. {@DebugInfo}", new { seriesId });
+				throw;
+			}
+		}
+	}
+}

Source/DtoImporterService/_importer notes.txt

@@ -0,0 +1,37 @@
+* Local (eg DbContext.Books.Local): indexes/hashes PK and nothing else. Local.Find(PK) is fast. All other searches (eg FirstOrDefault) have awful performance. It deceptively *feels* like we get this partially for free since added/modified entries live here.
+* live db: for all importers, fields used for lookup are indexed
+
+Using BookImporter as an example: since AudibleProductId is indexed, hitting the live db is much faster than using Local. Fastest is putting all in a local hash table
+
+Note: GetBook/GetBooks eager loads Series, category, et al
+
+for 1,200 iterations
+* load to LocalView
+  DbContext.Books.Local.FirstOrDefault(p => p.AudibleProductId == item.DtoItem.ProductId)
+  27,125 ms
+* read from live db
+  DbContext.Books.GetBook(item.DtoItem.ProductId)
+  12,224 ms
+* load to hash table: Dictionary<string, Book>
+  dictionary[item.DtoItem.ProductId];
+  1 ms (yes: ONE)
+
+With hashtable, somehow memory usage was not significantly affected
+
+-----------------------------------
+
+why were we using Local to begin with?
+
+articles suggest loading to Local with
+    context.Books.Load();
+this loads this table but not associated fields
+we want Books and associated fields
+    context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
+this is emulating Load() but with also getting associated fields
+			from: Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions
+			// Summary:
+			//     Enumerates the query. When using Entity Framework, this causes the results of
+			//     the query to be loaded into the associated context. This is equivalent to calling
+			//     ToList and then throwing away the list (without the overhead of actually creating
+			//     the list).
+			public static void Load<TSource>([NotNullAttribute] this IQueryable<TSource> source);