Compare commits

...

9 Commits

Author SHA1 Message Date
Robert McRackan
68ad627159 update dependencies 2022-04-30 21:03:40 -04:00
Robert McRackan
878a5dd36c Libary import got a complete overhaul. On a library of 1,200 titles: initial scan is 80-85% faster. Subsequent imports are 60-70% faster 2022-04-29 16:35:49 -04:00
Robert McRackan
7c144b8277 Bug fix #234 : chapters were no longer included in the m4b file 2022-04-27 11:31:13 -04:00
Robert McRackan
bca8c3865b Expose a way to insert ad hoc library books to grid 2022-04-26 16:37:13 -04:00
Robert McRackan
58102acd35 Trivial refactoring 2022-04-26 16:34:59 -04:00
Robert McRackan
5e577843f7 Fixing genre metatag is conditional upon AllowLibationFixup setting 2022-04-26 09:49:21 -04:00
Robert McRackan
e1d549cead update dependencies 2022-04-26 09:27:13 -04:00
Robert McRackan
323b8f2fb9 minor refactors 2022-04-26 08:18:35 -04:00
Robert McRackan
3dcbcf42ed Fix 2 for issue #202 2022-04-25 22:21:36 -04:00
41 changed files with 323 additions and 285 deletions

View File

@@ -5,8 +5,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AAXClean" Version="0.2.5" />
<PackageReference Include="AAXClean.Codecs" Version="0.1.2" />
<PackageReference Include="AAXClean" Version="0.3.2" />
<PackageReference Include="AAXClean.Codecs" Version="0.1.10" />
</ItemGroup>
<ItemGroup>

View File

@@ -6,6 +6,8 @@ namespace AaxDecrypter
{
public abstract class AaxcDownloadConvertBase : AudiobookDownloadBase
{
public event EventHandler<AppleTags> RetrievedMetadata;
protected OutputFormat OutputFormat { get; }
protected AaxFile AaxFile;
@@ -20,24 +22,21 @@ namespace AaxDecrypter
public override void SetCoverArt(byte[] coverArt)
{
base.SetCoverArt(coverArt);
if (coverArt is not null)
AaxFile?.AppleTags.SetCoverArt(coverArt);
if (coverArt is not null && AaxFile?.AppleTags is not null)
AaxFile.AppleTags.Cover = coverArt;
}
/// <summary>Optional step to run after Metadata is retrieved</summary>
public Action<AaxFile> UpdateMetadata { get; set; }
protected bool Step_GetMetadata()
{
AaxFile = new AaxFile(InputFileStream);
UpdateMetadata?.Invoke(AaxFile);
OnRetrievedTitle(AaxFile.AppleTags.TitleSansUnabridged);
OnRetrievedAuthors(AaxFile.AppleTags.FirstAuthor ?? "[unknown]");
OnRetrievedNarrators(AaxFile.AppleTags.Narrator ?? "[unknown]");
OnRetrievedCoverArt(AaxFile.AppleTags.Cover);
RetrievedMetadata?.Invoke(this, AaxFile.AppleTags);
return !IsCanceled;
}

View File

@@ -67,16 +67,18 @@ namespace AaxDecrypter
Serilog.Log.Logger.Error("Conversion failed");
return IsSuccess;
}
}
protected void OnRetrievedTitle(string title)
protected void OnRetrievedTitle(string title)
=> RetrievedTitle?.Invoke(this, title);
protected void OnRetrievedAuthors(string authors)
protected void OnRetrievedAuthors(string authors)
=> RetrievedAuthors?.Invoke(this, authors);
protected void OnRetrievedNarrators(string narrators)
protected void OnRetrievedNarrators(string narrators)
=> RetrievedNarrators?.Invoke(this, narrators);
protected void OnRetrievedCoverArt(byte[] coverArt)
protected void OnRetrievedCoverArt(byte[] coverArt)
=> RetrievedCoverArt?.Invoke(this, coverArt);
protected void OnDecryptProgressUpdate(DownloadProgress downloadProgress)
=> DecryptProgressUpdate?.Invoke(this, downloadProgress);
protected void OnDecryptTimeRemaining(TimeSpan timeRemaining)

View File

@@ -3,7 +3,7 @@
<PropertyGroup>
<TargetFramework>net6.0-windows</TargetFramework>
<Version>6.8.1.1</Version>
<Version>7.0.0.1</Version>
</PropertyGroup>
<ItemGroup>
@@ -11,7 +11,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Octokit" Version="0.50.0" />
<PackageReference Include="Octokit" Version="0.51.0" />
</ItemGroup>
<ItemGroup>

View File

@@ -6,7 +6,7 @@
<ItemGroup>
<PackageReference Include="CsvHelper" Version="27.2.1" />
<PackageReference Include="NPOI" Version="2.5.5" />
<PackageReference Include="NPOI" Version="2.5.6" />
</ItemGroup>
<ItemGroup>

View File

@@ -89,6 +89,9 @@ namespace ApplicationServices
var totalCount = importItems.Count;
Log.Logger.Information($"GetAllLibraryItems: Total count {totalCount}");
if (totalCount == 0)
return default;
Log.Logger.Information("Begin long-running import");
logTime($"pre {nameof(importIntoDbAsync)}");
var newCount = await importIntoDbAsync(importItems);

View File

@@ -5,7 +5,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AudibleApi" Version="2.7.2.1" />
<PackageReference Include="AudibleApi" Version="2.7.3.1" />
</ItemGroup>
<ItemGroup>

View File

@@ -9,8 +9,8 @@ namespace DataLayer.Configurations
{
entity.HasKey(bc => new { bc.BookId, bc.ContributorId, bc.Role });
entity.HasIndex(b => b.BookId);
entity.HasIndex(b => b.ContributorId);
entity.HasIndex(bc => bc.BookId);
entity.HasIndex(bc => bc.ContributorId);
entity
.HasOne(bc => bc.Book)

View File

@@ -21,12 +21,12 @@ namespace DataLayer.Configurations
// - update LibraryBook import code
// - would likely challenge assumptions throughout Libation which have been true up until now
entity.HasKey(b => b.BookId);
entity.HasKey(lb => lb.BookId);
entity
.HasOne(le => le.Book)
.HasOne(lb => lb.Book)
.WithOne()
.HasForeignKey<LibraryBook>(le => le.BookId);
.HasForeignKey<LibraryBook>(lb => lb.BookId);
}
}
}

View File

@@ -7,10 +7,10 @@ namespace DataLayer.Configurations
{
public void Configure(EntityTypeBuilder<SeriesBook> entity)
{
entity.HasKey(bc => new { bc.SeriesId, bc.BookId });
entity.HasKey(sb => new { sb.SeriesId, sb.BookId });
entity.HasIndex(b => b.SeriesId);
entity.HasIndex(b => b.BookId);
entity.HasIndex(sb => sb.SeriesId);
entity.HasIndex(sb => sb.BookId);
entity
.HasOne(sb => sb.Series)

View File

@@ -7,8 +7,8 @@ namespace DataLayer.Configurations
{
public void Configure(EntityTypeBuilder<Series> entity)
{
entity.HasKey(b => b.SeriesId);
entity.HasIndex(b => b.AudibleSeriesId);
entity.HasKey(s => s.SeriesId);
entity.HasIndex(s => s.AudibleSeriesId);
entity
.Metadata

View File

@@ -31,16 +31,12 @@ namespace DataLayer
public string AudibleContributorId { get; private set; }
private Contributor() { }
public Contributor(string name)
public Contributor(string name, string audibleContributorId = null)
{
ArgumentValidator.EnsureNotNullOrWhiteSpace(name, nameof(name));
Name = ArgumentValidator.EnsureNotNullOrWhiteSpace(name, nameof(name));
_booksLink = new HashSet<BookContributor>();
Name = name;
}
public Contributor(string name, string audibleContributorId) : this(name)
{
// don't overwrite with null or whitespace but not an error
if (!string.IsNullOrWhiteSpace(audibleContributorId))
AudibleContributorId = audibleContributorId;

View File

@@ -19,7 +19,8 @@ namespace DataLayer
public static Book GetBook(this IQueryable<Book> books, string productId)
=> books
.GetBooks()
.SingleOrDefault(b => b.AudibleProductId == productId);
// 'Single' is more accurate but 'First' is faster and less error prone
.FirstOrDefault(b => b.AudibleProductId == productId);
/// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary>
public static IQueryable<Book> GetBooks(this IQueryable<Book> books, Expression<Func<Book, bool>> predicate)

View File

@@ -4,62 +4,53 @@ using System.Linq;
using AudibleApi.Common;
using AudibleUtilities;
using DataLayer;
using Dinah.Core.Collections.Generic;
namespace DtoImporterService
{
public class BookImporter : ItemsImporterBase
{
public BookImporter(LibationContext context) : base(context) { }
protected override IValidator Validator => new BookValidator();
public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new BookValidator().Validate(importItems.Select(i => i.DtoItem));
public Dictionary<string, Book> Cache { get; private set; } = new();
private ContributorImporter contributorImporter { get; }
private SeriesImporter seriesImporter { get; }
private CategoryImporter categoryImporter { get; }
public BookImporter(LibationContext context) : base(context)
{
contributorImporter = new ContributorImporter(DbContext);
seriesImporter = new SeriesImporter(DbContext);
categoryImporter = new CategoryImporter(DbContext);
}
protected override int DoImport(IEnumerable<ImportItem> importItems)
{
// pre-req.s
new ContributorImporter(DbContext).Import(importItems);
new SeriesImporter(DbContext).Import(importItems);
new CategoryImporter(DbContext).Import(importItems);
// get distinct
var productIds = importItems.Select(i => i.DtoItem.ProductId).Distinct().ToList();
// load db existing => .Local
loadLocal_books(productIds);
contributorImporter.Import(importItems);
seriesImporter.Import(importItems);
categoryImporter.Import(importItems);
// load db existing => hash table
loadLocal_books(importItems);
// upsert
var qtyNew = upsertBooks(importItems);
return qtyNew;
}
private void loadLocal_books(List<string> productIds)
private void loadLocal_books(IEnumerable<ImportItem> importItems)
{
// if this context has already loaded books, don't need to reload them. vestige from when context was long-lived. in practice, we now typically use a fresh context. this is quick though so no harm in leaving it.
var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId).ToList();
var remainingProductIds = productIds
.Except(localProductIds)
// get distinct
var productIds = importItems
.Select(i => i.DtoItem.ProductId)
.Distinct()
.ToList();
#region // explanation of DbContext.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
/*
articles suggest loading to Local with
context.Books.Load();
we want Books and associated fields
context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
this is emulating Load() but with also getting associated fields
from: Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions
// Summary:
// Enumerates the query. When using Entity Framework, this causes the results of
// the query to be loaded into the associated context. This is equivalent to calling
// ToList and then throwing away the list (without the overhead of actually creating
// the list).
public static void Load<TSource>([NotNullAttribute] this IQueryable<TSource> source);
*/
#endregion
// GetBooks() eager loads Series, category, et al
if (remainingProductIds.Any())
DbContext.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
Cache = DbContext.Books
.GetBooks(b => productIds.Contains(b.AudibleProductId))
.ToDictionarySafe(b => b.AudibleProductId);
}
private int upsertBooks(IEnumerable<ImportItem> importItems)
@@ -68,8 +59,7 @@ namespace DtoImporterService
foreach (var item in importItems)
{
var book = DbContext.Books.Local.FirstOrDefault(p => p.AudibleProductId == item.DtoItem.ProductId);
if (book is null)
if (!Cache.TryGetValue(item.DtoItem.ProductId, out var book))
{
book = createNewBook(item);
qtyNew++;
@@ -94,8 +84,7 @@ namespace DtoImporterService
// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
var authors = item
.Authors
// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
.Select(a => DbContext.Contributors.Local.FirstOrDefault(c => a.Name == c.Name))
.Select(a => contributorImporter.Cache[a.Name])
.ToList();
var narrators
@@ -105,8 +94,7 @@ namespace DtoImporterService
// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
: item
.Narrators
// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
.Select(n => DbContext.Contributors.Local.FirstOrDefault(c => n.Name == c.Name))
.Select(n => contributorImporter.Cache[n.Name])
.ToList();
// categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd
@@ -120,8 +108,7 @@ namespace DtoImporterService
// 2+
: item.Categories[1].CategoryId;
// This should properly be SingleOrDefault() not FirstOrDefault(), but FirstOrDefault is defensive
var category = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == lastCategory);
var category = categoryImporter.Cache[lastCategory];
Book book;
try
@@ -137,6 +124,7 @@ namespace DtoImporterService
category,
importItem.LocaleName)
).Entity;
Cache.Add(book.AudibleProductId, book);
}
catch (Exception ex)
{
@@ -157,8 +145,7 @@ namespace DtoImporterService
var publisherName = item.Publisher;
if (!string.IsNullOrWhiteSpace(publisherName))
{
// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
var publisher = DbContext.Contributors.Local.FirstOrDefault(c => publisherName == c.Name);
var publisher = contributorImporter.Cache[publisherName];
book.ReplacePublisher(publisher);
}
@@ -189,7 +176,7 @@ namespace DtoImporterService
{
foreach (var seriesEntry in item.Series)
{
var series = DbContext.Series.Local.FirstOrDefault(s => seriesEntry.SeriesId == s.AudibleSeriesId);
var series = seriesImporter.Cache[seriesEntry.SeriesId];
book.UpsertSeries(series, seriesEntry.Sequence);
}
}

View File

@@ -4,14 +4,17 @@ using System.Linq;
using AudibleApi.Common;
using AudibleUtilities;
using DataLayer;
using Dinah.Core.Collections.Generic;
namespace DtoImporterService
{
public class CategoryImporter : ItemsImporterBase
{
public CategoryImporter(LibationContext context) : base(context) { }
protected override IValidator Validator => new CategoryValidator();
public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new CategoryValidator().Validate(importItems.Select(i => i.DtoItem));
public Dictionary<string, Category> Cache { get; private set; } = new();
public CategoryImporter(LibationContext context) : base(context) { }
protected override int DoImport(IEnumerable<ImportItem> importItems)
{
@@ -19,7 +22,9 @@ namespace DtoImporterService
var categoryIds = importItems
.Select(i => i.DtoItem)
.GetCategoriesDistinct()
.Select(c => c.CategoryId).ToList();
.Select(c => c.CategoryId)
.Distinct()
.ToList();
// load db existing => .Local
loadLocal_categories(categoryIds);
@@ -38,15 +43,10 @@ namespace DtoImporterService
// must include default/empty/missing
categoryIds.Add(Category.GetEmpty().AudibleCategoryId);
var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId).ToList();
var remainingCategoryIds = categoryIds
.Distinct()
.Except(localIds)
.ToList();
// load existing => local
if (remainingCategoryIds.Any())
DbContext.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId)).ToList();
Cache = DbContext.Categories
.Where(c => categoryIds.Contains(c.AudibleCategoryId))
.ToDictionarySafe(c => c.AudibleCategoryId);
}
// only use after loading contributors => local
@@ -67,22 +67,11 @@ namespace DtoImporterService
Category parentCategory = null;
if (i == 1)
// should be "Single()" but user is getting a strange error
parentCategory = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == pair[0].CategoryId);
Cache.TryGetValue(pair[0].CategoryId, out parentCategory);
// should be "SingleOrDefault()" but user is getting a strange error
var category = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == id);
if (category is null)
if (!Cache.TryGetValue(id, out var category))
{
try
{
category = DbContext.Categories.Add(new Category(new AudibleCategoryId(id), name)).Entity;
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding category. {@DebugInfo}", new { id, name });
throw;
}
category = addCategory(id, name);
qtyNew++;
}
@@ -92,5 +81,24 @@ namespace DtoImporterService
return qtyNew;
}
private Category addCategory(string id, string name)
{
try
{
var category = new Category(new AudibleCategoryId(id), name);
var entityEntry = DbContext.Categories.Add(category);
var entity = entityEntry.Entity;
Cache.Add(entity.AudibleCategoryId, entity);
return entity;
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding category. {@DebugInfo}", new { id, name });
throw;
}
}
}
}

View File

@@ -4,14 +4,17 @@ using System.Linq;
using AudibleApi.Common;
using AudibleUtilities;
using DataLayer;
using Dinah.Core.Collections.Generic;
namespace DtoImporterService
{
public class ContributorImporter : ItemsImporterBase
{
public ContributorImporter(LibationContext context) : base(context) { }
protected override IValidator Validator => new ContributorValidator();
public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new ContributorValidator().Validate(importItems.Select(i => i.DtoItem));
public Dictionary<string, Contributor> Cache { get; private set; } = new();
public ContributorImporter(LibationContext context) : base(context) { }
protected override int DoImport(IEnumerable<ImportItem> importItems)
{
@@ -50,78 +53,61 @@ namespace DtoImporterService
// must include default/empty/missing
contributorNames.Add(Contributor.GetEmpty().Name);
//// BAD: very inefficient
// var x = context.Contributors.Local.Where(c => !contribNames.Contains(c.Name));
// GOOD: Except() is efficient. Due to hashing, it's close to O(n)
var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList();
var remainingContribNames = contributorNames
.Distinct()
.Except(localNames)
.ToList();
// load existing => local
if (remainingContribNames.Any())
DbContext.Contributors.Where(c => remainingContribNames.Contains(c.Name)).ToList();
Cache = DbContext.Contributors
.Where(c => contributorNames.Contains(c.Name))
.ToDictionarySafe(c => c.Name);
}
// only use after loading contributors => local
private int upsertPeople(List<Person> people)
{
var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList();
var newPeople = people
.Select(p => p.Name)
.Distinct()
.Except(localNames)
.ToList();
var hash = people
// new people only
.Where(p => !Cache.ContainsKey(p.Name))
// remove duplicates by Name. first in wins
.ToDictionarySafe(p => p.Name);
var groupby = people.GroupBy(
p => p.Name,
p => p,
(key, g) => new { Name = key, People = g.ToList() }
);
foreach (var name in newPeople)
foreach (var kvp in hash)
{
// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
var p = groupby.FirstOrDefault(g => g.Name == name).People.First();
try
{
DbContext.Contributors.Add(new Contributor(p.Name, p.Asin));
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding person. {@DebugInfo}", new { p?.Name, p?.Asin });
throw;
}
var person = kvp.Value;
addContributor(person.Name, person.Asin);
}
return newPeople.Count;
return hash.Count;
}
// only use after loading contributors => local
private int upsertPublishers(List<string> publishers)
{
var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList();
var newPublishers = publishers
.Distinct()
.Except(localNames)
.ToList();
var hash = publishers
// new publishers only
.Where(p => !Cache.ContainsKey(p))
// remove duplicates
.ToHashSet();
foreach (var pub in newPublishers)
{
try
{
DbContext.Contributors.Add(new Contributor(pub));
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding publisher. {@DebugInfo}", new { pub });
throw;
}
}
foreach (var pub in hash)
addContributor(pub);
return newPublishers.Count;
return hash.Count;
}
}
private Contributor addContributor(string name, string id = null)
{
try
{
var newContrib = new Contributor(name);
var entityEntry = DbContext.Contributors.Add(newContrib);
var entity = entityEntry.Entity;
Cache.Add(entity.Name, entity);
return entity;
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding contributor. {@DebugInfo}", new { name, id });
throw;
}
}
}
}

View File

@@ -53,5 +53,9 @@ namespace DtoImporterService
public abstract class ItemsImporterBase : ImporterBase<IEnumerable<ImportItem>>
{
protected ItemsImporterBase(LibationContext context) : base(context) { }
protected abstract IValidator Validator { get; }
public sealed override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems)
=> Validator.Validate(importItems.Select(i => i.DtoItem));
}
}

View File

@@ -3,18 +3,24 @@ using System.Collections.Generic;
using System.Linq;
using AudibleUtilities;
using DataLayer;
using Dinah.Core.Collections.Generic;
namespace DtoImporterService
{
public class LibraryBookImporter : ItemsImporterBase
{
public LibraryBookImporter(LibationContext context) : base(context) { }
protected override IValidator Validator => new LibraryValidator();
public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new LibraryValidator().Validate(importItems.Select(i => i.DtoItem));
private BookImporter bookImporter { get; }
public LibraryBookImporter(LibationContext context) : base(context)
{
bookImporter = new BookImporter(DbContext);
}
protected override int DoImport(IEnumerable<ImportItem> importItems)
{
new BookImporter(DbContext).Import(importItems);
bookImporter.Import(importItems);
var qtyNew = upsertLibraryBooks(importItems);
return qtyNew;
@@ -36,25 +42,18 @@ namespace DtoImporterService
var currentLibraryProductIds = DbContext.LibraryBooks.Select(l => l.Book.AudibleProductId).ToList();
var newItems = importItems
.Where(dto => !currentLibraryProductIds
.Contains(dto.DtoItem.ProductId))
.Where(dto => !currentLibraryProductIds.Contains(dto.DtoItem.ProductId))
.ToList();
// if 2 accounts try to import the same book in the same transaction: error since we're only tracking and pulling by asin.
// just use the first
var groupby = newItems.GroupBy(
i => i.DtoItem.ProductId,
i => i,
(key, g) => new { ProductId = key, ImportItems = g.ToList() }
)
.ToList();
foreach (var gb in groupby)
{
var newItem = gb.ImportItems.First();
var hash = newItems.ToDictionarySafe(dto => dto.DtoItem.ProductId);
foreach (var kvp in hash)
{
var newItem = kvp.Value;
var libraryBook = new LibraryBook(
// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
DbContext.Books.Local.FirstOrDefault(b => b.AudibleProductId == newItem.DtoItem.ProductId),
bookImporter.Cache[newItem.DtoItem.ProductId],
newItem.DtoItem.DateAdded,
newItem.AccountId);
try
@@ -67,7 +66,7 @@ namespace DtoImporterService
}
}
var qtyNew = groupby.Count;
var qtyNew = hash.Count;
return qtyNew;
}
}

View File

@@ -8,8 +8,8 @@ namespace DtoImporterService
public record timeLogEntry(string msg, long totalElapsed, long delta);
public static class PerfLogger
{
private static Stopwatch sw = new Stopwatch();
private static List<timeLogEntry> __log { get; } = new List<timeLogEntry> { new("begin", 0, 0) };
private static Stopwatch sw { get; } = new();
private static List<timeLogEntry> __log { get; } = new() { new("begin", 0, 0) };
public static void logTime(string s)
{

View File

@@ -4,14 +4,17 @@ using System.Linq;
using AudibleApi.Common;
using AudibleUtilities;
using DataLayer;
using Dinah.Core.Collections.Generic;
namespace DtoImporterService
{
public class SeriesImporter : ItemsImporterBase
{
public SeriesImporter(LibationContext context) : base(context) { }
protected override IValidator Validator => new SeriesValidator();
public override IEnumerable<Exception> Validate(IEnumerable<ImportItem> importItems) => new SeriesValidator().Validate(importItems.Select(i => i.DtoItem));
public Dictionary<string, DataLayer.Series> Cache { get; private set; } = new();
public SeriesImporter(LibationContext context) : base(context) { }
protected override int DoImport(IEnumerable<ImportItem> importItems)
{
@@ -31,15 +34,12 @@ namespace DtoImporterService
private void loadLocal_series(List<AudibleApi.Common.Series> series)
{
var seriesIds = series.Select(s => s.SeriesId).ToList();
var localIds = DbContext.Series.Local.Select(s => s.AudibleSeriesId).ToList();
var remainingSeriesIds = seriesIds
.Distinct()
.Except(localIds)
.ToList();
var seriesIds = series.Select(s => s.SeriesId).Distinct().ToList();
if (remainingSeriesIds.Any())
DbContext.Series.Where(s => remainingSeriesIds.Contains(s.AudibleSeriesId)).ToList();
if (seriesIds.Any())
Cache = DbContext.Series
.Where(s => seriesIds.Contains(s.AudibleSeriesId))
.ToDictionarySafe(s => s.AudibleSeriesId);
}
private int upsertSeries(List<AudibleApi.Common.Series> requestedSeries)
@@ -48,18 +48,10 @@ namespace DtoImporterService
foreach (var s in requestedSeries)
{
var series = DbContext.Series.Local.FirstOrDefault(c => c.AudibleSeriesId == s.SeriesId);
if (series is null)
// AudibleApi.Common.Series.SeriesId == DataLayer.AudibleSeriesId
if (!Cache.TryGetValue(s.SeriesId, out var series))
{
try
{
series = DbContext.Series.Add(new DataLayer.Series(new AudibleSeriesId(s.SeriesId))).Entity;
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding series. {@DebugInfo}", new { s?.SeriesId });
throw;
}
series = addSeries(s.SeriesId);
qtyNew++;
}
series.UpdateName(s.SeriesName);
@@ -67,5 +59,24 @@ namespace DtoImporterService
return qtyNew;
}
private DataLayer.Series addSeries(string seriesId)
{
try
{
var series = new DataLayer.Series(new AudibleSeriesId(seriesId));
var entityEntry = DbContext.Series.Add(series);
var entity = entityEntry.Entity;
Cache.Add(entity.AudibleSeriesId, entity);
return entity;
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding series. {@DebugInfo}", new { seriesId });
throw;
}
}
}
}

View File

@@ -0,0 +1,37 @@
* Local (eg DbContext.Books.Local): indexes/hashes PK and nothing else. Local.Find(PK) is fast. All other searches (eg FirstOrDefault) have awful performance. It deceptively *feels* like we get this partially for free since added/modified entries live here.
* live db: for all importers, fields used for lookup are indexed
Using BookImporter as an example: since AudibleProductId is indexed, hitting the live db is much faster than using Local. Fastest is putting all in a local hash table
Note: GetBook/GetBooks eager loads Series, category, et al
for 1,200 iterations
* load to LocalView
DbContext.Books.Local.FirstOrDefault(p => p.AudibleProductId == item.DtoItem.ProductId)
27,125 ms
* read from live db
DbContext.Books.GetBook(item.DtoItem.ProductId)
12,224 ms
* load to hash table: Dictionary<string, Book>
dictionary[item.DtoItem.ProductId];
1 ms (yes: ONE)
With hashtable, somehow memory usage was not significantly affected
-----------------------------------
why were we using Local to begin with?
articles suggest loading to Local with
context.Books.Load();
this loads this table but not associated fields
we want Books and associated fields
context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
this is emulating Load() but with also getting associated fields
from: Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions
// Summary:
// Enumerates the query. When using Entity Framework, this causes the results of
// the query to be loaded into the associated context. This is equivalent to calling
// ToList and then throwing away the list (without the overhead of actually creating
// the list).
public static void Load<TSource>([NotNullAttribute] this IQueryable<TSource> source);

View File

@@ -11,30 +11,33 @@ namespace FileLiberator
public event EventHandler<byte[]> CoverImageDiscovered;
public abstract void Cancel();
protected void OnRequestCoverArt(Action<byte[]> setCoverArtDel)
{
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(RequestCoverArt) });
RequestCoverArt?.Invoke(this, setCoverArtDel);
}
protected void OnTitleDiscovered(string title)
protected void OnTitleDiscovered(string title) => OnTitleDiscovered(null, title);
protected void OnTitleDiscovered(object _, string title)
{
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(TitleDiscovered), Title = title });
TitleDiscovered?.Invoke(this, title);
}
protected void OnAuthorsDiscovered(string authors)
protected void OnAuthorsDiscovered(string authors) => OnAuthorsDiscovered(null, authors);
protected void OnAuthorsDiscovered(object _, string authors)
{
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(AuthorsDiscovered), Authors = authors });
AuthorsDiscovered?.Invoke(this, authors);
}
protected void OnNarratorsDiscovered(string narrators)
protected void OnNarratorsDiscovered(string narrators) => OnNarratorsDiscovered(null, narrators);
protected void OnNarratorsDiscovered(object _, string narrators)
{
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(NarratorsDiscovered), Narrators = narrators });
NarratorsDiscovered?.Invoke(this, narrators);
}
protected void OnRequestCoverArt(Action<byte[]> setCoverArtDel)
{
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(RequestCoverArt) });
RequestCoverArt?.Invoke(this, setCoverArtDel);
}
protected void OnCoverImageDiscovered(byte[] coverImage)
{
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(CoverImageDiscovered), CoverImageBytes = coverImage?.Length });

View File

@@ -73,8 +73,8 @@ namespace FileLiberator
private void M4bBook_ConversionProgressUpdate(object sender, ConversionProgressEventArgs e)
{
var duration = m4bBook.Duration;
double remainingSecsToProcess = (duration - e.ProcessPosition).TotalSeconds;
double estTimeRemaining = remainingSecsToProcess / e.ProcessSpeed;
var remainingSecsToProcess = (duration - e.ProcessPosition).TotalSeconds;
var estTimeRemaining = remainingSecsToProcess / e.ProcessSpeed;
if (double.IsNormal(estTimeRemaining))
OnStreamingTimeRemaining(TimeSpan.FromSeconds(estTimeRemaining));

View File

@@ -128,21 +128,19 @@ namespace FileLiberator
outFileName, cacheDir, audiobookDlLic, outputFormat,
AudibleFileStorage.Audio.CreateMultipartRenamerFunc(libraryBook))
: new AaxcDownloadSingleConverter(outFileName, cacheDir, audiobookDlLic, outputFormat);
converter.UpdateMetadata = aaxFile => aaxFile.AppleTags.Generes = string.Join(", ", libraryBook.Book.CategoriesNames);
if (Configuration.Instance.AllowLibationFixup)
converter.RetrievedMetadata += (_, tags) => tags.Generes = string.Join(", ", libraryBook.Book.CategoriesNames);
abDownloader = converter;
}
abDownloader
= Configuration.Instance.SplitFilesByChapter ? new AaxcDownloadMultiConverter(
outFileName, cacheDir, audiobookDlLic, outputFormat,
AudibleFileStorage.Audio.CreateMultipartRenamerFunc(libraryBook)
)
: new AaxcDownloadSingleConverter(outFileName, cacheDir, audiobookDlLic, outputFormat);
abDownloader.DecryptProgressUpdate += (_, progress) => OnStreamingProgressChanged(progress);
abDownloader.DecryptTimeRemaining += (_, remaining) => OnStreamingTimeRemaining(remaining);
abDownloader.RetrievedTitle += (_, title) => OnTitleDiscovered(title);
abDownloader.RetrievedAuthors += (_, authors) => OnAuthorsDiscovered(authors);
abDownloader.RetrievedNarrators += (_, narrators) => OnNarratorsDiscovered(narrators);
abDownloader.DecryptProgressUpdate += OnStreamingProgressChanged;
abDownloader.DecryptTimeRemaining += OnStreamingTimeRemaining;
abDownloader.RetrievedTitle += OnTitleDiscovered;
abDownloader.RetrievedAuthors += OnAuthorsDiscovered;
abDownloader.RetrievedNarrators += OnNarratorsDiscovered;
abDownloader.RetrievedCoverArt += AaxcDownloader_RetrievedCoverArt;
abDownloader.FileCreated += (_, path) => OnFileCreated(libraryBook, path);
@@ -178,7 +176,7 @@ namespace FileLiberator
throw new Exception(errorString("Locale"));
}
private void AaxcDownloader_RetrievedCoverArt(object sender, byte[] e)
private void AaxcDownloader_RetrievedCoverArt(object _, byte[] e)
{
if (e is not null)
OnCoverImageDiscovered(e);

View File

@@ -12,8 +12,7 @@ namespace FileLiberator
{
var client = new HttpClient();
var progress = new Progress<DownloadProgress>();
progress.ProgressChanged += (_, e) => OnStreamingProgressChanged(e);
var progress = new Progress<DownloadProgress>(OnStreamingProgressChanged);
OnStreamingBegin(proposedDownloadFilePath);

View File

@@ -62,8 +62,7 @@ namespace FileLiberator
var api = await libraryBook.GetApiAsync();
var downloadUrl = await api.GetPdfDownloadLinkAsync(libraryBook.Book.AudibleProductId);
var progress = new Progress<DownloadProgress>();
progress.ProgressChanged += (_, e) => OnStreamingProgressChanged(e);
var progress = new Progress<DownloadProgress>(OnStreamingProgressChanged);
var client = new HttpClient();

View File

@@ -18,12 +18,14 @@ namespace FileLiberator
StreamingBegin?.Invoke(this, filePath);
}
protected void OnStreamingProgressChanged(DownloadProgress progress)
protected void OnStreamingProgressChanged(DownloadProgress progress) => OnStreamingProgressChanged(null, progress);
protected void OnStreamingProgressChanged(object _, DownloadProgress progress)
{
StreamingProgressChanged?.Invoke(this, progress);
}
protected void OnStreamingTimeRemaining(TimeSpan timeRemaining)
protected void OnStreamingTimeRemaining(TimeSpan timeRemaining) => OnStreamingTimeRemaining(null, timeRemaining);
protected void OnStreamingTimeRemaining(object _, TimeSpan timeRemaining)
{
StreamingTimeRemaining?.Invoke(this, timeRemaining);
}

View File

@@ -5,7 +5,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dinah.Core" Version="4.0.4.1" />
<PackageReference Include="Dinah.Core" Version="4.0.6.1" />
<PackageReference Include="Polly" Version="7.2.3" />
</ItemGroup>

View File

@@ -35,7 +35,8 @@ namespace LibationCli
var (TotalBooksProcessed, NewBooksAdded) = await LibraryCommands.ImportAccountAsync((a) => ApiExtended.CreateAsync(a), _accounts);
Console.WriteLine("Scan complete.");
Console.WriteLine($"Total processed: {TotalBooksProcessed}\r\nNew: {NewBooksAdded}");
Console.WriteLine($"Total processed: {TotalBooksProcessed}");
Console.WriteLine($"New: {NewBooksAdded}");
}
private Account[] getAccounts()

View File

@@ -14,11 +14,6 @@ namespace LibationWinForms.BookLiberation
private Func<byte[]> GetCoverArtDelegate;
// book info
private string title;
private string authorNames;
private string narratorNames;
#region Processable event handler overrides
public override void Processable_Begin(object sender, LibraryBook libraryBook)
{
@@ -31,8 +26,8 @@ namespace LibationWinForms.BookLiberation
//Set default values from library
AudioDecodable_TitleDiscovered(sender, libraryBook.Book.Title);
AudioDecodable_AuthorsDiscovered(sender, string.Join(", ", libraryBook.Book.Authors));
AudioDecodable_NarratorsDiscovered(sender, string.Join(", ", libraryBook.Book.NarratorNames));
AudioDecodable_AuthorsDiscovered(sender, libraryBook.Book.AuthorNames);
AudioDecodable_NarratorsDiscovered(sender, libraryBook.Book.NarratorNames);
AudioDecodable_CoverImageDiscovered(sender,
PictureStorage.GetPicture(
new PictureDefinition(
@@ -60,14 +55,23 @@ namespace LibationWinForms.BookLiberation
updateRemainingTime((int)timeRemaining.TotalSeconds);
}
private void updateRemainingTime(int remaining)
=> remainingTimeLbl.UIThreadAsync(() => remainingTimeLbl.Text = $"ETA:\r\n{formatTime(remaining)}");
private string formatTime(int seconds)
{
var timeSpan = new TimeSpan(0, 0, seconds);
return
timeSpan.TotalHours >= 1 ? $"{timeSpan:%h}h {timeSpan:mm}m {timeSpan:ss}s"
: timeSpan.TotalMinutes >= 1 ? $"{timeSpan:%m}m {timeSpan:ss}s"
: $"{seconds} sec";
}
#endregion
#region AudioDecodable event handlers
public override void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate)
{
base.AudioDecodable_RequestCoverArt(sender, setCoverArtDelegate);
setCoverArtDelegate(GetCoverArtDelegate?.Invoke());
}
private string title;
private string authorNames;
private string narratorNames;
public override void AudioDecodable_TitleDiscovered(object sender, string title)
{
@@ -91,27 +95,20 @@ namespace LibationWinForms.BookLiberation
updateBookInfo();
}
private void updateBookInfo()
=> bookInfoLbl.UIThreadAsync(() => bookInfoLbl.Text = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}");
public override void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate)
{
base.AudioDecodable_RequestCoverArt(sender, setCoverArtDelegate);
setCoverArtDelegate(GetCoverArtDelegate?.Invoke());
}
public override void AudioDecodable_CoverImageDiscovered(object sender, byte[] coverArt)
{
base.AudioDecodable_CoverImageDiscovered(sender, coverArt);
pictureBox1.UIThreadAsync(() => pictureBox1.Image = Dinah.Core.Drawing.ImageReader.ToImage(coverArt));
}
#endregion
// thread-safe UI updates
private void updateBookInfo()
=> bookInfoLbl.UIThreadAsync(() => bookInfoLbl.Text = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}");
private void updateRemainingTime(int remaining)
=> remainingTimeLbl.UIThreadAsync(() => remainingTimeLbl.Text = $"ETA:\r\n{formatTime(remaining)}");
private string formatTime(int seconds)
{
var timeSpan = new TimeSpan(0, 0, seconds);
return
timeSpan.TotalHours >= 1 ? $"{timeSpan:%h}h {timeSpan:mm}m {timeSpan:ss}s"
: timeSpan.TotalMinutes >= 1 ? $"{timeSpan:%m}m {timeSpan:ss}s"
: $"{seconds} sec";
}
}
}

View File

@@ -152,11 +152,12 @@ namespace LibationWinForms.BookLiberation.BaseForms
#endregion
#region AudioDecodable event handlers
public virtual void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate) { }
public virtual void AudioDecodable_TitleDiscovered(object sender, string title) { }
public virtual void AudioDecodable_AuthorsDiscovered(object sender, string authors) { }
public virtual void AudioDecodable_NarratorsDiscovered(object sender, string narrators) { }
public virtual void AudioDecodable_CoverImageDiscovered(object sender, byte[] coverArt) { }
public virtual void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate) { }
#endregion
}
}

View File

@@ -31,7 +31,7 @@ namespace LibationWinForms.Dialogs
try
{
(TotalBooksProcessed, NewBooksAdded) = await LibraryCommands.ImportAccountAsync((account) => ApiExtended.CreateAsync(account, new WinformLoginChoiceEager(account)), _accounts);
(TotalBooksProcessed, NewBooksAdded) = await LibraryCommands.ImportAccountAsync(account => ApiExtended.CreateAsync(account, new WinformLoginChoiceEager(account)), _accounts);
}
catch (Exception ex)
{

View File

@@ -61,7 +61,7 @@ namespace LibationWinForms
// suppressed filter while init'ing UI
var prev_isProcessingGridSelect = isProcessingGridSelect;
isProcessingGridSelect = true;
this.UIThreadSync(() => setGrid());
this.UIThreadSync(setGrid);
isProcessingGridSelect = prev_isProcessingGridSelect;
// UI init complete. now we can apply filter

View File

@@ -28,7 +28,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.4.1" />
<PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.6.1" />
</ItemGroup>
<ItemGroup>

View File

@@ -107,6 +107,11 @@ namespace LibationWinForms
#endregion
private SortableBindingList<GridEntry> bindingList;
/// <summary>Insert ad hoc library books to top of grid</summary>
public void AddToTop(DataLayer.LibraryBook libraryBook) => bindingList.Insert(0, new GridEntry(libraryBook));
#region UI display functions
private bool hasBeenDisplayed = false;
@@ -145,7 +150,8 @@ namespace LibationWinForms
.ToList();
// BIND
gridEntryBindingSource.DataSource = new SortableBindingList<GridEntry>(orderedGridEntries);
bindingList = new SortableBindingList<GridEntry>(orderedGridEntries);
gridEntryBindingSource.DataSource = bindingList;
// FILTER
Filter();

View File

@@ -228,7 +228,6 @@ namespace LibationWinForms
var libhackFiles = Directory.EnumerateDirectories(config.Books, "*.libhack", SearchOption.AllDirectories);
using var context = ApplicationServices.DbContexts.GetContext();
context.Books.Load();
var jArr = JArray.Parse(File.ReadAllText(filePaths));
@@ -248,7 +247,7 @@ namespace LibationWinForms
if (fileType == FileType.Unknown || fileType == FileType.AAXC)
continue;
var book = context.Books.Local.FirstOrDefault(b => b.AudibleProductId == asin);
var book = context.Books.FirstOrDefault(b => b.AudibleProductId == asin);
if (book is null)
continue;

View File

@@ -10,8 +10,8 @@
<PackageReference Include="FluentAssertions" Version="6.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="Moq" Version="4.17.2" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.9" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.9" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
<PackageReference Include="coverlet.collector" Version="3.1.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

View File

@@ -9,8 +9,8 @@
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.9" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.9" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
<PackageReference Include="coverlet.collector" Version="3.1.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

View File

@@ -9,8 +9,8 @@
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.9" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.9" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
<PackageReference Include="coverlet.collector" Version="3.1.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

View File

@@ -9,8 +9,8 @@
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.9" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.9" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
<PackageReference Include="coverlet.collector" Version="3.1.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

View File

@@ -10,8 +10,8 @@
<PackageReference Include="FluentAssertions" Version="6.6.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
<PackageReference Include="Moq" Version="4.17.2" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.9" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.9" />
<PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
<PackageReference Include="coverlet.collector" Version="3.1.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>