* Much faster library scans

* Libraries of unlimited size now supported (prev limit was 10k)

Source/AaxDecrypter/NetworkFileStream.cs

@@ -217,40 +217,47 @@ namespace AaxDecrypter
 		{
 			var downloadPosition = WritePosition;
 			var nextFlush = downloadPosition + DATA_FLUSH_SZ;
-
 			var buff = new byte[DOWNLOAD_BUFF_SZ];
-			do
+
+			try
 			{
-				var bytesRead = _networkStream.Read(buff, 0, DOWNLOAD_BUFF_SZ);
-				_writeFile.Write(buff, 0, bytesRead);
-
-				downloadPosition += bytesRead;
-
-				if (downloadPosition > nextFlush)
+				do
 				{
-					_writeFile.Flush();
-					WritePosition = downloadPosition;
-					Update();
-					nextFlush = downloadPosition + DATA_FLUSH_SZ;
-					downloadedPiece.Set();
-				}
+					var bytesRead = _networkStream.Read(buff, 0, DOWNLOAD_BUFF_SZ);
+					_writeFile.Write(buff, 0, bytesRead);
 
-			} while (downloadPosition < ContentLength && !IsCancelled);
+					downloadPosition += bytesRead;
 
-			_writeFile.Close();
-			_networkStream.Close();
-			WritePosition = downloadPosition;
-			Update();
+					if (downloadPosition > nextFlush)
+					{
+						_writeFile.Flush();
+						WritePosition = downloadPosition;
+						Update();
+						nextFlush = downloadPosition + DATA_FLUSH_SZ;
+						downloadedPiece.Set();
+					}
 
-			downloadedPiece.Set();
-			downloadEnded.Set();
+				} while (downloadPosition < ContentLength && !IsCancelled);
 
-			if (!IsCancelled && WritePosition < ContentLength)
-				throw new WebException($"Downloaded size (0x{WritePosition:X10}) is less than {nameof(ContentLength)} (0x{ContentLength:X10}).");
+				_writeFile.Close();
+				_networkStream.Close();
+				WritePosition = downloadPosition;
+				Update();
 
-			if (WritePosition > ContentLength)
-				throw new WebException($"Downloaded size (0x{WritePosition:X10}) is greater than {nameof(ContentLength)} (0x{ContentLength:X10}).");
+				downloadedPiece.Set();
+				downloadEnded.Set();
 
+				if (!IsCancelled && WritePosition < ContentLength)
+					throw new WebException($"Downloaded size (0x{WritePosition:X10}) is less than {nameof(ContentLength)} (0x{ContentLength:X10}).");
+
+				if (WritePosition > ContentLength)
+					throw new WebException($"Downloaded size (0x{WritePosition:X10}) is greater than {nameof(ContentLength)} (0x{ContentLength:X10}).");
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Error(ex, "An error was encountered while downloading {Uri}", Uri);
+				IsCancelled = true;
+			}
 		}
 
 		#endregion

Source/AppScaffolding/AppScaffolding.csproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6.0-windows</TargetFramework>
-    <Version>7.9.0.1</Version>
+    <Version>7.10.0.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

Source/AudibleUtilities/ApiExtended.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using AudibleApi;
 using AudibleApi.Common;
@@ -118,31 +119,38 @@ namespace AudibleUtilities
 		private async Task<List<Item>> getItemsAsync(LibraryOptions libraryOptions, bool importEpisodes)
 		{
 			var items = new List<Item>();
-#if DEBUG
-//// this will not work for multi accounts
-//var library_json = "library.json";
-//library_json = System.IO.Path.GetFullPath(library_json);
-//if (System.IO.File.Exists(library_json))
-//{
-//    items = AudibleApi.Common.Converter.FromJson<List<Item>>(System.IO.File.ReadAllText(library_json));
-//}
-#endif
 
-			Serilog.Log.Logger.Debug("Begin initial library scan");
+			Serilog.Log.Logger.Debug("Beginning library scan.");
 
-			if (!items.Any())
-				items = await Api.GetAllLibraryItemsAsync(libraryOptions);
+			List<Task<List<Item>>> getChildEpisodesTasks = new();
 
-			Serilog.Log.Logger.Debug("Initial library scan complete. Begin episode scan");
+			int count = 0, maxConcurrentEpisodeScans = 5;
+			using SemaphoreSlim concurrencySemaphore = new(maxConcurrentEpisodeScans);
 
-			await manageEpisodesAsync(items, importEpisodes);
+			await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions))
+			{
+				if (item.IsEpisodes && importEpisodes)
+				{
+					//Get child episodes asynchronously and await all at the end
+					getChildEpisodesTasks.Add(getChildEpisodesAsync(concurrencySemaphore, item));
+				}
+				else if (!item.IsEpisodes)
+					items.Add(item);
 
-			Serilog.Log.Logger.Debug("Episode scan complete");
+				count++;
+			}
+
+			Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on {getChildEpisodesTasksCount} series episode scans to complete.", count, getChildEpisodesTasks.Count);
+
+			//await and add all episides from all parents
+			foreach (var epList in await Task.WhenAll(getChildEpisodesTasks))
+				items.AddRange(epList);
+
+			Serilog.Log.Logger.Debug("Completed library scan.");
 
 #if DEBUG
 //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
 #endif
-
 			var validators = new List<IValidator>();
 			validators.AddRange(getValidators());
 			foreach (var v in validators)
@@ -156,55 +164,21 @@ namespace AudibleUtilities
 		}
 
 		#region episodes and podcasts
-		private async Task manageEpisodesAsync(List<Item> items, bool importEpisodes)
+		
+		private async Task<List<Item>> getChildEpisodesAsync(SemaphoreSlim concurrencySemaphore, Item parent)
 		{
-			// add podcasts and episodes to list. If fail, don't let it de-rail the rest of the import
+			await concurrencySemaphore.WaitAsync();
+
 			try
 			{
-				// get parents
-				var parents = items.Where(i => i.IsEpisodes).ToList();
-#if DEBUG
-//var parentsDebug = parents.Select(i => i.ToJson()).Aggregate((a, b) => $"{a}\r\n\r\n{b}");
-//System.IO.File.WriteAllText("parents.json", parentsDebug);
-#endif
+				Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent);
 
-				if (!parents.Any())
-					return;
-
-				Serilog.Log.Logger.Information($"{parents.Count} series of shows/podcasts found");
-
-				// remove episode parents. even if the following stuff fails, these will still be removed from the collection
-				items.RemoveAll(i => i.IsEpisodes);
-
-				if (importEpisodes)
-				{
-					// add children
-					var children = await getEpisodesAsync(parents);
-					Serilog.Log.Logger.Information($"{children.Count} episodes of shows/podcasts found");
-					items.AddRange(children);
-				}
-			}
-			catch (Exception ex)
-			{
-				Serilog.Log.Logger.Error(ex, "Error adding podcasts and episodes");
-			}
-		}
-
-		private async Task<List<Item>> getEpisodesAsync(List<Item> parents)
-		{
-			var results = new List<Item>();
-
-			foreach (var parent in parents)
-			{
 				var children = await getEpisodeChildrenAsync(parent);
 
 				// actual individual episode, not the parent of a series.
 				// for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic
 				if (!children.Any())
-				{
-					results.Add(parent);
-					continue;
-				}
+					return new List<Item>() { parent };
 
 				foreach (var child in children)
 				{
@@ -232,10 +206,14 @@ namespace AudibleUtilities
 					};
 				}
 
-				results.AddRange(children);
-			}
+				Serilog.Log.Logger.Debug("Completed episode scan for {parent}", parent);
 
-			return results;
+				return children;
+			}
+			finally
+			{
+				concurrencySemaphore.Release();
+			}
 		}
 
 		private async Task<List<Item>> getEpisodeChildrenAsync(Item parent)
@@ -277,7 +255,7 @@ namespace AudibleUtilities
 					throw;
 				}
 
-				Serilog.Log.Logger.Debug($"Batch {i}: {childrenBatch.Count} results");
+				Serilog.Log.Logger.Debug($"Batch {i}: {childrenBatch.Count} results\t({{parent}})", parent);
 				// the service returned no results. probably indicates an error. stop running batches
 				if (!childrenBatch.Any())
 					break;
@@ -295,7 +273,7 @@ namespace AudibleUtilities
 			if (childrenIds.Count != results.Count)
 			{
 				var ex = new ApplicationException($"Mis-match: Children defined by parent={childrenIds.Count}. Children returned by batches={results.Count}");
-				Serilog.Log.Logger.Error(ex, "Quantity of series episodes defined by parent does not match quantity returned by batch fetching.");
+				Serilog.Log.Logger.Error(ex, "{parent} - Quantity of series episodes defined by parent does not match quantity returned by batch fetching.", parent);
 				throw ex;
 			}
 

Source/AudibleUtilities/AudibleUtilities.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AudibleApi" Version="2.8.4.1" />
+    <PackageReference Include="AudibleApi" Version="3.0.2.1" />
   </ItemGroup>
 
   <ItemGroup>

Source/FileLiberator/AudioDecodable.cs

@@ -4,7 +4,8 @@ namespace FileLiberator
 {
     public abstract class AudioDecodable : Processable
     {
-        public event EventHandler<Action<byte[]>> RequestCoverArt;
+        public delegate byte[] RequestCoverArtHandler(object sender, EventArgs eventArgs);
+        public event RequestCoverArtHandler RequestCoverArt;
         public event EventHandler<string> TitleDiscovered;
         public event EventHandler<string> AuthorsDiscovered;
         public event EventHandler<string> NarratorsDiscovered;
@@ -32,10 +33,10 @@ namespace FileLiberator
             NarratorsDiscovered?.Invoke(this, narrators);
 		}
 
-        protected void OnRequestCoverArt(Action<byte[]> setCoverArtDel)
+        protected byte[] OnRequestCoverArt()
 		{
             Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(RequestCoverArt) });
-            RequestCoverArt?.Invoke(this, setCoverArtDel);
+            return RequestCoverArt?.Invoke(this, new());
         }
 
         protected void OnCoverImageDiscovered(byte[] coverImage)

Source/FileLiberator/DownloadDecryptBook.cs

@@ -247,7 +247,7 @@ namespace FileLiberator
             if (e is not null)
                 OnCoverImageDiscovered(e);
             else if (Configuration.Instance.AllowLibationFixup)
-                OnRequestCoverArt(abDownloader.SetCoverArt);
+                abDownloader.SetCoverArt(OnRequestCoverArt());
         }
 
         /// <summary>Move new files to 'Books' directory</summary>

Source/LibationWinForms/GridView/GridEntry.cs

@@ -100,9 +100,9 @@ namespace LibationWinForms.GridView
 	{
 #nullable enable
 		public static IEnumerable<SeriesEntry> Series(this IEnumerable<GridEntry> gridEntries)
-			=> gridEntries.Where(i => i is SeriesEntry).Cast<SeriesEntry>();
+			=> gridEntries.OfType<SeriesEntry>();
 		public static IEnumerable<LibraryBookEntry> LibraryBooks(this IEnumerable<GridEntry> gridEntries)
-			=> gridEntries.Where(i => i is LibraryBookEntry).Cast<LibraryBookEntry>();
+			=> gridEntries.OfType<LibraryBookEntry>();
 		public static LibraryBookEntry? FindBookByAsin(this IEnumerable<LibraryBookEntry> gridEntries, string audibleProductID)
 			=> gridEntries.FirstOrDefault(i => i.AudibleProductId == audibleProductID);
 		public static SeriesEntry? FindBookSeriesEntry(this IEnumerable<GridEntry> gridEntries, IEnumerable<SeriesBook> matchSeries)

Source/LibationWinForms/GridView/LiberateDataGridViewImageButtonColumn.cs

@@ -30,10 +30,7 @@ namespace LibationWinForms.GridView
 
 				if (status.IsSeries)
 				{
-					var imageName = status.Expanded ? "minus" : "plus";
-
-					var bmp = (Bitmap)Properties.Resources.ResourceManager.GetObject(imageName);
-					DrawButtonImage(graphics, bmp, cellBounds);
+					DrawButtonImage(graphics, status.Expanded ? Properties.Resources.minus: Properties.Resources.plus, cellBounds);
 
 					ToolTipText = status.Expanded ? "Click to Collpase" : "Click to Expand";
 				}

Source/LibationWinForms/ProcessQueue/ProcessBook.cs

@@ -60,7 +60,6 @@ namespace LibationWinForms.ProcessQueue
 		private Processable CurrentProcessable => _currentProcessable ??= Processes.Dequeue().Invoke();
 		private Processable NextProcessable() => _currentProcessable = null;
 		private Processable _currentProcessable;
-		private Func<byte[]> GetCoverArtDelegate;
 		private readonly Queue<Func<Processable>> Processes = new();
 		private readonly LogMe Logger;
 
@@ -232,11 +231,14 @@ namespace LibationWinForms.ProcessQueue
 			BookText = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}";
 		}
 
-		public void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate)
+		private byte[] AudioDecodable_RequestCoverArt(object sender, EventArgs e)
 		{
-			byte[] coverData = GetCoverArtDelegate();
-			setCoverArtDelegate(coverData);
+			byte[] coverData = PictureStorage
+				.GetPictureSynchronously(
+				new PictureDefinition(LibraryBook.Book.PictureId, PictureSize._500x500));
+
 			AudioDecodable_CoverImageDiscovered(this, coverData);
+			return coverData;
 		}
 
 		private void AudioDecodable_CoverImageDiscovered(object sender, byte[] coverArt)
@@ -273,11 +275,6 @@ namespace LibationWinForms.ProcessQueue
 
 			Logger.Info($"{Environment.NewLine}{((Processable)sender).Name} Step, Begin: {libraryBook.Book}");
 
-			GetCoverArtDelegate = () => PictureStorage.GetPictureSynchronously(
-						new PictureDefinition(
-							libraryBook.Book.PictureId,
-							PictureSize._500x500));
-
 			title = libraryBook.Book.Title;
 			authorNames = libraryBook.Book.AuthorNames();
 			narratorNames = libraryBook.Book.NarratorNames();
@@ -286,7 +283,6 @@ namespace LibationWinForms.ProcessQueue
 
 		private async void Processable_Completed(object sender, LibraryBook libraryBook)
 		{
-
 			Logger.Info($"{((Processable)sender).Name} Step, Completed: {libraryBook.Book}");
 			UnlinkProcessable((Processable)sender);