Truncate too-long error message

- Improve book download retry

ApplicationServices/UNTESTED/LibraryCommands.cs

@@ -81,16 +81,13 @@ namespace ApplicationServices
 		{
 			ArgumentValidator.EnsureNotNull(account, nameof(account));
 
-			var localeName = account.Locale?.Name;
 			Log.Logger.Information("ImportLibraryAsync. {@DebugInfo}", new
 			{
-				AccountName = account.AccountName.ToMask(),
-				AccountId = account.AccountId.ToMask(),
-				LocaleName = localeName,
+				Account = account?.MaskedLogEntry ?? "[null]"
 			});
 
 			var dtoItems = await AudibleApiActions.GetLibraryValidatedAsync(api);
-			return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = localeName }).ToList();
+			return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = account.Locale?.Name }).ToList();
 		}
 
 		private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)

FileLiberator/UNTESTED/DecryptBook.cs

@@ -137,7 +137,7 @@ namespace FileLiberator
 			// create final directory. move each file into it. MOVE AUDIO FILE LAST
 			// new dir: safetitle_limit50char + " [" + productId + "]"
 
-			var destinationDir = getDestDir(product);
+			var destinationDir = AudibleFileStorage.Audio.GetDestDir(product.Title, product.AudibleProductId);
 			Directory.CreateDirectory(destinationDir);
 
 			var sortedFiles = getProductFilesSorted(product, outputAudioFilename);
@@ -158,18 +158,6 @@ namespace FileLiberator
             return destinationDir;
         }
 
-		private static string getDestDir(Book product)
-		{
-			// to prevent the paths from getting too long, we don't need after the 1st ":" for the folder
-			var underscoreIndex = product.Title.IndexOf(':');
-			var titleDir
-				= underscoreIndex < 4
-				? product.Title
-				: product.Title.Substring(0, underscoreIndex);
-			var finalDir = FileUtility.GetValidFilename(AudibleFileStorage.BooksDirectory, titleDir, null, product.AudibleProductId);
-			return finalDir;
-		}
-
 		private static List<FileInfo> getProductFilesSorted(Book product, string outputAudioFilename)
 		{
 			// files are: temp path\author\[asin].ext

FileLiberator/UNTESTED/DownloadBook.cs

@@ -72,7 +72,7 @@ namespace FileLiberator
 				libraryBook.Book.Title,
 				libraryBook.Book.AudibleProductId,
 				libraryBook.Book.Locale,
-				libraryBook.Account,
+				Account = libraryBook.Account?.ToMask() ?? "[empty]",
 				tempAaxFilename,
 				actualFilePath,
 				length,

FileLiberator/UNTESTED/IProcessableExt.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using ApplicationServices;
@@ -16,38 +17,37 @@ namespace FileLiberator
         //
 
 
-        /// <summary>Process the first valid product. Create default context</summary>
-        /// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
-        public static async Task<StatusHandler> ProcessFirstValidAsync(this IProcessable processable)
-        {
-            var libraryBook = processable.getNextValidBook();
-            if (libraryBook == null)
-                return null;
+        // when used in foreach: stateful. deferred execution
+        public static IEnumerable<LibraryBook> GetValidLibraryBooks(this IProcessable processable)
+            => DbContexts.GetContext()
+            .GetLibrary_Flat_NoTracking()
+            .Where(libraryBook => processable.Validate(libraryBook));
 
-            return await processBookAsync(processable, libraryBook);
-        }
-
-        /// <summary>Process the first valid product. Create default context</summary>
-        /// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
-        public static async Task<StatusHandler> ProcessSingleAsync(this IProcessable processable, string productId)
+        public static LibraryBook GetSingleLibraryBook(string productId)
         {
             using var context = DbContexts.GetContext();
             var libraryBook = context
                 .Library
                 .GetLibrary()
                 .SingleOrDefault(lb => lb.Book.AudibleProductId == productId);
+            return libraryBook;
+        }
 
+        /// <summary>Process the first valid product. Create default context</summary>
+        /// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
+        public static async Task<StatusHandler> ProcessSingleAsync(this IProcessable processable, LibraryBook libraryBook)
+        {
             if (libraryBook == null)
                 return null;
             if (!processable.Validate(libraryBook))
                 return new StatusHandler { "Validation failed" };
 
-            return await processBookAsync(processable, libraryBook);
+            return await processable.ProcessBookAsync_NoValidation(libraryBook);
         }
 
-        private static async Task<StatusHandler> processBookAsync(IProcessable processable, LibraryBook libraryBook)
+        public static async Task<StatusHandler> ProcessBookAsync_NoValidation(this IProcessable processable, LibraryBook libraryBook)
         {
-            Serilog.Log.Logger.Information("Begin " + nameof(processBookAsync) + " {@DebugInfo}", new
+            Serilog.Log.Logger.Information("Begin " + nameof(ProcessBookAsync_NoValidation) + " {@DebugInfo}", new
             {
                 libraryBook.Book.Title,
                 libraryBook.Book.AudibleProductId,
@@ -63,17 +63,6 @@ namespace FileLiberator
             return status;
         }
 
-        private static LibraryBook getNextValidBook(this IProcessable processable)
-        {
-            var libraryBooks = DbContexts.GetContext().GetLibrary_Flat_NoTracking();
-
-            foreach (var libraryBook in libraryBooks)
-                if (processable.Validate(libraryBook))
-                    return libraryBook;
-
-            return null;
-        }   
-
 		public static async Task<StatusHandler> TryProcessAsync(this IProcessable processable, LibraryBook libraryBook)
 			=> processable.Validate(libraryBook)
 			? await processable.ProcessAsync(libraryBook)

FileManager/UNTESTED/AudibleFileStorage.cs

@@ -11,76 +11,77 @@ namespace FileManager
     // could add images here, but for now images are stored in a well-known location
     public enum FileType { Unknown, Audio, AAX, PDF }
 
-    /// <summary>
-    /// Files are large. File contents are never read by app.
-    /// Paths are varied.
-    /// Files are written during download/decrypt/backup/liberate.
-    /// Paths are read at app launch and during download/decrypt/backup/liberate.
-    /// Many files are often looked up at once
-    /// </summary>
-    public sealed class AudibleFileStorage : Enumeration<AudibleFileStorage>
-    {
-        #region static
-        public static AudibleFileStorage Audio { get; }
-        public static AudibleFileStorage AAX { get; }
-        public static AudibleFileStorage PDF { get; }
+	/// <summary>
+	/// Files are large. File contents are never read by app.
+	/// Paths are varied.
+	/// Files are written during download/decrypt/backup/liberate.
+	/// Paths are read at app launch and during download/decrypt/backup/liberate.
+	/// Many files are often looked up at once
+	/// </summary>
+	public abstract class AudibleFileStorage : Enumeration<AudibleFileStorage>
+	{
+		public abstract string[] Extensions { get; }
+		public abstract string StorageDirectory { get; }
 
-        public static string DownloadsInProgress { get; }
-        public static string DecryptInProgress { get; }
-        public static string BooksDirectory => Configuration.Instance.Books;
-        // not customizable. don't move to config
-        public static string DownloadsFinal { get; }
-            = new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("DownloadsFinal").FullName;
+		#region static
+		public static AudioFileStorage Audio { get; } = new AudioFileStorage();
+		public static AudibleFileStorage AAX { get; } = new AaxFileStorage();
+		public static AudibleFileStorage PDF { get; } = new PdfFileStorage();
 
-        static AudibleFileStorage()
+		public static string DownloadsInProgress
         {
-            #region init DecryptInProgress
-            if (!Configuration.Instance.DecryptInProgressEnum.In("WinTemp", "LibationFiles"))
-                Configuration.Instance.DecryptInProgressEnum = "WinTemp";
-            var M4bRootDir
-                = Configuration.Instance.DecryptInProgressEnum == "WinTemp" // else "LibationFiles"
-                ? Configuration.WinTemp
-                : Configuration.Instance.LibationFiles;
-            DecryptInProgress = Path.Combine(M4bRootDir, "DecryptInProgress");
-            Directory.CreateDirectory(DecryptInProgress);
-            #endregion
+            get
+            {
+                if (!Configuration.Instance.DownloadsInProgressEnum.In("WinTemp", "LibationFiles"))
+                    Configuration.Instance.DownloadsInProgressEnum = "WinTemp";
+                var AaxRootDir
+                    = Configuration.Instance.DownloadsInProgressEnum == "WinTemp"
+                    ? Configuration.WinTemp
+                    : Configuration.Instance.LibationFiles;
 
-            #region init DownloadsInProgress
-            if (!Configuration.Instance.DownloadsInProgressEnum.In("WinTemp", "LibationFiles"))
-                Configuration.Instance.DownloadsInProgressEnum = "WinTemp";
-            var AaxRootDir
-                = Configuration.Instance.DownloadsInProgressEnum == "WinTemp" // else "LibationFiles"
-                ? Configuration.WinTemp
-                : Configuration.Instance.LibationFiles;
-            DownloadsInProgress = Path.Combine(AaxRootDir, "DownloadsInProgress");
-            Directory.CreateDirectory(DownloadsInProgress);
-            #endregion
+                return Directory.CreateDirectory(Path.Combine(AaxRootDir, "DownloadsInProgress")).FullName;
+            }
+        }
 
-            #region init BooksDirectory
-            if (string.IsNullOrWhiteSpace(Configuration.Instance.Books))
-                Configuration.Instance.Books = Path.Combine(Configuration.Instance.LibationFiles, "Books");
-            Directory.CreateDirectory(Configuration.Instance.Books);
-            #endregion
+        public static string DecryptInProgress
+        {
+            get
+            {
+                if (!Configuration.Instance.DecryptInProgressEnum.In("WinTemp", "LibationFiles"))
+                    Configuration.Instance.DecryptInProgressEnum = "WinTemp";
 
-            // must do this in static ctor, not w/inline properties
-            // static properties init before static ctor so these dir.s would still be null
-            Audio = new AudibleFileStorage(FileType.Audio, BooksDirectory, "m4b", "mp3", "aac", "mp4", "m4a", "ogg", "flac");
-            AAX = new AudibleFileStorage(FileType.AAX, DownloadsFinal, "aax");
-            PDF = new AudibleFileStorage(FileType.PDF, BooksDirectory, "pdf", "zip");
+                var M4bRootDir
+                    = Configuration.Instance.DecryptInProgressEnum == "WinTemp"
+                    ? Configuration.WinTemp
+                    : Configuration.Instance.LibationFiles;
+
+                return Directory.CreateDirectory(Path.Combine(M4bRootDir, "DecryptInProgress")).FullName;
+            }
+        }
+
+        // not customizable. don't move to config
+        public static string DownloadsFinal => new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("DownloadsFinal").FullName;
+
+        public static string BooksDirectory
+        {
+            get
+            {
+                if (string.IsNullOrWhiteSpace(Configuration.Instance.Books))
+                    Configuration.Instance.Books = Path.Combine(Configuration.Instance.LibationFiles, "Books");
+                return Directory.CreateDirectory(Configuration.Instance.Books).FullName;
+            }
         }
         #endregion
 
         #region instance
         public FileType FileType => (FileType)Value;
 
-        public string StorageDirectory => DisplayName;
-
 		private IEnumerable<string> extensions_noDots { get; }
 		private string extAggr { get; }
 
-        private AudibleFileStorage(FileType fileType, string storageDirectory, params string[] extensions) : base((int)fileType, storageDirectory)
+        protected AudibleFileStorage(FileType fileType) : base((int)fileType, fileType.ToString())
 		{
-			extensions_noDots = extensions.Select(ext => ext.Trim('.')).ToList();
+			extensions_noDots = Extensions.Select(ext => ext.Trim('.')).ToList();
 			extAggr = extensions_noDots.Aggregate((a, b) => $"{a}|{b}");
 		}
 
@@ -90,30 +91,84 @@ namespace FileManager
         /// - a directory name has the product id and an audio file is immediately inside
         /// - any audio filename contains the product id
         /// </summary>
-        public bool Exists(string productId)
-            => GetPath(productId) != null;
+        public bool Exists(string productId) => GetPath(productId) != null;
 
 		public string GetPath(string productId)
         {
-			{
-                var cachedFile = FilePathCache.GetPath(productId, FileType);
-                if (cachedFile != null)
-                    return cachedFile;
-            }
+            var cachedFile = FilePathCache.GetPath(productId, FileType);
+            if (cachedFile != null)
+                return cachedFile;
 
-			var firstOrNull =
+            var firstOrNull =
 				Directory
 				.EnumerateFiles(StorageDirectory, "*.*", SearchOption.AllDirectories)
 				.FirstOrDefault(s => Regex.IsMatch(s, $@"{productId}.*?\.({extAggr})$", RegexOptions.IgnoreCase));
 
 			if (firstOrNull is null)
 				return null;
+
 			FilePathCache.Upsert(productId, FileType, firstOrNull);
 			return firstOrNull;
         }
 
+        public string GetDestDir(string title, string asin)
+        {
+            // to prevent the paths from getting too long, we don't need after the 1st ":" for the folder
+            var underscoreIndex = title.IndexOf(':');
+            var titleDir
+                = underscoreIndex < 4
+                ? title
+                : title.Substring(0, underscoreIndex);
+            var finalDir = FileUtility.GetValidFilename(StorageDirectory, titleDir, null, asin);
+            return finalDir;
+        }
+
         public bool IsFileTypeMatch(FileInfo fileInfo)
             => extensions_noDots.ContainsInsensative(fileInfo.Extension.Trim('.'));
         #endregion
     }
+
+    public class AudioFileStorage : AudibleFileStorage
+    {
+        public const string SKIP_FILE_EXT = "libhack";
+
+		public override string[] Extensions { get; } = new[] { "m4b", "mp3", "aac", "mp4", "m4a", "ogg", "flac", SKIP_FILE_EXT };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => BooksDirectory;
+
+        public AudioFileStorage() : base(FileType.Audio) { }
+
+        public void CreateSkipFile(string title, string asin, string contents)
+        {
+            var path = FileUtility.GetValidFilename(GetDestDir(title, asin), title, SKIP_FILE_EXT, asin);
+            File.WriteAllText(path, contents);
+        }
+    }
+
+    public class AaxFileStorage : AudibleFileStorage
+    {
+        public override string[] Extensions { get; } = new[] { "aax" };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => DownloadsFinal;
+
+        public AaxFileStorage() : base(FileType.AAX) { }
+    }
+
+    public class PdfFileStorage : AudibleFileStorage
+    {
+		public override string[] Extensions { get; } = new[] { "pdf", "zip" };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => BooksDirectory;
+
+        public PdfFileStorage() : base(FileType.PDF) { }
+    }
 }

InternalUtilities/Account.cs

@@ -93,5 +93,11 @@ namespace InternalUtilities
 		}
 
 		public override string ToString() => $"{AccountId} - {Locale?.Name ?? "[empty]"}";
+
+		public string MaskedLogEntry => @$"AccountId={mask(AccountId)}|AccountName={mask(AccountName)}|Locale={Locale?.Name ?? "[empty]"}";
+		private static string mask(string str)
+			=> str is null ? "[null]"
+			: str == string.Empty ? "[empty]"
+			: str.ToMask();
 	}
 }

InternalUtilities/UNTESTED/AudibleApiActions.cs

@@ -32,7 +32,7 @@ namespace InternalUtilities
 		{
 			Serilog.Log.Logger.Information("GetApiAsync. {@DebugInfo}", new
 			{
-				AccountId = account?.AccountId.ToMask() ?? "[empty]",
+				Account = account?.MaskedLogEntry ?? "[null]",
 				LocaleName = account?.Locale?.Name
 			});
 			return EzApiCreator.GetApiAsync(

LibationLauncher/LibationLauncher.csproj

@@ -13,7 +13,7 @@
     <!-- <PublishSingleFile>true</PublishSingleFile> -->
     <RuntimeIdentifier>win-x64</RuntimeIdentifier>
 
-    <Version>4.1.2.1</Version>
+    <Version>4.1.6.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

LibationWinForms/UNTESTED/BookLiberation/ProcessorAutomationController.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Threading.Tasks;
 using DataLayer;
+using Dinah.Core;
 using Dinah.Core.ErrorHandling;
 using FileLiberator;
 
@@ -288,13 +289,22 @@ namespace LibationWinForms.BookLiberation
         {
             automatedBackupsForm.Show();
 
+            // processable.ProcessFirstValidAsync used to be encapsulated elsewhere. however, support for 'skip this time only' requires state. iterators provide this state for free. therefore: use foreach/iterator here
             try
             {
-                var shouldContinue = true;
-                while (shouldContinue)
+                foreach (var libraryBook in processable.GetValidLibraryBooks())
                 {
-                    var statusHandler = await processable.ProcessFirstValidAsync();
-                    shouldContinue = validateStatus(statusHandler, automatedBackupsForm);
+                    try
+                    {
+                        var statusHandler = await processable.ProcessBookAsync_NoValidation(libraryBook);
+                        if (!validateStatus(statusHandler, automatedBackupsForm))
+                            break;
+                    }
+                    catch (Exception e)
+                    {
+                        automatedBackupsForm.AppendError(e);
+                        DisplaySkipDialog(automatedBackupsForm, libraryBook, e);
+                    }
                 }
             }
             catch (Exception ex)
@@ -311,17 +321,69 @@ namespace LibationWinForms.BookLiberation
 
             try
             {
-                var statusHandler = await processable.ProcessSingleAsync(productId);
-                validateStatus(statusHandler, automatedBackupsForm);
+                var libraryBook = IProcessableExt.GetSingleLibraryBook(productId);
+
+                try
+                {
+                    var statusHandler = await processable.ProcessSingleAsync(libraryBook);
+                    validateStatus(statusHandler, automatedBackupsForm);
+                }
+                catch (Exception ex)
+                {
+                    automatedBackupsForm.AppendError(ex);
+                    DisplaySkipDialog(automatedBackupsForm, libraryBook, ex);
+                }
             }
-            catch (Exception ex)
+            catch (Exception e)
             {
-                automatedBackupsForm.AppendError(ex);
+                automatedBackupsForm.AppendError(e);
             }
 
             automatedBackupsForm.FinalizeUI();
         }
 
+        private static void DisplaySkipDialog(AutomatedBackupsForm automatedBackupsForm, LibraryBook libraryBook, Exception ex)
+        {
+
+            try
+            {
+                const int TRUNC = 150;
+
+                var errMsg = ex.Message.Truncate(TRUNC);
+                if (ex.Message.Length > TRUNC)
+                    errMsg += "...";
+
+                var text = @$"
+The below error occurred while trying to process this book. Skip this book permanently?
+
+- Click YES to skip this book permanently.
+
+- Click NO to skip the book this time only. We'll try again later.
+
+Error:
+{errMsg}
+".Trim();
+                var dialogResult = System.Windows.Forms.MessageBox.Show(
+                    text,
+                    "Skip importing this book?",
+                    System.Windows.Forms.MessageBoxButtons.YesNo,
+                    System.Windows.Forms.MessageBoxIcon.Question);
+
+                if (dialogResult != System.Windows.Forms.DialogResult.Yes)
+                {
+                    FileManager.AudibleFileStorage.Audio.CreateSkipFile(
+                        libraryBook.Book.Title,
+                        libraryBook.Book.AudibleProductId,
+                        ex.Message + "\r\n|\r\n" + ex.StackTrace);
+                }
+            }
+            catch (Exception exc)
+            {
+                automatedBackupsForm.AppendText($"Error attempting to display {nameof(DisplaySkipDialog)}");
+                automatedBackupsForm.AppendError(exc);
+            }
+        }
+
         private static bool validateStatus(StatusHandler statusHandler, AutomatedBackupsForm automatedBackupsForm)
         {
             if (statusHandler == null)

LibationWinForms/UNTESTED/Dialogs/AccountsDialog.cs

@@ -151,7 +151,7 @@ namespace LibationWinForms.Dialogs
 				if (string.IsNullOrWhiteSpace(dto.AccountId))
 					throw new Exception("Please enter an account id for all accounts");
 				if (string.IsNullOrWhiteSpace(dto.LocaleName))
-					throw new Exception("Please select a local name for all accounts");
+					throw new Exception("Please select a locale (i.e.: country or region) for all accounts");
 
 				var acct = accountsSettings.Upsert(dto.AccountId, dto.LocaleName);
 				acct.LibraryScan = dto.LibraryScan;