Improved logging for import errors

AppScaffolding/AppScaffolding.csproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
-    <Version>6.7.4.1</Version>
+    <Version>6.7.8.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

AppScaffolding/LibationScaffolding.cs

@@ -230,10 +230,10 @@ namespace AppScaffolding
 				config.InProgress,
 
 				AudibleFileStorage.DownloadsInProgressDirectory,
-				DownloadsInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
+				DownloadsInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
 
 				AudibleFileStorage.DecryptInProgressDirectory,
-				DecryptInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
+				DecryptInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
 			});
 		}
 

ApplicationServices/LibraryCommands.cs

@@ -164,25 +164,46 @@ namespace ApplicationServices
 		}
 
 		private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)
+        {
+            logTime("importIntoDbAsync -- pre db");
+            using var context = DbContexts.GetContext();
+            var libraryBookImporter = new LibraryBookImporter(context);
+            var newCount = await Task.Run(() => libraryBookImporter.Import(importItems));
+            logTime("importIntoDbAsync -- post Import()");
+            int qtyChanges = saveChanges(context);
+            logTime("importIntoDbAsync -- post SaveChanges");
+
+            if (qtyChanges > 0)
+                await Task.Run(() => finalizeLibrarySizeChange());
+            logTime("importIntoDbAsync -- post finalizeLibrarySizeChange");
+
+            return newCount;
+        }
+
+        private static int saveChanges(LibationContext context)
 		{
-			logTime("importIntoDbAsync -- pre db");
-			using var context = DbContexts.GetContext();
-			var libraryBookImporter = new LibraryBookImporter(context);
-			var newCount = await Task.Run(() => libraryBookImporter.Import(importItems));
-			logTime("importIntoDbAsync -- post Import()");
-			var qtyChanges = context.SaveChanges();
-			logTime("importIntoDbAsync -- post SaveChanges");
+			try
+			{
+				return context.SaveChanges();
+			}
+			catch (Microsoft.EntityFrameworkCore.DbUpdateException ex)
+			{
+				// DbUpdateException exceptions can wreck serilog. Condense it until we can find a better solution. I suspect the culpret is the "WithExceptionDetails" serilog extension
 
-			if (qtyChanges > 0)
-				await Task.Run(() => finalizeLibrarySizeChange());
-			logTime("importIntoDbAsync -- post finalizeLibrarySizeChange");
+				static string format(Exception ex) => $"\r\nMessage: {ex.Message}\r\nStack Trace:\r\n{ex.StackTrace}";
 
-			return newCount;
-		}
-		#endregion
+				var msg = "Microsoft.EntityFrameworkCore.DbUpdateException";
+				if (ex.InnerException is null)
+					throw new Exception($"{msg}{format(ex)}");
+				throw new Exception(
+					$"{msg}{format(ex)}",
+					new Exception($"Inner Exception{format(ex.InnerException)}"));
+			}
+        }
+        #endregion
 
-		#region remove books
-		public static Task<List<LibraryBook>> RemoveBooksAsync(List<string> idsToRemove) => Task.Run(() => removeBooks(idsToRemove));
+        #region remove books
+        public static Task<List<LibraryBook>> RemoveBooksAsync(List<string> idsToRemove) => Task.Run(() => removeBooks(idsToRemove));
 		private static List<LibraryBook> removeBooks(List<string> idsToRemove)
 		{
 			using var context = DbContexts.GetContext();

DataLayer/EfClasses/Category.cs

@@ -18,7 +18,7 @@ namespace DataLayer
     public class Category
     {
         // Empty is a special case. use private ctor w/o validation
-        public static Category GetEmpty() => new Category { CategoryId = -1, AudibleCategoryId = "", Name = "" };
+        public static Category GetEmpty() => new() { CategoryId = -1, AudibleCategoryId = "", Name = "" };
 
         internal int CategoryId { get; private set; }
         public string AudibleCategoryId { get; private set; }

DataLayer/EfClasses/Contributor.cs

@@ -7,7 +7,7 @@ namespace DataLayer
     public class Contributor
 	{
 		// Empty is a special case. use private ctor w/o validation
-		public static Contributor GetEmpty() => new Contributor { ContributorId = -1, Name = "" };
+		public static Contributor GetEmpty() => new() { ContributorId = -1, Name = "" };
 
 		// contributors search links are just name with url-encoding. space can be + or %20
 		//   author search link:   /search?searchAuthor=Robert+Bevan

DtoImporterService/CategoryImporter.cs

@@ -35,6 +35,9 @@ namespace DtoImporterService
 
 		private void loadLocal_categories(List<string> categoryIds)
 		{
+			// must include default/empty/missing
+			categoryIds.Add(Category.GetEmpty().AudibleCategoryId);
+
 			var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId).ToList();
 			var remainingCategoryIds = categoryIds
 				.Distinct()
@@ -42,10 +45,8 @@ namespace DtoImporterService
 				.ToList();
 
 			// load existing => local
-			// remember to include default/empty/missing
-			var emptyName = Contributor.GetEmpty().Name;
 			if (remainingCategoryIds.Any())
-				DbContext.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId) || c.Name == emptyName).ToList();
+				DbContext.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId)).ToList();
 		}
 
 		// only use after loading contributors => local

FileManager/BackgroundFileSystem.cs

@@ -43,7 +43,7 @@ namespace FileManager
             lock (fsCacheLocker)
             {
                 fsCache.Clear();
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
             }
         }
 
@@ -52,7 +52,7 @@ namespace FileManager
             Stop();
 
             lock (fsCacheLocker)
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
 
             directoryChangesEvents = new BlockingCollection<FileSystemEventArgs>();
 			fileSystemWatcher = new FileSystemWatcher(RootDirectory)
@@ -135,7 +135,7 @@ namespace FileManager
         private void AddPath(string path)
         {
             if (File.GetAttributes(path).HasFlag(FileAttributes.Directory))
-                AddUniqueFiles(Directory.EnumerateFiles(path, SearchPattern, SearchOption));
+                AddUniqueFiles(FileUtility.SaferEnumerateFiles(path, SearchPattern, SearchOption));
             else
                 AddUniqueFile(path);
         }

FileManager/FileUtility.cs

@@ -234,5 +234,39 @@ namespace FileManager
                     throw;
                 }
             });
+
+        /// <summary>
+        /// A safer way to get all the files in a directory and sub directory without crashing on UnauthorizedException or PathTooLongException
+        /// </summary>
+        /// <param name="rootPath">Starting directory</param>
+        /// <param name="patternMatch">Filename pattern match</param>
+        /// <param name="searchOption">Search subdirectories or only top level directory for files</param>
+        /// <returns>List of files</returns>
+        public static IEnumerable<string> SaferEnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly)
+        {
+            var foundFiles = Enumerable.Empty<string>();
+
+            if (searchOption == SearchOption.AllDirectories)
+            {
+                try
+                {
+                    IEnumerable<string> subDirs = Directory.EnumerateDirectories(path);
+                    // Add files in subdirectories recursively to the list
+                    foreach (string dir in subDirs)
+                        foundFiles = foundFiles.Concat(SaferEnumerateFiles(dir, searchPattern, searchOption));
+                }
+                catch (UnauthorizedAccessException) { }
+                catch (PathTooLongException) { }
+            }
+
+            try
+            {
+                // Add files from the current directory
+                foundFiles = foundFiles.Concat(Directory.EnumerateFiles(path, searchPattern));
+            }
+            catch (UnauthorizedAccessException) { }
+
+            return foundFiles;
+        }
     }
 }

LibationFileManager/AudibleFileStorage.cs

@@ -73,8 +73,8 @@ namespace LibationFileManager
         protected override string GetFilePathCustom(string productId)
         {
             var regex = GetBookSearchRegex(productId);
-            return Directory
-                .EnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
+            return FileUtility
+                .SaferEnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
                 .FirstOrDefault(s => regex.IsMatch(s));
         }
 

LibationFileManager/TagsPersistence.cs

@@ -54,11 +54,17 @@ namespace LibationFileManager
 
 		private static void ensureCache()
 		{
-			if (cache is null)
-				lock (locker)
-					cache = !File.Exists(TagsFile)
-						? new Dictionary<string, string>()
-						: JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
+			if (cache is not null)
+				return;
+
+			lock (locker)
+			{
+				if (File.Exists(TagsFile))
+					cache = JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
+
+				// if file doesn't exist. or if file is corrupt and deserialize returns null
+				cache ??= new Dictionary<string, string>();
+			}
 		}
 	}
 }

README.md

@@ -92,6 +92,26 @@ Or if you have multiple accounts, you'll get to choose whether to scan all accou
 
 ![Import which accounts](images/v40_import.png)
 
+If this is a new installation, or you're scanning an account you haven't scanned before, you'll be prompted to enter your password for the Audible account.
+
+![Login password](images/alt-login1.png)
+
+Enter the password and click Submit. Audible will prompt you with a CAPTCHA image. 
+
+![Login captcha](images/alt-login2.png)
+
+Enter the CAPTCHA answer characters and click Submit. If all has gone well, Libation will start scanning the account. 
+
+In rare instances, the Captcha image/response will fail in an endless loop. If this happens, delete the problem account, and then click Save. Re-add the account and click Save again. Now try to scan the account again. This time, instead of typing your password, click the link that says "Or click here". This will open the Audible External Login dialog shown below.
+
+![Login alternative setup](images/alt-login3.png)
+
+You can either copy the URL shown and paste it into your browser or launch the browser directly by clicking Launch in Browser. Audible will display its standard login page. Login, including answering the CAPTCHA on the next page. In some cases, you might have to approve the login from the email account associated with that login, but once the login is successful, you'll see an error message.
+
+![Login alternative login result](images/alt-login4.png)
+
+This actually means you've successfully logged in. Copy the entire URL shown in your browser and return to Libation. Paste that URL into the text box at the bottom of the Audible External Login window and click Submit.
+
 You'll see this window while it's scanning:
 
 ![Import step 2](images/Import2.png)