Bug fix #231 : Some books without categories are not getting imported

CLI manual change to settings file no longer needed

AppScaffolding/AppScaffolding.csproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
-    <Version>6.6.2.1</Version>
+    <Version>6.7.7.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

AppScaffolding/LibationScaffolding.cs

@@ -56,7 +56,7 @@ namespace AppScaffolding
 			// migrations go below here
 			//
 
-			Migrations.migrate_to_v6_5_2(config);
+			Migrations.migrate_to_v6_6_9(config);
 		}
 
 		public static void PopulateMissingConfigValues(Configuration config)
@@ -107,29 +107,12 @@ namespace AppScaffolding
 			if (config.GetObject("Serilog") != null)
 				return;
 
-			// "Serilog": {
-			//   "MinimumLevel": "Information"
-			//   "WriteTo": [
-			//     {
-			//       "Name": "Console"
-			//     },
-			//     {
-			//       "Name": "File",
-			//       "Args": {
-			//         "rollingInterval": "Day",
-			//         "outputTemplate": ...
-			//       }
-			//     }
-			//   ],
-			//   "Using": [ "Dinah.Core" ],
-			//   "Enrich": [ "WithCaller" ]
-			// }
 			var serilogObj = new JObject
 			{
 				{ "MinimumLevel", "Information" },
 				{ "WriteTo", new JArray
 					{
-						new JObject { {"Name", "Console" } },
+						// new JObject { {"Name", "Console" } }, // this has caused more problems than it's solved
 						new JObject
 						{
 							{ "Name", "File" },
@@ -144,14 +127,16 @@ namespace AppScaffolding
 									//   output example:             2019-11-26 08:48:40.224 -05:00 [DBG] Begin Libation
 									// - with class and method info: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception}";
 									//   output example:             2019-11-26 08:48:40.224 -05:00 [DBG] (at LibationWinForms.Program.init()) Begin Libation
-									{ "outputTemplate", "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception}" }
+									// {Properties:j} needed for expanded exception logging
+									{ "outputTemplate", "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception} {Properties:j}" }
 								}
 							}
 						}
 					}
 				},
-				{ "Using", new JArray{ "Dinah.Core" } }, // dll's name, NOT namespace
-				{ "Enrich", new JArray{ "WithCaller" } },
+				// better exception logging with: Serilog.Exceptions library -- WithExceptionDetails
+				{ "Using", new JArray{ "Dinah.Core", "Serilog.Exceptions" } }, // dll's name, NOT namespace
+				{ "Enrich", new JArray{ "WithCaller", "WithExceptionDetails" } },
 			};
 			config.SetObject("Serilog", serilogObj);
 		}
@@ -166,9 +151,9 @@ namespace AppScaffolding
 			// capture most Console.WriteLine() and write to serilog. See below tests for details.
 			// Some dependencies print helpful info via Console.WriteLine. We'd like to log it.
 			//
-			// Serilog also writes to Console so this might be asking for trouble. ie: infinite loops.
-			// SerilogTextWriter needs to be more robust and tested. Esp the Write() methods.
-			// Empirical testing so far has shown no issues.
+			// If Serilog also writes to Console, this might be asking for trouble. ie: infinite loops.
+			// To use that way, SerilogTextWriter needs to be more robust and tested. Esp the Write() methods.
+			// However, empirical testing so far has shown no issues.
 			Console.SetOut(new MultiTextWriter(origOut, new SerilogTextWriter()));
 
 			#region Console => Serilog tests
@@ -245,10 +230,10 @@ namespace AppScaffolding
 				config.InProgress,
 
 				AudibleFileStorage.DownloadsInProgressDirectory,
-				DownloadsInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
+				DownloadsInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
 
 				AudibleFileStorage.DecryptInProgressDirectory,
-				DecryptInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
+				DecryptInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
 			});
 		}
 
@@ -346,9 +331,51 @@ namespace AppScaffolding
 			};
 		#endregion
 
-		public static void migrate_to_v6_5_2(Configuration config)
+		public static void migrate_to_v6_6_9(Configuration config)
 		{
-			// example
+			var writeToPath = $"Serilog.WriteTo";
+
+			// remove WriteTo[].Name == Console
+			{
+				if (UNSAFE_MigrationHelper.Settings_TryGetArrayLength(writeToPath, out var length1))
+				{
+					for (var i = length1 - 1; i >= 0; i--)
+					{
+						var exists = UNSAFE_MigrationHelper.Settings_TryGetFromJsonPath($"{writeToPath}[{i}].Name", out var value);
+
+						if (exists && value == "Console")
+							UNSAFE_MigrationHelper.Settings_RemoveFromArray(writeToPath, i);
+					}
+				}
+			}
+
+			// add Serilog.Exceptions -- WithExceptionDetails
+			{
+				// outputTemplate should contain "{Properties:j}"
+				{
+					// re-calculate. previous loop may have changed the length
+					if (UNSAFE_MigrationHelper.Settings_TryGetArrayLength(writeToPath, out var length2))
+					{
+						var propertyName = "outputTemplate";
+						for (var i = 0; i < length2; i++)
+						{
+							var jsonPath = $"{writeToPath}[{i}].Args";
+							var exists = UNSAFE_MigrationHelper.Settings_TryGetFromJsonPath($"{jsonPath}.{propertyName}", out var value);
+
+							var newValue = "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception} {Properties:j}";
+
+							if (exists && value != newValue)
+								UNSAFE_MigrationHelper.Settings_SetWithJsonPath(jsonPath, propertyName, newValue);
+						}
+					}
+				}
+
+				// Serilog.Using must include "Serilog.Exceptions"
+				UNSAFE_MigrationHelper.Settings_AddUniqueToArray("Serilog.Using", "Serilog.Exceptions");
+
+				// Serilog.Enrich must include "WithExceptionDetails"
+				UNSAFE_MigrationHelper.Settings_AddUniqueToArray("Serilog.Enrich", "WithExceptionDetails");
+			}
 		}
 	}
 }

AppScaffolding/UNSAFE_MigrationHelper.cs

@@ -113,6 +113,108 @@ namespace AppScaffolding
 			return success;
 		}
 
+		public static bool Settings_JsonPathIsType(string jsonPath, JTokenType jTokenType)
+		{
+			JToken val = null;
+
+			process_SettingsJson(jObj => val = jObj.SelectToken(jsonPath), false);
+
+			return val?.Type == jTokenType;
+		}
+
+		public static bool Settings_TryGetFromJsonPath(string jsonPath, out string value)
+		{
+			JToken val = null;
+
+			process_SettingsJson(jObj => val = jObj.SelectToken(jsonPath), false);
+
+			if (val?.Type == JTokenType.String)
+			{
+				value = val.Value<string>();
+				return true;
+			}
+			else
+			{
+				value = null;
+				return false;
+			}
+		}
+
+		public static void Settings_SetWithJsonPath(string jsonPath, string propertyName, string newValue)
+		{
+			if (!Settings_TryGetFromJsonPath($"{jsonPath}.{propertyName}", out _))
+				return;
+
+			process_SettingsJson(jObj =>
+			{
+				var token = jObj.SelectToken(jsonPath);
+				if (token is null
+				|| token is not JObject o
+				|| o[propertyName] is null)
+					return;
+
+				var oldValue = token.Value<string>(propertyName);
+				if (oldValue != newValue)
+					token[propertyName] = newValue;
+			});
+		}
+
+		public static bool Settings_TryGetArrayLength(string jsonPath, out int length)
+		{
+			length = 0;
+
+			if (!Settings_JsonPathIsType(jsonPath, JTokenType.Array))
+				return false;
+
+			JArray array = null;
+			process_SettingsJson(jObj => array = (JArray)jObj.SelectToken(jsonPath));
+
+			length = array.Count;
+			return true;
+		}
+
+		public static void Settings_AddToArray(string jsonPath, string newValue)
+		{
+			if (!Settings_JsonPathIsType(jsonPath, JTokenType.Array))
+				return;
+
+			process_SettingsJson(jObj =>
+			{
+				var array = (JArray)jObj.SelectToken(jsonPath);
+				array.Add(newValue);
+			});
+		}
+
+		/// <summary>Do not add if already exists</summary>
+		public static void Settings_AddUniqueToArray(string arrayPath, string newValue)
+		{
+			if (!Settings_TryGetArrayLength(arrayPath, out var qty))
+				return;
+
+			for (var i = 0; i < qty; i++)
+			{
+				var exists = Settings_TryGetFromJsonPath($"{arrayPath}[{i}]", out var value);
+				if (exists && value == newValue)
+					return;
+			}
+
+			Settings_AddToArray(arrayPath, newValue);
+		}
+
+		/// <summary>only remove if not exists</summary>
+		public static void Settings_RemoveFromArray(string jsonPath, int position)
+		{
+			if (!Settings_JsonPathIsType(jsonPath, JTokenType.Array))
+				return;
+
+			process_SettingsJson(jObj =>
+			{
+				var array = (JArray)jObj.SelectToken(jsonPath);
+				if (position < array.Count)
+					array.RemoveAt(position);
+			});
+		}
+
 		/// <summary>only insert if not exists</summary>
 		public static void Settings_Insert(string key, string value)
 			=> process_SettingsJson(jObj => jObj.TryAdd(key, value));

ApplicationServices/LibraryExporter.cs

@@ -44,6 +44,9 @@ namespace ApplicationServices
 		[Name("Length In Minutes")]
 		public int LengthInMinutes { get; set; }
 
+		[Name("Description")]
+		public string Description { get; set; }
+
 		[Name("Publisher")]
 		public string Publisher { get; set; }
 
@@ -111,6 +114,7 @@ namespace ApplicationServices
 				AuthorNames = a.Book.AuthorNames,
 				NarratorNames = a.Book.NarratorNames,
 				LengthInMinutes = a.Book.LengthInMinutes,
+				Description = a.Book.Description,
 				Publisher = a.Book.Publisher,
 				HasPdf = a.Book.HasPdf,
 				SeriesNames = a.Book.SeriesNames,
@@ -180,6 +184,7 @@ namespace ApplicationServices
 				nameof (ExportDto.AuthorNames),
 				nameof (ExportDto.NarratorNames),
 				nameof (ExportDto.LengthInMinutes),
+				nameof (ExportDto.Description),
 				nameof (ExportDto.Publisher),
 				nameof (ExportDto.HasPdf),
 				nameof (ExportDto.SeriesNames),
@@ -233,6 +238,7 @@ namespace ApplicationServices
 				row.CreateCell(col++).SetCellValue(dto.AuthorNames);
 				row.CreateCell(col++).SetCellValue(dto.NarratorNames);
 				row.CreateCell(col++).SetCellValue(dto.LengthInMinutes);
+				row.CreateCell(col++).SetCellValue(dto.Description);
 				row.CreateCell(col++).SetCellValue(dto.Publisher);
 				row.CreateCell(col++).SetCellValue(dto.HasPdf);
 				row.CreateCell(col++).SetCellValue(dto.SeriesNames);

AudibleUtilities/ApiExtended.cs

@@ -127,11 +127,18 @@ namespace AudibleUtilities
 //    items = AudibleApi.Common.Converter.FromJson<List<Item>>(System.IO.File.ReadAllText(library_json));
 //}
 #endif
+
+			Serilog.Log.Logger.Debug("Begin initial library scan");
+
 			if (!items.Any())
 				items = await Api.GetAllLibraryItemsAsync(responseGroups);
 
+			Serilog.Log.Logger.Debug("Initial library scan complete. Begin episode scan");
+
 			await manageEpisodesAsync(items, importEpisodes);
 
+			Serilog.Log.Logger.Debug("Episode scan complete");
+
 #if DEBUG
 //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
 #endif
@@ -209,7 +216,8 @@ namespace AudibleUtilities
 						new Series
 						{
 							Asin = parent.Asin,
-							Sequence = parent.Relationships.Single(r => r.Asin == child.Asin).Sort.ToString(),
+							// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive for malformed data from audible
+							Sequence = parent.Relationships.FirstOrDefault(r => r.Asin == child.Asin).Sort.ToString(),
 							Title = parent.TitleWithSubtitle
 						}
 					};

AudibleUtilities/AudibleUtilities.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AudibleApi" Version="2.5.1.1" />
+    <PackageReference Include="AudibleApi" Version="2.7.2.1" />
   </ItemGroup>
 
   <ItemGroup>

DataLayer/DataLayer.csproj

@@ -13,12 +13,12 @@
   
   <ItemGroup>
     <PackageReference Include="Dinah.EntityFrameworkCore" Version="4.0.0.3" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.0">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.0" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.0">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.3" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

DtoImporterService/BookImporter.cs

@@ -94,7 +94,8 @@ namespace DtoImporterService
 			// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
 			var authors = item
 				.Authors
-				.Select(a => DbContext.Contributors.Local.Single(c => a.Name == c.Name))
+				// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
+				.Select(a => DbContext.Contributors.Local.FirstOrDefault(c => a.Name == c.Name))
 				.ToList();
 
 			var narrators
@@ -104,7 +105,8 @@ namespace DtoImporterService
 				// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
 				: item
 					.Narrators
-					.Select(n => DbContext.Contributors.Local.Single(c => n.Name == c.Name))
+					// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
+					.Select(n => DbContext.Contributors.Local.FirstOrDefault(c => n.Name == c.Name))
 					.ToList();
 
 			// categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd
@@ -118,7 +120,10 @@ namespace DtoImporterService
 				// 2+
 				: item.Categories[1].CategoryId;
 
-			var category = DbContext.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == lastCategory);
+			// This should properly be SingleOrDefault() not FirstOrDefault(), but FirstOrDefault is defensive
+			var category
+				= DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == lastCategory)
+				?? Category.GetEmpty();
 
 			Book book;
 			try
@@ -154,7 +159,8 @@ namespace DtoImporterService
 			var publisherName = item.Publisher;
 			if (!string.IsNullOrWhiteSpace(publisherName))
 			{
-				var publisher = DbContext.Contributors.Local.Single(c => publisherName == c.Name);
+				// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
+				var publisher = DbContext.Contributors.Local.FirstOrDefault(c => publisherName == c.Name);
 				book.ReplacePublisher(publisher);
 			}
 
@@ -171,7 +177,9 @@ namespace DtoImporterService
 			var item = importItem.DtoItem;
 
 			// set/update book-specific info which may have changed
-			book.PictureId = item.PictureId;
+			if (item.PictureId is not null)
+				book.PictureId = item.PictureId;
+
 			book.UpdateProductRating(item.Product_OverallStars, item.Product_PerformanceStars, item.Product_StoryStars);
 
 			// important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import

DtoImporterService/ContributorImporter.cs

@@ -82,7 +82,8 @@ namespace DtoImporterService
 				);
 			foreach (var name in newPeople)
 			{
-				var p = groupby.Single(g => g.Name == name).People.First();
+				// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
+				var p = groupby.FirstOrDefault(g => g.Name == name).People.First();
 
 				try
 				{

DtoImporterService/LibraryBookImporter.cs

@@ -53,7 +53,8 @@ namespace DtoImporterService
 				var newItem = gb.ImportItems.First();
 
 				var libraryBook = new LibraryBook(
-					DbContext.Books.Local.Single(b => b.AudibleProductId == newItem.DtoItem.ProductId),
+					// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive
+					DbContext.Books.Local.FirstOrDefault(b => b.AudibleProductId == newItem.DtoItem.ProductId),
 					newItem.DtoItem.DateAdded,
 					newItem.AccountId);
 				try

FileManager/BackgroundFileSystem.cs

@@ -43,7 +43,7 @@ namespace FileManager
             lock (fsCacheLocker)
             {
                 fsCache.Clear();
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
             }
         }
 
@@ -52,7 +52,7 @@ namespace FileManager
             Stop();
 
             lock (fsCacheLocker)
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
 
             directoryChangesEvents = new BlockingCollection<FileSystemEventArgs>();
 			fileSystemWatcher = new FileSystemWatcher(RootDirectory)
@@ -135,7 +135,7 @@ namespace FileManager
         private void AddPath(string path)
         {
             if (File.GetAttributes(path).HasFlag(FileAttributes.Directory))
-                AddUniqueFiles(Directory.EnumerateFiles(path, SearchPattern, SearchOption));
+                AddUniqueFiles(FileUtility.SaferEnumerateFiles(path, SearchPattern, SearchOption));
             else
                 AddUniqueFile(path);
         }

FileManager/FileManager.csproj

@@ -5,8 +5,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core" Version="4.0.3.1" />
-    <PackageReference Include="Polly" Version="7.2.2" />
+    <PackageReference Include="Dinah.Core" Version="4.0.4.1" />
+    <PackageReference Include="Polly" Version="7.2.3" />
   </ItemGroup>
 
 </Project>

FileManager/FileUtility.cs

@@ -139,7 +139,7 @@ namespace FileManager
             return path[0] + remainder;
         }
 
-        private static string removeInvalidWhitespace_pattern { get; } = $@"\s*\{Path.DirectorySeparatorChar}\s*";
+        private static string removeInvalidWhitespace_pattern { get; } = $@"[\s\.]*\{Path.DirectorySeparatorChar}\s*";
         private static Regex removeInvalidWhitespace_regex { get; } = new(removeInvalidWhitespace_pattern, RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
 
         /// <summary>no part of the path may begin or end in whitespace</summary>
@@ -149,14 +149,20 @@ namespace FileManager
             // replace whitespace around path slashes
             //    regex (with space added for clarity)
             //    \s*  \\  \s*    =>    \
-            fullfilename = fullfilename.Trim();
-            
-            fullfilename = removeInvalidWhitespace_regex.Replace(fullfilename, @"\");
+            // no ending dots. beginning dots are valid
+
+            // regex is easier by ending with separator
+            fullfilename += Path.DirectorySeparatorChar;
+            fullfilename = removeInvalidWhitespace_regex.Replace(fullfilename, Path.DirectorySeparatorChar.ToString());
+            // take seperator back off
+            fullfilename = RemoveLastCharacter(fullfilename);
 
             fullfilename = removeDoubleSlashes(fullfilename);
             return fullfilename;
         }
 
+        public static string RemoveLastCharacter(this string str) => string.IsNullOrEmpty(str) ? str : str[..^1];
+
         /// <summary>
         /// Move file.
         /// <br/>- Ensure valid file name path: remove invalid chars, ensure uniqueness, enforce max file length
@@ -183,15 +189,19 @@ namespace FileManager
             {
                 try
                 {
-                    if (File.Exists(source))
+                    if (!File.Exists(source))
                     {
-                        File.Delete(source);
-                        Serilog.Log.Logger.Information("File successfully deleted", new { source });
+                        Serilog.Log.Logger.Debug("No file to delete: {@DebugText}", new { source });
+                        return;
                     }
+
+                    Serilog.Log.Logger.Debug("Attempt to delete file: {@DebugText}", new { source });
+                    File.Delete(source);
+                    Serilog.Log.Logger.Information("File successfully deleted: {@DebugText}", new { source });
                 }
                 catch (Exception e)
                 {
-                    Serilog.Log.Logger.Error(e, "Failed to delete file", new { source });
+                    Serilog.Log.Logger.Error(e, "Failed to delete file: {@DebugText}", new { source });
                     throw;
                 }
             });
@@ -202,19 +212,61 @@ namespace FileManager
             {
                 try
                 {
-                    if (File.Exists(source))
+                    if (!File.Exists(source))
                     {
-                        SaferDelete(destination);
-                        Directory.CreateDirectory(Path.GetDirectoryName(destination));
-                        File.Move(source, destination);
-                        Serilog.Log.Logger.Information("File successfully moved", new { source, destination });
+                        Serilog.Log.Logger.Debug("No file to move: {@DebugText}", new { source });
+                        return;
                     }
+
+                    SaferDelete(destination);
+
+                    var dir = Path.GetDirectoryName(destination);
+                    Serilog.Log.Logger.Debug("Attempt to create directory: {@DebugText}", new { dir });
+                    Directory.CreateDirectory(dir);
+
+                    Serilog.Log.Logger.Debug("Attempt to move file: {@DebugText}", new { source, destination });
+                    File.Move(source, destination);
+                    Serilog.Log.Logger.Information("File successfully moved: {@DebugText}", new { source, destination });
                 }
                 catch (Exception e)
                 {
-                    Serilog.Log.Logger.Error(e, "Failed to move file", new { source, destination });
+                    Serilog.Log.Logger.Error(e, "Failed to move file: {@DebugText}", new { source, destination });
                     throw;
                 }
             });
+
+        /// <summary>
+        /// A safer way to get all the files in a directory and sub directory without crashing on UnauthorizedException or PathTooLongException
+        /// </summary>
+        /// <param name="rootPath">Starting directory</param>
+        /// <param name="patternMatch">Filename pattern match</param>
+        /// <param name="searchOption">Search subdirectories or only top level directory for files</param>
+        /// <returns>List of files</returns>
+        public static IEnumerable<string> SaferEnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly)
+        {
+            var foundFiles = Enumerable.Empty<string>();
+
+            if (searchOption == SearchOption.AllDirectories)
+            {
+                try
+                {
+                    IEnumerable<string> subDirs = Directory.EnumerateDirectories(path);
+                    // Add files in subdirectories recursively to the list
+                    foreach (string dir in subDirs)
+                        foundFiles = foundFiles.Concat(SaferEnumerateFiles(dir, searchPattern, searchOption));
+                }
+                catch (UnauthorizedAccessException) { }
+                catch (PathTooLongException) { }
+            }
+
+            try
+            {
+                // Add files from the current directory
+                foundFiles = foundFiles.Concat(Directory.EnumerateFiles(path, searchPattern));
+            }
+            catch (UnauthorizedAccessException) { }
+
+            return foundFiles;
+        }
     }
 }

FileManager/PersistentDictionary.cs

@@ -208,7 +208,7 @@ namespace FileManager
             {
                 var msg = "Unrecoverable error. Settings file cannot be found";
                 var ex = new FileNotFoundException(msg, Filepath);
-                Serilog.Log.Logger.Error(msg, ex);
+                Serilog.Log.Logger.Error(ex, msg);
                 throw ex;
             }
 
@@ -226,7 +226,7 @@ namespace FileManager
             {
                 var msg = "Unrecoverable error. Unable to read settings from Settings file";
                 var ex = new NullReferenceException(msg);
-                Serilog.Log.Logger.Error(msg, ex);
+                Serilog.Log.Logger.Error(ex, msg);
                 throw ex;
             }
 

LibationFileManager/AudibleFileStorage.cs

@@ -73,8 +73,8 @@ namespace LibationFileManager
         protected override string GetFilePathCustom(string productId)
         {
             var regex = GetBookSearchRegex(productId);
-            return Directory
-                .EnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
+            return FileUtility
+                .SaferEnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
                 .FirstOrDefault(s => regex.IsMatch(s));
         }
 

LibationFileManager/Configuration.cs

@@ -270,7 +270,7 @@ namespace LibationFileManager
                 var valueWasChanged = persistentDictionary.SetWithJsonPath("Serilog", "MinimumLevel", value.ToString());
                 if (!valueWasChanged)
                 {
-                    Log.Logger.Information("LogLevel.set attempt. No change");
+                    Log.Logger.Debug("LogLevel.set attempt. No change");
                     return;
                 }
 

LibationFileManager/FilePathCache.cs

@@ -26,6 +26,18 @@ namespace LibationFileManager
 			if (File.Exists(jsonFile))
 			{
 				var list = JsonConvert.DeserializeObject<List<CacheEntry>>(File.ReadAllText(jsonFile));
+
+				// file exists but deser is null. this will never happen when file is healthy
+				if (list is null)
+				{
+					lock (locker)
+					{
+						Serilog.Log.Logger.Error("Error deserializing file. Wrong format. Possibly corrupt. Deleting file. {@DebugInfo}", new { jsonFile });
+						File.Delete(jsonFile);
+						return;
+					}
+				}
+
 				cache = new Cache<CacheEntry>(list);
 			}
         }

LibationFileManager/LibationFileManager.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
+    <PackageReference Include="Serilog.Exceptions" Version="8.1.0" />
   </ItemGroup>
 
   <ItemGroup>

LibationFileManager/PictureStorage.cs

@@ -73,16 +73,10 @@ namespace LibationFileManager
 				if (!cache.ContainsKey(def) || cache[def] == null)
 				{
 					var path = getPath(def);
-					byte[] bytes;
-
-					if (File.Exists(path))
-						bytes = File.ReadAllBytes(path);
-					else
-					{
-						bytes = downloadBytes(def);
-						saveFile(def, bytes);
-					}
-
+					var bytes
+						= File.Exists(path)
+						? File.ReadAllBytes(path)
+						: downloadBytes(def);
 					cache[def] = bytes;
 				}
 				return cache[def];
@@ -104,7 +98,6 @@ namespace LibationFileManager
 					continue;
 
 				var bytes = downloadBytes(def);
-				saveFile(def, bytes);
 				lock (cacheLocker)
 					cache[def] = bytes;
 
@@ -115,14 +108,24 @@ namespace LibationFileManager
 		private static HttpClient imageDownloadClient { get; } = new HttpClient();
 		private static byte[] downloadBytes(PictureDefinition def)
 		{
-			var sz = (int)def.Size;
-			return imageDownloadClient.GetByteArrayAsync("ht" + $"tps://images-na.ssl-images-amazon.com/images/I/{def.PictureId}._SL{sz}_.jpg").Result;
-		}
+			if (def.PictureId is null)
+				return getDefaultImage(def.Size);
 
-		private static void saveFile(PictureDefinition def, byte[] bytes)
-		{
-			var path = getPath(def);
-			File.WriteAllBytes(path, bytes);
+			try
+			{
+				var sz = (int)def.Size;
+				var bytes = imageDownloadClient.GetByteArrayAsync("ht" + $"tps://images-na.ssl-images-amazon.com/images/I/{def.PictureId}._SL{sz}_.jpg").Result;
+
+				// save image file. make sure to not save default image
+				var path = getPath(def);
+				File.WriteAllBytes(path, bytes);
+
+				return bytes;
+			}
+			catch
+			{
+				return getDefaultImage(def.Size);
+			}
 		}
 	}
 }

LibationFileManager/TagsPersistence.cs

@@ -54,11 +54,17 @@ namespace LibationFileManager
 
 		private static void ensureCache()
 		{
-			if (cache is null)
-				lock (locker)
-					cache = !File.Exists(TagsFile)
-						? new Dictionary<string, string>()
-						: JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
+			if (cache is not null)
+				return;
+
+			lock (locker)
+			{
+				if (File.Exists(TagsFile))
+					cache = JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
+
+				// if file doesn't exist. or if file is corrupt and deserialize returns null
+				cache ??= new Dictionary<string, string>();
+			}
 		}
 	}
 }

LibationWinForms/LibationWinForms.csproj

@@ -28,7 +28,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.3.1" />
+    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.4.1" />
   </ItemGroup>
 
   <ItemGroup>

README.md

@@ -92,6 +92,26 @@ Or if you have multiple accounts, you'll get to choose whether to scan all accou
 
 ![Import which accounts](images/v40_import.png)
 
+If this is a new installation, or you're scanning an account you haven't scanned before, you'll be prompted to enter your password for the Audible account.
+
+![Login password](images/alt-login1.png)
+
+Enter the password and click Submit. Audible will prompt you with a CAPTCHA image. 
+
+![Login captcha](images/alt-login2.png)
+
+Enter the CAPTCHA answer characters and click Submit. If all has gone well, Libation will start scanning the account. 
+
+In rare instances, the Captcha image/response will fail in an endless loop. If this happens, delete the problem account, and then click Save. Re-add the account and click Save again. Now try to scan the account again. This time, instead of typing your password, click the link that says "Or click here". This will open the Audible External Login dialog shown below.
+
+![Login alternative setup](images/alt-login3.png)
+
+You can either copy the URL shown and paste it into your browser or launch the browser directly by clicking Launch in Browser. Audible will display its standard login page. Login, including answering the CAPTCHA on the next page. In some cases, you might have to approve the login from the email account associated with that login, but once the login is successful, you'll see an error message.
+
+![Login alternative login result](images/alt-login4.png)
+
+This actually means you've successfully logged in. Copy the entire URL shown in your browser and return to Libation. Paste that URL into the text box at the bottom of the Audible External Login window and click Submit.
+
 You'll see this window while it's scanning:
 
 ![Import step 2](images/Import2.png)
@@ -156,7 +176,6 @@ When you set up Libation, you'll specify a Books directory. Libation looks insid
 
 * .m4b: your audiobook in m4b format. This is the most pure version of your audiobook and retains the highest quality. Now that it's decrypted, you can play it on any audio player and put it on any device. If you'd like, you can also use 3rd party tools to turn it into an mp3. The freedom to do what you want with your files was the original inspiration for Libation.
 * .cue: this is a file which logs where chapter breaks occur. Many tools are able to use this if you want to split your book into files along chapter lines.
-* .nfo: This is just some general info about the book and includes some technical stats about the audiofile.
 
 ### Export your library
 
@@ -244,7 +263,7 @@ To make upgrades and reinstalls easier, Libation separates all of its responsibi
 
 ### Linux and Mac
 
-Although Libation only currently officially supports Windows, [some users](https://github.com/rmcrackan/Libation/issues/28#issuecomment-890594158) have had success with WINE.
+Although Libation only currently officially supports Windows, some users have had success with WINE. ([Linux](https://github.com/rmcrackan/Libation/issues/28#issuecomment-890594158), [OSX Crossover and WINE](https://github.com/rmcrackan/Libation/issues/150#issuecomment-1004918592))
 
 ### Settings
 
@@ -294,14 +313,3 @@ export library to file
   libationcli export -p "C:\foo\bar\my.xlsx" --xlsx
   libationcli export -p "C:\foo\bar\my.xlsx" -x
 ```
-
-Currently logs are written to Console and to file. This means they'll be printed in the CLI. To disable, find this in Settings.json and delete the 3 lines after `"WriteTo": [`
-
-```
-  "Serilog": {
-    "MinimumLevel": "Information",
-    "WriteTo": [
-      {
-        "Name": "Console"
-      },
-```

_Tests/AudibleUtilities.Tests/AudibleUtilities.Tests.csproj

@@ -7,12 +7,12 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.2.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
-    <PackageReference Include="Moq" Version="4.16.1" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="Moq" Version="4.17.2" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/FileLiberator.Tests/FileLiberator.Tests.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.2.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/FileManager.Tests/FileManager.Tests.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.2.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/FileManager.Tests/FileUtilityTests.cs

@@ -113,4 +113,51 @@ namespace FileUtilityTests
 		public void Tests(string input, string expected)
 			=> FileUtility.GetStandardizedExtension(input).Should().Be(expected);
 	}
+
+	[TestClass]
+    public class GetValidFilename
+	{
+		[TestMethod]
+		// dot-files
+		[DataRow(@"C:\a bc\x y z\.f i l e.txt")]
+		// dot-folders
+		[DataRow(@"C:\a bc\.x y z\f i l e.txt")]
+		public void Valid(string input) => Tests(input, input);
+
+		[TestMethod]
+		// folder spaces
+		[DataRow(@"C:\   a bc   \x y z   ", @"C:\a bc\x y z")]
+		// file spaces
+		[DataRow(@"C:\a bc\x y z\   f i l e.txt   ", @"C:\a bc\x y z\f i l e.txt")]
+		// eliminate beginning space and end dots and spaces
+		[DataRow(@"C:\a bc\   . . . x y z . . .   \f i l e.txt", @"C:\a bc\. . . x y z\f i l e.txt")]
+		// file end dots
+		[DataRow(@"C:\a bc\x y z\f i l e.txt . . .", @"C:\a bc\x y z\f i l e.txt")]
+		public void Tests(string input, string expected)
+			=> FileUtility.GetValidFilename(input).Should().Be(expected);
+	}
+
+	[TestClass]
+    public class RemoveLastCharacter
+	{
+		[TestMethod]
+		public void is_null() => Tests(null, null);
+
+		[TestMethod]
+		public void empty() => Tests("", "");
+
+		[TestMethod]
+		public void single_space() => Tests(" ", "");
+
+		[TestMethod]
+		public void multiple_space() => Tests("   ", "  ");
+
+		[TestMethod]
+		[DataRow("1", "")]
+		[DataRow("1 ", "1")]
+		[DataRow("12", "1")]
+		[DataRow("123", "12")]
+		public void Tests(string input, string expected)
+			=> FileUtility.RemoveLastCharacter(input).Should().Be(expected);
+	}
 }

_Tests/LibationFileManager.Tests/LibationFileManager.Tests.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.2.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj

@@ -7,12 +7,12 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.2.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
-    <PackageReference Include="Moq" Version="4.16.1" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="Moq" Version="4.17.2" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>