Search engine bug fix and unit tests

* add unit tests for search engine

ApplicationServices/UNTESTED/LibraryCommands.cs

@@ -81,16 +81,13 @@ namespace ApplicationServices
 		{
 			ArgumentValidator.EnsureNotNull(account, nameof(account));
 
-			var localeName = account.Locale?.Name;
 			Log.Logger.Information("ImportLibraryAsync. {@DebugInfo}", new
 			{
-				AccountName = account.AccountName.ToMask(),
-				AccountId = account.AccountId.ToMask(),
-				LocaleName = localeName,
+				Account = account?.MaskedLogEntry ?? "[null]"
 			});
 
 			var dtoItems = await AudibleApiActions.GetLibraryValidatedAsync(api);
-			return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = localeName }).ToList();
+			return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = account.Locale?.Name }).ToList();
 		}
 
 		private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)

ApplicationServices/UNTESTED/SearchEngineCommands.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System;
+using System.IO;
 using DataLayer;
 using LibationSearchEngine;
 
@@ -12,31 +13,43 @@ namespace ApplicationServices
 			engine.CreateNewIndex();
 		}
 
-		public static SearchResultSet Search(string searchString)
+		public static SearchResultSet Search(string searchString) => performSearchEngineFunc_safe(e =>
+			e.Search(searchString)
+		);
+
+		public static void UpdateBookTags(Book book) => performSearchEngineAction_safe(e =>
+			e.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags)
+		);
+
+		public static void UpdateIsLiberated(Book book) => performSearchEngineAction_safe(e =>
+			e.UpdateIsLiberated(book.AudibleProductId)
+		);
+
+		private static void performSearchEngineAction_safe(Action<SearchEngine> action)
 		{
 			var engine = new SearchEngine(DbContexts.GetContext());
 			try
 			{
-				return engine.Search(searchString);
+				action(engine);
 			}
 			catch (FileNotFoundException)
 			{
 				FullReIndex();
-				return engine.Search(searchString);
+				action(engine);
 			}
 		}
 
-		public static void UpdateBookTags(Book book)
+		private static T performSearchEngineFunc_safe<T>(Func<SearchEngine, T> action)
 		{
 			var engine = new SearchEngine(DbContexts.GetContext());
 			try
 			{
-				engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
+				return action(engine);
 			}
 			catch (FileNotFoundException)
 			{
 				FullReIndex();
-				engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
+				return action(engine);
 			}
 		}
 	}

FileLiberator/UNTESTED/DecryptBook.cs

@@ -78,11 +78,11 @@ namespace FileLiberator
                     Dinah.Core.IO.FileExt.SafeDelete(aaxFilename);
                 }
 
-                var statusHandler = new StatusHandler();
                 var finalAudioExists = AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId);
                 if (!finalAudioExists)
-                    statusHandler.AddError("Cannot find final audio file after decryption");
-                return statusHandler;
+                    return new StatusHandler { "Cannot find final audio file after decryption" };
+
+                return new StatusHandler();
             }
             finally
             {
@@ -137,7 +137,7 @@ namespace FileLiberator
 			// create final directory. move each file into it. MOVE AUDIO FILE LAST
 			// new dir: safetitle_limit50char + " [" + productId + "]"
 
-			var destinationDir = getDestDir(product);
+			var destinationDir = AudibleFileStorage.Audio.GetDestDir(product.Title, product.AudibleProductId);
 			Directory.CreateDirectory(destinationDir);
 
 			var sortedFiles = getProductFilesSorted(product, outputAudioFilename);
@@ -158,18 +158,6 @@ namespace FileLiberator
             return destinationDir;
         }
 
-		private static string getDestDir(Book product)
-		{
-			// to prevent the paths from getting too long, we don't need after the 1st ":" for the folder
-			var underscoreIndex = product.Title.IndexOf(':');
-			var titleDir
-				= underscoreIndex < 4
-				? product.Title
-				: product.Title.Substring(0, underscoreIndex);
-			var finalDir = FileUtility.GetValidFilename(AudibleFileStorage.BooksDirectory, titleDir, null, product.AudibleProductId);
-			return finalDir;
-		}
-
 		private static List<FileInfo> getProductFilesSorted(Book product, string outputAudioFilename)
 		{
 			// files are: temp path\author\[asin].ext

FileLiberator/UNTESTED/DownloadBook.cs

@@ -72,7 +72,7 @@ namespace FileLiberator
 				libraryBook.Book.Title,
 				libraryBook.Book.AudibleProductId,
 				libraryBook.Book.Locale,
-				libraryBook.Account,
+				Account = libraryBook.Account?.ToMask() ?? "[empty]",
 				tempAaxFilename,
 				actualFilePath,
 				length,

FileLiberator/UNTESTED/IProcessableExt.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using ApplicationServices;
@@ -16,38 +17,33 @@ namespace FileLiberator
         //
 
 
-        /// <summary>Process the first valid product. Create default context</summary>
-        /// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
-        public static async Task<StatusHandler> ProcessFirstValidAsync(this IProcessable processable)
-        {
-            var libraryBook = processable.getNextValidBook();
-            if (libraryBook == null)
-                return null;
+        // when used in foreach: stateful. deferred execution
+        public static IEnumerable<LibraryBook> GetValidLibraryBooks(this IProcessable processable)
+            => DbContexts.GetContext()
+            .GetLibrary_Flat_NoTracking()
+            .Where(libraryBook => processable.Validate(libraryBook));
 
-            return await processBookAsync(processable, libraryBook);
-        }
-
-        /// <summary>Process the first valid product. Create default context</summary>
-        /// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
-        public static async Task<StatusHandler> ProcessSingleAsync(this IProcessable processable, string productId)
+        public static LibraryBook GetSingleLibraryBook(string productId)
         {
             using var context = DbContexts.GetContext();
             var libraryBook = context
                 .Library
                 .GetLibrary()
                 .SingleOrDefault(lb => lb.Book.AudibleProductId == productId);
+            return libraryBook;
+        }
 
-            if (libraryBook == null)
-                return null;
+        public static async Task<StatusHandler> ProcessSingleAsync(this IProcessable processable, LibraryBook libraryBook)
+        {
             if (!processable.Validate(libraryBook))
                 return new StatusHandler { "Validation failed" };
 
-            return await processBookAsync(processable, libraryBook);
+            return await processable.ProcessBookAsync_NoValidation(libraryBook);
         }
 
-        private static async Task<StatusHandler> processBookAsync(IProcessable processable, LibraryBook libraryBook)
+        public static async Task<StatusHandler> ProcessBookAsync_NoValidation(this IProcessable processable, LibraryBook libraryBook)
         {
-            Serilog.Log.Logger.Information("Begin " + nameof(processBookAsync) + " {@DebugInfo}", new
+            Serilog.Log.Logger.Information("Begin " + nameof(ProcessBookAsync_NoValidation) + " {@DebugInfo}", new
             {
                 libraryBook.Book.Title,
                 libraryBook.Book.AudibleProductId,
@@ -55,25 +51,13 @@ namespace FileLiberator
                 Account = libraryBook.Account?.ToMask() ?? "[empty]"
             });
 
-            // this should never happen. check anyway. ProcessFirstValidAsync returning null is the signal that we're done. we can't let another IProcessable accidentally send this command
-            var status = await processable.ProcessAsync(libraryBook);
-            if (status == null)
-                throw new Exception("Processable should never return a null status");
+            var status
+                = (await processable.ProcessAsync(libraryBook))
+                ?? new StatusHandler { "Processable should never return a null status" };
 
             return status;
         }
 
-        private static LibraryBook getNextValidBook(this IProcessable processable)
-        {
-            var libraryBooks = DbContexts.GetContext().GetLibrary_Flat_NoTracking();
-
-            foreach (var libraryBook in libraryBooks)
-                if (processable.Validate(libraryBook))
-                    return libraryBook;
-
-            return null;
-        }   
-
 		public static async Task<StatusHandler> TryProcessAsync(this IProcessable processable, LibraryBook libraryBook)
 			=> processable.Validate(libraryBook)
 			? await processable.ProcessAsync(libraryBook)

FileManager/UNTESTED/AudibleFileStorage.cs

@@ -11,76 +11,77 @@ namespace FileManager
     // could add images here, but for now images are stored in a well-known location
     public enum FileType { Unknown, Audio, AAX, PDF }
 
-    /// <summary>
-    /// Files are large. File contents are never read by app.
-    /// Paths are varied.
-    /// Files are written during download/decrypt/backup/liberate.
-    /// Paths are read at app launch and during download/decrypt/backup/liberate.
-    /// Many files are often looked up at once
-    /// </summary>
-    public sealed class AudibleFileStorage : Enumeration<AudibleFileStorage>
-    {
-        #region static
-        public static AudibleFileStorage Audio { get; }
-        public static AudibleFileStorage AAX { get; }
-        public static AudibleFileStorage PDF { get; }
+	/// <summary>
+	/// Files are large. File contents are never read by app.
+	/// Paths are varied.
+	/// Files are written during download/decrypt/backup/liberate.
+	/// Paths are read at app launch and during download/decrypt/backup/liberate.
+	/// Many files are often looked up at once
+	/// </summary>
+	public abstract class AudibleFileStorage : Enumeration<AudibleFileStorage>
+	{
+		public abstract string[] Extensions { get; }
+		public abstract string StorageDirectory { get; }
 
-        public static string DownloadsInProgress { get; }
-        public static string DecryptInProgress { get; }
-        public static string BooksDirectory => Configuration.Instance.Books;
-        // not customizable. don't move to config
-        public static string DownloadsFinal { get; }
-            = new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("DownloadsFinal").FullName;
+		#region static
+		public static AudioFileStorage Audio { get; } = new AudioFileStorage();
+		public static AudibleFileStorage AAX { get; } = new AaxFileStorage();
+		public static AudibleFileStorage PDF { get; } = new PdfFileStorage();
 
-        static AudibleFileStorage()
+		public static string DownloadsInProgress
         {
-            #region init DecryptInProgress
-            if (!Configuration.Instance.DecryptInProgressEnum.In("WinTemp", "LibationFiles"))
-                Configuration.Instance.DecryptInProgressEnum = "WinTemp";
-            var M4bRootDir
-                = Configuration.Instance.DecryptInProgressEnum == "WinTemp" // else "LibationFiles"
-                ? Configuration.WinTemp
-                : Configuration.Instance.LibationFiles;
-            DecryptInProgress = Path.Combine(M4bRootDir, "DecryptInProgress");
-            Directory.CreateDirectory(DecryptInProgress);
-            #endregion
+            get
+            {
+                if (!Configuration.Instance.DownloadsInProgressEnum.In("WinTemp", "LibationFiles"))
+                    Configuration.Instance.DownloadsInProgressEnum = "WinTemp";
+                var AaxRootDir
+                    = Configuration.Instance.DownloadsInProgressEnum == "WinTemp"
+                    ? Configuration.WinTemp
+                    : Configuration.Instance.LibationFiles;
 
-            #region init DownloadsInProgress
-            if (!Configuration.Instance.DownloadsInProgressEnum.In("WinTemp", "LibationFiles"))
-                Configuration.Instance.DownloadsInProgressEnum = "WinTemp";
-            var AaxRootDir
-                = Configuration.Instance.DownloadsInProgressEnum == "WinTemp" // else "LibationFiles"
-                ? Configuration.WinTemp
-                : Configuration.Instance.LibationFiles;
-            DownloadsInProgress = Path.Combine(AaxRootDir, "DownloadsInProgress");
-            Directory.CreateDirectory(DownloadsInProgress);
-            #endregion
+                return Directory.CreateDirectory(Path.Combine(AaxRootDir, "DownloadsInProgress")).FullName;
+            }
+        }
 
-            #region init BooksDirectory
-            if (string.IsNullOrWhiteSpace(Configuration.Instance.Books))
-                Configuration.Instance.Books = Path.Combine(Configuration.Instance.LibationFiles, "Books");
-            Directory.CreateDirectory(Configuration.Instance.Books);
-            #endregion
+        public static string DecryptInProgress
+        {
+            get
+            {
+                if (!Configuration.Instance.DecryptInProgressEnum.In("WinTemp", "LibationFiles"))
+                    Configuration.Instance.DecryptInProgressEnum = "WinTemp";
 
-            // must do this in static ctor, not w/inline properties
-            // static properties init before static ctor so these dir.s would still be null
-            Audio = new AudibleFileStorage(FileType.Audio, BooksDirectory, "m4b", "mp3", "aac", "mp4", "m4a", "ogg", "flac");
-            AAX = new AudibleFileStorage(FileType.AAX, DownloadsFinal, "aax");
-            PDF = new AudibleFileStorage(FileType.PDF, BooksDirectory, "pdf", "zip");
+                var M4bRootDir
+                    = Configuration.Instance.DecryptInProgressEnum == "WinTemp"
+                    ? Configuration.WinTemp
+                    : Configuration.Instance.LibationFiles;
+
+                return Directory.CreateDirectory(Path.Combine(M4bRootDir, "DecryptInProgress")).FullName;
+            }
+        }
+
+        // not customizable. don't move to config
+        public static string DownloadsFinal => new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("DownloadsFinal").FullName;
+
+        public static string BooksDirectory
+        {
+            get
+            {
+                if (string.IsNullOrWhiteSpace(Configuration.Instance.Books))
+                    Configuration.Instance.Books = Path.Combine(Configuration.Instance.LibationFiles, "Books");
+                return Directory.CreateDirectory(Configuration.Instance.Books).FullName;
+            }
         }
         #endregion
 
         #region instance
         public FileType FileType => (FileType)Value;
 
-        public string StorageDirectory => DisplayName;
-
 		private IEnumerable<string> extensions_noDots { get; }
 		private string extAggr { get; }
 
-        private AudibleFileStorage(FileType fileType, string storageDirectory, params string[] extensions) : base((int)fileType, storageDirectory)
+        protected AudibleFileStorage(FileType fileType) : base((int)fileType, fileType.ToString())
 		{
-			extensions_noDots = extensions.Select(ext => ext.Trim('.')).ToList();
+			extensions_noDots = Extensions.Select(ext => ext.Trim('.')).ToList();
 			extAggr = extensions_noDots.Aggregate((a, b) => $"{a}|{b}");
 		}
 
@@ -90,30 +91,89 @@ namespace FileManager
         /// - a directory name has the product id and an audio file is immediately inside
         /// - any audio filename contains the product id
         /// </summary>
-        public bool Exists(string productId)
-            => GetPath(productId) != null;
+        public bool Exists(string productId) => GetPath(productId) != null;
 
 		public string GetPath(string productId)
         {
-			{
-                var cachedFile = FilePathCache.GetPath(productId, FileType);
-                if (cachedFile != null)
-                    return cachedFile;
-            }
+            var cachedFile = FilePathCache.GetPath(productId, FileType);
+            if (cachedFile != null)
+                return cachedFile;
 
-			var firstOrNull =
+            var firstOrNull =
 				Directory
 				.EnumerateFiles(StorageDirectory, "*.*", SearchOption.AllDirectories)
 				.FirstOrDefault(s => Regex.IsMatch(s, $@"{productId}.*?\.({extAggr})$", RegexOptions.IgnoreCase));
 
 			if (firstOrNull is null)
 				return null;
+
 			FilePathCache.Upsert(productId, FileType, firstOrNull);
 			return firstOrNull;
         }
 
+        public string GetDestDir(string title, string asin)
+        {
+            // to prevent the paths from getting too long, we don't need after the 1st ":" for the folder
+            var underscoreIndex = title.IndexOf(':');
+            var titleDir
+                = underscoreIndex < 4
+                ? title
+                : title.Substring(0, underscoreIndex);
+            var finalDir = FileUtility.GetValidFilename(StorageDirectory, titleDir, null, asin);
+            return finalDir;
+        }
+
         public bool IsFileTypeMatch(FileInfo fileInfo)
             => extensions_noDots.ContainsInsensative(fileInfo.Extension.Trim('.'));
         #endregion
     }
+
+    public class AudioFileStorage : AudibleFileStorage
+    {
+        public const string SKIP_FILE_EXT = "libhack";
+
+		public override string[] Extensions { get; } = new[] { "m4b", "mp3", "aac", "mp4", "m4a", "ogg", "flac", SKIP_FILE_EXT };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => BooksDirectory;
+
+        public AudioFileStorage() : base(FileType.Audio) { }
+
+        public string CreateSkipFile(string title, string asin, string contents = null)
+        {
+            var destinationDir = GetDestDir(title, asin);
+            Directory.CreateDirectory(destinationDir);
+
+            var path = FileUtility.GetValidFilename(destinationDir, title, SKIP_FILE_EXT, asin);
+            File.WriteAllText(path, contents ?? string.Empty);
+
+            return path;
+        }
+    }
+
+    public class AaxFileStorage : AudibleFileStorage
+    {
+        public override string[] Extensions { get; } = new[] { "aax" };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => DownloadsFinal;
+
+        public AaxFileStorage() : base(FileType.AAX) { }
+    }
+
+    public class PdfFileStorage : AudibleFileStorage
+    {
+		public override string[] Extensions { get; } = new[] { "pdf", "zip" };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => BooksDirectory;
+
+        public PdfFileStorage() : base(FileType.PDF) { }
+    }
 }

InternalUtilities/Account.cs

@@ -93,5 +93,11 @@ namespace InternalUtilities
 		}
 
 		public override string ToString() => $"{AccountId} - {Locale?.Name ?? "[empty]"}";
+
+		public string MaskedLogEntry => @$"AccountId={mask(AccountId)}|AccountName={mask(AccountName)}|Locale={Locale?.Name ?? "[empty]"}";
+		private static string mask(string str)
+			=> str is null ? "[null]"
+			: str == string.Empty ? "[empty]"
+			: str.ToMask();
 	}
 }

InternalUtilities/UNTESTED/AudibleApiActions.cs

@@ -32,7 +32,7 @@ namespace InternalUtilities
 		{
 			Serilog.Log.Logger.Information("GetApiAsync. {@DebugInfo}", new
 			{
-				AccountId = account?.AccountId.ToMask() ?? "[empty]",
+				Account = account?.MaskedLogEntry ?? "[null]",
 				LocaleName = account?.Locale?.Name
 			});
 			return EzApiCreator.GetApiAsync(

Libation.sln

@@ -86,6 +86,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "0 Libation Tests", "0 Libat
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "InternalUtilities.Tests", "_Tests\InternalUtilities.Tests\InternalUtilities.Tests.csproj", "{8447C956-B03E-4F59-9DD4-877793B849D9}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibationSearchEngine.Tests", "_Tests\LibationSearchEngine.Tests\LibationSearchEngine.Tests.csproj", "{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -208,6 +210,10 @@ Global
 		{8447C956-B03E-4F59-9DD4-877793B849D9}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8447C956-B03E-4F59-9DD4-877793B849D9}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8447C956-B03E-4F59-9DD4-877793B849D9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -242,6 +248,7 @@ Global
 		{E7EFD64D-6630-4426-B09C-B6862A92E3FD} = {F0CBB7A7-D3FB-41FF-8F47-CF3F6A592249}
 		{F3B04A3A-20C8-4582-A54A-715AF6A5D859} = {8679CAC8-9164-4007-BDD2-F004810EDA14}
 		{8447C956-B03E-4F59-9DD4-877793B849D9} = {67E66E82-5532-4440-AFB3-9FB1DF9DEF53}
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98} = {67E66E82-5532-4440-AFB3-9FB1DF9DEF53}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

LibationLauncher/LibationLauncher.csproj

@@ -13,7 +13,7 @@
     <!-- <PublishSingleFile>true</PublishSingleFile> -->
     <RuntimeIdentifier>win-x64</RuntimeIdentifier>
 
-    <Version>4.1.0.1</Version>
+    <Version>4.2.3.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

LibationSearchEngine/UNTESTED/LuceneRegex.cs

@@ -42,5 +42,58 @@ namespace LibationSearchEngine
         //   positive look behind: beginning  space  {  [  :
         //   positive look ahead: end  space  ]  }
         public static Regex NumbersRegex { get; } = new Regex(@"(?<=^|\s|\{|\[|:)(\d+\.?\d*)(?=$|\s|\]|\})", RegexOptions.Compiled);
+
+        /// <summary>
+        /// proper bools are single keywords which are turned into keyword:True
+        /// if bordered by colons or inside brackets, they are not stand-alone bool keywords
+        /// the negative lookbehind and lookahead patterns prevent bugs where a bool keyword is also a user-defined tag:
+        ///   [israted]
+        ///     parseTag => tags:israted
+        ///     replaceBools => tags:israted:True
+        ///   or
+        ///     [israted]
+        ///       replaceBools => israted:True
+        ///         parseTag => [israted:True]
+        /// also don't want to apply :True where the value already exists:
+        ///   israted:false => israted:false:True
+        ///   
+        /// despite using parans, lookahead and lookbehind are zero-length assertions which do not capture. therefore the bool search keyword is still $1 since it's the first and only capture
+        /// </summary>
+        private static string boolPattern_parameterized { get; }
+            = @"
+### IMPORTANT: 'ignore whitespace' is only partially honored in character sets
+### - new lines are ok
+### - ANY leading whitespace is treated like actual matching spaces  :(
+
+                    ### can't begin with colon. incorrect syntax
+                    ### can't begin with open bracket: this signals the start of a tag
+(?<!                # begin negative lookbehind
+  [:\[]             #   char set: colon and open bracket, escaped
+  \s*               #   optional space
+)                   # end negative lookbehind
+
+\b                  # word boundary
+  ({0})             #   captured bool search keyword. this is the $1 reference used in regex.Replace
+\b                  # word boundary
+
+                    ### can't end with colon. this signals that the bool's value already exists
+                    ### can't begin with close bracket: this signals the end of a tag
+(?!                 # begin negative lookahead
+  \s*               #   optional space
+  [:\]]             #   char set: colon and close bracket, escaped
+)                   # end negative lookahead
+";
+        private static Dictionary<string, Regex> boolRegexDic { get; } = new Dictionary<string, Regex>();
+        public static Regex GetBoolRegex(string boolSearch)
+        {
+            if (boolRegexDic.TryGetValue(boolSearch, out var regex))
+                return regex;
+
+            var boolPattern = string.Format(boolPattern_parameterized, boolSearch);
+            regex = new Regex(boolPattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase | RegexOptions.Compiled);
+            boolRegexDic.Add(boolSearch, regex);
+
+            return regex;
+        }
     }
 }

LibationSearchEngine/UNTESTED/SearchEngine.cs

@@ -26,9 +26,16 @@ namespace LibationSearchEngine
 
         public const string _ID_ = "_ID_";
         public const string TAGS = "tags";
+        // special field for each book which includes all major parts of the book's metadata. enables non-targetting searching
         public const string ALL = "all";
 
-        private static ReadOnlyDictionary<string, Func<LibraryBook, string>> idIndexRules { get; }
+        // the workaround which allows displaying all books when query is empty
+        public const string ALL_QUERY = "*:*";
+
+        public SearchEngine(LibationContext context) => this.context = context;
+
+		#region index rules
+		private static ReadOnlyDictionary<string, Func<LibraryBook, string>> idIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, string>>(
                 new Dictionary<string, Func<LibraryBook, string>>
                 {
@@ -38,6 +45,7 @@ namespace LibationSearchEngine
                     ["ASIN"] = lb => lb.Book.AudibleProductId
                 }
                 );
+
         private static ReadOnlyDictionary<string, Func<LibraryBook, string>> stringIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, string>>(
                 new Dictionary<string, Func<LibraryBook, string>>
@@ -98,6 +106,7 @@ namespace LibationSearchEngine
                     ["MyRating"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating.ToLuceneString()
                 }
                 );
+
         private static ReadOnlyDictionary<string, Func<LibraryBook, bool>> boolIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, bool>>(
                 new Dictionary<string, Func<LibraryBook, bool>>
@@ -114,12 +123,17 @@ namespace LibationSearchEngine
                     ["IsRated"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating > 0f,
                     ["Rated"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating > 0f,
 
-                    ["IsAuthorNarrated"] = lb => isAuthorNarrated(lb),
-                    ["AuthorNarrated"] = lb => isAuthorNarrated(lb),
+                    ["IsAuthorNarrated"] = isAuthorNarrated,
+                    ["AuthorNarrated"] = isAuthorNarrated,
 
                     [nameof(Book.IsAbridged)] = lb => lb.Book.IsAbridged,
                     ["Abridged"] = lb => lb.Book.IsAbridged,
-                });
+
+                    // this will only be evaluated at time of re-index. ie: state of files moved later will be out of sync until next re-index
+                    ["IsLiberated"] = lb => isLiberated(lb.Book.AudibleProductId),
+                    ["Liberated"] = lb => isLiberated(lb.Book.AudibleProductId),
+                }
+                );
 
         private static bool isAuthorNarrated(LibraryBook lb)
         {
@@ -128,6 +142,8 @@ namespace LibationSearchEngine
             return authors.Intersect(narrators).Any();
         }
 
+        private static bool isLiberated(string id) => AudibleFileStorage.Audio.Exists(id);
+
         // use these common fields in the "all" default search field
         private static IEnumerable<Func<LibraryBook, string>> allFieldIndexRules { get; }
             = new List<Func<LibraryBook, string>>
@@ -137,8 +153,10 @@ namespace LibationSearchEngine
                 stringIndexRules[nameof(Book.AuthorNames)],
                 stringIndexRules[nameof(Book.NarratorNames)]
             };
+		#endregion
 
-        public static IEnumerable<string> GetSearchIdFields()
+		#region get search fields. used for display in help
+		public static IEnumerable<string> GetSearchIdFields()
         {
             foreach (var key in idIndexRules.Keys)
                 yield return key;
@@ -173,11 +191,13 @@ namespace LibationSearchEngine
             foreach (var key in numberIndexRules.Keys)
                 yield return key;
         }
+		#endregion
 
-        private Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);
-
-        public SearchEngine(LibationContext context) => this.context = context;
-
+		#region create and update index
+        /// <summary>
+        /// create new. ie: full re-index
+        /// </summary>
+        /// <param name="overwrite"></param>
         public void CreateNewIndex(bool overwrite = true)
         {
             // 300 products
@@ -211,6 +231,22 @@ namespace LibationSearchEngine
             log();
         }
 
+		/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
+		public void UpdateBook(string productId)
+        {
+            var libraryBook = context.GetLibraryBook_Flat_NoTracking(productId);
+            var term = new Term(_ID_, productId);
+
+            var document = createBookIndexDocument(libraryBook);
+            var createNewIndex = false;
+
+			using var index = getIndex();
+			using var analyzer = new StandardAnalyzer(Version);
+			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
+			ixWriter.DeleteDocuments(term);
+			ixWriter.AddDocument(document);
+		}
+
         private static Document createBookIndexDocument(LibraryBook libraryBook)
         {
             var doc = new Document();
@@ -248,77 +284,100 @@ namespace LibationSearchEngine
             return doc;
         }
 
-		/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
-		public void UpdateBook(string productId)
-        {
-            var libraryBook = context.GetLibraryBook_Flat_NoTracking(productId);
-            var term = new Term(_ID_, productId);
+        // update single document entry
+        // all fields, including 'tags' are case-specific
+        public void UpdateTags(string productId, string tags) => updateAnalyzedField(productId, TAGS, tags);
 
-            var document = createBookIndexDocument(libraryBook);
-            var createNewIndex = false;
+        // all fields are case-specific
+        private static void updateAnalyzedField(string productId, string fieldName, string newValue)
+            => updateDocument(
+                productId,
+                d =>
+                {
+                    // fields are key value pairs. MULTIPLE FIELDS CAN POTENTIALLY HAVE THE SAME KEY.
+                    // ie: must remove old before adding new else will create unwanted duplicates.
+                    d.RemoveField(fieldName);
+                    d.AddAnalyzed(fieldName, newValue);
+                });
 
-			using var index = getIndex();
-			using var analyzer = new StandardAnalyzer(Version);
-			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
-			ixWriter.DeleteDocuments(term);
-			ixWriter.AddDocument(document);
-		}
+        // update single document entry
+        public void UpdateIsLiberated(string productId)
+            => updateDocument(
+                productId,
+                d =>
+                {
+                    // fields are key value pairs. MULTIPLE FIELDS CAN POTENTIALLY HAVE THE SAME KEY.
+                    // ie: must remove old before adding new else will create unwanted duplicates.
+                    var v = isLiberated(productId);
+                    d.RemoveField("IsLiberated");
+                    d.AddBool("IsLiberated", v);
+                    d.RemoveField("Liberated");
+                    d.AddBool("Liberated", v);
+                });
 
-        public void UpdateTags(string productId, string tags)
+        private static void updateDocument(string productId, Action<Document> action)
         {
             var productTerm = new Term(_ID_, productId);
 
-			using var index = getIndex();
+            using var index = getIndex();
 
-			// get existing document
-			using var searcher = new IndexSearcher(index);
-			var query = new TermQuery(productTerm);
-			var docs = searcher.Search(query, 1);
-			var scoreDoc = docs.ScoreDocs.SingleOrDefault();
-			if (scoreDoc == null)
-				throw new Exception("document not found");
-			var document = searcher.Doc(scoreDoc.Doc);
+            // get existing document
+            using var searcher = new IndexSearcher(index);
+            var query = new TermQuery(productTerm);
+            var docs = searcher.Search(query, 1);
+            var scoreDoc = docs.ScoreDocs.SingleOrDefault();
+            if (scoreDoc == null)
+                throw new Exception("document not found");
+            var document = searcher.Doc(scoreDoc.Doc);
 
 
-			// update document entry with new tags
-			// fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new
-			// REMEMBER: all fields, including 'tags' are case-specific
-			document.RemoveField(TAGS);
-			document.AddAnalyzed(TAGS, tags);
+            // perform update
+            action(document);
 
-			// update index
-			var createNewIndex = false;
-			using var analyzer = new StandardAnalyzer(Version);
-			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
-			ixWriter.UpdateDocument(productTerm, document, analyzer);
-		}
 
+            // update index
+            var createNewIndex = false;
+            using var analyzer = new StandardAnalyzer(Version);
+            using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
+            ixWriter.UpdateDocument(productTerm, document, analyzer);
+        }
+		#endregion
+
+		#region search
         public SearchResultSet Search(string searchString)
         {
-            if (string.IsNullOrWhiteSpace(searchString))
-                searchString = "*:*";
+            Serilog.Log.Logger.Debug("original search string: {@DebugInfo}", new { searchString });
+            searchString = FormatSearchQuery(searchString);
+            Serilog.Log.Logger.Debug("formatted search string: {@DebugInfo}", new { searchString });
 
-            #region apply formatting
-            searchString = parseTag(searchString);
+            var results = generalSearch(searchString);
+            Serilog.Log.Logger.Debug("Hit(s): {@DebugInfo}", new { count = results.Docs.Count() });
+            displayResults(results);
+
+            return results;
+        }
+
+        public static string FormatSearchQuery(string searchString)
+        {
+            if (string.IsNullOrWhiteSpace(searchString))
+                return ALL_QUERY;
 
             searchString = replaceBools(searchString);
 
+            searchString = parseTag(searchString);
+
             // in ranges " TO " must be uppercase
             searchString = searchString.Replace(" to ", " TO ");
 
             searchString = padNumbers(searchString);
 
             searchString = lowerFieldNames(searchString);
-            #endregion
 
-            var results = generalSearch(searchString);
-
-            displayResults(results);
-
-            return results;
+            return searchString;
         }
 
-        private static string parseTag(string tagSearchString)
+		#region format query string
+		private static string parseTag(string tagSearchString)
         {
             var allMatches = LuceneRegex
                 .TagRegex
@@ -337,9 +396,10 @@ namespace LibationSearchEngine
 
         private static string replaceBools(string searchString)
         {
-            // negative look-ahead for optional spaces then colon. don't want to double-up. eg:"israted:false" => "israted:false:True"
             foreach (var boolSearch in boolIndexRules.Keys)
-                searchString = Regex.Replace(searchString, $@"\b({boolSearch})\b(?!\s*:)", @"$1:True", RegexOptions.IgnoreCase);
+                searchString =
+                    LuceneRegex.GetBoolRegex(boolSearch)
+                    .Replace(searchString, @"$1:True");
 
             return searchString;
         }
@@ -376,13 +436,10 @@ namespace LibationSearchEngine
 
             return searchString;
         }
-
-        public int MaxSearchResultsToReturn { get; set; } = 999;
+        #endregion
 
         private SearchResultSet generalSearch(string searchString)
         {
-            Console.WriteLine($"searchString: {searchString}");
-
             var defaultField = ALL;
 
 			using var index = getIndex();
@@ -403,14 +460,14 @@ namespace LibationSearchEngine
 					boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
 			}
 
-			Console.WriteLine($"  query: {query}");
-
 			var docs = searcher
-				.Search(query, MaxSearchResultsToReturn)
+				.Search(query, searcher.MaxDoc + 1)
 				.ScoreDocs
 				.Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score))
 				.ToList();
-			return new SearchResultSet(query.ToString(), docs);
+            var queryString = query.ToString();
+            Serilog.Log.Logger.Debug("query: {@DebugInfo}", new { queryString });
+			return new SearchResultSet(queryString, docs);
 		}
 
         private IEnumerable<Occur> getOccurs_recurs(BooleanQuery query)
@@ -430,7 +487,6 @@ namespace LibationSearchEngine
 
         private void displayResults(SearchResultSet docs)
         {
-            Console.WriteLine($"Hit(s): {docs.Docs.Count()}");
             //for (int i = 0; i < docs.Docs.Count(); i++)
             //{
             //    var sde = docs.Docs.First();
@@ -438,13 +494,14 @@ namespace LibationSearchEngine
             //    Document doc = sde.Doc;
             //    float score = sde.Score;
 
-            //    Console.WriteLine($"{(i + 1)}) score={score}. Fields:");
+            //    Serilog.Log.Logger.Debug($"{(i + 1)}) score={score}. Fields:");
             //    var allFields = doc.GetFields();
             //    foreach (var f in allFields)
-            //        Console.WriteLine($"   [{f.Name}]={f.StringValue}");
+            //        Serilog.Log.Logger.Debug($"   [{f.Name}]={f.StringValue}");
             //}
-
-            //Console.WriteLine();
         }
+		#endregion
+
+		private static Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);
     }
 }

LibationWinForms/UNTESTED/BookLiberation/AutomatedBackupsForm.cs

@@ -24,24 +24,13 @@ namespace LibationWinForms.BookLiberation
             InitializeComponent();
         }
 
-		public void AppendError(Exception ex)
-		{
-			Serilog.Log.Logger.Error(ex, "Automated backup: error");
-			appendText("ERROR: " + ex.Message);
-		}
-		public void AppendText(string text)
-		{
-			Serilog.Log.Logger.Information($"Automated backup: {text}");
-			appendText(text);
-		}
-		private void appendText(string text)
+		public void WriteLine(string text)
 			=> logTb.UIThread(() => logTb.AppendText($"{DateTime.Now} {text}{Environment.NewLine}"));
 
 		public void FinalizeUI()
         {
             keepGoingCb.Enabled = false;
             logTb.AppendText("");
-            AppendText("DONE");
         }
 
         private void AutomatedBackupsForm_FormClosing(object sender, FormClosingEventArgs e) => keepGoingCb.Checked = false;

LibationWinForms/UNTESTED/BookLiberation/ProcessorAutomationController.cs

@@ -1,11 +1,45 @@
 using System;
+using System.Linq;
 using System.Threading.Tasks;
+using System.Windows.Forms;
 using DataLayer;
 using Dinah.Core.ErrorHandling;
 using FileLiberator;
 
 namespace LibationWinForms.BookLiberation
 {
+    // decouple serilog and form. include convenience factory method
+    public class LogMe
+    {
+        public event EventHandler<string> LogInfo;
+        public event EventHandler<string> LogErrorString;
+        public event EventHandler<(Exception, string)> LogError;
+
+        public static LogMe RegisterForm(AutomatedBackupsForm form)
+        {
+            var logMe = new LogMe();
+
+            logMe.LogInfo += (_, text) => Serilog.Log.Logger.Information($"Automated backup: {text}");
+            logMe.LogInfo += (_, text) => form.WriteLine(text);
+
+            logMe.LogErrorString += (_, text) => Serilog.Log.Logger.Error(text);
+            logMe.LogErrorString += (_, text) => form.WriteLine(text);
+
+			logMe.LogError += (_, tuple) => Serilog.Log.Logger.Error(tuple.Item1, tuple.Item2 ?? "Automated backup: error");
+            logMe.LogError += (_, tuple) =>
+            {
+                form.WriteLine(tuple.Item2 ?? "Automated backup: error");
+                form.WriteLine("ERROR: " + tuple.Item1.Message);
+            };
+
+            return logMe;
+        }
+
+        public void Info(string text) => LogInfo?.Invoke(this, text);
+        public void Error(string text) => LogErrorString?.Invoke(this, text);
+        public void Error(Exception ex, string text = null) => LogError?.Invoke(this, (ex, text));
+    }
+
     public static class ProcessorAutomationController
     {
         public static async Task BackupSingleBookAsync(string productId, EventHandler<LibraryBook> completedAction = null)
@@ -14,10 +48,12 @@ namespace LibationWinForms.BookLiberation
 
             var backupBook = getWiredUpBackupBook(completedAction);
 
-            var automatedBackupsForm = attachToBackupsForm(backupBook);
+            (AutomatedBackupsForm automatedBackupsForm, LogMe logMe) = attachToBackupsForm(backupBook);
             automatedBackupsForm.KeepGoingVisible = false;
 
-            await runSingleBackupAsync(backupBook, automatedBackupsForm, productId);
+            var libraryBook = IProcessableExt.GetSingleLibraryBook(productId);
+            // continue even if libraryBook is null. we'll display even that in the processing box
+            await new BackupSingle(logMe, backupBook, automatedBackupsForm, libraryBook).RunBackupAsync();
         }
 
         public static async Task BackupAllBooksAsync(EventHandler<LibraryBook> completedAction = null)
@@ -26,9 +62,8 @@ namespace LibationWinForms.BookLiberation
 
             var backupBook = getWiredUpBackupBook(completedAction);
 
-            var automatedBackupsForm = attachToBackupsForm(backupBook);
-
-            await runBackupLoopAsync(backupBook, automatedBackupsForm);
+            (AutomatedBackupsForm automatedBackupsForm, LogMe logMe) = attachToBackupsForm(backupBook);
+            await new BackupLoop(logMe, backupBook, automatedBackupsForm).RunBackupAsync();
         }
 
         private static BackupBook getWiredUpBackupBook(EventHandler<LibraryBook> completedAction)
@@ -37,7 +72,17 @@ namespace LibationWinForms.BookLiberation
 
             backupBook.DownloadBook.Begin += (_, __) => wireUpEvents(backupBook.DownloadBook);
             backupBook.DecryptBook.Begin += (_, __) => wireUpEvents(backupBook.DecryptBook);
-			backupBook.DownloadPdf.Begin += (_, __) => wireUpEvents(backupBook.DownloadPdf);
+            backupBook.DownloadPdf.Begin += (_, __) => wireUpEvents(backupBook.DownloadPdf);
+
+            // must occur before completedAction. A common use case is:
+            // - filter by -liberated
+            // - liberate only that book
+            //   completedAction is to refresh grid
+            // - want to see that book disappear from grid
+            // also for this to work, updateIsLiberated can NOT be async
+            backupBook.DownloadBook.Completed += updateIsLiberated;
+            backupBook.DecryptBook.Completed += updateIsLiberated;
+            backupBook.DownloadPdf.Completed += updateIsLiberated;
 
             if (completedAction != null)
             {
@@ -49,22 +94,25 @@ namespace LibationWinForms.BookLiberation
             return backupBook;
         }
 
-        private static AutomatedBackupsForm attachToBackupsForm(BackupBook backupBook)
+        private static void updateIsLiberated(object sender, LibraryBook e) => ApplicationServices.SearchEngineCommands.UpdateIsLiberated(e.Book);
+
+        private static (AutomatedBackupsForm, LogMe) attachToBackupsForm(BackupBook backupBook)
         {
-            #region create form
+            #region create form and logger
             var automatedBackupsForm = new AutomatedBackupsForm();
+            var logMe = LogMe.RegisterForm(automatedBackupsForm);
             #endregion
 
             #region define how model actions will affect form behavior
-            void downloadBookBegin(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"Download Step, Begin: {libraryBook.Book}");
-            void statusUpdate(object _, string str) => automatedBackupsForm.AppendText("- " + str);
-            void downloadBookCompleted(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"Download Step, Completed: {libraryBook.Book}");
-            void decryptBookBegin(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"Decrypt Step, Begin: {libraryBook.Book}");
+            void downloadBookBegin(object _, LibraryBook libraryBook) => logMe.Info($"Download Step, Begin: {libraryBook.Book}");
+            void statusUpdate(object _, string str) => logMe.Info("- " + str);
+            void downloadBookCompleted(object _, LibraryBook libraryBook) => logMe.Info($"Download Step, Completed: {libraryBook.Book}");
+            void decryptBookBegin(object _, LibraryBook libraryBook) => logMe.Info($"Decrypt Step, Begin: {libraryBook.Book}");
             // extra line after book is completely finished
-            void decryptBookCompleted(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"Decrypt Step, Completed: {libraryBook.Book}{Environment.NewLine}");
-            void downloadPdfBegin(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"PDF Step, Begin: {libraryBook.Book}");
+            void decryptBookCompleted(object _, LibraryBook libraryBook) => logMe.Info($"Decrypt Step, Completed: {libraryBook.Book}{Environment.NewLine}");
+            void downloadPdfBegin(object _, LibraryBook libraryBook) => logMe.Info($"PDF Step, Begin: {libraryBook.Book}");
             // extra line after book is completely finished
-            void downloadPdfCompleted(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"PDF Step, Completed: {libraryBook.Book}{Environment.NewLine}");
+            void downloadPdfCompleted(object _, LibraryBook libraryBook) => logMe.Info($"PDF Step, Completed: {libraryBook.Book}{Environment.NewLine}");
             #endregion
 
             #region subscribe new form to model's events
@@ -95,7 +143,7 @@ namespace LibationWinForms.BookLiberation
             };
             #endregion
 
-            return automatedBackupsForm;
+            return (automatedBackupsForm, logMe);
         }
 
         public static async Task BackupAllPdfsAsync(EventHandler<LibraryBook> completedAction = null)
@@ -104,9 +152,8 @@ namespace LibationWinForms.BookLiberation
 
             var downloadPdf = getWiredUpDownloadPdf(completedAction);
 
-            var automatedBackupsForm = attachToBackupsForm(downloadPdf);
-
-            await runBackupLoopAsync(downloadPdf, automatedBackupsForm);
+			(AutomatedBackupsForm automatedBackupsForm, LogMe logMe) = attachToBackupsForm(downloadPdf);
+            await new BackupLoop(logMe, downloadPdf, automatedBackupsForm).RunBackupAsync();
         }
 
         private static DownloadPdf getWiredUpDownloadPdf(EventHandler<LibraryBook> completedAction)
@@ -250,17 +297,18 @@ namespace LibationWinForms.BookLiberation
             #endregion
         }
 
-        private static AutomatedBackupsForm attachToBackupsForm(IDownloadableProcessable downloadable)
+        private static (AutomatedBackupsForm, LogMe) attachToBackupsForm(IDownloadableProcessable downloadable)
         {
-            #region create form
+            #region create form and logger
             var automatedBackupsForm = new AutomatedBackupsForm();
+            var logMe = LogMe.RegisterForm(automatedBackupsForm);
             #endregion
 
             #region define how model actions will affect form behavior
-            void begin(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"Begin: {libraryBook.Book}");
-            void statusUpdate(object _, string str) => automatedBackupsForm.AppendText("- " + str);
+            void begin(object _, LibraryBook libraryBook) => logMe.Info($"Begin: {libraryBook.Book}");
+            void statusUpdate(object _, string str) => logMe.Info("- " + str);
             // extra line after book is completely finished
-            void completed(object _, LibraryBook libraryBook) => automatedBackupsForm.AppendText($"Completed: {libraryBook.Book}{Environment.NewLine}");
+            void completed(object _, LibraryBook libraryBook) => logMe.Info($"Completed: {libraryBook.Book}{Environment.NewLine}");
             #endregion
 
             #region subscribe new form to model's events
@@ -279,73 +327,150 @@ namespace LibationWinForms.BookLiberation
             };
             #endregion
 
-            return automatedBackupsForm;
+            return (automatedBackupsForm, logMe);
+        }
+    }
+
+    abstract class BackupRunner
+    {
+        protected LogMe LogMe { get; }
+        protected IProcessable Processable { get; }
+        protected AutomatedBackupsForm AutomatedBackupsForm { get; }
+
+        protected BackupRunner(LogMe logMe, IProcessable processable, AutomatedBackupsForm automatedBackupsForm)
+        {
+            LogMe = logMe;
+            Processable = processable;
+            AutomatedBackupsForm = automatedBackupsForm;
         }
 
-        // automated backups looper feels like a composible IProcessable: logic, UI, begin + process child + end
-        // however the process step doesn't follow the pattern: Validate(product) + Process(product)
-        private static async Task runBackupLoopAsync(IProcessable processable, AutomatedBackupsForm automatedBackupsForm)
+        protected abstract Task RunAsync();
+
+        protected abstract string SkipDialogText { get; }
+        protected abstract MessageBoxButtons SkipDialogButtons { get; }
+        protected abstract DialogResult CreateSkipFileResult { get; }
+
+        public async Task RunBackupAsync()
         {
-            automatedBackupsForm.Show();
+            AutomatedBackupsForm.Show();
 
             try
             {
-                var shouldContinue = true;
-                while (shouldContinue)
-                {
-                    var statusHandler = await processable.ProcessFirstValidAsync();
-                    shouldContinue = validateStatus(statusHandler, automatedBackupsForm);
-                }
+                await RunAsync();
             }
             catch (Exception ex)
             {
-                automatedBackupsForm.AppendError(ex);
+                LogMe.Error(ex);
             }
 
-            automatedBackupsForm.FinalizeUI();
+            AutomatedBackupsForm.FinalizeUI();
+            LogMe.Info("DONE");
         }
 
-        private static async Task runSingleBackupAsync(IProcessable processable, AutomatedBackupsForm automatedBackupsForm, string productId)
+        protected async Task<bool> ProcessOneAsync(Func<LibraryBook, Task<StatusHandler>> func, LibraryBook libraryBook)
         {
-            automatedBackupsForm.Show();
+            string logMessage;
 
             try
             {
-                var statusHandler = await processable.ProcessSingleAsync(productId);
-                validateStatus(statusHandler, automatedBackupsForm);
-            }
-            catch (Exception ex)
-            {
-                automatedBackupsForm.AppendError(ex);
-            }
+                var statusHandler = await func(libraryBook);
 
-            automatedBackupsForm.FinalizeUI();
-        }
+                if (statusHandler.IsSuccess)
+                    return true;
 
-        private static bool validateStatus(StatusHandler statusHandler, AutomatedBackupsForm automatedBackupsForm)
-        {
-            if (statusHandler == null)
-            {
-                automatedBackupsForm.AppendText("Done. All books have been processed");
-                return false;
-            }
-
-            if (statusHandler.HasErrors)
-            {
-                automatedBackupsForm.AppendText("ERROR. All books have not been processed. Most recent valid book: processing failed");
                 foreach (var errorMessage in statusHandler.Errors)
-                    automatedBackupsForm.AppendText(errorMessage);
-                return false;
+                    LogMe.Error(errorMessage);
+
+                logMessage = statusHandler.Errors.Aggregate((a, b) => $"{a}\r\n{b}");
+            }
+            catch (Exception ex)
+            {
+                LogMe.Error(ex);
+
+                logMessage = ex.Message + "\r\n|\r\n" + ex.StackTrace;
             }
 
-            if (!automatedBackupsForm.KeepGoing)
-            {
-                if (automatedBackupsForm.KeepGoingVisible && !automatedBackupsForm.KeepGoingChecked)
-                    automatedBackupsForm.AppendText("'Keep going' is unchecked");
+            LogMe.Error("ERROR. All books have not been processed. Most recent book: processing failed");
+
+            var dialogResult = MessageBox.Show(SkipDialogText, "Skip importing this book?", SkipDialogButtons, MessageBoxIcon.Question);
+
+            if (dialogResult == DialogResult.Abort)
                 return false;
+
+            if (dialogResult == CreateSkipFileResult)
+            {
+                var path = FileManager.AudibleFileStorage.Audio.CreateSkipFile(libraryBook.Book.Title, libraryBook.Book.AudibleProductId, logMessage);
+                LogMe.Info($@"
+Created new 'skip' file
+  [{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}
+  {path}
+".Trim());
             }
 
             return true;
         }
     }
+    class BackupSingle : BackupRunner
+    {
+        private LibraryBook _libraryBook { get; }
+
+		protected override string SkipDialogText => @"
+An error occurred while trying to process this book. Skip this book permanently?
+
+- Click YES to skip this book permanently.
+
+- Click NO to skip the book this time only. We'll try again later.
+".Trim();
+		protected override MessageBoxButtons SkipDialogButtons => MessageBoxButtons.YesNo;
+		protected override DialogResult CreateSkipFileResult => DialogResult.Yes;
+
+		public BackupSingle(LogMe logMe, IProcessable processable, AutomatedBackupsForm automatedBackupsForm, LibraryBook libraryBook)
+            : base(logMe, processable, automatedBackupsForm)
+        {
+            _libraryBook = libraryBook;
+        }
+
+		protected override async Task RunAsync()
+        {
+            if (_libraryBook is not null)
+                await ProcessOneAsync(Processable.ProcessSingleAsync, _libraryBook);
+        }
+    }
+    class BackupLoop : BackupRunner
+    {
+		protected override string SkipDialogText => @"
+An error occurred while trying to process this book
+
+- ABORT: stop processing books.
+
+- RETRY: retry this book later. Just skip it for now. Continue processing books. (Will try this book again later.)
+
+- IGNORE: Permanently ignore this book. Continue processing books. (Will not try this book again later.)
+".Trim();
+        protected override MessageBoxButtons SkipDialogButtons => MessageBoxButtons.AbortRetryIgnore;
+        protected override DialogResult CreateSkipFileResult => DialogResult.Ignore;
+
+        public BackupLoop(LogMe logMe, IProcessable processable, AutomatedBackupsForm automatedBackupsForm)
+            : base(logMe, processable, automatedBackupsForm) { }
+
+        protected override async Task RunAsync()
+        {
+            // support for 'skip this time only' requires state. iterators provide this state for free. therefore: use foreach/iterator here
+            foreach (var libraryBook in Processable.GetValidLibraryBooks())
+			{
+				var keepGoing = await ProcessOneAsync(Processable.ProcessBookAsync_NoValidation, libraryBook);
+				if (!keepGoing)
+					return;
+
+				if (!AutomatedBackupsForm.KeepGoing)
+				{
+					if (AutomatedBackupsForm.KeepGoingVisible && !AutomatedBackupsForm.KeepGoingChecked)
+						LogMe.Info("'Keep going' is unchecked");
+					return;
+				}
+			}
+
+			LogMe.Info("Done. All books have been processed");
+        }
+	}
 }

LibationWinForms/UNTESTED/Dialogs/AccountsDialog.cs

@@ -151,7 +151,7 @@ namespace LibationWinForms.Dialogs
 				if (string.IsNullOrWhiteSpace(dto.AccountId))
 					throw new Exception("Please enter an account id for all accounts");
 				if (string.IsNullOrWhiteSpace(dto.LocaleName))
-					throw new Exception("Please select a local name for all accounts");
+					throw new Exception("Please select a locale (i.e.: country or region) for all accounts");
 
 				var acct = accountsSettings.Upsert(dto.AccountId, dto.LocaleName);
 				acct.LibraryScan = dto.LibraryScan;

LibationWinForms/UNTESTED/ProductsGrid.cs

@@ -201,6 +201,9 @@ namespace LibationWinForms
             // update cells incl Liberate button text
             dataGridView.InvalidateRow(rowId);
 
+            // needed in case filtering by -IsLiberated and it gets changed to Liberated. want to immediately show the change
+            filter();
+
             BackupCountsChanged?.Invoke(this, EventArgs.Empty);
         }
 
@@ -396,8 +399,6 @@ namespace LibationWinForms
             }
             currencyManager.ResumeBinding();
 			VisibleCountChanged?.Invoke(this, dataGridView.AsEnumerable().Count(r => r.Visible));
-
-			var luceneSearchString_debug = searchResults.SearchString;
         }
         #endregion
 

_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.4" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="coverlet.collector" Version="3.0.3">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\Dinah.Core\_Tests\TestCommon\TestCommon.csproj" />
+    <ProjectReference Include="..\..\LibationSearchEngine\LibationSearchEngine.csproj" />
+  </ItemGroup>
+
+</Project>

_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Dinah.Core;
+using FluentAssertions;
+using FluentAssertions.Common;
+using LibationSearchEngine;
+using Microsoft.VisualStudio.TestPlatform.Common.Filtering;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+using Moq.Protected;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using TestCommon;
+
+namespace SearchEngineTests
+{
+	[TestClass]
+	public class FormatSearchQuery
+	{
+		[TestMethod]
+		// null, empty, whitespace -- *:*
+		[DataRow(null, "*:*")]
+		[DataRow("", "*:*")]
+		[DataRow("   ", "*:*")]
+
+		// tag surrounded by spaces
+		[DataRow("[foo]", "tags:foo")]
+		[DataRow("  [foo]", "  tags:foo")]
+		[DataRow("[foo]  ", "tags:foo  ")]
+		[DataRow("  [foo]  ", "  tags:foo  ")]
+		[DataRow("-[foo]", "-tags:foo")]
+		[DataRow("  -[foo]", "  -tags:foo")]
+		[DataRow("-[foo]  ", "-tags:foo  ")]
+		[DataRow("  -[foo]  ", "  -tags:foo  ")]
+
+		// tag case irrelevant
+		[DataRow("[FoO]", "tags:FoO")]
+
+		// bool keyword surrounded by spaces
+		[DataRow("israted", "israted:True")]
+		[DataRow("  israted", "  israted:True")]
+		[DataRow("israted  ", "israted:True  ")]
+		[DataRow("  israted  ", "  israted:True  ")]
+		[DataRow("-israted", "-israted:True")]
+		[DataRow("  -israted", "  -israted:True")]
+		[DataRow("-israted  ", "-israted:True  ")]
+		[DataRow("  -israted  ", "  -israted:True  ")]
+
+		// bool keyword. Append :True
+		[DataRow("israted", "israted:True")]
+
+		// bool keyword with [:bool]. Do not add :True
+		[DataRow("israted:True", "israted:True")]
+		[DataRow("isRated:false", "israted:false")]
+
+		// tag which happens to be a bool keyword >> parse as tag
+		[DataRow("[israted]", "tags:israted")]
+
+		// numbers with "to". TO all caps, numbers [8.2] format
+		[DataRow("1 to 10", "00000001.00 TO 00000010.00")]
+		[DataRow("19990101 to 20001231", "19990101.00 TO 20001231.00")]
+
+		// field to lowercase
+		[DataRow("Author:Doyle", "author:Doyle")]
+		// bool field to lowercase
+		[DataRow("IsRated", "israted:True")]
+		[DataRow("-isRATED", "-israted:True")]
+
+		public void FormattingTest(string input, string output)
+			=> SearchEngine.FormatSearchQuery(input).Should().Be(output);
+	}
+}