* search engine: refactoring and improved logging

* bug fix: after book is liberated, filter should immediately honor new "is liberated" status

ApplicationServices/UNTESTED/SearchEngineCommands.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System;
+using System.IO;
 using DataLayer;
 using LibationSearchEngine;
 
@@ -12,31 +13,43 @@ namespace ApplicationServices
 			engine.CreateNewIndex();
 		}
 
-		public static SearchResultSet Search(string searchString)
+		public static SearchResultSet Search(string searchString) => performSearchEngineFunc_safe(e =>
+			e.Search(searchString)
+		);
+
+		public static void UpdateBookTags(Book book) => performSearchEngineAction_safe(e =>
+			e.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags)
+		);
+
+		public static void UpdateIsLiberated(Book book) => performSearchEngineAction_safe(e =>
+			e.UpdateIsLiberated(book.AudibleProductId)
+		);
+
+		private static void performSearchEngineAction_safe(Action<SearchEngine> action)
 		{
 			var engine = new SearchEngine(DbContexts.GetContext());
 			try
 			{
-				return engine.Search(searchString);
+				action(engine);
 			}
 			catch (FileNotFoundException)
 			{
 				FullReIndex();
-				return engine.Search(searchString);
+				action(engine);
 			}
 		}
 
-		public static void UpdateBookTags(Book book)
+		private static T performSearchEngineFunc_safe<T>(Func<SearchEngine, T> action)
 		{
 			var engine = new SearchEngine(DbContexts.GetContext());
 			try
 			{
-				engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
+				return action(engine);
 			}
 			catch (FileNotFoundException)
 			{
 				FullReIndex();
-				engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
+				return action(engine);
 			}
 		}
 	}

FileLiberator/UNTESTED/DecryptBook.cs

@@ -78,11 +78,11 @@ namespace FileLiberator
                     Dinah.Core.IO.FileExt.SafeDelete(aaxFilename);
                 }
 
-                var statusHandler = new StatusHandler();
                 var finalAudioExists = AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId);
                 if (!finalAudioExists)
-                    statusHandler.AddError("Cannot find final audio file after decryption");
-                return statusHandler;
+                    return new StatusHandler { "Cannot find final audio file after decryption" };
+
+                return new StatusHandler();
             }
             finally
             {

LibationLauncher/LibationLauncher.csproj

@@ -13,7 +13,7 @@
     <!-- <PublishSingleFile>true</PublishSingleFile> -->
     <RuntimeIdentifier>win-x64</RuntimeIdentifier>
 
-    <Version>4.1.7.1</Version>
+    <Version>4.2.1.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

LibationSearchEngine/UNTESTED/SearchEngine.cs

@@ -26,9 +26,16 @@ namespace LibationSearchEngine
 
         public const string _ID_ = "_ID_";
         public const string TAGS = "tags";
+        // special field for each book which includes all major parts of the book's metadata. enables non-targetting searching
         public const string ALL = "all";
 
-        private static ReadOnlyDictionary<string, Func<LibraryBook, string>> idIndexRules { get; }
+        // the workaround which allows displaying all books when query is empty
+        public const string ALL_QUERY = "*:*";
+
+        public SearchEngine(LibationContext context) => this.context = context;
+
+		#region index rules
+		private static ReadOnlyDictionary<string, Func<LibraryBook, string>> idIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, string>>(
                 new Dictionary<string, Func<LibraryBook, string>>
                 {
@@ -38,6 +45,7 @@ namespace LibationSearchEngine
                     ["ASIN"] = lb => lb.Book.AudibleProductId
                 }
                 );
+
         private static ReadOnlyDictionary<string, Func<LibraryBook, string>> stringIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, string>>(
                 new Dictionary<string, Func<LibraryBook, string>>
@@ -98,6 +106,7 @@ namespace LibationSearchEngine
                     ["MyRating"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating.ToLuceneString()
                 }
                 );
+
         private static ReadOnlyDictionary<string, Func<LibraryBook, bool>> boolIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, bool>>(
                 new Dictionary<string, Func<LibraryBook, bool>>
@@ -114,12 +123,17 @@ namespace LibationSearchEngine
                     ["IsRated"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating > 0f,
                     ["Rated"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating > 0f,
 
-                    ["IsAuthorNarrated"] = lb => isAuthorNarrated(lb),
-                    ["AuthorNarrated"] = lb => isAuthorNarrated(lb),
+                    ["IsAuthorNarrated"] = isAuthorNarrated,
+                    ["AuthorNarrated"] = isAuthorNarrated,
 
                     [nameof(Book.IsAbridged)] = lb => lb.Book.IsAbridged,
                     ["Abridged"] = lb => lb.Book.IsAbridged,
-                });
+
+                    // this will only be evaluated at time of re-index. ie: state of files moved later will be out of sync until next re-index
+                    ["IsLiberated"] = lb => isLiberated(lb.Book.AudibleProductId),
+                    ["Liberated"] = lb => isLiberated(lb.Book.AudibleProductId),
+                }
+                );
 
         private static bool isAuthorNarrated(LibraryBook lb)
         {
@@ -128,6 +142,8 @@ namespace LibationSearchEngine
             return authors.Intersect(narrators).Any();
         }
 
+        private static bool isLiberated(string id) => AudibleFileStorage.Audio.Exists(id);
+
         // use these common fields in the "all" default search field
         private static IEnumerable<Func<LibraryBook, string>> allFieldIndexRules { get; }
             = new List<Func<LibraryBook, string>>
@@ -137,8 +153,10 @@ namespace LibationSearchEngine
                 stringIndexRules[nameof(Book.AuthorNames)],
                 stringIndexRules[nameof(Book.NarratorNames)]
             };
+		#endregion
 
-        public static IEnumerable<string> GetSearchIdFields()
+		#region get search fields. used for display in help
+		public static IEnumerable<string> GetSearchIdFields()
         {
             foreach (var key in idIndexRules.Keys)
                 yield return key;
@@ -173,11 +191,13 @@ namespace LibationSearchEngine
             foreach (var key in numberIndexRules.Keys)
                 yield return key;
         }
+		#endregion
 
-        private Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);
-
-        public SearchEngine(LibationContext context) => this.context = context;
-
+		#region create and update index
+        /// <summary>
+        /// create new. ie: full re-index
+        /// </summary>
+        /// <param name="overwrite"></param>
         public void CreateNewIndex(bool overwrite = true)
         {
             // 300 products
@@ -211,6 +231,22 @@ namespace LibationSearchEngine
             log();
         }
 
+		/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
+		public void UpdateBook(string productId)
+        {
+            var libraryBook = context.GetLibraryBook_Flat_NoTracking(productId);
+            var term = new Term(_ID_, productId);
+
+            var document = createBookIndexDocument(libraryBook);
+            var createNewIndex = false;
+
+			using var index = getIndex();
+			using var analyzer = new StandardAnalyzer(Version);
+			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
+			ixWriter.DeleteDocuments(term);
+			ixWriter.AddDocument(document);
+		}
+
         private static Document createBookIndexDocument(LibraryBook libraryBook)
         {
             var doc = new Document();
@@ -248,55 +284,72 @@ namespace LibationSearchEngine
             return doc;
         }
 
-		/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
-		public void UpdateBook(string productId)
-        {
-            var libraryBook = context.GetLibraryBook_Flat_NoTracking(productId);
-            var term = new Term(_ID_, productId);
+        // update single document entry
+        // all fields, including 'tags' are case-specific
+        public void UpdateTags(string productId, string tags) => updateAnalyzedField(productId, TAGS, tags);
 
-            var document = createBookIndexDocument(libraryBook);
-            var createNewIndex = false;
+        // all fields are case-specific
+        private static void updateAnalyzedField(string productId, string fieldName, string newValue)
+            => updateDocument(
+                productId,
+                d =>
+                {
+                    // fields are key value pairs. MULTIPLE FIELDS CAN POTENTIALLY HAVE THE SAME KEY.
+                    // ie: must remove old before adding new else will create unwanted duplicates.
+                    d.RemoveField(fieldName);
+                    d.AddAnalyzed(fieldName, newValue);
+                });
 
-			using var index = getIndex();
-			using var analyzer = new StandardAnalyzer(Version);
-			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
-			ixWriter.DeleteDocuments(term);
-			ixWriter.AddDocument(document);
-		}
+        // update single document entry
+        public void UpdateIsLiberated(string productId)
+            => updateDocument(
+                productId,
+                d =>
+                {
+                    // fields are key value pairs. MULTIPLE FIELDS CAN POTENTIALLY HAVE THE SAME KEY.
+                    // ie: must remove old before adding new else will create unwanted duplicates.
+                    var v = isLiberated(productId);
+                    d.RemoveField("IsLiberated");
+                    d.AddBool("IsLiberated", v);
+                    d.RemoveField("Liberated");
+                    d.AddBool("Liberated", v);
+                });
 
-        public void UpdateTags(string productId, string tags)
+        private static void updateDocument(string productId, Action<Document> action)
         {
             var productTerm = new Term(_ID_, productId);
 
-			using var index = getIndex();
+            using var index = getIndex();
 
-			// get existing document
-			using var searcher = new IndexSearcher(index);
-			var query = new TermQuery(productTerm);
-			var docs = searcher.Search(query, 1);
-			var scoreDoc = docs.ScoreDocs.SingleOrDefault();
-			if (scoreDoc == null)
-				throw new Exception("document not found");
-			var document = searcher.Doc(scoreDoc.Doc);
+            // get existing document
+            using var searcher = new IndexSearcher(index);
+            var query = new TermQuery(productTerm);
+            var docs = searcher.Search(query, 1);
+            var scoreDoc = docs.ScoreDocs.SingleOrDefault();
+            if (scoreDoc == null)
+                throw new Exception("document not found");
+            var document = searcher.Doc(scoreDoc.Doc);
 
 
-			// update document entry with new tags
-			// fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new
-			// REMEMBER: all fields, including 'tags' are case-specific
-			document.RemoveField(TAGS);
-			document.AddAnalyzed(TAGS, tags);
+            // perform update
+            action(document);
 
-			// update index
-			var createNewIndex = false;
-			using var analyzer = new StandardAnalyzer(Version);
-			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
-			ixWriter.UpdateDocument(productTerm, document, analyzer);
-		}
 
+            // update index
+            var createNewIndex = false;
+            using var analyzer = new StandardAnalyzer(Version);
+            using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
+            ixWriter.UpdateDocument(productTerm, document, analyzer);
+        }
+		#endregion
+
+		#region search
         public SearchResultSet Search(string searchString)
         {
+            Serilog.Log.Logger.Debug("original search string: {@DebugInfo}", new { searchString });
+
             if (string.IsNullOrWhiteSpace(searchString))
-                searchString = "*:*";
+                searchString = ALL_QUERY;
 
             #region apply formatting
             searchString = parseTag(searchString);
@@ -311,14 +364,19 @@ namespace LibationSearchEngine
             searchString = lowerFieldNames(searchString);
             #endregion
 
+            Serilog.Log.Logger.Debug("formatted search string: {@DebugInfo}", new { searchString });
+
             var results = generalSearch(searchString);
 
+            Serilog.Log.Logger.Debug("Hit(s): {@DebugInfo}", new { count = results.Docs.Count() });
+
             displayResults(results);
 
             return results;
         }
 
-        private static string parseTag(string tagSearchString)
+		#region format query string
+		private static string parseTag(string tagSearchString)
         {
             var allMatches = LuceneRegex
                 .TagRegex
@@ -376,13 +434,10 @@ namespace LibationSearchEngine
 
             return searchString;
         }
-
-        public int MaxSearchResultsToReturn { get; set; } = 999;
+		#endregion
 
         private SearchResultSet generalSearch(string searchString)
         {
-            Console.WriteLine($"searchString: {searchString}");
-
             var defaultField = ALL;
 
 			using var index = getIndex();
@@ -403,14 +458,14 @@ namespace LibationSearchEngine
 					boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
 			}
 
-			Console.WriteLine($"  query: {query}");
-
 			var docs = searcher
-				.Search(query, MaxSearchResultsToReturn)
+				.Search(query, searcher.MaxDoc + 1)
 				.ScoreDocs
 				.Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score))
 				.ToList();
-			return new SearchResultSet(query.ToString(), docs);
+            var queryString = query.ToString();
+            Serilog.Log.Logger.Debug("query: {@DebugInfo}", new { queryString });
+			return new SearchResultSet(queryString, docs);
 		}
 
         private IEnumerable<Occur> getOccurs_recurs(BooleanQuery query)
@@ -430,7 +485,6 @@ namespace LibationSearchEngine
 
         private void displayResults(SearchResultSet docs)
         {
-            Console.WriteLine($"Hit(s): {docs.Docs.Count()}");
             //for (int i = 0; i < docs.Docs.Count(); i++)
             //{
             //    var sde = docs.Docs.First();
@@ -438,13 +492,14 @@ namespace LibationSearchEngine
             //    Document doc = sde.Doc;
             //    float score = sde.Score;
 
-            //    Console.WriteLine($"{(i + 1)}) score={score}. Fields:");
+            //    Serilog.Log.Logger.Debug($"{(i + 1)}) score={score}. Fields:");
             //    var allFields = doc.GetFields();
             //    foreach (var f in allFields)
-            //        Console.WriteLine($"   [{f.Name}]={f.StringValue}");
+            //        Serilog.Log.Logger.Debug($"   [{f.Name}]={f.StringValue}");
             //}
-
-            //Console.WriteLine();
         }
+		#endregion
+
+		private static Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);
     }
 }

LibationWinForms/UNTESTED/BookLiberation/ProcessorAutomationController.cs

@@ -74,6 +74,16 @@ namespace LibationWinForms.BookLiberation
             backupBook.DecryptBook.Begin += (_, __) => wireUpEvents(backupBook.DecryptBook);
             backupBook.DownloadPdf.Begin += (_, __) => wireUpEvents(backupBook.DownloadPdf);
 
+            // must occur before completedAction. A common use case is:
+            // - filter by -liberated
+            // - liberate only that book
+            //   completedAction is to refresh grid
+            // - want to see that book disappear from grid
+            // also for this to work, updateIsLiberated can NOT be async
+            backupBook.DownloadBook.Completed += updateIsLiberated;
+            backupBook.DecryptBook.Completed += updateIsLiberated;
+            backupBook.DownloadPdf.Completed += updateIsLiberated;
+
             if (completedAction != null)
             {
                 backupBook.DownloadBook.Completed += completedAction;
@@ -84,6 +94,8 @@ namespace LibationWinForms.BookLiberation
             return backupBook;
         }
 
+        private static void updateIsLiberated(object sender, LibraryBook e) => ApplicationServices.SearchEngineCommands.UpdateIsLiberated(e.Book);
+
         private static (AutomatedBackupsForm, LogMe) attachToBackupsForm(BackupBook backupBook)
         {
             #region create form and logger
@@ -334,6 +346,10 @@ namespace LibationWinForms.BookLiberation
 
         protected abstract Task RunAsync();
 
+        protected abstract string SkipDialogText { get; }
+        protected abstract MessageBoxButtons SkipDialogButtons { get; }
+        protected abstract DialogResult CreateSkipFileResult { get; }
+
         public async Task RunBackupAsync()
         {
             AutomatedBackupsForm.Show();
@@ -351,46 +367,47 @@ namespace LibationWinForms.BookLiberation
             LogMe.Info("DONE");
         }
 
-        protected abstract string SkipDialogText { get; }
-        protected abstract MessageBoxButtons SkipDialogButtons { get; }
-        protected abstract DialogResult CreateSkipFileResult { get; }
-
-        protected bool ValidateStatusAsync(StatusHandler statusHandler, LibraryBook libraryBook)
+        protected async Task<bool> ProcessOneAsync(Func<LibraryBook, Task<StatusHandler>> func, LibraryBook libraryBook)
         {
-            if (!statusHandler.HasErrors)
-                return true;
-
-            LogMe.Error("ERROR. All books have not been processed. Most recent valid book: processing failed");
-            foreach (var errorMessage in statusHandler.Errors)
-                LogMe.Error(errorMessage);
+            string logMessage;
 
             try
             {
-                var dialogResult = MessageBox.Show(SkipDialogText, "Skip importing this book?", SkipDialogButtons, MessageBoxIcon.Question);
+                var statusHandler = await func(libraryBook);
 
-                if (dialogResult == DialogResult.Abort)
-                    return false;
+                if (statusHandler.IsSuccess)
+                    return true;
 
-                if (dialogResult == CreateSkipFileResult)
-                {
-                    var path = FileManager.AudibleFileStorage.Audio.CreateSkipFile(
-                        libraryBook.Book.Title,
-                        libraryBook.Book.AudibleProductId,
-                        statusHandler.Errors.Aggregate((a, b) => $"{a}\r\n{b}"));
-                    LogMe.Info($@"
-Created new skip file
-  [{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}
-  {path}
-".Trim());
-                }
+                foreach (var errorMessage in statusHandler.Errors)
+                    LogMe.Error(errorMessage);
 
-                return true;
+                logMessage = statusHandler.Errors.Aggregate((a, b) => $"{a}\r\n{b}");
             }
             catch (Exception ex)
             {
-                LogMe.Error(ex, "Error attempting to display skip option box");
-                return false;
+                LogMe.Error(ex);
+
+                logMessage = ex.Message + "\r\n|\r\n" + ex.StackTrace;
             }
+
+            LogMe.Error("ERROR. All books have not been processed. Most recent book: processing failed");
+
+            var dialogResult = MessageBox.Show(SkipDialogText, "Skip importing this book?", SkipDialogButtons, MessageBoxIcon.Question);
+
+            if (dialogResult == DialogResult.Abort)
+                return false;
+
+            if (dialogResult == CreateSkipFileResult)
+            {
+                var path = FileManager.AudibleFileStorage.Audio.CreateSkipFile(libraryBook.Book.Title, libraryBook.Book.AudibleProductId, logMessage);
+                LogMe.Info($@"
+Created new 'skip' file
+  [{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}
+  {path}
+".Trim());
+            }
+
+            return true;
         }
     }
     class BackupSingle : BackupRunner
@@ -415,11 +432,8 @@ An error occurred while trying to process this book. Skip this book permanently?
 
 		protected override async Task RunAsync()
         {
-            if (_libraryBook is null)
-                return;
-
-            var statusHandler = await Processable.ProcessSingleAsync(_libraryBook);
-            ValidateStatusAsync(statusHandler, _libraryBook);
+            if (_libraryBook is not null)
+                await ProcessOneAsync(Processable.ProcessSingleAsync, _libraryBook);
         }
     }
     class BackupLoop : BackupRunner
@@ -443,29 +457,20 @@ An error occurred while trying to process this book
         {
             // support for 'skip this time only' requires state. iterators provide this state for free. therefore: use foreach/iterator here
             foreach (var libraryBook in Processable.GetValidLibraryBooks())
-            {
-                try
-                {
-                    var statusHandler = await Processable.ProcessBookAsync_NoValidation(libraryBook);
+			{
+				var keepGoing = await ProcessOneAsync(Processable.ProcessBookAsync_NoValidation, libraryBook);
+				if (!keepGoing)
+					return;
 
-                    var keepGoing = ValidateStatusAsync(statusHandler, libraryBook);
-                    if (!keepGoing)
-                        return;
+				if (!AutomatedBackupsForm.KeepGoing)
+				{
+					if (AutomatedBackupsForm.KeepGoingVisible && !AutomatedBackupsForm.KeepGoingChecked)
+						LogMe.Info("'Keep going' is unchecked");
+					return;
+				}
+			}
 
-                    if (!AutomatedBackupsForm.KeepGoing)
-                    {
-                        if (AutomatedBackupsForm.KeepGoingVisible && !AutomatedBackupsForm.KeepGoingChecked)
-                            LogMe.Info("'Keep going' is unchecked");
-                        return;
-                    }
-                }
-                catch (Exception exc)
-                {
-                    LogMe.Error(exc);
-                }
-            }
-
-            LogMe.Info("Done. All books have been processed");
+			LogMe.Info("Done. All books have been processed");
         }
-    }
+	}
 }

LibationWinForms/UNTESTED/ProductsGrid.cs

@@ -201,6 +201,9 @@ namespace LibationWinForms
             // update cells incl Liberate button text
             dataGridView.InvalidateRow(rowId);
 
+            // needed in case filtering by -IsLiberated and it gets changed to Liberated. want to immediately show the change
+            filter();
+
             BackupCountsChanged?.Invoke(this, EventArgs.Empty);
         }
 
@@ -396,8 +399,6 @@ namespace LibationWinForms
             }
             currencyManager.ResumeBinding();
 			VisibleCountChanged?.Invoke(this, dataGridView.AsEnumerable().Count(r => r.Visible));
-
-			var luceneSearchString_debug = searchResults.SearchString;
         }
         #endregion