update version

Added support for downloaded chapter titles.

AaxDecrypter/AAXChapters.cs

@@ -0,0 +1,40 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.Linq;
+using Dinah.Core.Diagnostics;
+
+namespace AaxDecrypter
+{
+    public class AAXChapters : Chapters
+    {
+        public AAXChapters(string file)
+        {
+            var info = new ProcessStartInfo
+            {
+                FileName = DecryptSupportLibraries.ffprobePath,
+                Arguments = "-loglevel panic -show_chapters -print_format xml \"" + file + "\""
+            };
+            var xml = info.RunHidden().Output;
+
+            var xmlDocument = new System.Xml.XmlDocument();
+            xmlDocument.LoadXml(xml);
+            var chaptersXml = xmlDocument.SelectNodes("/ffprobe/chapters/chapter")
+                .Cast<System.Xml.XmlNode>()
+                .Where(n => n.Name == "chapter");
+
+            foreach (var cnode in chaptersXml)
+            {
+                double startTime = double.Parse(cnode.Attributes["start_time"].Value.Replace(",", "."), CultureInfo.InvariantCulture);
+                double endTime = double.Parse(cnode.Attributes["end_time"].Value.Replace(",", "."), CultureInfo.InvariantCulture);
+
+                string chapterTitle = cnode.ChildNodes
+                    .Cast<System.Xml.XmlNode>()
+                    .Where(childnode => childnode.Attributes["key"].Value == "title")
+                    .Select(childnode => childnode.Attributes["value"].Value)
+                    .FirstOrDefault();
+
+                AddChapter(new Chapter(startTime, endTime, chapterTitle));
+            }
+        }
+    }
+}

AaxDecrypter/AaxDecrypter.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

AaxDecrypter/AaxToM4bConverter.cs

@@ -62,9 +62,10 @@ namespace AaxDecrypter
         public Tags tags { get; private set; }
         public EncodingInfo encodingInfo { get; private set; }
 
-        public static async Task<AaxToM4bConverter> CreateAsync(string inputFile, string decryptKey)
+        public static async Task<AaxToM4bConverter> CreateAsync(string inputFile, string decryptKey, Chapters chapters = null)
         {
             var converter = new AaxToM4bConverter(inputFile, decryptKey);
+            converter.chapters = chapters ?? new AAXChapters(inputFile);
             await converter.prelimProcessing();
             converter.printPrelim();
 
@@ -98,12 +99,11 @@ namespace AaxDecrypter
         {
             tags = new Tags(inputFileName);
             encodingInfo = new EncodingInfo(inputFileName);
-            chapters = new Chapters(inputFileName, tags.duration.TotalSeconds);
 
             var defaultFilename = Path.Combine(
                 Path.GetDirectoryName(inputFileName),
-                getASCIITag(tags.author),
-                getASCIITag(tags.title) + ".m4b"
+                PathLib.ToPathSafeString(tags.author),
+                PathLib.ToPathSafeString(tags.title) + ".m4b"
                 );
 
 			// set default name
@@ -111,12 +111,6 @@ namespace AaxDecrypter
 
             await Task.Run(() => saveCover(inputFileName));
         }
-        private string getASCIITag(string property)
-        {
-            foreach (char ch in new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars()))
-                property = property.Replace(ch.ToString(), "");
-            return property;
-        }
 
         private void saveCover(string aaxFile)
         {
@@ -126,19 +120,14 @@ namespace AaxDecrypter
 
         private void printPrelim()
         {
-            Console.WriteLine("Audible Book ID = " + tags.id);
+            Console.WriteLine($"Audible Book ID = {tags.id}");
 
-            Console.WriteLine("Book: " + tags.title);
-            Console.WriteLine("Author: " + tags.author);
-            Console.WriteLine("Narrator: " + tags.narrator);
-            Console.WriteLine("Year: " + tags.year);
-            Console.WriteLine("Total Time: "
-                + tags.duration.GetTotalTimeFormatted()
-                + " in " + chapters.Count() + " chapters");
-            Console.WriteLine("WARNING-Source is "
-                + encodingInfo.originalBitrate + " kbits @ "
-                + encodingInfo.sampleRate + "Hz, "
-                + encodingInfo.channels + " channels");
+            Console.WriteLine($"Book: {tags.title}");
+            Console.WriteLine($"Author: {tags.author}");
+            Console.WriteLine($"Narrator: {tags.narrator}");
+            Console.WriteLine($"Year: {tags.year}");
+            Console.WriteLine($"Total Time: {tags.duration.GetTotalTimeFormatted()} in {chapters.Count} chapters");
+            Console.WriteLine($"WARNING-Source is {encodingInfo.originalBitrate} kbits @ {encodingInfo.sampleRate}Hz, {encodingInfo.channels} channels");
         }
 
         public bool Run()
@@ -159,19 +148,14 @@ namespace AaxDecrypter
 
         public void SetOutputFilename(string outFileName)
         {
-            outputFileName = outFileName;
-
-            if (Path.GetExtension(outputFileName) != ".m4b")
-                outputFileName = outputFileWithNewExt(".m4b");
-
-			if (File.Exists(outputFileName))
-				File.Delete(outputFileName);
-
+            outputFileName = PathLib.ReplaceExtension(outFileName, ".m4b");
             outDir = Path.GetDirectoryName(outputFileName);
+
+            if (File.Exists(outputFileName))
+                File.Delete(outputFileName);
         }
 
-        private string outputFileWithNewExt(string extension)
-            => Path.Combine(outDir, Path.GetFileNameWithoutExtension(outputFileName) + '.' + extension.Trim('.'));
+        private string outputFileWithNewExt(string extension) => PathLib.ReplaceExtension(outputFileName, extension);
 
         public bool Step1_CreateDir()
         {
@@ -294,9 +278,9 @@ namespace AaxDecrypter
         public bool Step3_Chapterize()
         {
             var str1 = "";
-            if (chapters.FirstChapterStart != 0.0)
+            if (chapters.FirstChapter.StartTime != 0.0)
             {
-                str1 = " -ss " + chapters.FirstChapterStart.ToString("0.000", CultureInfo.InvariantCulture) + " -t " + (chapters.LastChapterStart - 1.0).ToString("0.000", CultureInfo.InvariantCulture) + " ";
+                str1 = " -ss " + chapters.FirstChapter.StartTime.ToString("0.000", CultureInfo.InvariantCulture) + " -t " + chapters.LastChapter.EndTime.ToString("0.000", CultureInfo.InvariantCulture) + " ";
             }
 
             var ffmpegTags = tags.GenerateFfmpegTags();
@@ -349,13 +333,13 @@ namespace AaxDecrypter
 
         public bool End_CreateCue()
         {
-            File.WriteAllText(outputFileWithNewExt(".cue"), chapters.GetCuefromChapters(Path.GetFileName(outputFileName)));
+            File.WriteAllText(outputFileWithNewExt(".cue"), Cue.CreateContents(Path.GetFileName(outputFileName), chapters));
             return true;
         }
 
         public bool End_CreateNfo()
         {
-            File.WriteAllText(outputFileWithNewExt(".nfo"), NFO.CreateNfoContents(AppName, tags, encodingInfo, chapters));
+            File.WriteAllText(outputFileWithNewExt(".nfo"), NFO.CreateContents(AppName, tags, encodingInfo, chapters));
             return true;
         }
     }

AaxDecrypter/Chapter.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace AaxDecrypter
+{
+    public class Chapter
+    {
+        public Chapter(double startTime, double endTime, string title)
+        {
+            StartTime = startTime;
+            EndTime = endTime;
+            Title = title;
+        }
+        /// <summary>
+        /// Chapter start time, in seconds.
+        /// </summary>
+        public double StartTime { get; private set; }
+        /// <summary>
+        /// Chapter end time, in seconds.
+        /// </summary>
+        public double EndTime { get; private set; }
+        public string Title { get; private set; }
+    }
+}

AaxDecrypter/Chapters.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace AaxDecrypter
+{
+    public abstract class Chapters
+    {
+        private List<Chapter> _chapterList = new();
+        public int Count => _chapterList.Count;
+        public Chapter FirstChapter => _chapterList[0];
+        public Chapter LastChapter => _chapterList[Count - 1];
+        public IEnumerable<Chapter> ChapterList => _chapterList.AsEnumerable();
+        public IEnumerable<TimeSpan> GetBeginningTimes() => ChapterList.Select(c => TimeSpan.FromSeconds(c.StartTime));
+        protected void AddChapter(Chapter chapter)
+        {
+            _chapterList.Add(chapter);
+        }
+        protected void AddChapters(IEnumerable<Chapter> chapters)
+        {
+            _chapterList.AddRange(chapters);
+        }
+        public string GenerateFfmpegChapters()
+        {
+            var stringBuilder = new StringBuilder();
+
+            foreach (Chapter c in ChapterList)
+            {
+                stringBuilder.Append("[CHAPTER]\n");
+                stringBuilder.Append("TIMEBASE=1/1000\n");
+                stringBuilder.Append("START=" + c.StartTime * 1000 + "\n");
+                stringBuilder.Append("END=" + c.EndTime * 1000 + "\n");
+                stringBuilder.Append("title=" + c.Title + "\n");
+            }
+
+            return stringBuilder.ToString();
+        }
+    }
+}

AaxDecrypter/Cue.cs

@@ -0,0 +1,60 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Dinah.Core;
+
+namespace AaxDecrypter
+{
+    public static class Cue
+    {
+        public static string CreateContents(string filePath, Chapters chapters)
+        {
+            var stringBuilder = new StringBuilder();
+
+            stringBuilder.AppendLine(GetFileLine(filePath, "MP3"));
+
+            var trackCount = 0;
+            foreach (Chapter c in chapters.ChapterList)
+            {
+                trackCount++;
+                var startTime = TimeSpan.FromSeconds(c.StartTime);
+
+                stringBuilder.AppendLine($"TRACK {trackCount} AUDIO");
+                stringBuilder.AppendLine($"  TITLE \"{c.Title}\"");
+                stringBuilder.AppendLine($"  INDEX 01 {(int)startTime.TotalMinutes}:{startTime:ss\\:ff}");
+            }
+
+            return stringBuilder.ToString();
+        }
+
+        public static void UpdateFileName(FileInfo cueFileInfo, string audioFilePath)
+            => UpdateFileName(cueFileInfo.FullName, audioFilePath);
+
+        public static void UpdateFileName(string cueFilePath, FileInfo audioFileInfo)
+            => UpdateFileName(cueFilePath, audioFileInfo.FullName);
+
+        public static void UpdateFileName(FileInfo cueFileInfo, FileInfo audioFileInfo)
+            => UpdateFileName(cueFileInfo.FullName, audioFileInfo.FullName);
+
+        public static void UpdateFileName(string cueFilePath, string audioFilePath)
+        {
+            var cueContents = File.ReadAllLines(cueFilePath);
+
+            for (var i = 0; i < cueContents.Length; i++)
+            {
+                var line = cueContents[i];
+                if (!line.Trim().StartsWith("FILE") || !line.Contains(" "))
+                    continue;
+
+                var fileTypeBegins = line.LastIndexOf(" ") + 1;
+                cueContents[i] = GetFileLine(audioFilePath, line[fileTypeBegins..]);
+                break;
+            }
+
+            File.WriteAllLines(cueFilePath, cueContents);
+        }
+
+        private static string GetFileLine(string filePath, string audioType) => $"FILE {Path.GetFileName(filePath).SurroundWithQuotes()} {audioType}";
+    }
+}

AaxDecrypter/NFO.cs

@@ -0,0 +1,54 @@
+namespace AaxDecrypter
+{
+    public static class NFO
+    {
+        public static string CreateContents(string ripper, Tags tags, EncodingInfo encodingInfo, Chapters chapters)
+        {
+            var _hours = (int)tags.duration.TotalHours;
+            var myDuration
+                = (_hours > 0 ? _hours + " hours, " : "")
+                + tags.duration.Minutes + " minutes, "
+                + tags.duration.Seconds + " seconds";
+
+            var header
+                = "General Information\r\n"
+                + "===================\r\n"
+                + $" Title:                  {tags.title}\r\n"
+                + $" Author:                 {tags.author}\r\n"
+                + $" Read By:                {tags.narrator}\r\n"
+                + $" Copyright:              {tags.year}\r\n"
+                + $" Audiobook Copyright:    {tags.year}\r\n";
+            if (tags.genre != "")
+                header += $" Genre:                  {tags.genre}\r\n";
+
+            var s
+                = header
+                + $" Publisher:              {tags.publisher}\r\n"
+                + $" Duration:               {myDuration}\r\n"
+                + $" Chapters:               {chapters.Count}\r\n"
+                + "\r\n"
+                + "\r\n"
+                + "Media Information\r\n"
+                + "=================\r\n"
+                + " Source Format:          Audible AAX\r\n"
+                + $" Source Sample Rate:     {encodingInfo.sampleRate} Hz\r\n"
+                + $" Source Channels:        {encodingInfo.channels}\r\n"
+                + $" Source Bitrate:         {encodingInfo.originalBitrate} kbits\r\n"
+                + "\r\n"
+                + " Lossless Encode:        Yes\r\n"
+                + " Encoded Codec:          AAC / M4B\r\n"
+                + $" Encoded Sample Rate:    {encodingInfo.sampleRate} Hz\r\n"
+                + $" Encoded Channels:       {encodingInfo.channels}\r\n"
+                + $" Encoded Bitrate:        {encodingInfo.originalBitrate} kbits\r\n"
+                + "\r\n"
+                + $" Ripper:                 {ripper}\r\n"
+                + "\r\n"
+                + "\r\n"
+                + "Book Description\r\n"
+                + "================\r\n"
+                + tags.comments;
+
+            return s;
+        }
+    }
+}

AaxDecrypter/UNTESTED/Chapters.cs

@@ -1,95 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Globalization;
-using System.Linq;
-using System.Text;
-using Dinah.Core.Diagnostics;
-
-namespace AaxDecrypter
-{
-    public class Chapters
-    {
-        private List<double> markers { get; }
-
-        public double FirstChapterStart => markers[0];
-        public double LastChapterStart => markers[markers.Count - 1];
-
-        public Chapters(string file, double totalTime)
-        {
-            markers = getAAXChapters(file);
-
-            // add end time
-            markers.Add(totalTime);
-        }
-
-        private static List<double> getAAXChapters(string file)
-        {
-            var info = new ProcessStartInfo
-            {
-                FileName = DecryptSupportLibraries.ffprobePath,
-                Arguments = "-loglevel panic -show_chapters -print_format xml \"" + file + "\""
-            };
-            var xml = info.RunHidden().Output;
-
-            var xmlDocument = new System.Xml.XmlDocument();
-            xmlDocument.LoadXml(xml);
-            var chapters = xmlDocument.SelectNodes("/ffprobe/chapters/chapter")
-                .Cast<System.Xml.XmlNode>()
-                .Select(xmlNode => double.Parse(xmlNode.Attributes["start_time"].Value.Replace(",", "."), CultureInfo.InvariantCulture))
-                .ToList();
-            return chapters;
-        }
-
-        // subtract 1 b/c end time marker is a real entry but isn't a real chapter
-        public int Count() => markers.Count - 1;
-
-        public string GetCuefromChapters(string fileName)
-        {
-            var stringBuilder = new StringBuilder();
-            if (fileName != "")
-            {
-                stringBuilder.Append("FILE \"" + fileName + "\" MP4\n");
-            }
-
-            for (var i = 0; i < Count(); i++)
-            {
-                var chapter = i + 1;
-
-                var timeSpan = TimeSpan.FromSeconds(markers[i]);
-                var minutes = Math.Floor(timeSpan.TotalMinutes).ToString();
-                var seconds = timeSpan.Seconds.ToString("D2");
-                var milliseconds = (timeSpan.Milliseconds / 10).ToString("D2");
-                string str = minutes + ":" + seconds + ":" + milliseconds;
-
-                stringBuilder.Append("TRACK " + chapter + " AUDIO\n");
-                stringBuilder.Append("  TITLE \"Chapter " + chapter.ToString("D2") + "\"\n");
-                stringBuilder.Append("  INDEX 01 " + str + "\n");
-            }
-
-            return stringBuilder.ToString();
-        }
-
-        public string GenerateFfmpegChapters()
-        {
-            var stringBuilder = new StringBuilder();
-
-            for (var i = 0; i < Count(); i++)
-            {
-                var chapter = i + 1;
-
-                var start = markers[i] * 1000.0;
-                var end = markers[i + 1] * 1000.0;
-                var chapterName = chapter.ToString("D3");
-
-                stringBuilder.Append("[CHAPTER]\n");
-                stringBuilder.Append("TIMEBASE=1/1000\n");
-                stringBuilder.Append("START=" + start + "\n");
-                stringBuilder.Append("END=" + end + "\n");
-                stringBuilder.Append("title=" + chapterName + "\n");
-            }
-
-            return stringBuilder.ToString();
-        }
-    }
-}

AaxDecrypter/UNTESTED/NFO.cs

@@ -1,56 +0,0 @@
-namespace AaxDecrypter
-{
-    public static class NFO
-    {
-        public static string CreateNfoContents(string ripper, Tags tags, EncodingInfo encodingInfo, Chapters chapters)
-        {
-            int _hours = (int)tags.duration.TotalHours;
-            string myDuration
-                = (_hours > 0 ? _hours + " hours, " : "")
-                + tags.duration.Minutes + " minutes, "
-                + tags.duration.Seconds + " seconds";
-
-            string str4
-                = "General Information\r\n"
-                + "===================\r\n"
-                + " Title:                  " + tags.title + "\r\n"
-                + " Author:                 " + tags.author + "\r\n"
-                + " Read By:                " + tags.narrator + "\r\n"
-                + " Copyright:              " + tags.year + "\r\n"
-                + " Audiobook Copyright:    " + tags.year + "\r\n";
-            if (tags.genre != "")
-            {
-                str4 = str4 + " Genre:                  " + tags.genre + "\r\n";
-            }
-
-            string s
-                = str4
-                + " Publisher:              " + tags.publisher + "\r\n"
-                + " Duration:               " + myDuration + "\r\n"
-                + " Chapters:               " + chapters.Count() + "\r\n"
-                + "\r\n"
-                + "\r\n"
-                + "Media Information\r\n"
-                + "=================\r\n"
-                + " Source Format:          Audible AAX\r\n"
-                + " Source Sample Rate:     " + encodingInfo.sampleRate + " Hz\r\n"
-                + " Source Channels:        " + encodingInfo.channels + "\r\n"
-                + " Source Bitrate:         " + encodingInfo.originalBitrate + " kbits\r\n"
-                + "\r\n"
-                + " Lossless Encode:        Yes\r\n"
-                + " Encoded Codec:          AAC / M4B\r\n"
-                + " Encoded Sample Rate:    " + encodingInfo.sampleRate + " Hz\r\n"
-                + " Encoded Channels:       " + encodingInfo.channels + "\r\n"
-                + " Encoded Bitrate:        " + encodingInfo.originalBitrate + " kbits\r\n"
-                + "\r\n"
-                + " Ripper:                 " + ripper + "\r\n"
-                + "\r\n"
-                + "\r\n"
-                + "Book Description\r\n"
-                + "================\r\n"
-                + tags.comments;
-
-            return s;
-        }
-    }
-}

ApplicationServices/ApplicationServices.csproj

@@ -1,9 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="CsvHelper" Version="27.0.2" />
+    <PackageReference Include="NPOI" Version="2.5.3" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApiDTOs\AudibleApiDTOs.csproj" />
     <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />

ApplicationServices/LibraryCommands.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Threading.Tasks;
 using AudibleApi;
 using DataLayer;
+using Dinah.Core;
 using DtoImporterService;
 using InternalUtilities;
 using Serilog;
@@ -24,7 +25,7 @@ namespace ApplicationServices
 				var totalCount = importItems.Count;
 				Log.Logger.Information($"GetAllLibraryItems: Total count {totalCount}");
 
-				var newCount = await getNewCountAsync(importItems);
+				var newCount = await importIntoDbAsync(importItems);
 				Log.Logger.Information($"Import: New count {newCount}");
 
 				await Task.Run(() => SearchEngineCommands.FullReIndex());
@@ -32,6 +33,23 @@ namespace ApplicationServices
 
 				return (totalCount, newCount);
 			}
+			catch (AudibleApi.Authentication.LoginFailedException lfEx)
+			{
+				lfEx.MoveResponseBodyFile(FileManager.Configuration.Instance.LibationFiles);
+
+				// nuget Serilog.Exceptions would automatically log custom properties
+				//   However, it comes with a scary warning when used with EntityFrameworkCore which I'm not yet ready to implement:
+				//   https://github.com/RehanSaeed/Serilog.Exceptions
+				// work-around: use 3rd param. don't just put exception object in 3rd param -- info overload: stack trace, etc
+				Log.Logger.Error(lfEx, "Error importing library. Login failed. {@DebugInfo}", new {
+					lfEx.RequestUrl,
+					ResponseStatusCodeNumber = (int)lfEx.ResponseStatusCode,
+					ResponseStatusCodeDesc = lfEx.ResponseStatusCode,
+					lfEx.ResponseInputFields,
+					lfEx.ResponseBodyFilePath
+				});
+				throw;
+			}
 			catch (Exception ex)
 			{
 				Log.Logger.Error(ex, "Error importing library");
@@ -61,21 +79,18 @@ namespace ApplicationServices
 
 		private static async Task<List<ImportItem>> scanAccountAsync(Api api, Account account)
 		{
-			Dinah.Core.ArgumentValidator.EnsureNotNull(account, nameof(account));
+			ArgumentValidator.EnsureNotNull(account, nameof(account));
 
-			var localeName = account.Locale?.Name;
 			Log.Logger.Information("ImportLibraryAsync. {@DebugInfo}", new
 			{
-				account.AccountName,
-				account.AccountId,
-				LocaleName = localeName,
+				Account = account?.MaskedLogEntry ?? "[null]"
 			});
 
 			var dtoItems = await AudibleApiActions.GetLibraryValidatedAsync(api);
-			return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = localeName }).ToList();
+			return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = account.Locale?.Name }).ToList();
 		}
 
-		private static async Task<int> getNewCountAsync(List<ImportItem> importItems)
+		private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)
 		{
 			using var context = DbContexts.GetContext();
 			var libraryImporter = new LibraryImporter(context);

ApplicationServices/LibraryExporter.cs

@@ -0,0 +1,268 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using CsvHelper;
+using CsvHelper.Configuration.Attributes;
+using DataLayer;
+using NPOI.XSSF.UserModel;
+using Serilog;
+
+namespace ApplicationServices
+{
+	public class ExportDto
+	{
+		public static string GetName(string fieldName)
+		{
+			var property = typeof(ExportDto).GetProperty(fieldName);
+			var attribute = property.GetCustomAttributes(typeof(NameAttribute), true)[0];
+			var description = (NameAttribute)attribute;
+			var text = description.Names;
+			return text[0];
+		}
+
+		[Name("Account")]
+		public string Account { get; set; }
+
+		[Name("Date Added to library")]
+		public DateTime DateAdded { get; set; }
+
+		[Name("Audible Product Id")]
+		public string AudibleProductId { get; set; }
+
+		[Name("Locale")]
+		public string Locale { get; set; }
+
+		[Name("Title")]
+		public string Title { get; set; }
+
+		[Name("Authors")]
+		public string AuthorNames { get; set; }
+
+		[Name("Narrators")]
+		public string NarratorNames { get; set; }
+
+		[Name("Length In Minutes")]
+		public int LengthInMinutes { get; set; }
+
+		[Name("Publisher")]
+		public string Publisher { get; set; }
+
+		[Name("Pdf url")]
+		public string PdfUrl { get; set; }
+
+		[Name("Series Names")]
+		public string SeriesNames { get; set; }
+
+		[Name("Series Order")]
+		public string SeriesOrder { get; set; }
+
+		[Name("Community Rating: Overall")]
+		public float? CommunityRatingOverall { get; set; }
+
+		[Name("Community Rating: Performance")]
+		public float? CommunityRatingPerformance { get; set; }
+
+		[Name("Community Rating: Story")]
+		public float? CommunityRatingStory { get; set; }
+
+		[Name("Cover Id")]
+		public string PictureId { get; set; }
+
+		[Name("Is Abridged?")]
+		public bool IsAbridged { get; set; }
+
+		[Name("Date Published")]
+		public DateTime? DatePublished { get; set; }
+
+		[Name("Categories")]
+		public string CategoriesNames { get; set; }
+
+		[Name("My Rating: Overall")]
+		public float? MyRatingOverall { get; set; }
+
+		[Name("My Rating: Performance")]
+		public float? MyRatingPerformance { get; set; }
+
+		[Name("My Rating: Story")]
+		public float? MyRatingStory { get; set; }
+
+		[Name("My Libation Tags")]
+		public string MyLibationTags { get; set; }
+	}
+	public static class LibToDtos
+	{
+		public static List<ExportDto> ToDtos(this IEnumerable<LibraryBook> library)
+			=> library.Select(a => new ExportDto
+			{
+				Account = a.Account,
+				DateAdded = a.DateAdded,
+				AudibleProductId = a.Book.AudibleProductId,
+				Locale = a.Book.Locale,
+				Title = a.Book.Title,
+				AuthorNames = a.Book.AuthorNames,
+				NarratorNames = a.Book.NarratorNames,
+				LengthInMinutes = a.Book.LengthInMinutes,
+				Publisher = a.Book.Publisher,
+				PdfUrl = a.Book.Supplements?.FirstOrDefault()?.Url,
+				SeriesNames = a.Book.SeriesNames,
+				SeriesOrder = a.Book.SeriesLink.Any() ? a.Book.SeriesLink?.Select(sl => $"{sl.Index} : {sl.Series.Name}").Aggregate((a, b) => $"{a}, {b}") : "",
+				CommunityRatingOverall = a.Book.Rating?.OverallRating,
+				CommunityRatingPerformance = a.Book.Rating?.PerformanceRating,
+				CommunityRatingStory = a.Book.Rating?.StoryRating,
+				PictureId = a.Book.PictureId,
+				IsAbridged = a.Book.IsAbridged,
+				DatePublished = a.Book.DatePublished,
+				CategoriesNames = a.Book.CategoriesNames.Any() ? a.Book.CategoriesNames.Aggregate((a, b) => $"{a}, {b}") : "",
+				MyRatingOverall = a.Book.UserDefinedItem.Rating.OverallRating,
+				MyRatingPerformance = a.Book.UserDefinedItem.Rating.PerformanceRating,
+				MyRatingStory = a.Book.UserDefinedItem.Rating.StoryRating,
+				MyLibationTags = a.Book.UserDefinedItem.Tags
+			}).ToList();
+	}
+	public static class LibraryExporter
+	{
+		public static void ToCsv(string saveFilePath)
+		{
+			using var context = DbContexts.GetContext();
+			var dtos = context.GetLibrary_Flat_NoTracking().ToDtos();
+
+			if (!dtos.Any())
+				return;
+
+			using var writer = new System.IO.StreamWriter(saveFilePath);
+			using var csv = new CsvWriter(writer, System.Globalization.CultureInfo.CurrentCulture);
+
+			csv.WriteHeader(typeof(ExportDto));
+			csv.NextRecord();
+			csv.WriteRecords(dtos);
+		}
+
+		public static void ToJson(string saveFilePath)
+		{
+			using var context = DbContexts.GetContext();
+			var dtos = context.GetLibrary_Flat_NoTracking().ToDtos();
+
+			var json = Newtonsoft.Json.JsonConvert.SerializeObject(dtos, Newtonsoft.Json.Formatting.Indented);
+			System.IO.File.WriteAllText(saveFilePath, json);
+		}
+
+		public static void ToXlsx(string saveFilePath)
+		{
+			using var context = DbContexts.GetContext();
+			var dtos = context.GetLibrary_Flat_NoTracking().ToDtos();
+
+			var workbook = new XSSFWorkbook();
+			var sheet = workbook.CreateSheet("Library");
+
+			var detailSubtotalFont = workbook.CreateFont();
+			detailSubtotalFont.IsBold = true;
+
+			var detailSubtotalCellStyle = workbook.CreateCellStyle();
+			detailSubtotalCellStyle.SetFont(detailSubtotalFont);
+
+			// headers
+			var rowIndex = 0;
+			var row = sheet.CreateRow(rowIndex);
+
+			var columns = new[] {
+				nameof (ExportDto.Account),
+				nameof (ExportDto.DateAdded),
+				nameof (ExportDto.AudibleProductId),
+				nameof (ExportDto.Locale),
+				nameof (ExportDto.Title),
+				nameof (ExportDto.AuthorNames),
+				nameof (ExportDto.NarratorNames),
+				nameof (ExportDto.LengthInMinutes),
+				nameof (ExportDto.Publisher),
+				nameof (ExportDto.PdfUrl),
+				nameof (ExportDto.SeriesNames),
+				nameof (ExportDto.SeriesOrder),
+				nameof (ExportDto.CommunityRatingOverall),
+				nameof (ExportDto.CommunityRatingPerformance),
+				nameof (ExportDto.CommunityRatingStory),
+				nameof (ExportDto.PictureId),
+				nameof (ExportDto.IsAbridged),
+				nameof (ExportDto.DatePublished),
+				nameof (ExportDto.CategoriesNames),
+				nameof (ExportDto.MyRatingOverall),
+				nameof (ExportDto.MyRatingPerformance),
+				nameof (ExportDto.MyRatingStory),
+				nameof (ExportDto.MyLibationTags)
+			};
+			var col = 0;
+			foreach (var c in columns)
+			{
+				var cell = row.CreateCell(col++);
+				var name = ExportDto.GetName(c);
+				cell.SetCellValue(name);
+				cell.CellStyle = detailSubtotalCellStyle;
+			}
+
+			var dateFormat = workbook.CreateDataFormat();
+			var dateStyle = workbook.CreateCellStyle();
+			dateStyle.DataFormat = dateFormat.GetFormat("MM/dd/yyyy HH:mm:ss");
+
+			rowIndex++;
+
+			// Add data rows
+			foreach (var dto in dtos)
+			{
+				col = 0;
+
+				row = sheet.CreateRow(rowIndex);
+
+				row.CreateCell(col++).SetCellValue(dto.Account);
+
+				var dateAddedCell = row.CreateCell(col++);
+				dateAddedCell.CellStyle = dateStyle;
+				dateAddedCell.SetCellValue(dto.DateAdded);
+
+				row.CreateCell(col++).SetCellValue(dto.AudibleProductId);
+				row.CreateCell(col++).SetCellValue(dto.Locale);
+				row.CreateCell(col++).SetCellValue(dto.Title);
+				row.CreateCell(col++).SetCellValue(dto.AuthorNames);
+				row.CreateCell(col++).SetCellValue(dto.NarratorNames);
+				row.CreateCell(col++).SetCellValue(dto.LengthInMinutes);
+				row.CreateCell(col++).SetCellValue(dto.Publisher);
+				row.CreateCell(col++).SetCellValue(dto.PdfUrl);
+				row.CreateCell(col++).SetCellValue(dto.SeriesNames);
+				row.CreateCell(col++).SetCellValue(dto.SeriesOrder);
+
+				col = createCell(row, col, dto.CommunityRatingOverall);
+				col = createCell(row, col, dto.CommunityRatingPerformance);
+				col = createCell(row, col, dto.CommunityRatingStory);
+
+				row.CreateCell(col++).SetCellValue(dto.PictureId);
+				row.CreateCell(col++).SetCellValue(dto.IsAbridged);
+
+				var datePubCell = row.CreateCell(col++);
+				datePubCell.CellStyle = dateStyle;
+				if (dto.DatePublished.HasValue)
+					datePubCell.SetCellValue(dto.DatePublished.Value);
+				else
+					datePubCell.SetCellValue("");
+
+				row.CreateCell(col++).SetCellValue(dto.CategoriesNames);
+
+				col = createCell(row, col, dto.MyRatingOverall);
+				col = createCell(row, col, dto.MyRatingPerformance);
+				col = createCell(row, col, dto.MyRatingStory);
+
+				row.CreateCell(col++).SetCellValue(dto.MyLibationTags);
+
+				rowIndex++;
+			}
+
+			using var fileData = new System.IO.FileStream(saveFilePath, System.IO.FileMode.Create);
+			workbook.Write(fileData);
+		}
+		private static int createCell(NPOI.SS.UserModel.IRow row, int col, float? nullableFloat)
+		{
+			if (nullableFloat.HasValue)
+				row.CreateCell(col++).SetCellValue(nullableFloat.Value);
+			else
+				row.CreateCell(col++).SetCellValue("");
+			return col;
+		}
+	}
+}

ApplicationServices/SearchEngineCommands.cs

@@ -0,0 +1,56 @@
+using System;
+using System.IO;
+using DataLayer;
+using LibationSearchEngine;
+
+namespace ApplicationServices
+{
+	public static class SearchEngineCommands
+	{
+		public static void FullReIndex()
+		{
+			var engine = new SearchEngine(DbContexts.GetContext());
+			engine.CreateNewIndex();
+		}
+
+		public static SearchResultSet Search(string searchString) => performSearchEngineFunc_safe(e =>
+			e.Search(searchString)
+		);
+
+		public static void UpdateBookTags(Book book) => performSearchEngineAction_safe(e =>
+			e.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags)
+		);
+
+		public static void UpdateIsLiberated(Book book) => performSearchEngineAction_safe(e =>
+			e.UpdateIsLiberated(book.AudibleProductId)
+		);
+
+		private static void performSearchEngineAction_safe(Action<SearchEngine> action)
+		{
+			var engine = new SearchEngine(DbContexts.GetContext());
+			try
+			{
+				action(engine);
+			}
+			catch (FileNotFoundException)
+			{
+				FullReIndex();
+				action(engine);
+			}
+		}
+
+		private static T performSearchEngineFunc_safe<T>(Func<SearchEngine, T> action)
+		{
+			var engine = new SearchEngine(DbContexts.GetContext());
+			try
+			{
+				return action(engine);
+			}
+			catch (FileNotFoundException)
+			{
+				FullReIndex();
+				return action(engine);
+			}
+		}
+	}
+}

ApplicationServices/UNTESTED/SearchEngineCommands.cs

@@ -1,43 +0,0 @@
-using System.IO;
-using DataLayer;
-using LibationSearchEngine;
-
-namespace ApplicationServices
-{
-	public static class SearchEngineCommands
-	{
-		public static void FullReIndex()
-		{
-			var engine = new SearchEngine(DbContexts.GetContext());
-			engine.CreateNewIndex();
-		}
-
-		public static SearchResultSet Search(string searchString)
-		{
-			var engine = new SearchEngine(DbContexts.GetContext());
-			try
-			{
-				return engine.Search(searchString);
-			}
-			catch (FileNotFoundException)
-			{
-				FullReIndex();
-				return engine.Search(searchString);
-			}
-		}
-
-		public static void UpdateBookTags(Book book)
-		{
-			var engine = new SearchEngine(DbContexts.GetContext());
-			try
-			{
-				engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
-			}
-			catch (FileNotFoundException)
-			{
-				FullReIndex();
-				engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
-			}
-		}
-	}
-}

DataLayer/DataLayer.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netcoreapp3.1;netstandard2.1</TargetFrameworks>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -12,13 +12,12 @@
   </PropertyGroup>
   
   <ItemGroup>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.7">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="5.0.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="3.1.7" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="3.1.7" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="3.1.7">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="5.0.5" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="5.0.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

DataLayer/LibationContext.cs

@@ -56,9 +56,10 @@ namespace DataLayer
             modelBuilder.ApplyConfiguration(new SeriesBookConfig());
             modelBuilder.ApplyConfiguration(new CategoryConfig());
 
-			// seeds go here. examples in scratch pad
-			modelBuilder
-				.Entity<Category>()
+            // seeds go here. examples in Dinah.EntityFrameworkCore.Tests\DbContextFactoryExample.cs
+
+            modelBuilder
+                .Entity<Category>()
 				.HasData(Category.GetEmpty());
 			modelBuilder
 				.Entity<Contributor>()

DataLayer/LibationContextFactory.cs

@@ -6,9 +6,6 @@ namespace DataLayer
     public class LibationContextFactory : DesignTimeDbContextFactoryBase<LibationContext>
     {
         protected override LibationContext CreateNewInstance(DbContextOptions<LibationContext> options) => new LibationContext(options);
-        protected override void UseDatabaseEngine(DbContextOptionsBuilder optionsBuilder, string connectionString) => optionsBuilder
-			//.UseSqlServer
-			.UseSqlite
-			(connectionString);
+        protected override void UseDatabaseEngine(DbContextOptionsBuilder optionsBuilder, string connectionString) => optionsBuilder.UseSqlite(connectionString);
     }
 }

DataLayer/UNTESTED/_scratch pad/ScratchPad.cs

@@ -1,122 +0,0 @@
-using System;
-using Dinah.EntityFrameworkCore;
-using Microsoft.EntityFrameworkCore;
-
-namespace _scratch_pad
-{
-    ////// to use this as a console, open properties and change from class library => console
-    //// DON'T FORGET TO REVERT IT
-    //public class Program
-    //{
-    //    public static void Main(string[] args)
-    //    {
-    //        var user = new Student() { Name = "Dinah Cheshire" };
-    //        var udi = new UserDef { UserDefId = 1, TagsRaw = "my,tags" };
-
-    //        using var context = new MyTestContextDesignTimeDbContextFactory().Create();
-    //        context.Add(user);
-    //        //context.Add(udi);
-    //        context.Update(udi);
-    //        context.SaveChanges();
-
-    //        Console.WriteLine($"Student was saved in the database with id: {user.Id}");
-    //    }
-    //}
-
-    public class MyTestContextDesignTimeDbContextFactory : DesignTimeDbContextFactoryBase<MyTestContext>
-    {
-        protected override MyTestContext CreateNewInstance(DbContextOptions<MyTestContext> options) => new MyTestContext(options);
-        protected override void UseDatabaseEngine(DbContextOptionsBuilder optionsBuilder, string connectionString) => optionsBuilder.UseSqlite(connectionString);
-    }
-
-    public class MyTestContext : DbContext
-    {
-        // see DesignTimeDbContextFactoryBase for info about ctors and connection strings/OnConfiguring()
-        public MyTestContext(DbContextOptions<MyTestContext> options) : base(options) { }
-
-        #region classes for OnModelCreating() seed example
-        class Blog
-        {
-            public int BlogId { get; set; }
-            public string Url { get; set; }
-            public System.Collections.Generic.ICollection<Post> Posts { get; set; }
-        }
-        class Post
-        {
-            public int PostId { get; set; }
-            public string Content { get; set; }
-            public string Title { get; set; }
-            public int BlogId { get; set; }
-            public Blog Blog { get; set; }
-            public Name AuthorName { get; set; }
-        }
-        class Name
-        {
-            public string First { get; set; }
-            public string Last { get; set; }
-        }
-        #endregion
-        protected override void OnModelCreating(ModelBuilder modelBuilder)
-        {
-            base.OnModelCreating(modelBuilder);
-
-            // config
-            modelBuilder.Entity<Blog>(entity => entity.Property(e => e.Url).IsRequired());
-            modelBuilder.Entity<Order>().OwnsOne(p => p.OrderDetails, cb =>
-            {
-                cb.OwnsOne(c => c.BillingAddress);
-                cb.OwnsOne(c => c.ShippingAddress);
-            });
-            modelBuilder.Entity<Post>(entity =>
-                entity
-                    .HasOne(d => d.Blog)
-                    .WithMany(p => p.Posts)
-                    .HasForeignKey("BlogId"));
-
-            // BlogSeed
-            modelBuilder.Entity<Blog>().HasData(new Blog { BlogId = 1, Url = "http://sample.com" });
-
-            // PostSeed
-            modelBuilder.Entity<Post>().HasData(new Post() { BlogId = 1, PostId = 1, Title = "First post", Content = "Test 1" });
-
-            // AnonymousPostSeed
-            modelBuilder.Entity<Post>().HasData(new { BlogId = 1, PostId = 2, Title = "Second post", Content = "Test 2" });
-
-            // OwnedTypeSeed
-            modelBuilder.Entity<Post>().OwnsOne(p => p.AuthorName).HasData(
-                new { PostId = 1, First = "Andriy", Last = "Svyryd" },
-                new { PostId = 2, First = "Diego", Last = "Vega" });
-        }
-
-        public DbSet<Student> Students { get; set; }
-        public DbSet<UserDef> UserDefs { get; set; }
-        public DbSet<Order> Orders { get; set; }
-    }
-
-    public class Student
-    {
-        public int Id { get; set; }
-        public string Name { get; set; }
-    }
-    public class UserDef
-    {
-        public int UserDefId { get; set; }
-        public string TagsRaw { get; set; }
-    }
-
-    public class Order
-    {
-        public int Id { get; set; }
-        public OrderDetails OrderDetails { get; set; }
-    }
-    public class OrderDetails
-    {
-        public StreetAddress BillingAddress { get; set; }
-        public StreetAddress ShippingAddress { get; set; }
-    }
-    public class StreetAddress
-    {
-        public string Street { get; set; }
-        public string City { get; set; }
-    }
-}

DataLayer/appsettings.json

@@ -1,14 +1,6 @@
 {
   "ConnectionStrings": {
-    "LibationContext_sqlserver": "Server=(LocalDb)\\MSSQLLocalDB;Database=DataLayer.LibationContext;Integrated Security=true;",
-
     "// this connection string is ONLY used for DataLayer's Migrations. this appsettings.json file is NOT used at all by application; it is overwritten":  "",
-    "LibationContext": "Data Source=LibationContext.db;Foreign Keys=False;",
-
-    "// sqlite notes": "",
-    "// absolute path example": "Data Source=C:/foo/bar/sample.db",
-    "// relative path example": "Data Source=sample.db",
-    "// on windows: sqlite paths accept windows and/or unix slashes": "",
-    "MyTestContext": "Data Source=%DESKTOP%/sample.db"
+    "LibationContext": "Data Source=LibationContext.db;Foreign Keys=False;"
   }
 }

DtoImporterService/DtoImporterService.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

FileLiberator/DecryptBook.cs

@@ -56,21 +56,35 @@ namespace FileLiberator
                 if (AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId))
                     return new StatusHandler { "Cannot find decrypt. Final audio file already exists" };
 
-                var outputAudioFilename = await aaxToM4bConverterDecrypt(aaxFilename, libraryBook);
+                var chapters = await downloadChapterNames(libraryBook);
+
+                var outputAudioFilename = await aaxToM4bConverterDecrypt(aaxFilename, libraryBook, chapters);
 
                 // decrypt failed
                 if (outputAudioFilename == null)
                     return new StatusHandler { "Decrypt failed" };
 
-                moveFilesToBooksDir(libraryBook.Book, outputAudioFilename);
+                var destinationDir = moveFilesToBooksDir(libraryBook.Book, outputAudioFilename);
 
-                Dinah.Core.IO.FileExt.SafeDelete(aaxFilename);
+                var config = Configuration.Instance;
+                if (config.RetainAaxFiles)
+                {
+                    var newAaxFilename = FileUtility.GetValidFilename(
+                        destinationDir,
+                        Path.GetFileNameWithoutExtension(aaxFilename),
+                        "aax");
+                    File.Move(aaxFilename, newAaxFilename);
+                }
+                else
+                {
+                    Dinah.Core.IO.FileExt.SafeDelete(aaxFilename);
+                }
 
-                var statusHandler = new StatusHandler();
                 var finalAudioExists = AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId);
                 if (!finalAudioExists)
-                    statusHandler.AddError("Cannot find final audio file after decryption");
-                return statusHandler;
+                    return new StatusHandler { "Cannot find final audio file after decryption" };
+
+                return new StatusHandler();
             }
             finally
             {
@@ -78,7 +92,7 @@ namespace FileLiberator
             }
         }
 
-        private async Task<string> aaxToM4bConverterDecrypt(string aaxFilename, LibraryBook libraryBook)
+        private async Task<string> aaxToM4bConverterDecrypt(string aaxFilename, LibraryBook libraryBook, Chapters chapters = null)
         {
             DecryptBegin?.Invoke(this, $"Begin decrypting {aaxFilename}");
 
@@ -90,7 +104,7 @@ namespace FileLiberator
                     .AccountsSettings
                     .GetAccount(libraryBook.Account, libraryBook.Book.Locale);
 
-                var converter = await AaxToM4bConverter.CreateAsync(aaxFilename, account.DecryptKey);
+                var converter = await AaxToM4bConverter.CreateAsync(aaxFilename, account.DecryptKey, chapters);
                 converter.AppName = "Libation";
 
                 TitleDiscovered?.Invoke(this, converter.tags.title);
@@ -120,41 +134,54 @@ namespace FileLiberator
             }
         }
 
-        private static void moveFilesToBooksDir(Book product, string outputAudioFilename)
+        private async Task<Chapters> downloadChapterNames(LibraryBook libraryBook)
+        {
+            try
+            {
+                var api = await AudibleApiActions.GetApiAsync(libraryBook.Account, libraryBook.Book.Locale);
+                var contentMetadata = await api.GetLibraryBookMetadataAsync(libraryBook.Book.AudibleProductId);
+
+                if (contentMetadata?.ChapterInfo != null)
+                    return new DownloadedChapters(contentMetadata.ChapterInfo);
+                return null;
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        private static string moveFilesToBooksDir(Book product, string outputAudioFilename)
 		{
 			// create final directory. move each file into it. MOVE AUDIO FILE LAST
 			// new dir: safetitle_limit50char + " [" + productId + "]"
 
-			var destinationDir = getDestDir(product);
+			var destinationDir = AudibleFileStorage.Audio.GetDestDir(product.Title, product.AudibleProductId);
 			Directory.CreateDirectory(destinationDir);
 
 			var sortedFiles = getProductFilesSorted(product, outputAudioFilename);
 
 			var musicFileExt = Path.GetExtension(outputAudioFilename).Trim('.');
 
-			foreach (var f in sortedFiles)
-			{
-				var dest = AudibleFileStorage.Audio.IsFileTypeMatch(f)
-					// audio filename: safetitle_limit50char + " [" + productId + "]." + audio_ext
-					? FileUtility.GetValidFilename(destinationDir, product.Title, musicFileExt, product.AudibleProductId)
-					// non-audio filename: safetitle_limit50char + " [" + productId + "][" + audio_ext +"]." + non_audio_ext
-					: FileUtility.GetValidFilename(destinationDir, product.Title, f.Extension, product.AudibleProductId, musicFileExt);
+            // audio filename: safetitle_limit50char + " [" + productId + "]." + audio_ext
+            var audioFileName = FileUtility.GetValidFilename(destinationDir, product.Title, musicFileExt, product.AudibleProductId);
 
-				File.Move(f.FullName, dest);
+            foreach (var f in sortedFiles)
+            {
+                var dest
+                    = AudibleFileStorage.Audio.IsFileTypeMatch(f)
+                    ? audioFileName
+                    // non-audio filename: safetitle_limit50char + " [" + productId + "][" + audio_ext +"]." + non_audio_ext
+                    : FileUtility.GetValidFilename(destinationDir, product.Title, f.Extension, product.AudibleProductId, musicFileExt);
+
+                if (Path.GetExtension(dest).Trim('.').ToLower() == "cue")
+                    Cue.UpdateFileName(f, audioFileName);
+
+                File.Move(f.FullName, dest);
 			}
-		}
 
-		private static string getDestDir(Book product)
-		{
-			// to prevent the paths from getting too long, we don't need after the 1st ":" for the folder
-			var underscoreIndex = product.Title.IndexOf(':');
-			var titleDir
-				= underscoreIndex < 4
-				? product.Title
-				: product.Title.Substring(0, underscoreIndex);
-			var finalDir = FileUtility.GetValidFilename(AudibleFileStorage.BooksDirectory, titleDir, null, product.AudibleProductId);
-			return finalDir;
-		}
+            return destinationDir;
+        }
 
 		private static List<FileInfo> getProductFilesSorted(Book product, string outputAudioFilename)
 		{

FileLiberator/DownloadBook.cs

@@ -45,7 +45,7 @@ namespace FileLiberator
 		{
 			validate(libraryBook);
 
-			var api = await AudibleApiActions.GetApiAsync(libraryBook.Account, libraryBook.Book.Locale);
+			var api = await GetApiAsync(libraryBook);
 
 			var actualFilePath = await PerformDownloadAsync(
 				tempAaxFilename,
@@ -72,7 +72,7 @@ namespace FileLiberator
 				libraryBook.Book.Title,
 				libraryBook.Book.AudibleProductId,
 				libraryBook.Book.Locale,
-				libraryBook.Account,
+				Account = libraryBook.Account?.ToMask() ?? "[empty]",
 				tempAaxFilename,
 				actualFilePath,
 				length,

FileLiberator/DownloadPdf.cs

@@ -31,17 +31,24 @@ namespace FileLiberator
 		private static string getProposedDownloadFilePath(LibraryBook libraryBook)
 		{
 			// if audio file exists, get it's dir. else return base Book dir
-			var destinationDir =
-				// this is safe b/c GetDirectoryName(null) == null
-				Path.GetDirectoryName(AudibleFileStorage.Audio.GetPath(libraryBook.Book.AudibleProductId))
-				?? AudibleFileStorage.PDF.StorageDirectory;
+			var existingPath = Path.GetDirectoryName(AudibleFileStorage.Audio.GetPath(libraryBook.Book.AudibleProductId));
+			var file = getdownloadUrl(libraryBook);
 
-			return Path.Combine(destinationDir, Path.GetFileName(getdownloadUrl(libraryBook)));
+			if (existingPath != null)
+				return Path.Combine(existingPath, Path.GetFileName(file));
+
+			var full = FileUtility.GetValidFilename(
+				AudibleFileStorage.PDF.StorageDirectory,
+				libraryBook.Book.Title,
+				Path.GetExtension(file),
+				libraryBook.Book.AudibleProductId);
+			return full;
 		}
 
 		private async Task downloadPdfAsync(LibraryBook libraryBook, string proposedDownloadFilePath)
 		{
-			var downloadUrl = getdownloadUrl(libraryBook);
+			var api = await GetApiAsync(libraryBook);
+			var downloadUrl = await api.GetPdfDownloadLinkAsync(libraryBook.Book.AudibleProductId);
 
 			var client = new HttpClient();
 			var actualDownloadedFilePath = await PerformDownloadAsync(

FileLiberator/DownloadableBase.cs

@@ -38,6 +38,9 @@ namespace FileLiberator
             }
         }
 
+		protected static Task<AudibleApi.Api> GetApiAsync(LibraryBook libraryBook)
+			=> InternalUtilities.AudibleApiActions.GetApiAsync(libraryBook.Account, libraryBook.Book.Locale);
+
 		protected async Task<string> PerformDownloadAsync(string proposedDownloadFilePath, Func<Progress<DownloadProgress>, Task<string>> func)
 		{
 			var progress = new Progress<DownloadProgress>();

FileLiberator/DownloadedChapters.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using AaxDecrypter;
+using AudibleApiDTOs;
+using Dinah.Core.Diagnostics;
+
+
+namespace FileLiberator
+{
+    public class DownloadedChapters : Chapters
+    {
+        public DownloadedChapters(ChapterInfo chapterInfo)
+        {
+            AddChapters(chapterInfo.Chapters
+                .Select(c => new AaxDecrypter.Chapter(c.StartOffsetMs / 1000d, (c.StartOffsetMs + c.LengthMs) / 1000d, c.Title)));
+        }
+    }
+}

FileLiberator/FileLiberator.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

FileLiberator/IProcessableExt.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using ApplicationServices;
+using DataLayer;
+using Dinah.Core;
+using Dinah.Core.ErrorHandling;
+
+namespace FileLiberator
+{
+    public static class IProcessableExt
+    {
+        //
+        // DO NOT USE ConfigureAwait(false) WITH ProcessAsync() unless ensuring ProcessAsync() implementation is cross-thread compatible
+        // ProcessAsync() often does a lot with forms in the UI context
+        //
+
+
+        // when used in foreach: stateful. deferred execution
+        public static IEnumerable<LibraryBook> GetValidLibraryBooks(this IProcessable processable)
+            => DbContexts.GetContext()
+            .GetLibrary_Flat_NoTracking()
+            .Where(libraryBook => processable.Validate(libraryBook));
+
+        public static LibraryBook GetSingleLibraryBook(string productId)
+        {
+            using var context = DbContexts.GetContext();
+            var libraryBook = context
+                .Library
+                .GetLibrary()
+                .SingleOrDefault(lb => lb.Book.AudibleProductId == productId);
+            return libraryBook;
+        }
+
+        public static async Task<StatusHandler> ProcessSingleAsync(this IProcessable processable, LibraryBook libraryBook)
+        {
+            if (!processable.Validate(libraryBook))
+                return new StatusHandler { "Validation failed" };
+
+            return await processable.ProcessBookAsync_NoValidation(libraryBook);
+        }
+
+        public static async Task<StatusHandler> ProcessBookAsync_NoValidation(this IProcessable processable, LibraryBook libraryBook)
+        {
+            Serilog.Log.Logger.Information("Begin " + nameof(ProcessBookAsync_NoValidation) + " {@DebugInfo}", new
+            {
+                libraryBook.Book.Title,
+                libraryBook.Book.AudibleProductId,
+                libraryBook.Book.Locale,
+                Account = libraryBook.Account?.ToMask() ?? "[empty]"
+            });
+
+            var status
+                = (await processable.ProcessAsync(libraryBook))
+                ?? new StatusHandler { "Processable should never return a null status" };
+
+            return status;
+        }
+
+		public static async Task<StatusHandler> TryProcessAsync(this IProcessable processable, LibraryBook libraryBook)
+			=> processable.Validate(libraryBook)
+			? await processable.ProcessAsync(libraryBook)
+			: new StatusHandler();
+	}
+}

FileLiberator/UNTESTED/IProcessableExt.cs

@@ -1,81 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using ApplicationServices;
-using DataLayer;
-using Dinah.Core.ErrorHandling;
-
-namespace FileLiberator
-{
-    public static class IProcessableExt
-    {
-        //
-        // DO NOT USE ConfigureAwait(false) WITH ProcessAsync() unless ensuring ProcessAsync() implementation is cross-thread compatible
-        // ProcessAsync() often does a lot with forms in the UI context
-        //
-
-
-        /// <summary>Process the first valid product. Create default context</summary>
-        /// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
-        public static async Task<StatusHandler> ProcessFirstValidAsync(this IProcessable processable)
-        {
-            var libraryBook = processable.getNextValidBook();
-            if (libraryBook == null)
-                return null;
-
-            return await processBookAsync(processable, libraryBook);
-        }
-
-        /// <summary>Process the first valid product. Create default context</summary>
-        /// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
-        public static async Task<StatusHandler> ProcessSingleAsync(this IProcessable processable, string productId)
-        {
-            using var context = DbContexts.GetContext();
-            var libraryBook = context
-                .Library
-                .GetLibrary()
-                .SingleOrDefault(lb => lb.Book.AudibleProductId == productId);
-
-            if (libraryBook == null)
-                return null;
-            if (!processable.Validate(libraryBook))
-                return new StatusHandler { "Validation failed" };
-
-            return await processBookAsync(processable, libraryBook);
-        }
-
-        private static async Task<StatusHandler> processBookAsync(IProcessable processable, LibraryBook libraryBook)
-        {
-            Serilog.Log.Information("Begin " + nameof(processBookAsync) + " {@DebugInfo}", new
-            {
-                libraryBook.Book.Title,
-                libraryBook.Book.AudibleProductId,
-                libraryBook.Book.Locale,
-                libraryBook.Account
-            });
-
-            // this should never happen. check anyway. ProcessFirstValidAsync returning null is the signal that we're done. we can't let another IProcessable accidentally send this command
-            var status = await processable.ProcessAsync(libraryBook);
-            if (status == null)
-                throw new Exception("Processable should never return a null status");
-
-            return status;
-        }
-
-        private static LibraryBook getNextValidBook(this IProcessable processable)
-        {
-            var libraryBooks = DbContexts.GetContext().GetLibrary_Flat_NoTracking();
-
-            foreach (var libraryBook in libraryBooks)
-                if (processable.Validate(libraryBook))
-                    return libraryBook;
-
-            return null;
-        }   
-
-		public static async Task<StatusHandler> TryProcessAsync(this IProcessable processable, LibraryBook libraryBook)
-			=> processable.Validate(libraryBook)
-			? await processable.ProcessAsync(libraryBook)
-			: new StatusHandler();
-	}
-}

FileManager/AudibleFileStorage.cs

@@ -0,0 +1,179 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Dinah.Core;
+using Dinah.Core.Collections.Generic;
+
+namespace FileManager
+{
+    // could add images here, but for now images are stored in a well-known location
+    public enum FileType { Unknown, Audio, AAX, PDF }
+
+	/// <summary>
+	/// Files are large. File contents are never read by app.
+	/// Paths are varied.
+	/// Files are written during download/decrypt/backup/liberate.
+	/// Paths are read at app launch and during download/decrypt/backup/liberate.
+	/// Many files are often looked up at once
+	/// </summary>
+	public abstract class AudibleFileStorage : Enumeration<AudibleFileStorage>
+	{
+		public abstract string[] Extensions { get; }
+		public abstract string StorageDirectory { get; }
+
+		#region static
+		public static AudioFileStorage Audio { get; } = new AudioFileStorage();
+		public static AudibleFileStorage AAX { get; } = new AaxFileStorage();
+		public static AudibleFileStorage PDF { get; } = new PdfFileStorage();
+
+		public static string DownloadsInProgress
+        {
+            get
+            {
+                if (!Configuration.Instance.DownloadsInProgressEnum.In("WinTemp", "LibationFiles"))
+                    Configuration.Instance.DownloadsInProgressEnum = "WinTemp";
+                var AaxRootDir
+                    = Configuration.Instance.DownloadsInProgressEnum == "WinTemp"
+                    ? Configuration.WinTemp
+                    : Configuration.Instance.LibationFiles;
+
+                return Directory.CreateDirectory(Path.Combine(AaxRootDir, "DownloadsInProgress")).FullName;
+            }
+        }
+
+        public static string DecryptInProgress
+        {
+            get
+            {
+                if (!Configuration.Instance.DecryptInProgressEnum.In("WinTemp", "LibationFiles"))
+                    Configuration.Instance.DecryptInProgressEnum = "WinTemp";
+
+                var M4bRootDir
+                    = Configuration.Instance.DecryptInProgressEnum == "WinTemp"
+                    ? Configuration.WinTemp
+                    : Configuration.Instance.LibationFiles;
+
+                return Directory.CreateDirectory(Path.Combine(M4bRootDir, "DecryptInProgress")).FullName;
+            }
+        }
+
+        // not customizable. don't move to config
+        public static string DownloadsFinal => new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("DownloadsFinal").FullName;
+
+        public static string BooksDirectory
+        {
+            get
+            {
+                if (string.IsNullOrWhiteSpace(Configuration.Instance.Books))
+                    Configuration.Instance.Books = Path.Combine(Configuration.Instance.LibationFiles, "Books");
+                return Directory.CreateDirectory(Configuration.Instance.Books).FullName;
+            }
+        }
+        #endregion
+
+        #region instance
+        public FileType FileType => (FileType)Value;
+
+		private IEnumerable<string> extensions_noDots { get; }
+		private string extAggr { get; }
+
+        protected AudibleFileStorage(FileType fileType) : base((int)fileType, fileType.ToString())
+		{
+			extensions_noDots = Extensions.Select(ext => ext.Trim('.')).ToList();
+			extAggr = extensions_noDots.Aggregate((a, b) => $"{a}|{b}");
+		}
+
+        /// <summary>
+        /// Example for full books:
+        /// Search recursively in _books directory. Full book exists if either are true
+        /// - a directory name has the product id and an audio file is immediately inside
+        /// - any audio filename contains the product id
+        /// </summary>
+        public bool Exists(string productId) => GetPath(productId) != null;
+
+		public string GetPath(string productId)
+        {
+            var cachedFile = FilePathCache.GetPath(productId, FileType);
+            if (cachedFile != null)
+                return cachedFile;
+
+            var firstOrNull =
+				Directory
+				.EnumerateFiles(StorageDirectory, "*.*", SearchOption.AllDirectories)
+				.FirstOrDefault(s => Regex.IsMatch(s, $@"{productId}.*?\.({extAggr})$", RegexOptions.IgnoreCase));
+
+			if (firstOrNull is null)
+				return null;
+
+			FilePathCache.Upsert(productId, FileType, firstOrNull);
+			return firstOrNull;
+        }
+
+        public string GetDestDir(string title, string asin)
+        {
+            // to prevent the paths from getting too long, we don't need after the 1st ":" for the folder
+            var underscoreIndex = title.IndexOf(':');
+            var titleDir
+                = underscoreIndex < 4
+                ? title
+                : title.Substring(0, underscoreIndex);
+            var finalDir = FileUtility.GetValidFilename(StorageDirectory, titleDir, null, asin);
+            return finalDir;
+        }
+
+        public bool IsFileTypeMatch(FileInfo fileInfo)
+            => extensions_noDots.ContainsInsensative(fileInfo.Extension.Trim('.'));
+        #endregion
+    }
+
+    public class AudioFileStorage : AudibleFileStorage
+    {
+        public const string SKIP_FILE_EXT = "libhack";
+
+		public override string[] Extensions { get; } = new[] { "m4b", "mp3", "aac", "mp4", "m4a", "ogg", "flac", SKIP_FILE_EXT };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => BooksDirectory;
+
+        public AudioFileStorage() : base(FileType.Audio) { }
+
+        public string CreateSkipFile(string title, string asin, string contents = null)
+        {
+            var destinationDir = GetDestDir(title, asin);
+            Directory.CreateDirectory(destinationDir);
+
+            var path = FileUtility.GetValidFilename(destinationDir, title, SKIP_FILE_EXT, asin);
+            File.WriteAllText(path, contents ?? string.Empty);
+
+            return path;
+        }
+    }
+
+    public class AaxFileStorage : AudibleFileStorage
+    {
+        public override string[] Extensions { get; } = new[] { "aax" };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => DownloadsFinal;
+
+        public AaxFileStorage() : base(FileType.AAX) { }
+    }
+
+    public class PdfFileStorage : AudibleFileStorage
+    {
+		public override string[] Extensions { get; } = new[] { "pdf", "zip" };
+
+        // we always want to use the latest config value, therefore
+        // - DO use 'get' arrow "=>"
+        // - do NOT use assign "="
+        public override string StorageDirectory => BooksDirectory;
+
+        public PdfFileStorage() : base(FileType.PDF) { }
+    }
+}

FileManager/Configuration.cs

@@ -83,6 +83,13 @@ namespace FileManager
             set => persistentDictionary.Set(nameof(DecryptInProgressEnum), value);
         }
 
+        [Description("Retain .aax files after decrypting?")]
+        public bool RetainAaxFiles
+        {
+            get => persistentDictionary.Get<bool>(nameof(RetainAaxFiles));
+            set => persistentDictionary.Set(nameof(RetainAaxFiles), value);
+        }
+
         // note: any potential file manager static ctors can't compensate if storage dir is changed at run time via settings. this is partly bad architecture. but the side effect is desirable. if changing LibationFiles location: restart app
 
         // singleton stuff

FileManager/FileManager.csproj

@@ -1,11 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Polly" Version="7.2.1" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.33" />
+    <PackageReference Include="Octokit" Version="0.50.0" />
+    <PackageReference Include="Polly" Version="7.2.2" />
   </ItemGroup>
 
   <ItemGroup>

FileManager/PersistentDictionary.cs

@@ -40,8 +40,13 @@ namespace FileManager
             return stringCache[propertyName];
         }
 
-        public T Get<T>(string propertyName) where T : class
-            => GetObject(propertyName) is T obj ? obj : default;
+        public T Get<T>(string propertyName)
+        {
+            var o = GetObject(propertyName);
+            if (o is null) return default;
+            if (o is JToken jt) return jt.Value<T>();
+            return (T)o;
+        }
 
         public object GetObject(string propertyName)
         {

FileManager/UNTESTED/AudibleFileStorage.cs

@@ -1,119 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text.RegularExpressions;
-using Dinah.Core;
-using Dinah.Core.Collections.Generic;
-
-namespace FileManager
-{
-    // could add images here, but for now images are stored in a well-known location
-    public enum FileType { Unknown, Audio, AAX, PDF }
-
-    /// <summary>
-    /// Files are large. File contents are never read by app.
-    /// Paths are varied.
-    /// Files are written during download/decrypt/backup/liberate.
-    /// Paths are read at app launch and during download/decrypt/backup/liberate.
-    /// Many files are often looked up at once
-    /// </summary>
-    public sealed class AudibleFileStorage : Enumeration<AudibleFileStorage>
-    {
-        #region static
-        public static AudibleFileStorage Audio { get; }
-        public static AudibleFileStorage AAX { get; }
-        public static AudibleFileStorage PDF { get; }
-
-        public static string DownloadsInProgress { get; }
-        public static string DecryptInProgress { get; }
-        public static string BooksDirectory => Configuration.Instance.Books;
-        // not customizable. don't move to config
-        public static string DownloadsFinal { get; }
-            = new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("DownloadsFinal").FullName;
-
-        static AudibleFileStorage()
-        {
-            #region init DecryptInProgress
-            if (!Configuration.Instance.DecryptInProgressEnum.In("WinTemp", "LibationFiles"))
-                Configuration.Instance.DecryptInProgressEnum = "WinTemp";
-            var M4bRootDir
-                = Configuration.Instance.DecryptInProgressEnum == "WinTemp" // else "LibationFiles"
-                ? Configuration.WinTemp
-                : Configuration.Instance.LibationFiles;
-            DecryptInProgress = Path.Combine(M4bRootDir, "DecryptInProgress");
-            Directory.CreateDirectory(DecryptInProgress);
-            #endregion
-
-            #region init DownloadsInProgress
-            if (!Configuration.Instance.DownloadsInProgressEnum.In("WinTemp", "LibationFiles"))
-                Configuration.Instance.DownloadsInProgressEnum = "WinTemp";
-            var AaxRootDir
-                = Configuration.Instance.DownloadsInProgressEnum == "WinTemp" // else "LibationFiles"
-                ? Configuration.WinTemp
-                : Configuration.Instance.LibationFiles;
-            DownloadsInProgress = Path.Combine(AaxRootDir, "DownloadsInProgress");
-            Directory.CreateDirectory(DownloadsInProgress);
-            #endregion
-
-            #region init BooksDirectory
-            if (string.IsNullOrWhiteSpace(Configuration.Instance.Books))
-                Configuration.Instance.Books = Path.Combine(Configuration.Instance.LibationFiles, "Books");
-            Directory.CreateDirectory(Configuration.Instance.Books);
-            #endregion
-
-            // must do this in static ctor, not w/inline properties
-            // static properties init before static ctor so these dir.s would still be null
-            Audio = new AudibleFileStorage(FileType.Audio, BooksDirectory, "m4b", "mp3", "aac", "mp4", "m4a", "ogg", "flac");
-            AAX = new AudibleFileStorage(FileType.AAX, DownloadsFinal, "aax");
-            PDF = new AudibleFileStorage(FileType.PDF, BooksDirectory, "pdf", "zip");
-        }
-        #endregion
-
-        #region instance
-        public FileType FileType => (FileType)Value;
-
-        public string StorageDirectory => DisplayName;
-
-		private IEnumerable<string> extensions_noDots { get; }
-		private string extAggr { get; }
-
-        private AudibleFileStorage(FileType fileType, string storageDirectory, params string[] extensions) : base((int)fileType, storageDirectory)
-		{
-			extensions_noDots = extensions.Select(ext => ext.Trim('.')).ToList();
-			extAggr = extensions_noDots.Aggregate((a, b) => $"{a}|{b}");
-		}
-
-        /// <summary>
-        /// Example for full books:
-        /// Search recursively in _books directory. Full book exists if either are true
-        /// - a directory name has the product id and an audio file is immediately inside
-        /// - any audio filename contains the product id
-        /// </summary>
-        public bool Exists(string productId)
-            => GetPath(productId) != null;
-
-		public string GetPath(string productId)
-        {
-			{
-                var cachedFile = FilePathCache.GetPath(productId, FileType);
-                if (cachedFile != null)
-                    return cachedFile;
-            }
-
-			var firstOrNull =
-				Directory
-				.EnumerateFiles(StorageDirectory, "*.*", SearchOption.AllDirectories)
-				.FirstOrDefault(s => Regex.IsMatch(s, $@"{productId}.*?\.({extAggr})$", RegexOptions.IgnoreCase));
-
-			if (firstOrNull is null)
-				return null;
-			FilePathCache.Upsert(productId, FileType, firstOrNull);
-			return firstOrNull;
-        }
-
-        public bool IsFileTypeMatch(FileInfo fileInfo)
-            => extensions_noDots.ContainsInsensative(fileInfo.Extension.Trim('.'));
-        #endregion
-    }
-}

InternalUtilities/Account.cs

@@ -93,5 +93,11 @@ namespace InternalUtilities
 		}
 
 		public override string ToString() => $"{AccountId} - {Locale?.Name ?? "[empty]"}";
+
+		public string MaskedLogEntry => @$"AccountId={mask(AccountId)}|AccountName={mask(AccountName)}|Locale={Locale?.Name ?? "[empty]"}";
+		private static string mask(string str)
+			=> str is null ? "[null]"
+			: str == string.Empty ? "[empty]"
+			: str.ToMask();
 	}
 }

InternalUtilities/AudibleApiActions.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Threading.Tasks;
 using AudibleApi;
 using AudibleApiDTOs;
+using Dinah.Core;
 using Polly;
 using Polly.Retry;
 
@@ -16,7 +17,7 @@ namespace InternalUtilities
 		{
 			Serilog.Log.Logger.Information("GetApiAsync. {@DebugInfo}", new
 			{
-				username,
+				Username = username.ToMask(),
 				LocaleName = localeName,
 			});
 			return EzApiCreator.GetApiAsync(
@@ -31,7 +32,7 @@ namespace InternalUtilities
 		{
 			Serilog.Log.Logger.Information("GetApiAsync. {@DebugInfo}", new
 			{
-				AccountId = account?.AccountId ?? "[empty]",
+				Account = account?.MaskedLogEntry ?? "[null]",
 				LocaleName = account?.Locale?.Name
 			});
 			return EzApiCreator.GetApiAsync(

InternalUtilities/InternalUtilities.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

Libation.sln

@@ -86,6 +86,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "0 Libation Tests", "0 Libat
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "InternalUtilities.Tests", "_Tests\InternalUtilities.Tests\InternalUtilities.Tests.csproj", "{8447C956-B03E-4F59-9DD4-877793B849D9}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LibationSearchEngine.Tests", "_Tests\LibationSearchEngine.Tests\LibationSearchEngine.Tests.csproj", "{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.EntityFrameworkCore.Tests", "..\Dinah.Core\_Tests\Dinah.EntityFrameworkCore.Tests\Dinah.EntityFrameworkCore.Tests.csproj", "{6F5131A0-09AE-4707-B82B-5E53CB74688E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -208,6 +212,14 @@ Global
 		{8447C956-B03E-4F59-9DD4-877793B849D9}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8447C956-B03E-4F59-9DD4-877793B849D9}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8447C956-B03E-4F59-9DD4-877793B849D9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6F5131A0-09AE-4707-B82B-5E53CB74688E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6F5131A0-09AE-4707-B82B-5E53CB74688E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6F5131A0-09AE-4707-B82B-5E53CB74688E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6F5131A0-09AE-4707-B82B-5E53CB74688E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -242,6 +254,8 @@ Global
 		{E7EFD64D-6630-4426-B09C-B6862A92E3FD} = {F0CBB7A7-D3FB-41FF-8F47-CF3F6A592249}
 		{F3B04A3A-20C8-4582-A54A-715AF6A5D859} = {8679CAC8-9164-4007-BDD2-F004810EDA14}
 		{8447C956-B03E-4F59-9DD4-877793B849D9} = {67E66E82-5532-4440-AFB3-9FB1DF9DEF53}
+		{C5B21768-C7C9-4FCB-AC1E-187B223D5A98} = {67E66E82-5532-4440-AFB3-9FB1DF9DEF53}
+		{6F5131A0-09AE-4707-B82B-5E53CB74688E} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

LibationLauncher/LibationLauncher.csproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <OutputType>WinExe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net5.0-windows</TargetFramework>
     <UseWindowsForms>true</UseWindowsForms>
     <ApplicationIcon>libation.ico</ApplicationIcon>
     <AssemblyName>Libation</AssemblyName>
@@ -13,7 +13,7 @@
     <!-- <PublishSingleFile>true</PublishSingleFile> -->
     <RuntimeIdentifier>win-x64</RuntimeIdentifier>
 
-    <Version>4.0.2.3</Version>
+    <Version>4.3.0.4</Version>
   </PropertyGroup>
 
   <ItemGroup>
@@ -21,7 +21,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Octokit" Version="0.48.0" />
+    <PackageReference Include="Octokit" Version="0.50.0" />
   </ItemGroup>
 
   <ItemGroup>

LibationLauncher/Program.cs

@@ -27,7 +27,8 @@ namespace LibationLauncher
 
 			AudibleApiStorage.EnsureAccountsSettingsFileExists();
 
-			migrate_v3_to_v4();
+			migrate_to_v4_0_0();
+			migrate_to_v4_0_3(); // add setting for whether to delete/retain aax
 
 			ensureLoggingConfig();
 			ensureSerilogConfig();
@@ -83,7 +84,7 @@ namespace LibationLauncher
 		#region v3 => v4 migration
 		static string AccountsSettingsFileLegacy30 => Path.Combine(Configuration.Instance.LibationFiles, "IdentityTokens.json");
 
-		private static void migrate_v3_to_v4()
+		private static void migrate_to_v4_0_0()
 		{
 			migrateLegacyIdentityFile();
 
@@ -205,6 +206,27 @@ namespace LibationLauncher
 		}
 		#endregion
 
+		#region migrate_to_v4_0_3 add setting for whether to delete/retain aax
+		private static void migrate_to_v4_0_3()
+		{
+			if (!File.Exists(Configuration.Instance.SettingsFilePath))
+				return;
+
+			// use JObject to remove decrypt key and locale from Settings.json
+			var settingsContents = File.ReadAllText(Configuration.Instance.SettingsFilePath);
+			var jObj = JObject.Parse(settingsContents);
+
+			var jRetainAaxFiles = jObj.Property("RetainAaxFiles");
+			if (jRetainAaxFiles is null)
+			{
+				jObj.Add("RetainAaxFiles", false);
+			
+				var newContents = jObj.ToString(Formatting.Indented);
+				File.WriteAllText(Configuration.Instance.SettingsFilePath, newContents);
+			}
+		}
+		#endregion
+
 		private static string defaultLoggingLevel { get; } = "Information";
 		private static void ensureLoggingConfig()
 		{
@@ -301,8 +323,8 @@ namespace LibationLauncher
 				.AddJsonFile(config.SettingsFilePath)
 				.Build();
 			Log.Logger = new LoggerConfiguration()
-			  .ReadFrom.Configuration(configuration)
-			  .CreateLogger();
+				.ReadFrom.Configuration(configuration)
+				.CreateLogger();
 
 			//// MANUAL HARD CODED
 			//Log.Logger = new LoggerConfiguration()
@@ -358,19 +380,25 @@ namespace LibationLauncher
 				try
 				{
 					LibationWinForms.BookLiberation.ProcessorAutomationController.DownloadFileAsync(zipUrl, selectedPath).GetAwaiter().GetResult();
-					MessageBox.Show($"File downloaded");
+					MessageBox.Show("File downloaded");
 				}
 				catch (Exception ex)
 				{
-					MessageBox.Show($"ERROR: {ex.Message}\r\n{ex.StackTrace}");
+					Error(ex, "Error downloading update");
 				}
 			}
 			catch (Exception ex)
 			{
-				MessageBox.Show($"Error checking for update. ERROR: {ex.Message}\r\n{ex.StackTrace}");
+				Error(ex, "Error checking for update");
 			}
 		}
 
+		private static void Error(Exception ex, string message)
+		{
+			Log.Logger.Error(ex, message);
+			MessageBox.Show($"{message}\r\nSee log for details");
+		}
+
 		private static void logStartupState()
 		{
 			var config = Configuration.Instance;

LibationSearchEngine/LibationSearchEngine.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

LibationSearchEngine/LuceneRegex.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+namespace LibationSearchEngine
+{
+    internal static class LuceneRegex
+    {
+        #region pattern pieces
+        //  negative lookbehind: cannot be preceeded by an escaping \
+        const string NOT_ESCAPED = @"(?<!\\)";
+
+        // disallow spaces and lucene reserved characters
+        //     + - && || ! ( ) { } [ ] ^ " ~ * ? : \
+        // define chars
+        // escape and concat
+        // create regex. also disallow spaces
+        private static char[] disallowedChars { get; } = new[] {
+            '+', '-', '&', '|', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\' };
+        private static string disallowedCharsEscaped { get; } = disallowedChars.Select(c => $@"\{c}").Aggregate((a, b) => a + b);
+        private static string WORD_CAPTURE { get; } = $@"([^\s{disallowedCharsEscaped}]+)";
+
+        // : with optional preceeding spaces. capture these so i don't accidentally replace a non-field name
+        const string FIELD_END = @"(\s*:)";
+
+        const string BEGIN_TAG = @"\[";
+        const string END_TAG = @"\]";
+
+        // space is forgiven at beginning and end of tag but not in the middle
+        // literal space character only. do NOT allow new lines, tabs, ...
+        const string OPTIONAL_SPACE_LITERAL = @"\u0020*";
+        #endregion
+
+        private static string tagPattern { get; } = NOT_ESCAPED + BEGIN_TAG + OPTIONAL_SPACE_LITERAL + WORD_CAPTURE + OPTIONAL_SPACE_LITERAL + END_TAG;
+        public static Regex TagRegex { get; } = new Regex(tagPattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);
+
+        private static string fieldPattern { get; } = NOT_ESCAPED + WORD_CAPTURE + FIELD_END;
+        public static Regex FieldRegex { get; } = new Regex(fieldPattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);
+
+        // auto-pad numbers to 8 char.s. This will match int.s and dates (yyyyMMdd)
+        //   positive look behind: beginning  space  {  [  :
+        //   positive look ahead: end  space  ]  }
+        public static Regex NumbersRegex { get; } = new Regex(@"(?<=^|\s|\{|\[|:)(\d+\.?\d*)(?=$|\s|\]|\})", RegexOptions.Compiled);
+
+        /// <summary>
+        /// proper bools are single keywords which are turned into keyword:True
+        /// if bordered by colons or inside brackets, they are not stand-alone bool keywords
+        /// the negative lookbehind and lookahead patterns prevent bugs where a bool keyword is also a user-defined tag:
+        ///   [israted]
+        ///     parseTag => tags:israted
+        ///     replaceBools => tags:israted:True
+        ///   or
+        ///     [israted]
+        ///       replaceBools => israted:True
+        ///         parseTag => [israted:True]
+        /// also don't want to apply :True where the value already exists:
+        ///   israted:false => israted:false:True
+        ///   
+        /// despite using parans, lookahead and lookbehind are zero-length assertions which do not capture. therefore the bool search keyword is still $1 since it's the first and only capture
+        /// </summary>
+        private static string boolPattern_parameterized { get; }
+            = @"
+### IMPORTANT: 'ignore whitespace' is only partially honored in character sets
+### - new lines are ok
+### - ANY leading whitespace is treated like actual matching spaces  :(
+
+                    ### can't begin with colon. incorrect syntax
+                    ### can't begin with open bracket: this signals the start of a tag
+(?<!                # begin negative lookbehind
+  [:\[]             #   char set: colon and open bracket, escaped
+  \s*               #   optional space
+)                   # end negative lookbehind
+
+\b                  # word boundary
+  ({0})             #   captured bool search keyword. this is the $1 reference used in regex.Replace
+\b                  # word boundary
+
+                    ### can't end with colon. this signals that the bool's value already exists
+                    ### can't begin with close bracket: this signals the end of a tag
+(?!                 # begin negative lookahead
+  \s*               #   optional space
+  [:\]]             #   char set: colon and close bracket, escaped
+)                   # end negative lookahead
+";
+        private static Dictionary<string, Regex> boolRegexDic { get; } = new Dictionary<string, Regex>();
+        public static Regex GetBoolRegex(string boolSearch)
+        {
+            if (boolRegexDic.TryGetValue(boolSearch, out var regex))
+                return regex;
+
+            var boolPattern = string.Format(boolPattern_parameterized, boolSearch);
+            regex = new Regex(boolPattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase | RegexOptions.Compiled);
+            boolRegexDic.Add(boolSearch, regex);
+
+            return regex;
+        }
+    }
+}

LibationSearchEngine/SearchEngine.cs

@@ -26,9 +26,16 @@ namespace LibationSearchEngine
 
         public const string _ID_ = "_ID_";
         public const string TAGS = "tags";
+        // special field for each book which includes all major parts of the book's metadata. enables non-targetting searching
         public const string ALL = "all";
 
-        private static ReadOnlyDictionary<string, Func<LibraryBook, string>> idIndexRules { get; }
+        // the workaround which allows displaying all books when query is empty
+        public const string ALL_QUERY = "*:*";
+
+        public SearchEngine(LibationContext context) => this.context = context;
+
+		#region index rules
+		private static ReadOnlyDictionary<string, Func<LibraryBook, string>> idIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, string>>(
                 new Dictionary<string, Func<LibraryBook, string>>
                 {
@@ -38,6 +45,7 @@ namespace LibationSearchEngine
                     ["ASIN"] = lb => lb.Book.AudibleProductId
                 }
                 );
+
         private static ReadOnlyDictionary<string, Func<LibraryBook, string>> stringIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, string>>(
                 new Dictionary<string, Func<LibraryBook, string>>
@@ -98,6 +106,7 @@ namespace LibationSearchEngine
                     ["MyRating"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating.ToLuceneString()
                 }
                 );
+
         private static ReadOnlyDictionary<string, Func<LibraryBook, bool>> boolIndexRules { get; }
             = new ReadOnlyDictionary<string, Func<LibraryBook, bool>>(
                 new Dictionary<string, Func<LibraryBook, bool>>
@@ -114,12 +123,17 @@ namespace LibationSearchEngine
                     ["IsRated"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating > 0f,
                     ["Rated"] = lb => lb.Book.UserDefinedItem.Rating.OverallRating > 0f,
 
-                    ["IsAuthorNarrated"] = lb => isAuthorNarrated(lb),
-                    ["AuthorNarrated"] = lb => isAuthorNarrated(lb),
+                    ["IsAuthorNarrated"] = isAuthorNarrated,
+                    ["AuthorNarrated"] = isAuthorNarrated,
 
                     [nameof(Book.IsAbridged)] = lb => lb.Book.IsAbridged,
                     ["Abridged"] = lb => lb.Book.IsAbridged,
-                });
+
+                    // this will only be evaluated at time of re-index. ie: state of files moved later will be out of sync until next re-index
+                    ["IsLiberated"] = lb => isLiberated(lb.Book.AudibleProductId),
+                    ["Liberated"] = lb => isLiberated(lb.Book.AudibleProductId),
+                }
+                );
 
         private static bool isAuthorNarrated(LibraryBook lb)
         {
@@ -128,6 +142,8 @@ namespace LibationSearchEngine
             return authors.Intersect(narrators).Any();
         }
 
+        private static bool isLiberated(string id) => AudibleFileStorage.Audio.Exists(id);
+
         // use these common fields in the "all" default search field
         private static IEnumerable<Func<LibraryBook, string>> allFieldIndexRules { get; }
             = new List<Func<LibraryBook, string>>
@@ -137,8 +153,10 @@ namespace LibationSearchEngine
                 stringIndexRules[nameof(Book.AuthorNames)],
                 stringIndexRules[nameof(Book.NarratorNames)]
             };
+		#endregion
 
-        public static IEnumerable<string> GetSearchIdFields()
+		#region get search fields. used for display in help
+		public static IEnumerable<string> GetSearchIdFields()
         {
             foreach (var key in idIndexRules.Keys)
                 yield return key;
@@ -173,11 +191,13 @@ namespace LibationSearchEngine
             foreach (var key in numberIndexRules.Keys)
                 yield return key;
         }
+		#endregion
 
-        private Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);
-
-        public SearchEngine(LibationContext context) => this.context = context;
-
+		#region create and update index
+        /// <summary>
+        /// create new. ie: full re-index
+        /// </summary>
+        /// <param name="overwrite"></param>
         public void CreateNewIndex(bool overwrite = true)
         {
             // 300 products
@@ -211,6 +231,22 @@ namespace LibationSearchEngine
             log();
         }
 
+		/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
+		public void UpdateBook(string productId)
+        {
+            var libraryBook = context.GetLibraryBook_Flat_NoTracking(productId);
+            var term = new Term(_ID_, productId);
+
+            var document = createBookIndexDocument(libraryBook);
+            var createNewIndex = false;
+
+			using var index = getIndex();
+			using var analyzer = new StandardAnalyzer(Version);
+			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
+			ixWriter.DeleteDocuments(term);
+			ixWriter.AddDocument(document);
+		}
+
         private static Document createBookIndexDocument(LibraryBook libraryBook)
         {
             var doc = new Document();
@@ -248,77 +284,100 @@ namespace LibationSearchEngine
             return doc;
         }
 
-		/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
-		public void UpdateBook(string productId)
-        {
-            var libraryBook = context.GetLibraryBook_Flat_NoTracking(productId);
-            var term = new Term(_ID_, productId);
+        // update single document entry
+        // all fields, including 'tags' are case-specific
+        public void UpdateTags(string productId, string tags) => updateAnalyzedField(productId, TAGS, tags);
 
-            var document = createBookIndexDocument(libraryBook);
-            var createNewIndex = false;
+        // all fields are case-specific
+        private static void updateAnalyzedField(string productId, string fieldName, string newValue)
+            => updateDocument(
+                productId,
+                d =>
+                {
+                    // fields are key value pairs. MULTIPLE FIELDS CAN POTENTIALLY HAVE THE SAME KEY.
+                    // ie: must remove old before adding new else will create unwanted duplicates.
+                    d.RemoveField(fieldName);
+                    d.AddAnalyzed(fieldName, newValue);
+                });
 
-			using var index = getIndex();
-			using var analyzer = new StandardAnalyzer(Version);
-			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
-			ixWriter.DeleteDocuments(term);
-			ixWriter.AddDocument(document);
-		}
+        // update single document entry
+        public void UpdateIsLiberated(string productId)
+            => updateDocument(
+                productId,
+                d =>
+                {
+                    // fields are key value pairs. MULTIPLE FIELDS CAN POTENTIALLY HAVE THE SAME KEY.
+                    // ie: must remove old before adding new else will create unwanted duplicates.
+                    var v = isLiberated(productId);
+                    d.RemoveField("IsLiberated");
+                    d.AddBool("IsLiberated", v);
+                    d.RemoveField("Liberated");
+                    d.AddBool("Liberated", v);
+                });
 
-        public void UpdateTags(string productId, string tags)
+        private static void updateDocument(string productId, Action<Document> action)
         {
             var productTerm = new Term(_ID_, productId);
 
-			using var index = getIndex();
+            using var index = getIndex();
 
-			// get existing document
-			using var searcher = new IndexSearcher(index);
-			var query = new TermQuery(productTerm);
-			var docs = searcher.Search(query, 1);
-			var scoreDoc = docs.ScoreDocs.SingleOrDefault();
-			if (scoreDoc == null)
-				throw new Exception("document not found");
-			var document = searcher.Doc(scoreDoc.Doc);
+            // get existing document
+            using var searcher = new IndexSearcher(index);
+            var query = new TermQuery(productTerm);
+            var docs = searcher.Search(query, 1);
+            var scoreDoc = docs.ScoreDocs.SingleOrDefault();
+            if (scoreDoc == null)
+                throw new Exception("document not found");
+            var document = searcher.Doc(scoreDoc.Doc);
 
 
-			// update document entry with new tags
-			// fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new
-			// REMEMBER: all fields, including 'tags' are case-specific
-			document.RemoveField(TAGS);
-			document.AddAnalyzed(TAGS, tags);
+            // perform update
+            action(document);
 
-			// update index
-			var createNewIndex = false;
-			using var analyzer = new StandardAnalyzer(Version);
-			using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
-			ixWriter.UpdateDocument(productTerm, document, analyzer);
-		}
 
+            // update index
+            var createNewIndex = false;
+            using var analyzer = new StandardAnalyzer(Version);
+            using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
+            ixWriter.UpdateDocument(productTerm, document, analyzer);
+        }
+		#endregion
+
+		#region search
         public SearchResultSet Search(string searchString)
         {
-            if (string.IsNullOrWhiteSpace(searchString))
-                searchString = "*:*";
+            Serilog.Log.Logger.Debug("original search string: {@DebugInfo}", new { searchString });
+            searchString = FormatSearchQuery(searchString);
+            Serilog.Log.Logger.Debug("formatted search string: {@DebugInfo}", new { searchString });
 
-            #region apply formatting
-            searchString = parseTag(searchString);
+            var results = generalSearch(searchString);
+            Serilog.Log.Logger.Debug("Hit(s): {@DebugInfo}", new { count = results.Docs.Count() });
+            displayResults(results);
+
+            return results;
+        }
+
+        public static string FormatSearchQuery(string searchString)
+        {
+            if (string.IsNullOrWhiteSpace(searchString))
+                return ALL_QUERY;
 
             searchString = replaceBools(searchString);
 
+            searchString = parseTag(searchString);
+
             // in ranges " TO " must be uppercase
             searchString = searchString.Replace(" to ", " TO ");
 
             searchString = padNumbers(searchString);
 
             searchString = lowerFieldNames(searchString);
-            #endregion
 
-            var results = generalSearch(searchString);
-
-            displayResults(results);
-
-            return results;
+            return searchString;
         }
 
-        private static string parseTag(string tagSearchString)
+		#region format query string
+		private static string parseTag(string tagSearchString)
         {
             var allMatches = LuceneRegex
                 .TagRegex
@@ -337,9 +396,10 @@ namespace LibationSearchEngine
 
         private static string replaceBools(string searchString)
         {
-            // negative look-ahead for optional spaces then colon. don't want to double-up. eg:"israted:false" => "israted:false:True"
             foreach (var boolSearch in boolIndexRules.Keys)
-                searchString = Regex.Replace(searchString, $@"\b({boolSearch})\b(?!\s*:)", @"$1:True", RegexOptions.IgnoreCase);
+                searchString =
+                    LuceneRegex.GetBoolRegex(boolSearch)
+                    .Replace(searchString, @"$1:True");
 
             return searchString;
         }
@@ -376,13 +436,10 @@ namespace LibationSearchEngine
 
             return searchString;
         }
-
-        public int MaxSearchResultsToReturn { get; set; } = 999;
+        #endregion
 
         private SearchResultSet generalSearch(string searchString)
         {
-            Console.WriteLine($"searchString: {searchString}");
-
             var defaultField = ALL;
 
 			using var index = getIndex();
@@ -403,14 +460,14 @@ namespace LibationSearchEngine
 					boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
 			}
 
-			Console.WriteLine($"  query: {query}");
-
 			var docs = searcher
-				.Search(query, MaxSearchResultsToReturn)
+				.Search(query, searcher.MaxDoc + 1)
 				.ScoreDocs
 				.Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score))
 				.ToList();
-			return new SearchResultSet(query.ToString(), docs);
+            var queryString = query.ToString();
+            Serilog.Log.Logger.Debug("query: {@DebugInfo}", new { queryString });
+			return new SearchResultSet(queryString, docs);
 		}
 
         private IEnumerable<Occur> getOccurs_recurs(BooleanQuery query)
@@ -430,7 +487,6 @@ namespace LibationSearchEngine
 
         private void displayResults(SearchResultSet docs)
         {
-            Console.WriteLine($"Hit(s): {docs.Docs.Count()}");
             //for (int i = 0; i < docs.Docs.Count(); i++)
             //{
             //    var sde = docs.Docs.First();
@@ -438,13 +494,14 @@ namespace LibationSearchEngine
             //    Document doc = sde.Doc;
             //    float score = sde.Score;
 
-            //    Console.WriteLine($"{(i + 1)}) score={score}. Fields:");
+            //    Serilog.Log.Logger.Debug($"{(i + 1)}) score={score}. Fields:");
             //    var allFields = doc.GetFields();
             //    foreach (var f in allFields)
-            //        Console.WriteLine($"   [{f.Name}]={f.StringValue}");
+            //        Serilog.Log.Logger.Debug($"   [{f.Name}]={f.StringValue}");
             //}
-
-            //Console.WriteLine();
         }
+		#endregion
+
+		private static Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);
     }
 }

LibationSearchEngine/UNTESTED/LuceneRegex.cs

@@ -1,46 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text.RegularExpressions;
-
-namespace LibationSearchEngine
-{
-    internal static class LuceneRegex
-    {
-        #region pattern pieces
-        //  negative lookbehind: cannot be preceeded by an escaping \
-        const string NOT_ESCAPED = @"(?<!\\)";
-
-        // disallow spaces and lucene reserved characters
-        //     + - && || ! ( ) { } [ ] ^ " ~ * ? : \
-        // define chars
-        // escape and concat
-        // create regex. also disallow spaces
-        private static char[] disallowedChars { get; } = new[] {
-            '+', '-', '&', '|', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\' };
-        private static string disallowedCharsEscaped { get; } = disallowedChars.Select(c => $@"\{c}").Aggregate((a, b) => a + b);
-        private static string WORD_CAPTURE { get; } = $@"([^\s{disallowedCharsEscaped}]+)";
-
-        // : with optional preceeding spaces. capture these so i don't accidentally replace a non-field name
-        const string FIELD_END = @"(\s*:)";
-
-        const string BEGIN_TAG = @"\[";
-        const string END_TAG = @"\]";
-
-        // space is forgiven at beginning and end of tag but not in the middle
-        // literal space character only. do NOT allow new lines, tabs, ...
-        const string OPTIONAL_SPACE_LITERAL = @"\u0020*";
-        #endregion
-
-        private static string tagPattern { get; } = NOT_ESCAPED + BEGIN_TAG + OPTIONAL_SPACE_LITERAL + WORD_CAPTURE + OPTIONAL_SPACE_LITERAL + END_TAG;
-        public static Regex TagRegex { get; } = new Regex(tagPattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);
-
-        private static string fieldPattern { get; } = NOT_ESCAPED + WORD_CAPTURE + FIELD_END;
-        public static Regex FieldRegex { get; } = new Regex(fieldPattern, RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);
-
-        // auto-pad numbers to 8 char.s. This will match int.s and dates (yyyyMMdd)
-        //   positive look behind: beginning  space  {  [  :
-        //   positive look ahead: end  space  ]  }
-        public static Regex NumbersRegex { get; } = new Regex(@"(?<=^|\s|\{|\[|:)(\d+\.?\d*)(?=$|\s|\]|\})", RegexOptions.Compiled);
-    }
-}

LibationWinForms/BookLiberation/AutomatedBackupsForm.cs

@@ -24,24 +24,13 @@ namespace LibationWinForms.BookLiberation
             InitializeComponent();
         }
 
-		public void AppendError(Exception ex)
-		{
-			Serilog.Log.Logger.Error(ex, "Automated backup: error");
-			appendText("ERROR: " + ex.Message);
-		}
-		public void AppendText(string text)
-		{
-			Serilog.Log.Logger.Information($"Automated backup: {text}");
-			appendText(text);
-		}
-		private void appendText(string text)
+		public void WriteLine(string text)
 			=> logTb.UIThread(() => logTb.AppendText($"{DateTime.Now} {text}{Environment.NewLine}"));
 
 		public void FinalizeUI()
         {
             keepGoingCb.Enabled = false;
             logTb.AppendText("");
-            AppendText("DONE");
         }
 
         private void AutomatedBackupsForm_FormClosing(object sender, FormClosingEventArgs e) => keepGoingCb.Checked = false;