Bug fix #210 : if attempting to paginate more than 10,000 titles : "Implied library size is unsupported"

CLI manual change to settings file no longer needed

AppScaffolding/AppScaffolding.csproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
-    <Version>6.6.7.1</Version>
+    <Version>6.7.2.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

AppScaffolding/LibationScaffolding.cs

@@ -56,7 +56,7 @@ namespace AppScaffolding
 			// migrations go below here
 			//
 
-			Migrations.migrate_to_v6_5_2(config);
+			Migrations.migrate_to_v6_6_9(config);
 		}
 
 		public static void PopulateMissingConfigValues(Configuration config)
@@ -107,29 +107,12 @@ namespace AppScaffolding
 			if (config.GetObject("Serilog") != null)
 				return;
 
-			// "Serilog": {
-			//   "MinimumLevel": "Information"
-			//   "WriteTo": [
-			//     {
-			//       "Name": "Console"
-			//     },
-			//     {
-			//       "Name": "File",
-			//       "Args": {
-			//         "rollingInterval": "Day",
-			//         "outputTemplate": ...
-			//       }
-			//     }
-			//   ],
-			//   "Using": [ "Dinah.Core" ],
-			//   "Enrich": [ "WithCaller" ]
-			// }
 			var serilogObj = new JObject
 			{
 				{ "MinimumLevel", "Information" },
 				{ "WriteTo", new JArray
 					{
-						new JObject { {"Name", "Console" } },
+						// new JObject { {"Name", "Console" } }, // this has caused more problems than it's solved
 						new JObject
 						{
 							{ "Name", "File" },
@@ -144,14 +127,16 @@ namespace AppScaffolding
 									//   output example:             2019-11-26 08:48:40.224 -05:00 [DBG] Begin Libation
 									// - with class and method info: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception}";
 									//   output example:             2019-11-26 08:48:40.224 -05:00 [DBG] (at LibationWinForms.Program.init()) Begin Libation
-									{ "outputTemplate", "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception}" }
+									// {Properties:j} needed for expanded exception logging
+									{ "outputTemplate", "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception} {Properties:j}" }
 								}
 							}
 						}
 					}
 				},
-				{ "Using", new JArray{ "Dinah.Core" } }, // dll's name, NOT namespace
-				{ "Enrich", new JArray{ "WithCaller" } },
+				// better exception logging with: Serilog.Exceptions library -- WithExceptionDetails
+				{ "Using", new JArray{ "Dinah.Core", "Serilog.Exceptions" } }, // dll's name, NOT namespace
+				{ "Enrich", new JArray{ "WithCaller", "WithExceptionDetails" } },
 			};
 			config.SetObject("Serilog", serilogObj);
 		}
@@ -166,9 +151,9 @@ namespace AppScaffolding
 			// capture most Console.WriteLine() and write to serilog. See below tests for details.
 			// Some dependencies print helpful info via Console.WriteLine. We'd like to log it.
 			//
-			// Serilog also writes to Console so this might be asking for trouble. ie: infinite loops.
-			// SerilogTextWriter needs to be more robust and tested. Esp the Write() methods.
-			// Empirical testing so far has shown no issues.
+			// If Serilog also writes to Console, this might be asking for trouble. ie: infinite loops.
+			// To use that way, SerilogTextWriter needs to be more robust and tested. Esp the Write() methods.
+			// However, empirical testing so far has shown no issues.
 			Console.SetOut(new MultiTextWriter(origOut, new SerilogTextWriter()));
 
 			#region Console => Serilog tests
@@ -346,9 +331,51 @@ namespace AppScaffolding
 			};
 		#endregion
 
-		public static void migrate_to_v6_5_2(Configuration config)
+		public static void migrate_to_v6_6_9(Configuration config)
 		{
-			// example
+			var writeToPath = $"Serilog.WriteTo";
+
+			// remove WriteTo[].Name == Console
+			{
+				if (UNSAFE_MigrationHelper.Settings_TryGetArrayLength(writeToPath, out var length1))
+				{
+					for (var i = length1 - 1; i >= 0; i--)
+					{
+						var exists = UNSAFE_MigrationHelper.Settings_TryGetFromJsonPath($"{writeToPath}[{i}].Name", out var value);
+
+						if (exists && value == "Console")
+							UNSAFE_MigrationHelper.Settings_RemoveFromArray(writeToPath, i);
+					}
+				}
+			}
+
+			// add Serilog.Exceptions -- WithExceptionDetails
+			{
+				// outputTemplate should contain "{Properties:j}"
+				{
+					// re-calculate. previous loop may have changed the length
+					if (UNSAFE_MigrationHelper.Settings_TryGetArrayLength(writeToPath, out var length2))
+					{
+						var propertyName = "outputTemplate";
+						for (var i = 0; i < length2; i++)
+						{
+							var jsonPath = $"{writeToPath}[{i}].Args";
+							var exists = UNSAFE_MigrationHelper.Settings_TryGetFromJsonPath($"{jsonPath}.{propertyName}", out var value);
+
+							var newValue = "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] (at {Caller}) {Message:lj}{NewLine}{Exception} {Properties:j}";
+
+							if (exists && value != newValue)
+								UNSAFE_MigrationHelper.Settings_SetWithJsonPath(jsonPath, propertyName, newValue);
+						}
+					}
+				}
+
+				// Serilog.Using must include "Serilog.Exceptions"
+				UNSAFE_MigrationHelper.Settings_AddUniqueToArray("Serilog.Using", "Serilog.Exceptions");
+
+				// Serilog.Enrich must include "WithExceptionDetails"
+				UNSAFE_MigrationHelper.Settings_AddUniqueToArray("Serilog.Enrich", "WithExceptionDetails");
+			}
 		}
 	}
 }

AppScaffolding/UNSAFE_MigrationHelper.cs

@@ -113,6 +113,108 @@ namespace AppScaffolding
 			return success;
 		}
 
+		public static bool Settings_JsonPathIsType(string jsonPath, JTokenType jTokenType)
+		{
+			JToken val = null;
+
+			process_SettingsJson(jObj => val = jObj.SelectToken(jsonPath), false);
+
+			return val?.Type == jTokenType;
+		}
+
+		public static bool Settings_TryGetFromJsonPath(string jsonPath, out string value)
+		{
+			JToken val = null;
+
+			process_SettingsJson(jObj => val = jObj.SelectToken(jsonPath), false);
+
+			if (val?.Type == JTokenType.String)
+			{
+				value = val.Value<string>();
+				return true;
+			}
+			else
+			{
+				value = null;
+				return false;
+			}
+		}
+
+		public static void Settings_SetWithJsonPath(string jsonPath, string propertyName, string newValue)
+		{
+			if (!Settings_TryGetFromJsonPath($"{jsonPath}.{propertyName}", out _))
+				return;
+
+			process_SettingsJson(jObj =>
+			{
+				var token = jObj.SelectToken(jsonPath);
+				if (token is null
+				|| token is not JObject o
+				|| o[propertyName] is null)
+					return;
+
+				var oldValue = token.Value<string>(propertyName);
+				if (oldValue != newValue)
+					token[propertyName] = newValue;
+			});
+		}
+
+		public static bool Settings_TryGetArrayLength(string jsonPath, out int length)
+		{
+			length = 0;
+
+			if (!Settings_JsonPathIsType(jsonPath, JTokenType.Array))
+				return false;
+
+			JArray array = null;
+			process_SettingsJson(jObj => array = (JArray)jObj.SelectToken(jsonPath));
+
+			length = array.Count;
+			return true;
+		}
+
+		public static void Settings_AddToArray(string jsonPath, string newValue)
+		{
+			if (!Settings_JsonPathIsType(jsonPath, JTokenType.Array))
+				return;
+
+			process_SettingsJson(jObj =>
+			{
+				var array = (JArray)jObj.SelectToken(jsonPath);
+				array.Add(newValue);
+			});
+		}
+
+		/// <summary>Do not add if already exists</summary>
+		public static void Settings_AddUniqueToArray(string arrayPath, string newValue)
+		{
+			if (!Settings_TryGetArrayLength(arrayPath, out var qty))
+				return;
+
+			for (var i = 0; i < qty; i++)
+			{
+				var exists = Settings_TryGetFromJsonPath($"{arrayPath}[{i}]", out var value);
+				if (exists && value == newValue)
+					return;
+			}
+
+			Settings_AddToArray(arrayPath, newValue);
+		}
+
+		/// <summary>only remove if not exists</summary>
+		public static void Settings_RemoveFromArray(string jsonPath, int position)
+		{
+			if (!Settings_JsonPathIsType(jsonPath, JTokenType.Array))
+				return;
+
+			process_SettingsJson(jObj =>
+			{
+				var array = (JArray)jObj.SelectToken(jsonPath);
+				if (position < array.Count)
+					array.RemoveAt(position);
+			});
+		}
+
 		/// <summary>only insert if not exists</summary>
 		public static void Settings_Insert(string key, string value)
 			=> process_SettingsJson(jObj => jObj.TryAdd(key, value));

AudibleUtilities/ApiExtended.cs

@@ -127,11 +127,18 @@ namespace AudibleUtilities
 //    items = AudibleApi.Common.Converter.FromJson<List<Item>>(System.IO.File.ReadAllText(library_json));
 //}
 #endif
+
+			Serilog.Log.Logger.Debug("Begin initial library scan");
+
 			if (!items.Any())
 				items = await Api.GetAllLibraryItemsAsync(responseGroups);
 
+			Serilog.Log.Logger.Debug("Initial library scan complete. Begin episode scan");
+
 			await manageEpisodesAsync(items, importEpisodes);
 
+			Serilog.Log.Logger.Debug("Episode scan complete");
+
 #if DEBUG
 //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
 #endif

AudibleUtilities/AudibleUtilities.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AudibleApi" Version="2.6.1.1" />
+    <PackageReference Include="AudibleApi" Version="2.7.2.1" />
   </ItemGroup>
 
   <ItemGroup>

DataLayer/DataLayer.csproj

@@ -13,12 +13,12 @@
   
   <ItemGroup>
     <PackageReference Include="Dinah.EntityFrameworkCore" Version="4.0.0.3" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.0">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.0" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.0">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.3" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

FileManager/FileManager.csproj

@@ -5,8 +5,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core" Version="4.0.3.1" />
-    <PackageReference Include="Polly" Version="7.2.2" />
+    <PackageReference Include="Dinah.Core" Version="4.0.4.1" />
+    <PackageReference Include="Polly" Version="7.2.3" />
   </ItemGroup>
 
 </Project>

FileManager/PersistentDictionary.cs

@@ -208,7 +208,7 @@ namespace FileManager
             {
                 var msg = "Unrecoverable error. Settings file cannot be found";
                 var ex = new FileNotFoundException(msg, Filepath);
-                Serilog.Log.Logger.Error(msg, ex);
+                Serilog.Log.Logger.Error(ex, msg);
                 throw ex;
             }
 
@@ -226,7 +226,7 @@ namespace FileManager
             {
                 var msg = "Unrecoverable error. Unable to read settings from Settings file";
                 var ex = new NullReferenceException(msg);
-                Serilog.Log.Logger.Error(msg, ex);
+                Serilog.Log.Logger.Error(ex, msg);
                 throw ex;
             }
 

LibationFileManager/Configuration.cs

@@ -270,7 +270,7 @@ namespace LibationFileManager
                 var valueWasChanged = persistentDictionary.SetWithJsonPath("Serilog", "MinimumLevel", value.ToString());
                 if (!valueWasChanged)
                 {
-                    Log.Logger.Information("LogLevel.set attempt. No change");
+                    Log.Logger.Debug("LogLevel.set attempt. No change");
                     return;
                 }
 

LibationFileManager/FilePathCache.cs

@@ -26,6 +26,18 @@ namespace LibationFileManager
 			if (File.Exists(jsonFile))
 			{
 				var list = JsonConvert.DeserializeObject<List<CacheEntry>>(File.ReadAllText(jsonFile));
+
+				// file exists but deser is null. this will never happen when file is healthy
+				if (list is null)
+				{
+					lock (locker)
+					{
+						Serilog.Log.Logger.Error("Error deserializing file. Wrong format. Possibly corrupt. Deleting file. {@DebugInfo}", new { jsonFile });
+						File.Delete(jsonFile);
+						return;
+					}
+				}
+
 				cache = new Cache<CacheEntry>(list);
 			}
         }

LibationFileManager/LibationFileManager.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
+    <PackageReference Include="Serilog.Exceptions" Version="8.1.0" />
   </ItemGroup>
 
   <ItemGroup>

LibationWinForms/LibationWinForms.csproj

@@ -28,7 +28,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.3.1" />
+    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.4.1" />
   </ItemGroup>
 
   <ItemGroup>

README.md

@@ -293,14 +293,3 @@ export library to file
   libationcli export -p "C:\foo\bar\my.xlsx" --xlsx
   libationcli export -p "C:\foo\bar\my.xlsx" -x
 ```
-
-Currently logs are written to Console and to file. This means they'll be printed in the CLI. To disable, find this in Settings.json and delete the 3 lines after `"WriteTo": [`
-
-```
-  "Serilog": {
-    "MinimumLevel": "Information",
-    "WriteTo": [
-      {
-        "Name": "Console"
-      },
-```

_Tests/AudibleUtilities.Tests/AudibleUtilities.Tests.csproj

@@ -7,12 +7,12 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.3.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
-    <PackageReference Include="Moq" Version="4.16.1" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="Moq" Version="4.17.2" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/FileLiberator.Tests/FileLiberator.Tests.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.3.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/FileManager.Tests/FileManager.Tests.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.3.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/LibationFileManager.Tests/LibationFileManager.Tests.csproj

@@ -7,11 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.3.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj

@@ -7,12 +7,12 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.3.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.0.0" />
-    <PackageReference Include="Moq" Version="4.16.1" />
+    <PackageReference Include="FluentAssertions" Version="6.5.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="Moq" Version="4.17.2" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
-    <PackageReference Include="coverlet.collector" Version="3.1.0">
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>