Bug fix #225 : SaferEnumerateFiles will skip files with UnauthorizedAccessException

CLI manual change to settings file no longer needed

AppScaffolding/AppScaffolding.csproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
-    <Version>6.6.9.1</Version>
+    <Version>6.7.5.1</Version>
   </PropertyGroup>
 
   <ItemGroup>

AppScaffolding/LibationScaffolding.cs

@@ -230,10 +230,10 @@ namespace AppScaffolding
 				config.InProgress,
 
 				AudibleFileStorage.DownloadsInProgressDirectory,
-				DownloadsInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
+				DownloadsInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DownloadsInProgressDirectory).Count(),
 
 				AudibleFileStorage.DecryptInProgressDirectory,
-				DecryptInProgressFiles = Directory.EnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
+				DecryptInProgressFiles = FileManager.FileUtility.SaferEnumerateFiles(AudibleFileStorage.DecryptInProgressDirectory).Count(),
 			});
 		}
 

AudibleUtilities/ApiExtended.cs

@@ -127,11 +127,18 @@ namespace AudibleUtilities
 //    items = AudibleApi.Common.Converter.FromJson<List<Item>>(System.IO.File.ReadAllText(library_json));
 //}
 #endif
+
+			Serilog.Log.Logger.Debug("Begin initial library scan");
+
 			if (!items.Any())
 				items = await Api.GetAllLibraryItemsAsync(responseGroups);
 
+			Serilog.Log.Logger.Debug("Initial library scan complete. Begin episode scan");
+
 			await manageEpisodesAsync(items, importEpisodes);
 
+			Serilog.Log.Logger.Debug("Episode scan complete");
+
 #if DEBUG
 //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
 #endif

AudibleUtilities/AudibleUtilities.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AudibleApi" Version="2.6.2.1" />
+    <PackageReference Include="AudibleApi" Version="2.7.2.1" />
   </ItemGroup>
 
   <ItemGroup>

DataLayer/DataLayer.csproj

@@ -13,12 +13,12 @@
   
   <ItemGroup>
     <PackageReference Include="Dinah.EntityFrameworkCore" Version="4.0.0.3" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.2">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.2" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.2">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="6.0.3" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="6.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

FileManager/BackgroundFileSystem.cs

@@ -43,7 +43,7 @@ namespace FileManager
             lock (fsCacheLocker)
             {
                 fsCache.Clear();
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
             }
         }
 
@@ -52,7 +52,7 @@ namespace FileManager
             Stop();
 
             lock (fsCacheLocker)
-                fsCache.AddRange(Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption));
+                fsCache.AddRange(FileUtility.SaferEnumerateFiles(RootDirectory, SearchPattern, SearchOption));
 
             directoryChangesEvents = new BlockingCollection<FileSystemEventArgs>();
 			fileSystemWatcher = new FileSystemWatcher(RootDirectory)
@@ -135,7 +135,7 @@ namespace FileManager
         private void AddPath(string path)
         {
             if (File.GetAttributes(path).HasFlag(FileAttributes.Directory))
-                AddUniqueFiles(Directory.EnumerateFiles(path, SearchPattern, SearchOption));
+                AddUniqueFiles(FileUtility.SaferEnumerateFiles(path, SearchPattern, SearchOption));
             else
                 AddUniqueFile(path);
         }

FileManager/FileManager.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core" Version="4.0.3.1" />
+    <PackageReference Include="Dinah.Core" Version="4.0.4.1" />
     <PackageReference Include="Polly" Version="7.2.3" />
   </ItemGroup>
 

FileManager/FileUtility.cs

@@ -139,7 +139,7 @@ namespace FileManager
             return path[0] + remainder;
         }
 
-        private static string removeInvalidWhitespace_pattern { get; } = $@"\s*\{Path.DirectorySeparatorChar}\s*";
+        private static string removeInvalidWhitespace_pattern { get; } = $@"[\s\.]*\{Path.DirectorySeparatorChar}\s*";
         private static Regex removeInvalidWhitespace_regex { get; } = new(removeInvalidWhitespace_pattern, RegexOptions.Compiled | RegexOptions.IgnorePatternWhitespace);
 
         /// <summary>no part of the path may begin or end in whitespace</summary>
@@ -149,14 +149,20 @@ namespace FileManager
             // replace whitespace around path slashes
             //    regex (with space added for clarity)
             //    \s*  \\  \s*    =>    \
-            fullfilename = fullfilename.Trim();
-            
-            fullfilename = removeInvalidWhitespace_regex.Replace(fullfilename, @"\");
+            // no ending dots. beginning dots are valid
+
+            // regex is easier by ending with separator
+            fullfilename += Path.DirectorySeparatorChar;
+            fullfilename = removeInvalidWhitespace_regex.Replace(fullfilename, Path.DirectorySeparatorChar.ToString());
+            // take seperator back off
+            fullfilename = RemoveLastCharacter(fullfilename);
 
             fullfilename = removeDoubleSlashes(fullfilename);
             return fullfilename;
         }
 
+        public static string RemoveLastCharacter(this string str) => string.IsNullOrEmpty(str) ? str : str[..^1];
+
         /// <summary>
         /// Move file.
         /// <br/>- Ensure valid file name path: remove invalid chars, ensure uniqueness, enforce max file length
@@ -183,15 +189,19 @@ namespace FileManager
             {
                 try
                 {
-                    if (File.Exists(source))
+                    if (!File.Exists(source))
                     {
-                        File.Delete(source);
-                        Serilog.Log.Logger.Information("File successfully deleted", new { source });
+                        Serilog.Log.Logger.Debug("No file to delete: {@DebugText}", new { source });
+                        return;
                     }
+
+                    Serilog.Log.Logger.Debug("Attempt to delete file: {@DebugText}", new { source });
+                    File.Delete(source);
+                    Serilog.Log.Logger.Information("File successfully deleted: {@DebugText}", new { source });
                 }
                 catch (Exception e)
                 {
-                    Serilog.Log.Logger.Error(e, "Failed to delete file", new { source });
+                    Serilog.Log.Logger.Error(e, "Failed to delete file: {@DebugText}", new { source });
                     throw;
                 }
             });
@@ -202,19 +212,61 @@ namespace FileManager
             {
                 try
                 {
-                    if (File.Exists(source))
+                    if (!File.Exists(source))
                     {
-                        SaferDelete(destination);
-                        Directory.CreateDirectory(Path.GetDirectoryName(destination));
-                        File.Move(source, destination);
-                        Serilog.Log.Logger.Information("File successfully moved", new { source, destination });
+                        Serilog.Log.Logger.Debug("No file to move: {@DebugText}", new { source });
+                        return;
                     }
+
+                    SaferDelete(destination);
+
+                    var dir = Path.GetDirectoryName(destination);
+                    Serilog.Log.Logger.Debug("Attempt to create directory: {@DebugText}", new { dir });
+                    Directory.CreateDirectory(dir);
+
+                    Serilog.Log.Logger.Debug("Attempt to move file: {@DebugText}", new { source, destination });
+                    File.Move(source, destination);
+                    Serilog.Log.Logger.Information("File successfully moved: {@DebugText}", new { source, destination });
                 }
                 catch (Exception e)
                 {
-                    Serilog.Log.Logger.Error(e, "Failed to move file", new { source, destination });
+                    Serilog.Log.Logger.Error(e, "Failed to move file: {@DebugText}", new { source, destination });
                     throw;
                 }
             });
+
+        /// <summary>
+        /// A safer way to get all the files in a directory and sub directory without crashing on UnauthorizedException or PathTooLongException
+        /// </summary>
+        /// <param name="rootPath">Starting directory</param>
+        /// <param name="patternMatch">Filename pattern match</param>
+        /// <param name="searchOption">Search subdirectories or only top level directory for files</param>
+        /// <returns>List of files</returns>
+        public static IEnumerable<string> SaferEnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly)
+        {
+            var foundFiles = Enumerable.Empty<string>();
+
+            if (searchOption == SearchOption.AllDirectories)
+            {
+                try
+                {
+                    IEnumerable<string> subDirs = Directory.EnumerateDirectories(path);
+                    // Add files in subdirectories recursively to the list
+                    foreach (string dir in subDirs)
+                        foundFiles = foundFiles.Concat(SaferEnumerateFiles(dir, searchPattern, searchOption));
+                }
+                catch (UnauthorizedAccessException) { }
+                catch (PathTooLongException) { }
+            }
+
+            try
+            {
+                // Add files from the current directory
+                foundFiles = foundFiles.Concat(Directory.EnumerateFiles(path, searchPattern));
+            }
+            catch (UnauthorizedAccessException) { }
+
+            return foundFiles;
+        }
     }
 }

LibationFileManager/AudibleFileStorage.cs

@@ -73,8 +73,8 @@ namespace LibationFileManager
         protected override string GetFilePathCustom(string productId)
         {
             var regex = GetBookSearchRegex(productId);
-            return Directory
-                .EnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
+            return FileUtility
+                .SaferEnumerateFiles(DownloadsInProgressDirectory, "*.*", SearchOption.AllDirectories)
                 .FirstOrDefault(s => regex.IsMatch(s));
         }
 

LibationWinForms/LibationWinForms.csproj

@@ -28,7 +28,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.3.1" />
+    <PackageReference Include="Dinah.Core.WindowsDesktop" Version="4.0.4.1" />
   </ItemGroup>
 
   <ItemGroup>

README.md

@@ -293,14 +293,3 @@ export library to file
   libationcli export -p "C:\foo\bar\my.xlsx" --xlsx
   libationcli export -p "C:\foo\bar\my.xlsx" -x
 ```
-
-Currently logs are written to Console and to file. This means they'll be printed in the CLI. To disable, find this in Settings.json and delete the 3 lines after `"WriteTo": [`
-
-```
-  "Serilog": {
-    "MinimumLevel": "Information",
-    "WriteTo": [
-      {
-        "Name": "Console"
-      },
-```

_Tests/AudibleUtilities.Tests/AudibleUtilities.Tests.csproj

@@ -9,7 +9,7 @@
   <ItemGroup>
     <PackageReference Include="FluentAssertions" Version="6.5.1" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="Moq" Version="4.17.1" />
+    <PackageReference Include="Moq" Version="4.17.2" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
     <PackageReference Include="coverlet.collector" Version="3.1.2">

_Tests/FileManager.Tests/FileUtilityTests.cs

@@ -113,4 +113,51 @@ namespace FileUtilityTests
 		public void Tests(string input, string expected)
 			=> FileUtility.GetStandardizedExtension(input).Should().Be(expected);
 	}
+
+	[TestClass]
+    public class GetValidFilename
+	{
+		[TestMethod]
+		// dot-files
+		[DataRow(@"C:\a bc\x y z\.f i l e.txt")]
+		// dot-folders
+		[DataRow(@"C:\a bc\.x y z\f i l e.txt")]
+		public void Valid(string input) => Tests(input, input);
+
+		[TestMethod]
+		// folder spaces
+		[DataRow(@"C:\   a bc   \x y z   ", @"C:\a bc\x y z")]
+		// file spaces
+		[DataRow(@"C:\a bc\x y z\   f i l e.txt   ", @"C:\a bc\x y z\f i l e.txt")]
+		// eliminate beginning space and end dots and spaces
+		[DataRow(@"C:\a bc\   . . . x y z . . .   \f i l e.txt", @"C:\a bc\. . . x y z\f i l e.txt")]
+		// file end dots
+		[DataRow(@"C:\a bc\x y z\f i l e.txt . . .", @"C:\a bc\x y z\f i l e.txt")]
+		public void Tests(string input, string expected)
+			=> FileUtility.GetValidFilename(input).Should().Be(expected);
+	}
+
+	[TestClass]
+    public class RemoveLastCharacter
+	{
+		[TestMethod]
+		public void is_null() => Tests(null, null);
+
+		[TestMethod]
+		public void empty() => Tests("", "");
+
+		[TestMethod]
+		public void single_space() => Tests(" ", "");
+
+		[TestMethod]
+		public void multiple_space() => Tests("   ", "  ");
+
+		[TestMethod]
+		[DataRow("1", "")]
+		[DataRow("1 ", "1")]
+		[DataRow("12", "1")]
+		[DataRow("123", "12")]
+		public void Tests(string input, string expected)
+			=> FileUtility.RemoveLastCharacter(input).Should().Be(expected);
+	}
 }

_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj

@@ -9,7 +9,7 @@
   <ItemGroup>
     <PackageReference Include="FluentAssertions" Version="6.5.1" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="Moq" Version="4.17.1" />
+    <PackageReference Include="Moq" Version="4.17.2" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
     <PackageReference Include="MSTest.TestFramework" Version="2.2.8" />
     <PackageReference Include="coverlet.collector" Version="3.1.2">