Libation/Source/FileManager/BackgroundFileSystem.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace FileManager;

/// <summary>
/// Tracks actual locations of files.
/// </summary>
public class BackgroundFileSystem : IDisposable
{
	public LongPath? RootDirectory { get; private set; }
	public string SearchPattern { get; private set; }
	public SearchOption SearchOption { get; private set; }

	private FileSystemWatcher? fileSystemWatcher { get; set; }
	private BlockingCollection<FileSystemEventArgs>? directoryChangesEvents { get; set; }
	private Task? backgroundScanner { get; set; }

	private Lock fsCacheLocker { get; } = new();
	private List<LongPath> fsCache { get; } = new();

	public BackgroundFileSystem(LongPath rootDirectory, string searchPattern, SearchOption searchOptions)
	{
		RootDirectory = rootDirectory;
		SearchPattern = searchPattern;
		SearchOption = searchOptions;

		Init();
	}

	public LongPath? FindFile(System.Text.RegularExpressions.Regex regex)
	{
		lock (fsCacheLocker)
			return fsCache.FirstOrDefault(s => regex.IsMatch(s));
	}

	public List<LongPath> FindFiles(System.Text.RegularExpressions.Regex regex)
	{
		lock (fsCacheLocker)
			return fsCache.Where(s => regex.IsMatch(s)).ToList();
	}

	public void RefreshFiles()
	{
		lock (fsCacheLocker)
		{
			fsCache.Clear();
			if (Directory.Exists(RootDirectory))
				fsCache.AddRange(SafestEnumerateFiles(RootDirectory));
		}
	}

	private void Init()
	{
		Stop();

		lock (fsCacheLocker)
		{
			if (!Directory.Exists(RootDirectory))
			{
				RootDirectory = null;
				return;
			}
			fsCache.AddRange(SafestEnumerateFiles(RootDirectory));
		}

		try
		{
			directoryChangesEvents = new BlockingCollection<FileSystemEventArgs>();
			fileSystemWatcher = new FileSystemWatcher(RootDirectory)
			{
				IncludeSubdirectories = true,
				EnableRaisingEvents = true
			};
			fileSystemWatcher.Created += FileSystemWatcher_Changed;
			fileSystemWatcher.Deleted += FileSystemWatcher_Changed;
			fileSystemWatcher.Renamed += FileSystemWatcher_Changed;
			fileSystemWatcher.Error += FileSystemWatcher_Error;

			backgroundScanner = new Task(BackgroundScanner);
			backgroundScanner.Start();
		}
		// Watching a directory fails for the same reasons reading one does, and a removable drive can be pulled
		// between the listing above and this. Constructing this type happens inside the static initializer that
		// builds the Books file cache, and the runtime caches a failed initializer for the life of the process:
		// one throw here would be rethrown at every later reader of the Books directory. Give up the live
		// updates instead. Dropping RootDirectory has the owner rebuild this once the directory works again.
		catch (Exception ex)
		{
			Serilog.Log.Logger.Error(ex, "Could not watch a directory for changes, so its file cache was abandoned: {@DebugText}", new { path = (string?)RootDirectory });
			Stop();
			RootDirectory = null;
		}
	}
	private void Stop()
	{
		//Stop raising events. Detach first so a handler cannot run after this returns, and clear the field so a
		//second Stop() is harmless.
		var watcher = fileSystemWatcher;
		fileSystemWatcher = null;
		if (watcher is not null)
		{
			watcher.Created -= FileSystemWatcher_Changed;
			watcher.Deleted -= FileSystemWatcher_Changed;
			watcher.Renamed -= FileSystemWatcher_Changed;
			watcher.Error -= FileSystemWatcher_Error;
			watcher.Dispose();
		}

		try
		{
			//Calling CompleteAdding() will cause background scanner to terminate.
			directoryChangesEvents?.CompleteAdding();
		}
		// if directoryChangesEvents is non-null and isDisposed, this exception is thrown. there's no other way to check >:(
		catch (ObjectDisposedException) { }

		//Wait for background scanner to terminate before reinitializing.
		try
		{
			backgroundScanner?.Wait();
		}
		// A scanner that died still has to be waited on, but its failure is not this caller's to raise: Stop() is
		// reached from Refresh() and from Dispose(), neither of which has anything to do with whatever went wrong,
		// and both of which are about to replace the scanner anyway.
		catch (AggregateException ex)
		{
			Serilog.Log.Logger.Error(ex, "The file cache's background scanner had already stopped on an error.");
		}

		//Dispose of directoryChangesEvents after backgroundScanner exists. Clear the field first so a late event
		//has nothing to add to. CompleteAdding has to have happened while it was still set, or the scanner would
		//still be blocked waiting for it.
		var events = directoryChangesEvents;
		directoryChangesEvents = null;
		events?.Dispose();

		lock (fsCacheLocker)
			fsCache.Clear();
	}

	private void FileSystemWatcher_Error(object sender, ErrorEventArgs e)
	{
		Init();
	}

	private void FileSystemWatcher_Changed(object sender, FileSystemEventArgs e)
	{
		try
		{
			directoryChangesEvents?.Add(e);
		}
		// Stop() completes and disposes the collection, but events the OS had already buffered still arrive after
		// that. On Windows they arrive on a native completion callback, where an exception does not fail a call -
		// it takes the process down. Dropping them is correct: whoever called Stop() is either reinitializing,
		// which rebuilds the cache from disk, or disposing.
		//
		// Covers both ways the collection refuses: completed for additions, and disposed, whose
		// ObjectDisposedException derives from this.
		catch (InvalidOperationException) { }
	}

	#region Background Thread
	private void BackgroundScanner()
	{
		while (directoryChangesEvents?.TryTake(out var change, -1) is true)
		{
			try
			{
				lock (fsCacheLocker)
					UpdateLocalCache(change);
			}
			// One path this cannot read must not end the scan. An exception here used to be terminal twice over:
			// it stopped the cache tracking anything further, and it was stored on the task, so the next Stop()
			// rethrew it as an AggregateException at whoever had called Refresh() - a caller with nothing to do
			// with the file that went missing.
			catch (Exception ex)
			{
				Serilog.Log.Logger.Debug(ex, "Could not apply a file system change to the file cache: {@DebugText}", new { change.ChangeType, change.FullPath });
			}
		}
	}

	private void UpdateLocalCache(FileSystemEventArgs change)
	{
		if (change.ChangeType == WatcherChangeTypes.Deleted)
		{
			RemovePath(change.FullPath);
		}
		else if (change.ChangeType == WatcherChangeTypes.Created)
		{
			AddPath(change.FullPath);
		}
		else if (change.ChangeType == WatcherChangeTypes.Renamed && change is RenamedEventArgs renameChange)
		{
			RemovePath(renameChange.OldFullPath);
			AddPath(renameChange.FullPath);
		}
	}

	private void RemovePath(LongPath path)
	{
		path = path.LongPathName;
		var pathsToRemove = fsCache.Where(p => ((string)p).StartsWith(path)).ToArray();

		foreach (var p in pathsToRemove)
			fsCache.Remove(p);
	}

	private void AddPath(LongPath path)
	{
		path = path.LongPathName;
		//Temporary files created when updating the db will disappear before their attributes can be read.
		if (Path.GetFileName(path).Contains("LibationContext.db"))
			return;

		// Whether it exists and what it is were two questions, and the answer to the first could stop being true
		// before the second was asked: a download's temp file, or a folder the user has just moved or deleted, is
		// gone by the time its attributes are read. One question instead, and its failure means the same thing
		// the existence check meant - there is nothing here to add.
		if (TryGetAttributes(path) is not FileAttributes attributes)
			return;

		if (attributes.HasFlag(FileAttributes.Directory))
			AddUniqueFiles(SafestEnumerateFiles(path));
		else
			AddUniqueFile(path);
	}

	/// <returns>What the path is, or null when it cannot be read and so has nothing to add.</returns>
	internal static FileAttributes? TryGetAttributes(LongPath path)
	{
		try
		{
			return File.GetAttributes(path);
		}
		catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException)
		{
			Serilog.Log.Logger.Debug(ex, "Nothing to add to the file cache for {@DebugText}", new { path = (string)path });
			return null;
		}
	}

	private IEnumerable<LongPath> SafestEnumerateFiles(string path)
	{
		try
		{
			return FileUtility.SaferEnumerateFiles(path, SearchPattern, SearchOption);
		}
		catch
		{
			return [];
		}
	}

	private void AddUniqueFiles(IEnumerable<LongPath> newFiles)
	{
		foreach (var file in newFiles)
			AddUniqueFile(file);
	}

	private void AddUniqueFile(LongPath newFile)
	{
		if (!fsCache.Contains(newFile))
			fsCache.Add(newFile);
	}

	#endregion

	public void Dispose()
	{
		Stop();
		GC.SuppressFinalize(this);
	}
}