Libation/Source/LibationSearchEngine/SearchEngine.cs

using DataLayer;
using Dinah.Core;
using FileManager;
using LibationFileManager;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Store;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;

namespace LibationSearchEngine;

public class SearchEngine
{
	public const Lucene.Net.Util.Version Version = Lucene.Net.Util.Version.LUCENE_30;

	public const string _ID_ = "_ID_";
	public const string TAGS = "tags";
	// special field for each book which includes all major parts of the book's metadata. enables non-targetting searching
	public const string ALL = "all";

	#region index rules

	private static bool isAuthorNarrated(Book book)
	{
		var authors = book.Authors.Select(a => a.Name).ToArray();
		var narrators = book.Narrators.Select(a => a.Name).ToArray();
		return authors.Intersect(narrators).Any();
	}

	// use these common fields in the "all" default search field
	public static IndexRuleCollection FieldIndexRules { get; } = new IndexRuleCollection
		{
			{ FieldType.ID, lb => lb.Book.AudibleProductId.ToLowerInvariant(), nameof(Book.AudibleProductId), "ProductId", "Id", "ASIN" },
			{ FieldType.Raw, lb => lb.Book.AudibleProductId, _ID_ },
			{ FieldType.String, lb => lb.Book.TitleWithSubtitle, "Title", "ProductId", "Id", "ASIN" },
			{ FieldType.String, lb => lb.Book.AuthorNames, "AuthorNames", "Author", "Authors" },
			{ FieldType.String, lb => lb.Book.NarratorNames, "NarratorNames", "Narrator", "Narrators" },
			{ FieldType.String, lb => lb.Book.Publisher, nameof(Book.Publisher) },
			{ FieldType.String, lb => lb.Book.SeriesNames(), "SeriesNames", "Narrator", "Series" },
			{ FieldType.String, lb => string.Join(", ", lb.Book.SeriesLink.Select(s => s.Series.AudibleSeriesId)), "SeriesId" },
			{ FieldType.String, lb => lb.Book.AllCategoryIds() is not { } categories ? null : string.Join(", ", categories), "CategoriesId", "CategoryId" },
			{ FieldType.String, lb => lb.Book.AllCategoryNames() is null ? null : string.Join(", ", lb.Book.AllCategoryNames()), "Category", "Categories", "CategoriesNames" },
			{ FieldType.String, lb => lb.Book.UserDefinedItem.Tags, TAGS.FirstCharToUpper() },
			{ FieldType.String, lb => lb.Book.Locale, "Locale", "Region" },
			{ FieldType.String, lb => lb.Account, "Account", "Email" },
			{ FieldType.String, lb => lb.Book.UserDefinedItem.LastDownloadedFormat?.CodecString, "Codec", "DownloadedCodec" },
			{ FieldType.Bool, lb => lb.Book.HasPdf.ToString(), "HasDownloads", "HasDownload", "Downloads" , "Download", "HasPDFs", "HasPDF" , "PDFs", "PDF" },
			{ FieldType.Bool, lb => (lb.Book.UserDefinedItem.Rating.OverallRating > 0f).ToString(), "IsRated", "Rated" },
			{ FieldType.Bool, lb => isAuthorNarrated(lb.Book).ToString(), "IsAuthorNarrated", "AuthorNarrated" },
			{ FieldType.Bool, lb => lb.Book.IsAbridged.ToString(), nameof(Book.IsAbridged), "Abridged" },
			{ FieldType.Bool, lb => lb.Book.IsSpatial.ToString(), nameof(Book.IsSpatial), "Spatial" },
			{ FieldType.Bool, lb => (lb.Book.UserDefinedItem.BookStatus == LiberatedStatus.Liberated).ToString(), "IsLiberated", "Liberated" },
			{ FieldType.Bool, lb => (lb.Book.UserDefinedItem.BookStatus == LiberatedStatus.Error).ToString(), "LiberatedError" },
			{ FieldType.Bool, lb => lb.Book.IsEpisodeChild().ToString(), "Podcast", "Podcasts", "IsPodcast", "Episode", "Episodes", "IsEpisode" },
			{ FieldType.Bool, lb => lb.AbsentFromLastScan.ToString(), "AbsentFromLastScan", "Absent" },
			{ FieldType.Bool, lb => (!string.IsNullOrWhiteSpace(lb.Book.SeriesNames())).ToString(), "IsInSeries", "InSeries" },
			{ FieldType.Bool, lb => lb.Book.UserDefinedItem.IsFinished.ToString(), nameof(UserDefinedItem.IsFinished), "Finished", "IsFinished" },
			{ FieldType.Bool, lb => lb.IsAudiblePlus.ToString(), nameof(LibraryBook.IsAudiblePlus), "AudiblePlus", "Plus" },
            // all numbers are padded to 8 char.s
            // This will allow a single method to auto-pad numbers. The method will match these as well as date: yyyymmdd
            { FieldType.Number, lb => lb.Book.LengthInMinutes.ToLuceneString(), nameof(Book.LengthInMinutes), "Length", "Minutes" },
			{ FieldType.Number, lb => (lb.Book.LengthInMinutes / 60).ToLuceneString(), "Hours" },
			{ FieldType.Number, lb => lb.Book.Rating.OverallRating.ToLuceneString(), "ProductRating", "Rating" },
			{ FieldType.Number, lb => lb.Book.UserDefinedItem.Rating.OverallRating.ToLuceneString(), "UserRating", "MyRating" },
			{ FieldType.Number, lb => lb.Book.DatePublished?.ToLuceneString() ?? "", nameof(Book.DatePublished) },
			{ FieldType.Number, lb => lb.Book.UserDefinedItem.LastDownloaded.ToLuceneString(), nameof(UserDefinedItem.LastDownloaded), "LastDownload" },
		{ FieldType.Number, lb => lb.Book.UserDefinedItem.LastDownloadedFormat?.BitRate.ToLuceneString(), "Bitrate", "DownloadedBitrate" },
		{ FieldType.Number, lb => lb.Book.UserDefinedItem.LastDownloadedFormat?.SampleRate.ToLuceneString(), "SampleRate", "DownloadedSampleRate" },
		{ FieldType.Number, lb => lb.DateAdded.ToLuceneString(), nameof(LibraryBook.DateAdded) }
	};
	#endregion

	#region create and update index
	/// <summary>create new. ie: full re-index</summary>
	public void CreateNewIndex(IEnumerable<LibraryBook> library, bool overwrite = true)
	{
		var libraryList = library.ToList();

		// location of index/create the index
		using var index = getIndex();

		// analyzer for tokenizing text. same analyzer should be used for indexing and searching
		using var analyzer = new StandardAnalyzer(Version);
		using var ixWriter = openWriterForRebuild(index, analyzer, overwrite);

		foreach (var libraryBook in libraryList)
		{
			var doc = createBookIndexDocument(libraryBook);
			ixWriter.AddDocument(doc);
		}
	}

	/// <summary>
	/// Opens the writer for a full re-index, waiting out lock conflicts and repairing an index Lucene cannot open.
	/// </summary>
	private IndexWriter openWriterForRebuild(Lucene.Net.Store.Directory index, StandardAnalyzer analyzer, bool overwrite)
	{
		// Exponential backoff for lock conflicts: 400 ms, 800 ms, 1600 ms, 3200 ms. 6.4 sec total before giving up.
		const int maxAttempts = 5;
		const int baseDelayMs = 400;
		var repairsRemaining = 1;

		for (var attempt = 1; ; attempt++)
		{
			try
			{
				var createNewIndex = overwrite || !IndexReader.IndexExists(index);
				return new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
			}
			catch (Exception ex) when (isTransientLockConflict(ex) && attempt < maxAttempts)
			{
				var delayMs = baseDelayMs * (1 << (attempt - 1));
				Serilog.Log.Logger.Warning(ex, "Search index lock conflict (attempt {Attempt}/{Max}), retrying in {Delay}ms", attempt, maxAttempts, delayMs);
				Thread.Sleep(delayMs);
			}
			catch (Exception ex) when (!isTransientLockConflict(ex) && repairsRemaining-- > 0)
			{
				// Passing overwrite/create to IndexWriter does not repair an unreadable index: its IndexFileDeleter
				// reads every segments_* file in the directory and only tolerates missing ones, so a single unreadable
				// segments file -- even a stale one from an older commit -- keeps the whole directory unusable until it
				// is deleted. This is a full re-index, so nothing on disk is worth preserving.
				Serilog.Log.Logger.Error(ex, "Search index at {Path} could not be opened. Deleting it and rebuilding from the library.", SearchEngineDirectory);

				if (deleteAllSearchIndexFiles(SearchEngineDirectory) is { Count: > 0 } undeletable)
					throw new IOException($"The search index at '{SearchEngineDirectory}' is damaged and could not be deleted automatically. Close Libation, delete that folder, then restart. Undeletable file(s): {string.Join(", ", undeletable)}", ex);

				// the repair, not the failed open, gets a fresh allowance of lock retries
				attempt = 0;
			}
		}
	}

	/// <summary>
	/// True when the index could not be opened because something else is holding it, which is worth waiting out.
	/// <see cref="LockObtainFailedException"/> derives from <see cref="IOException"/>, so a bare
	/// <c>IOException</c> check cannot tell a lock conflict apart from a damaged index.
	/// </summary>
	private static bool isTransientLockConflict(Exception ex)
		=> ex is LockObtainFailedException
		// Windows may report "file in use" as UnauthorizedAccessException
		|| ex is UnauthorizedAccessException;

	/// <summary>
	/// True when Lucene cannot read the index and the only cure is deleting it and rebuilding from the database.
	/// <list type="bullet">
	/// <item><see cref="CorruptIndexException"/>: eg. checksum mismatch in the segments file.</item>
	/// <item>A truncated or zero-length <c>segments_*</c> file, which Lucene 3 reports as a plain
	/// <see cref="IOException"/> from <c>BufferedIndexInput.Refill</c> rather than as a
	/// <see cref="CorruptIndexException"/>. Matched by message because the type is shared with
	/// <see cref="LockObtainFailedException"/>, which must keep retrying instead.</item>
	/// <item>Lucene 3 parses <c>segments_*</c> filenames in the index directory. Cloud sync (eg. OneDrive) can leave
	/// debris or conflict copies whose names break that parser, throwing <see cref="ArgumentException"/> with a
	/// message like "Invalid or unsupported character in number", hence this string check.</item>
	/// </list>
	/// </summary>
	public static bool IsRecoverableCorruptIndexException(Exception ex)
		=> ex is CorruptIndexException
		|| (ex is IOException && !isTransientLockConflict(ex) && ex.Message.Contains("read past EOF", StringComparison.OrdinalIgnoreCase))
		|| (ex is ArgumentException aex && aex.Message.Contains("character in number", StringComparison.OrdinalIgnoreCase));

	/// <summary>
	/// Best-effort delete of everything under the index directory. Returns the <c>segments_*</c> files that survived:
	/// Lucene rebuilds happily in an empty directory, and ignores leftover segment data, but any segments file it
	/// cannot read keeps the directory permanently unusable.
	/// </summary>
	private static List<string> deleteAllSearchIndexFiles(string searchEngineDirectory)
	{
		if (!System.IO.Directory.Exists(searchEngineDirectory))
			return [];

		foreach (var file in System.IO.Directory.GetFiles(searchEngineDirectory, "*", SearchOption.AllDirectories))
			FileUtility.TrySaferDelete(file);

		foreach (var dir in System.IO.Directory.GetDirectories(searchEngineDirectory, "*", SearchOption.AllDirectories).OrderByDescending(d => d.Length))
		{
			try
			{
				System.IO.Directory.Delete(dir);
			}
			catch (Exception ex)
			{
				Serilog.Log.Logger.Warning(ex, "Could not remove search index subdirectory {Dir}", dir);
			}
		}

		return System.IO.Directory.Exists(searchEngineDirectory)
			? [.. System.IO.Directory.GetFiles(searchEngineDirectory, "segments*", SearchOption.AllDirectories)]
			: [];
	}

	public SearchEngine(string? directory = null)
	{
		SearchEngineDirectory = directory ?? new DirectoryInfo(Configuration.Instance.LibationFiles.Location).CreateSubdirectoryEx("SearchEngine").FullName;
	}

	/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
	public void UpdateBook(LibationContext context, string productId)
	{
		if (context.GetLibraryBook_Flat_NoTracking(productId) is not { } libraryBook)
			return;

		var document = createBookIndexDocument(libraryBook);
		var createNewIndex = false;

		var term = new Term(_ID_, productId);
		using var index = getIndex();
		using var analyzer = new StandardAnalyzer(Version);
		using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
		ixWriter.DeleteDocuments(term);
		ixWriter.AddDocument(document);
	}

	private static Document createBookIndexDocument(LibraryBook libraryBook)
	{
		var doc = new Document();

		// concat all common fields for the default 'all' field
		var allConcat =
			FieldIndexRules
			.Select(rule => rule.GetValue(libraryBook))
			.OfType<string>()
		.Aggregate((a, b) => $"{a} {b}");
		doc.AddAnalyzed(ALL, allConcat);

		foreach (var rule in FieldIndexRules)
			doc.AddIndexRule(rule, libraryBook);

		return doc;
	}

	// update single document entry
	// all fields, including 'tags' are case-specific
	public void UpdateTags(string productId, string tags) => updateAnalyzedField(productId, TAGS, tags);

	// all fields are case-specific
	private void updateAnalyzedField(string productId, string fieldName, string newValue)
		=> updateDocument(
			productId,
			d =>
			{
				d.RemoveField(fieldName.ToLower());
				d.AddAnalyzed(fieldName, newValue);
			});

	// update single document entry
	public void UpdateLiberatedStatus(LibraryBook book)
		=> updateDocument(
			book.Book.AudibleProductId,
			d =>
			{
				if (FieldIndexRules.GetRuleByFieldName("IsLiberated") is { } lib)
				{
					d.RemoveRule(lib);
					d.AddIndexRule(lib, book);
				}
				if (FieldIndexRules.GetRuleByFieldName("LiberatedError") is { } libError)
				{
					d.RemoveRule(libError);
					d.AddIndexRule(libError, book);
				}
				if (FieldIndexRules.GetRuleByFieldName(nameof(UserDefinedItem.LastDownloaded)) is { } lastDl)
				{
					d.RemoveRule(lastDl);
					d.AddIndexRule(lastDl, book);
				}
			});

	public void UpdateUserRatings(LibraryBook book)
		=> updateDocument(
			book.Book.AudibleProductId,
			d =>
		{
			if (FieldIndexRules.GetRuleByFieldName("UserRating") is { } rating)
			{
				d.RemoveRule(rating);
				d.AddIndexRule(rating, book);
			}
		});

	private void updateDocument(string productId, Action<Document> action)
	{
		var productTerm = new Term(_ID_, productId);

		using var index = getIndex();

		// get existing document
		using var searcher = new IndexSearcher(index);
		var query = new TermQuery(productTerm);
		var docs = searcher.Search(query, 1);
		var scoreDoc = docs.ScoreDocs.SingleOrDefault();
		if (scoreDoc is null)
			return;
		var document = searcher.Doc(scoreDoc.Doc);

		// perform update
		action(document);

		// update index
		var createNewIndex = false;
		using var analyzer = new StandardAnalyzer(Version);
		using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
		ixWriter.UpdateDocument(productTerm, document, analyzer);
	}
	#endregion

	// the workaround which allows displaying all books when query is empty
	public const string ALL_QUERY = "*:*";

	#region search
	public SearchResultSet Search(string searchString)
	{
		using var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);

		Serilog.Log.Logger.Debug("original search string: {@DebugInfo}", new { searchString });
		searchString = QuerySanitizer.Sanitize(searchString, analyzer);
		Serilog.Log.Logger.Debug("formatted search string: {@DebugInfo}", new { searchString });

		var results = generalSearch(searchString, analyzer);
		Serilog.Log.Logger.Debug("Hit(s): {@DebugInfo}", new { count = results.Docs.Count() });
		displayResults(results);

		return results;
	}

	private SearchResultSet generalSearch(string searchString, StandardAnalyzer analyzer)
	{
		var defaultField = ALL;

		using var index = getIndex();
		using var searcher = new IndexSearcher(index);
		var query = analyzer.GetQuery(defaultField, searchString);

		// lucene doesn't allow only negations. eg this returns nothing:
		//     -tags:hidden
		// work arounds: https://kb.ucla.edu/articles/pure-negation-query-in-lucene
		// HOWEVER, doing this to any other type of query can cause EVERYTHING to be a match unless "Occur" is carefully set
		// this should really check that all leaf nodes are MUST_NOT
		if (query is BooleanQuery boolQuery)
		{
			var occurs = getOccurs_recurs(boolQuery);
			if (occurs.Any() && occurs.All(o => o == Occur.MUST_NOT))
				boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
		}

		var docs = searcher
			.Search(query, searcher.MaxDoc + 1)
			.ScoreDocs
			.Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score))
			.ToList();
		var queryString = query.ToString();
		Serilog.Log.Logger.Debug("query: {@DebugInfo}", new { queryString });
		return new SearchResultSet(queryString, docs);
	}

	private IEnumerable<Occur> getOccurs_recurs(BooleanQuery query)
	{
		var returnList = new List<Occur>();

		foreach (var clause in query)
		{
			returnList.Add(clause.Occur);

			if (clause.Query is BooleanQuery boolQuery)
				returnList.AddRange(getOccurs_recurs(boolQuery));
		}

		return returnList;
	}

	private void displayResults(SearchResultSet docs)
	{
		//for (int i = 0; i < docs.Docs.Count(); i++)
		//{
		//    var sde = docs.Docs.First();

		//    Document doc = sde.Doc;
		//    float score = sde.Score;

		//    Serilog.Log.Logger.Debug($"{(i + 1)}) score={score}. Fields:");
		//    var allFields = doc.GetFields();
		//    foreach (var f in allFields)
		//        Serilog.Log.Logger.Debug($"   [{f.Name}]={f.StringValue}");
		//}
	}
	#endregion

	private Lucene.Net.Store.Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);

	//Defaults  to "LibationFiles/SearchEngine, but can be overridden
	//in constructor for use in TrashBinDialog search
	private string SearchEngineDirectory { get; }
}