diff --git a/Scripts/seed-demo-library.cs b/Scripts/seed-demo-library.cs index df895a8f..cb20d520 100644 --- a/Scripts/seed-demo-library.cs +++ b/Scripts/seed-demo-library.cs @@ -3,7 +3,8 @@ using Microsoft.Data.Sqlite; using System.Text.Json; -// Seeds a Libation library with one book per Liberate-column state, for manual UI testing. +// Seeds a Libation library with one book per Liberate-column state, plus a few titles that +// subtitle removal changes, for manual UI testing. // // dotnet run Scripts/seed-demo-library.cs [--clean] [path-to-Libation-files-folder] // @@ -76,9 +77,9 @@ foreach (var book in books) insert into Books (AudibleProductId, ContentType, Description, IsAbridged, IsSpatial, LengthInMinutes, Locale, Rating_OverallRating, Rating_PerformanceRating, Rating_StoryRating, Subtitle, Title) - values ($asin, $contentType, 'Seeded by seed-demo-library.cs', 0, 0, 600, 'us', 0, 0, 0, '', $title) + values ($asin, $contentType, 'Seeded by seed-demo-library.cs', 0, 0, 600, 'us', 0, 0, 0, $subtitle, $title) """, - ("$asin", book.Asin), ("$contentType", book.ContentType), ("$title", book.Title)); + ("$asin", book.Asin), ("$contentType", book.ContentType), ("$title", book.Title), ("$subtitle", book.Subtitle)); var bookId = Scalar("select BookId from Books where AudibleProductId = $asin", ("$asin", book.Asin)); @@ -263,6 +264,27 @@ static List BuildDemoBooks() Expectation: expectation, IsAbsent: true)); + // Titles that subtitle removal changes. stops at the first colon, so it cuts a colon in + // Audible's own title just as readily as it drops Audible's subtitle field, and the grid's Title column + // shows both the same way. The HasSubtitle and TitleHasColon filters are the only way to tell them apart. + foreach (var (title, subtitle, match) in new[] + { + ("A Book Series Omnibus", "Volume One", "HasSubtitle"), + ("A Book Series Omnibus", "Volume Two", "HasSubtitle"), + ("Star Trek: The Next Generation", "", "TitleHasColon"), + ("Dune: Book One", "The Graphic Novel", "both HasSubtitle and TitleHasColon"), + }) + books.Add(new DemoBook( + Asin: $"{AsinPrefix}{++n:000}", + Title: title, + ContentType: Product, + BookStatus: NotLiberated, + PdfStatus: null, + IsPlus: false, + NeedsPartialDownload: false, + Expectation: $"red lamp; filter match: {match}", + Subtitle: subtitle)); + return books; } @@ -373,4 +395,5 @@ record DemoBook( string Expectation, string? SeriesAsin = null, string? SeriesOrder = null, - bool IsAbsent = false); + bool IsAbsent = false, + string Subtitle = ""); diff --git a/Source/LibationFileManager/Templates/Templates.cs b/Source/LibationFileManager/Templates/Templates.cs index d6c57085..12dec1df 100644 --- a/Source/LibationFileManager/Templates/Templates.cs +++ b/Source/LibationFileManager/Templates/Templates.cs @@ -392,7 +392,8 @@ public abstract class Templates #region Tag Formatters - private static string? GetTitleShort(string? title) + /// <summary>Backs the <c><title short></c> tag. Public so the search index can flag the titles it shortens.</summary> + public static string? GetTitleShort(string? title) => title != null && title.IndexOf(':') is var i && i >= 0 ? title[..i] : title; diff --git a/Source/LibationSearchEngine/SearchEngine.cs b/Source/LibationSearchEngine/SearchEngine.cs index 5ab1d4a0..41bf10c2 100644 --- a/Source/LibationSearchEngine/SearchEngine.cs +++ b/Source/LibationSearchEngine/SearchEngine.cs @@ -33,6 +33,14 @@ public class SearchEngine return authors.Intersect(narrators).Any(); } + /// <summary> + /// True when <c><title short></c> cuts the title itself rather than merely dropping Audible's subtitle + /// field. Audible ships plenty of titles with a colon in them, and those are the ones where shortening loses + /// something the user may need: "Omnibus: Volume One" and "Omnibus: Volume Two" both shorten to "Omnibus". + /// </summary> + private static bool titleIsShortened(Book book) + => LibationFileManager.Templates.Templates.GetTitleShort(book.Title) != book.Title; + // use these common fields in the "all" default search field public static IndexRuleCollection FieldIndexRules { get; } = new IndexRuleCollection { @@ -60,6 +68,8 @@ public class SearchEngine { FieldType.Bool, lb => lb.Book.IsEpisodeChild().ToString(), "Podcast", "Podcasts", "IsPodcast", "Episode", "Episodes", "IsEpisode" }, { FieldType.Bool, lb => lb.AbsentFromLastScan.ToString(), "AbsentFromLastScan", "Absent" }, { FieldType.Bool, lb => (!string.IsNullOrWhiteSpace(lb.Book.SeriesNames())).ToString(), "IsInSeries", "InSeries" }, + { FieldType.Bool, lb => (!string.IsNullOrWhiteSpace(lb.Book.Subtitle)).ToString(), "HasSubtitle", "HasSubtitles" }, + { FieldType.Bool, lb => titleIsShortened(lb.Book).ToString(), "TitleHasColon", "ColonInTitle" }, { FieldType.Bool, lb => lb.Book.UserDefinedItem.IsFinished.ToString(), nameof(UserDefinedItem.IsFinished), "Finished", "IsFinished" }, { FieldType.Bool, lb => lb.IsAudiblePlus.ToString(), nameof(LibraryBook.IsAudiblePlus), "AudiblePlus", "Plus" }, // all numbers are padded to 8 char.s diff --git a/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs b/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs index d2ab0a1c..6b7991dc 100644 --- a/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs +++ b/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs @@ -65,6 +65,11 @@ public class FormatSearchQuery [DataRow("1 to 10", "00000001.00 TO 00000010.00")] [DataRow("19990101 to 20001231", "19990101.00 TO 20001231.00")] + // subtitle keywords are bool fields, not text fields + [DataRow("HasSubtitle", "hassubtitle:True")] + [DataRow("-TitleHasColon", "-titlehascolon:True")] + [DataRow("HasSubtitle OR TitleHasColon", "hassubtitle:True OR titlehascolon:True")] + // field to lowercase [DataRow("Author:Doyle", "author:Doyle")] // bool field to lowercase diff --git a/Source/_Tests/LibationSearchEngine.Tests/SubtitleSearchFieldTests.cs b/Source/_Tests/LibationSearchEngine.Tests/SubtitleSearchFieldTests.cs new file mode 100644 index 00000000..38f71de2 --- /dev/null +++ b/Source/_Tests/LibationSearchEngine.Tests/SubtitleSearchFieldTests.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using AssertionHelper; +using DataLayer; +using LibationSearchEngine; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Directory = System.IO.Directory; + +namespace SearchEngineTests; + +/// <summary> +/// Subtitle removal is all-or-nothing: <c><title short></c> stops at the first colon, so "Omnibus: Volume One" +/// and "Omnibus: Volume Two" both become "Omnibus". Finding the affected books used to be impossible -- the analyzer +/// throws punctuation away and Lucene reads a colon in a query as a field separator -- so the index flags the two +/// separate ways a name can lose part of its title. +/// </summary> +[TestClass] +public class SubtitleSearchFieldTests +{ + private const string PLAIN = "B0PLAIN0001"; + private const string SUBTITLE = "B0SUBTITL01"; + private const string COLON = "B0COLON0001"; + private const string BOTH = "B0BOTH00001"; + + private string indexDirectory = null!; + + [TestInitialize] + public void Initialize() + { + indexDirectory = Path.Combine(Path.GetTempPath(), "LibationSearchEngineTests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(indexDirectory); + new SearchEngine(indexDirectory).CreateNewIndex(library); + } + + [TestCleanup] + public void Cleanup() + { + try + { + if (Directory.Exists(indexDirectory)) + Directory.Delete(indexDirectory, recursive: true); + } + catch (IOException) + { + // Windows refuses to delete a file Lucene still holds open, and a leftover temp directory is not + // worth failing a test over + } + } + + private static LibraryBook book(string asin, string title, string? subtitle) + { + var contributor = Contributor.GetEmpty(); + var b = new Book(new AudibleProductId(asin), title, subtitle, null, 1, ContentType.Product, [contributor], [contributor], "us"); + return new LibraryBook(b, new DateTime(2026, 8, 15), "account"); + } + + private static readonly List<LibraryBook> library = + [ + book(PLAIN, "Sign of the Four", null), + book(SUBTITLE, "A Book Series Omnibus", "Volume One"), + book(COLON, "Star Trek: The Next Generation", null), + book(BOTH, "Dune: Book One", "The Graphic Novel") + ]; + + private string[] search(string query) + => [.. new SearchEngine(indexDirectory).Search(query).Docs.Select(d => d.ProductId)]; + + /// <summary>Audible's own subtitle field, which every template except <c><title></c> leaves out.</summary> + [TestMethod] + [DataRow("HasSubtitle")] + [DataRow("HasSubtitles")] + [DataRow("hassubtitle")] + public void books_with_an_audible_subtitle_are_found(string query) + => search(query).Should().BeEquivalentTo([SUBTITLE, BOTH]); + + /// <summary>The riskier case: the colon is inside Audible's title, so shortening cuts the title itself.</summary> + [TestMethod] + [DataRow("TitleHasColon")] + [DataRow("ColonInTitle")] + [DataRow("titlehascolon")] + public void books_whose_title_contains_a_colon_are_found(string query) + => search(query).Should().BeEquivalentTo([COLON, BOTH]); + + [TestMethod] + public void the_two_fields_are_independent() + { + search("HasSubtitle AND TitleHasColon").Should().BeEquivalentTo([BOTH]); + search("HasSubtitle OR TitleHasColon").Should().BeEquivalentTo([SUBTITLE, COLON, BOTH]); + } + + /// <summary>The complement is what makes this useful: everything shortening cannot damage.</summary> + [TestMethod] + public void books_a_short_title_leaves_alone_are_found_by_negation() + { + search("-HasSubtitle").Should().BeEquivalentTo([PLAIN, COLON]); + search("-TitleHasColon").Should().BeEquivalentTo([PLAIN, SUBTITLE]); + search("-HasSubtitle AND -TitleHasColon").Should().BeEquivalentTo([PLAIN]); + } + + /// <summary>Bool fields combine with the rest of the syntax, which is the point: filter, then liberate.</summary> + [TestMethod] + public void the_fields_combine_with_other_search_terms() + => search("TitleHasColon AND title:dune").Should().BeEquivalentTo([BOTH]); + + /// <summary>A subtitle is still part of the title text, so searching for its words keeps working.</summary> + [TestMethod] + public void subtitle_text_remains_searchable() + => search("title:\"volume one\"").Should().BeEquivalentTo([SUBTITLE]); +} diff --git a/docs/features/naming-templates.md b/docs/features/naming-templates.md index 2ad0a523..f92a9173 100644 --- a/docs/features/naming-templates.md +++ b/docs/features/naming-templates.md @@ -67,6 +67,8 @@ These tags will be replaced in the template with the audiobook's values. To change how these properties are displayed, [read about custom formatters](#tag-formatters) +`<title short>` stops at the first colon wherever that colon came from, so it also cuts titles such as "Star Trek: The Next Generation" that Audible sent without a subtitle. `<audible title>` leaves out Audible's subtitle without cutting the title. To see which of your books are affected either way, filter on `HasSubtitle` and `TitleHasColon`; see [searching and filtering](searching-and-filtering.md#subtitles-and-short-titles). + ### Conditional Tags Anything between the opening tag (`<tagname->`) and closing tag (`<-tagname>`) will only appear in the name if the condition evaluates to true. diff --git a/docs/features/searching-and-filtering.md b/docs/features/searching-and-filtering.md index a62d3c35..77a52613 100644 --- a/docs/features/searching-and-filtering.md +++ b/docs/features/searching-and-filtering.md @@ -54,6 +54,38 @@ I tagged autobiographies as auto_bio and biographies written by someone else as ![Search example: [bio]](../images/SearchExampleBio.png) ![Search example: [auto_bio]](../images/SearchExampleAutoBio.png) +## Subtitles and short titles + +The `<title short>` tag keeps everything before the first colon, which is what keeps the default folder name short. That is usually what you want, but not always. "A Book Series Omnibus: Volume One" and "A Book Series Omnibus: Volume Two" both shorten to "A Book Series Omnibus", so the two books land in the same folder and can no longer be told apart by name. + +A colon is not something you can search for: the search engine throws punctuation away when it indexes your library, and Lucene reads a colon in a query as the separator between a field and its value. Two boolean fields find these books instead. + +| Field | Matches | +|----------------------------------|---------------------------------------------------------------------------------------------------------------| +| `HasSubtitle` (`HasSubtitles`) | Audible sent a separate subtitle, which every title tag except `<title>` leaves out | +| `TitleHasColon` (`ColonInTitle`) | Audible's title itself contains a colon, so `<title short>` cuts into the title rather than dropping a subtitle | + +Some searches worth keeping as quick filters: + +- `TitleHasColon` - every book whose title `<title short>` cuts +- `TitleHasColon AND -IsLiberated` - the same, limited to books you have not downloaded yet +- `HasSubtitle OR TitleHasColon` - everything shortening changes in any way +- `-HasSubtitle AND -TitleHasColon` - the books shortening cannot change + +If these fields find nothing at all, your search index was built before they existed. Scanning your library rebuilds it, as does closing Libation and deleting the `SearchEngine` folder in your Libation files folder. The index is only a cache of your library, so deleting it is safe. + +Once you can see the affected books, you can decide what to do about them. If the only problem is colons inside Audible's titles, switching `<title short>` to `<audible title>` in Settings > Download/Decrypt fixes every one of them at once: it still leaves out Audible's subtitle, but it never cuts the title. If instead two books share a title and differ only by subtitle, use `<title>` for those books, or keep `<id>` in the template so their names stay unique. Either way, filter to the books you want handled differently, liberate them with one template, then restore your usual template for the rest. + +### Auditing titles in a spreadsheet + +The filter tells you which books are shortened. It cannot tell you which ones actually collide, and a colon on its own is harmless - the damage is done when two books end up with the same name. Export (Export in the menu bar) writes `Title` and `Subtitle` as separate columns, where `Title` is Audible's title exactly as `<title short>` sees it, so a spreadsheet can answer the question the filter cannot. + +Put this beside the Title column to see the name each book would be shortened to, then use `COUNTIF` on the result to find the ones that repeat: + +`=LEFT(A2, IFERROR(FIND(":", A2) - 1, LEN(A2)))` + +To carry the result back into Libation, paste the affected ids into the filter box joined by OR, eg. `id:B015D78L0U OR id:B01LYFDNZM`, which selects exactly those books in the grid. + ## Filters If you have a search you want to save, click Add To Quick Filters to save it in your Quick Filters list. To use it again, select it from the Quick Filters list.