From 4187712c8d6744bb9b29fc6589fd2e611ecb2e49 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 23:43:53 +0000 Subject: [PATCH] Add HasSubtitle and TitleHasColon search fields stops at the first colon, so it shortens Audible titles that contain one just as readily as it drops Audible's subtitle, and distinct books then collapse onto the same name. A colon cannot be searched for: the analyzer discards punctuation and Lucene reads a colon in a query as a field separator. Two bool index fields find the affected books instead. Document how the two title tags differ, since <audible title> already drops Audible's subtitle without ever cutting a title, and how to audit for names that actually collide in a spreadsheet export. Co-authored-by: rmcrackan <rmcrackan@gmail.com> --- Scripts/seed-demo-library.cs | 31 ++++- .../Templates/Templates.cs | 3 +- Source/LibationSearchEngine/SearchEngine.cs | 10 ++ .../SearchEngineTests.cs | 5 + .../SubtitleSearchFieldTests.cs | 111 ++++++++++++++++++ docs/features/naming-templates.md | 2 + docs/features/searching-and-filtering.md | 32 +++++ 7 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 Source/_Tests/LibationSearchEngine.Tests/SubtitleSearchFieldTests.cs diff --git a/Scripts/seed-demo-library.cs b/Scripts/seed-demo-library.cs index df895a8f..cb20d520 100644 --- a/Scripts/seed-demo-library.cs +++ b/Scripts/seed-demo-library.cs @@ -3,7 +3,8 @@ using Microsoft.Data.Sqlite; using System.Text.Json; -// Seeds a Libation library with one book per Liberate-column state, for manual UI testing. +// Seeds a Libation library with one book per Liberate-column state, plus a few titles that +// subtitle removal changes, for manual UI testing. // // dotnet run Scripts/seed-demo-library.cs [--clean] [path-to-Libation-files-folder] // @@ -76,9 +77,9 @@ foreach (var book in books) insert into Books (AudibleProductId, ContentType, Description, IsAbridged, IsSpatial, LengthInMinutes, Locale, Rating_OverallRating, Rating_PerformanceRating, Rating_StoryRating, Subtitle, Title) - values ($asin, $contentType, 'Seeded by seed-demo-library.cs', 0, 0, 600, 'us', 0, 0, 0, '', $title) + values ($asin, $contentType, 'Seeded by seed-demo-library.cs', 0, 0, 600, 'us', 0, 0, 0, $subtitle, $title) """, - ("$asin", book.Asin), ("$contentType", book.ContentType), ("$title", book.Title)); + ("$asin", book.Asin), ("$contentType", book.ContentType), ("$title", book.Title), ("$subtitle", book.Subtitle)); var bookId = Scalar("select BookId from Books where AudibleProductId = $asin", ("$asin", book.Asin)); @@ -263,6 +264,27 @@ static List<DemoBook> BuildDemoBooks() Expectation: expectation, IsAbsent: true)); + // Titles that subtitle removal changes. <title short> stops at the first colon, so it cuts a colon in + // Audible's own title just as readily as it drops Audible's subtitle field, and the grid's Title column + // shows both the same way. The HasSubtitle and TitleHasColon filters are the only way to tell them apart. + foreach (var (title, subtitle, match) in new[] + { + ("A Book Series Omnibus", "Volume One", "HasSubtitle"), + ("A Book Series Omnibus", "Volume Two", "HasSubtitle"), + ("Star Trek: The Next Generation", "", "TitleHasColon"), + ("Dune: Book One", "The Graphic Novel", "both HasSubtitle and TitleHasColon"), + }) + books.Add(new DemoBook( + Asin: $"{AsinPrefix}{++n:000}", + Title: title, + ContentType: Product, + BookStatus: NotLiberated, + PdfStatus: null, + IsPlus: false, + NeedsPartialDownload: false, + Expectation: $"red lamp; filter match: {match}", + Subtitle: subtitle)); + return books; } @@ -373,4 +395,5 @@ record DemoBook( string Expectation, string? SeriesAsin = null, string? SeriesOrder = null, - bool IsAbsent = false); + bool IsAbsent = false, + string Subtitle = ""); diff --git a/Source/LibationFileManager/Templates/Templates.cs b/Source/LibationFileManager/Templates/Templates.cs index d6c57085..12dec1df 100644 --- a/Source/LibationFileManager/Templates/Templates.cs +++ b/Source/LibationFileManager/Templates/Templates.cs @@ -392,7 +392,8 @@ public abstract class Templates #region Tag Formatters - private static string? GetTitleShort(string? title) + /// <summary>Backs the <c><title short></c> tag. Public so the search index can flag the titles it shortens.</summary> + public static string? GetTitleShort(string? title) => title != null && title.IndexOf(':') is var i && i >= 0 ? title[..i] : title; diff --git a/Source/LibationSearchEngine/SearchEngine.cs b/Source/LibationSearchEngine/SearchEngine.cs index 5ab1d4a0..41bf10c2 100644 --- a/Source/LibationSearchEngine/SearchEngine.cs +++ b/Source/LibationSearchEngine/SearchEngine.cs @@ -33,6 +33,14 @@ public class SearchEngine return authors.Intersect(narrators).Any(); } + /// <summary> + /// True when <c><title short></c> cuts the title itself rather than merely dropping Audible's subtitle + /// field. Audible ships plenty of titles with a colon in them, and those are the ones where shortening loses + /// something the user may need: "Omnibus: Volume One" and "Omnibus: Volume Two" both shorten to "Omnibus". + /// </summary> + private static bool titleIsShortened(Book book) + => LibationFileManager.Templates.Templates.GetTitleShort(book.Title) != book.Title; + // use these common fields in the "all" default search field public static IndexRuleCollection FieldIndexRules { get; } = new IndexRuleCollection { @@ -60,6 +68,8 @@ public class SearchEngine { FieldType.Bool, lb => lb.Book.IsEpisodeChild().ToString(), "Podcast", "Podcasts", "IsPodcast", "Episode", "Episodes", "IsEpisode" }, { FieldType.Bool, lb => lb.AbsentFromLastScan.ToString(), "AbsentFromLastScan", "Absent" }, { FieldType.Bool, lb => (!string.IsNullOrWhiteSpace(lb.Book.SeriesNames())).ToString(), "IsInSeries", "InSeries" }, + { FieldType.Bool, lb => (!string.IsNullOrWhiteSpace(lb.Book.Subtitle)).ToString(), "HasSubtitle", "HasSubtitles" }, + { FieldType.Bool, lb => titleIsShortened(lb.Book).ToString(), "TitleHasColon", "ColonInTitle" }, { FieldType.Bool, lb => lb.Book.UserDefinedItem.IsFinished.ToString(), nameof(UserDefinedItem.IsFinished), "Finished", "IsFinished" }, { FieldType.Bool, lb => lb.IsAudiblePlus.ToString(), nameof(LibraryBook.IsAudiblePlus), "AudiblePlus", "Plus" }, // all numbers are padded to 8 char.s diff --git a/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs b/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs index d2ab0a1c..6b7991dc 100644 --- a/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs +++ b/Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs @@ -65,6 +65,11 @@ public class FormatSearchQuery [DataRow("1 to 10", "00000001.00 TO 00000010.00")] [DataRow("19990101 to 20001231", "19990101.00 TO 20001231.00")] + // subtitle keywords are bool fields, not text fields + [DataRow("HasSubtitle", "hassubtitle:True")] + [DataRow("-TitleHasColon", "-titlehascolon:True")] + [DataRow("HasSubtitle OR TitleHasColon", "hassubtitle:True OR titlehascolon:True")] + // field to lowercase [DataRow("Author:Doyle", "author:Doyle")] // bool field to lowercase diff --git a/Source/_Tests/LibationSearchEngine.Tests/SubtitleSearchFieldTests.cs b/Source/_Tests/LibationSearchEngine.Tests/SubtitleSearchFieldTests.cs new file mode 100644 index 00000000..38f71de2 --- /dev/null +++ b/Source/_Tests/LibationSearchEngine.Tests/SubtitleSearchFieldTests.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using AssertionHelper; +using DataLayer; +using LibationSearchEngine; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Directory = System.IO.Directory; + +namespace SearchEngineTests; + +/// <summary> +/// Subtitle removal is all-or-nothing: <c><title short></c> stops at the first colon, so "Omnibus: Volume One" +/// and "Omnibus: Volume Two" both become "Omnibus". Finding the affected books used to be impossible -- the analyzer +/// throws punctuation away and Lucene reads a colon in a query as a field separator -- so the index flags the two +/// separate ways a name can lose part of its title. +/// </summary> +[TestClass] +public class SubtitleSearchFieldTests +{ + private const string PLAIN = "B0PLAIN0001"; + private const string SUBTITLE = "B0SUBTITL01"; + private const string COLON = "B0COLON0001"; + private const string BOTH = "B0BOTH00001"; + + private string indexDirectory = null!; + + [TestInitialize] + public void Initialize() + { + indexDirectory = Path.Combine(Path.GetTempPath(), "LibationSearchEngineTests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(indexDirectory); + new SearchEngine(indexDirectory).CreateNewIndex(library); + } + + [TestCleanup] + public void Cleanup() + { + try + { + if (Directory.Exists(indexDirectory)) + Directory.Delete(indexDirectory, recursive: true); + } + catch (IOException) + { + // Windows refuses to delete a file Lucene still holds open, and a leftover temp directory is not + // worth failing a test over + } + } + + private static LibraryBook book(string asin, string title, string? subtitle) + { + var contributor = Contributor.GetEmpty(); + var b = new Book(new AudibleProductId(asin), title, subtitle, null, 1, ContentType.Product, [contributor], [contributor], "us"); + return new LibraryBook(b, new DateTime(2026, 8, 15), "account"); + } + + private static readonly List<LibraryBook> library = + [ + book(PLAIN, "Sign of the Four", null), + book(SUBTITLE, "A Book Series Omnibus", "Volume One"), + book(COLON, "Star Trek: The Next Generation", null), + book(BOTH, "Dune: Book One", "The Graphic Novel") + ]; + + private string[] search(string query) + => [.. new SearchEngine(indexDirectory).Search(query).Docs.Select(d => d.ProductId)]; + + /// <summary>Audible's own subtitle field, which every template except <c><title></c> leaves out.</summary> + [TestMethod] + [DataRow("HasSubtitle")] + [DataRow("HasSubtitles")] + [DataRow("hassubtitle")] + public void books_with_an_audible_subtitle_are_found(string query) + => search(query).Should().BeEquivalentTo([SUBTITLE, BOTH]); + + /// <summary>The riskier case: the colon is inside Audible's title, so shortening cuts the title itself.</summary> + [TestMethod] + [DataRow("TitleHasColon")] + [DataRow("ColonInTitle")] + [DataRow("titlehascolon")] + public void books_whose_title_contains_a_colon_are_found(string query) + => search(query).Should().BeEquivalentTo([COLON, BOTH]); + + [TestMethod] + public void the_two_fields_are_independent() + { + search("HasSubtitle AND TitleHasColon").Should().BeEquivalentTo([BOTH]); + search("HasSubtitle OR TitleHasColon").Should().BeEquivalentTo([SUBTITLE, COLON, BOTH]); + } + + /// <summary>The complement is what makes this useful: everything shortening cannot damage.</summary> + [TestMethod] + public void books_a_short_title_leaves_alone_are_found_by_negation() + { + search("-HasSubtitle").Should().BeEquivalentTo([PLAIN, COLON]); + search("-TitleHasColon").Should().BeEquivalentTo([PLAIN, SUBTITLE]); + search("-HasSubtitle AND -TitleHasColon").Should().BeEquivalentTo([PLAIN]); + } + + /// <summary>Bool fields combine with the rest of the syntax, which is the point: filter, then liberate.</summary> + [TestMethod] + public void the_fields_combine_with_other_search_terms() + => search("TitleHasColon AND title:dune").Should().BeEquivalentTo([BOTH]); + + /// <summary>A subtitle is still part of the title text, so searching for its words keeps working.</summary> + [TestMethod] + public void subtitle_text_remains_searchable() + => search("title:\"volume one\"").Should().BeEquivalentTo([SUBTITLE]); +} diff --git a/docs/features/naming-templates.md b/docs/features/naming-templates.md index 2ad0a523..f92a9173 100644 --- a/docs/features/naming-templates.md +++ b/docs/features/naming-templates.md @@ -67,6 +67,8 @@ These tags will be replaced in the template with the audiobook's values. To change how these properties are displayed, [read about custom formatters](#tag-formatters) +`<title short>` stops at the first colon wherever that colon came from, so it also cuts titles such as "Star Trek: The Next Generation" that Audible sent without a subtitle. `<audible title>` leaves out Audible's subtitle without cutting the title. To see which of your books are affected either way, filter on `HasSubtitle` and `TitleHasColon`; see [searching and filtering](searching-and-filtering.md#subtitles-and-short-titles). + ### Conditional Tags Anything between the opening tag (`<tagname->`) and closing tag (`<-tagname>`) will only appear in the name if the condition evaluates to true. diff --git a/docs/features/searching-and-filtering.md b/docs/features/searching-and-filtering.md index a62d3c35..77a52613 100644 --- a/docs/features/searching-and-filtering.md +++ b/docs/features/searching-and-filtering.md @@ -54,6 +54,38 @@ I tagged autobiographies as auto_bio and biographies written by someone else as ![Search example: [bio]](../images/SearchExampleBio.png) ![Search example: [auto_bio]](../images/SearchExampleAutoBio.png) +## Subtitles and short titles + +The `<title short>` tag keeps everything before the first colon, which is what keeps the default folder name short. That is usually what you want, but not always. "A Book Series Omnibus: Volume One" and "A Book Series Omnibus: Volume Two" both shorten to "A Book Series Omnibus", so the two books land in the same folder and can no longer be told apart by name. + +A colon is not something you can search for: the search engine throws punctuation away when it indexes your library, and Lucene reads a colon in a query as the separator between a field and its value. Two boolean fields find these books instead. + +| Field | Matches | +|----------------------------------|---------------------------------------------------------------------------------------------------------------| +| `HasSubtitle` (`HasSubtitles`) | Audible sent a separate subtitle, which every title tag except `<title>` leaves out | +| `TitleHasColon` (`ColonInTitle`) | Audible's title itself contains a colon, so `<title short>` cuts into the title rather than dropping a subtitle | + +Some searches worth keeping as quick filters: + +- `TitleHasColon` - every book whose title `<title short>` cuts +- `TitleHasColon AND -IsLiberated` - the same, limited to books you have not downloaded yet +- `HasSubtitle OR TitleHasColon` - everything shortening changes in any way +- `-HasSubtitle AND -TitleHasColon` - the books shortening cannot change + +If these fields find nothing at all, your search index was built before they existed. Scanning your library rebuilds it, as does closing Libation and deleting the `SearchEngine` folder in your Libation files folder. The index is only a cache of your library, so deleting it is safe. + +Once you can see the affected books, you can decide what to do about them. If the only problem is colons inside Audible's titles, switching `<title short>` to `<audible title>` in Settings > Download/Decrypt fixes every one of them at once: it still leaves out Audible's subtitle, but it never cuts the title. If instead two books share a title and differ only by subtitle, use `<title>` for those books, or keep `<id>` in the template so their names stay unique. Either way, filter to the books you want handled differently, liberate them with one template, then restore your usual template for the rest. + +### Auditing titles in a spreadsheet + +The filter tells you which books are shortened. It cannot tell you which ones actually collide, and a colon on its own is harmless - the damage is done when two books end up with the same name. Export (Export in the menu bar) writes `Title` and `Subtitle` as separate columns, where `Title` is Audible's title exactly as `<title short>` sees it, so a spreadsheet can answer the question the filter cannot. + +Put this beside the Title column to see the name each book would be shortened to, then use `COUNTIF` on the result to find the ones that repeat: + +`=LEFT(A2, IFERROR(FIND(":", A2) - 1, LEN(A2)))` + +To carry the result back into Libation, paste the affected ids into the filter box joined by OR, eg. `id:B015D78L0U OR id:B01LYFDNZM`, which selects exactly those books in the grid. + ## Filters If you have a search you want to save, click Add To Quick Filters to save it in your Quick Filters list. To use it again, select it from the Quick Filters list.