using ApplicationServices; using AudibleApi; using CommandLine; using DataLayer; using FileLiberator; using LibationFileManager; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; namespace LibationCli; public abstract class ProcessableOptionsBase : OptionsBase { [Value(0, MetaName = "[asins]", HelpText = "Optional product IDs (ASINs) of books to process.")] public IEnumerable? Asins { get; set; } [Option('i', "id", Required = false, HelpText = "Product ID (ASIN) of a book to process. Repeatable. Same as positional [asins].")] public IEnumerable? Ids { get; set; } protected IEnumerable GetProductIds() => (Asins ?? []).Concat(Ids ?? []).Select(NormalizeProductId).Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase); private static string NormalizeProductId(string id) => id.Trim().TrimStart('[').TrimEnd(']'); protected static TProcessable CreateProcessable(EventHandler? completedAction = null) where TProcessable : Processable, IProcessable { var strProc = TProcessable.Create(Configuration.Instance); LibraryBook? currentLibraryBook = null; if (Environment.UserInteractive && !Console.IsOutputRedirected && !Console.IsErrorRedirected) { var progressBar = new ConsoleProgressBar(Console.Out); strProc.Completed += (_, e) => progressBar.Clear(); strProc.StreamingTimeRemaining += (_, e) => progressBar.RemainingTime = e; strProc.StreamingProgressChanged += (_, e) => progressBar.Progress = e.ProgressPercentage; } strProc.Begin += (o, e) => { currentLibraryBook = e; Console.WriteLine($"{typeof(TProcessable).Name} Begin: {e}"); }; strProc.Completed += (o, e) => { Console.WriteLine($"{typeof(TProcessable).Name} Completed: {e}"); }; strProc.Completed += (s, e) => { try { completedAction?.Invoke(s, e); } catch (Exception ex) { Console.Error.WriteLine("CLI error. See log for more details."); Serilog.Log.Logger.Error(ex, "CLI error"); } }; if (strProc is AudioDecodable audDec) { audDec.RequestCoverArt += (_, _) => { if (currentLibraryBook is null) return null; var pictureId = Configuration.Instance.FileDownloadQuality == Configuration.DownloadQuality.High ? currentLibraryBook.Book.PictureLarge ?? currentLibraryBook.Book.PictureId : currentLibraryBook.Book.PictureId; return pictureId is null ? null : PictureStorage.GetPictureSynchronously(new PictureDefinition(pictureId, PictureSize.Native)); }; } return strProc; } /// How much this run may download before it stops, or null for a verb without a per-run limit. protected virtual RunDownloadLimit? RunLimit => null; /// /// Whether this run should leave alone the titles Audible recently refused. False for a run that names /// its titles or passes --force: an explicit request is always attempted. /// internal virtual bool HonorsDeferredRetries => false; /// /// Whether a bulk run should leave alone the titles the last library scan did not find, which is what every /// multi-title path in the app already does through . /// /// False by default: convert-to-mp3 and the Audiobookshelf upload work on the files already on disk, and a /// title being absent from Audible says nothing about those. Only a verb that asks Audible for something has /// anything to gain by skipping them. /// /// internal virtual bool SkipsTitlesAbsentFromLastScan => false; /// /// A second pass over the library, run after , for the titles that pass its /// own Validate but were not selected by the first. liberate uses this to back-fill PDFs for titles /// whose audio it already has: the first pass only selects titles that need downloading, so on its own it /// never reaches a title that needs nothing but its PDF. /// /// Bulk runs only. A run that names its titles already gets every step each title needs, because the first /// pass re-downloads a named title and its PDF follows from that. /// /// protected async Task RunAsync(Processable Processable, Action? config = null, Action? notFound = null, Processable? bulkFollowUp = null) { var skippedForDailyLimit = 0; var skippedAsAbsent = 0; var deferredThisRun = new List(); var runLimitReached = false; // Needs no guard against pdf or convert runs, unlike the daily limit below: the tracker counts only // audiobook downloads this run recorded, and those verbs record none. var runLimit = RunLimit is RunDownloadLimit limit ? new RunLimitTracker(limit, DateTimeOffset.Now) : null; var productIds = GetProductIds().ToArray(); if (productIds.Length > 0) { foreach (var asin in productIds) { if (DbContexts.GetLibraryBook_Flat_NoTracking(asin, caseSensative: false) is LibraryBook lb) { if (!await ProcessOrStopAsync(Processable, lb, true)) break; } else { var msg = $"Book with ASIN '{asin}' not found in library. Skipping."; Console.Error.WriteLine(msg); Serilog.Log.Logger.Error(msg); notFound?.Invoke(asin); } } } else { // Read once, before the first book: a run that spends hours downloading must not start skipping // titles because of failures it recorded itself a moment ago. var deferrals = HonorsDeferredRetries ? DownloadDeferrals.Load(DateTimeOffset.Now) : DownloadDeferrals.None; var libraryBooks = DbContexts.GetLibrary_Flat_NoTracking(); // Filtered once, ahead of both passes, so the two agree about which titles this run may attempt - // and by the same rule the app's own multi-title paths use. if (SkipsTitlesAbsentFromLastScan) { var absent = libraryBooks.Where(lb => !lb.Downloadable).ToList(); libraryBooks = libraryBooks.Where(lb => lb.Downloadable).ToList(); // Counted as the titles a pass would otherwise have attempted, not as every absent title in the // library. Most of a large library's absent titles need nothing at all, and counting those would // put a number in front of the user that no run was ever going to act on. skippedAsAbsent = Processable.GetValidLibraryBooks(absent) .Concat(bulkFollowUp?.GetValidLibraryBooks(absent) ?? []) .DistinctBy(lb => lb.Book.AudibleProductId, StringComparer.OrdinalIgnoreCase) .Count(); } // Titles the follow-up pass must leave alone, because the first pass has already dealt with them: // the ones it attempted, whose own steps followed on from the attempt, and the ones it left waiting, // which are counted and explained once here rather than again below. Recorded by product id rather // than re-derived, because neither question can be answered from a title's state afterwards - a step // that just failed still validates, and a title being waited on looks like any other title that // needs downloading. var settledByFirstPass = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var lb in Processable.GetValidLibraryBooks(libraryBooks)) { settledByFirstPass.Add(lb.Book.AudibleProductId); if (IsDeferred(lb)) continue; if (!await ProcessOrStopAsync(Processable, lb, false)) break; } // Skipped when the first pass stopped early, so a run cut short by its download limit does not // carry on doing other work. if (bulkFollowUp is not null && !runLimitReached) { foreach (var lb in bulkFollowUp.GetValidLibraryBooks(libraryBooks)) { // The follow-up pass waits on a refused title just as the first does. It fetches a PDF, // which comes from the same license request the audiobook does, so a title being waited on // would be refused here for the reason the wait exists to stop asking about. if (settledByFirstPass.Contains(lb.Book.AudibleProductId) || IsDeferred(lb)) continue; if (!await ProcessOrStopAsync(bulkFollowUp, lb, false)) break; } } bool IsDeferred(LibraryBook libraryBook) { if (deferrals.Find(libraryBook) is not DeferredDownload deferred) return false; deferredThisRun.Add(deferred); Serilog.Log.Logger.Information( "Not attempting {libraryBook} yet. {@DebugInfo}", libraryBook.LogFriendly(), new { deferred.Kind, deferred.ConsecutiveFailures, deferred.Reason, RetryAfter = deferred.RetryAfter.ToLocalTime() }); return true; } } if (skippedAsAbsent > 0) { var summary = AbsentFromLastScanUserMessage.BuildCliSkippedSummary(skippedAsAbsent); Console.WriteLine(summary); Serilog.Log.Logger.Information(summary); } if (deferredThisRun.Count > 0) { var now = DateTimeOffset.Now; foreach (var line in DeferredDownloadUserMessage.BuildCliSkippedLines(deferredThisRun, now)) Console.WriteLine(line); Serilog.Log.Logger.Information( "Skipped {deferredCount} titles that recently failed to download. Skipped: {skipped}", deferredThisRun.Count, DeferredDownloadUserMessage.BuildLogBreakdown(deferredThisRun)); } if (skippedForDailyLimit > 0) { var summary = DailyDownloadLimitUserMessage.BuildCliSkippedSummary(skippedForDailyLimit); Console.WriteLine(summary); Serilog.Log.Logger.Information(summary); } var done = runLimitReached ? "Done. Stopped early: this run's download limit was reached." : "Done. All books have been processed"; Console.WriteLine(done); Serilog.Log.Logger.Information(done); // False ends the run. The limit is checked here rather than at the top of the run so that a run whose // books happen to end exactly at the limit says nothing: nothing was cut short. async Task ProcessOrStopAsync(Processable processable, LibraryBook libraryBook, bool validate) { if (runLimit is not null && runLimit.TryStop(out var stopMessage)) { runLimitReached = true; Console.WriteLine(stopMessage); Serilog.Log.Logger.Information(stopMessage); return false; } config?.Invoke(libraryBook); if (IsSkippedByDailyLimit(processable, libraryBook)) { skippedForDailyLimit++; return true; } runLimit?.Attempting(libraryBook.Book.AudibleProductId); await ProcessOneAsync(processable, libraryBook, validate); return true; } } protected bool announcedDailyLimit; /// /// True when the user's daily download limit covers this title and has been reached. Unlike the GUI queue the /// CLI never waits: a command-line run must not sit idle for hours, and in Docker the entrypoint has to return /// so its own sleep loop keeps working. Skipped titles stay un-liberated and are retried on the next run. /// protected bool IsSkippedByDailyLimit(Processable processable, LibraryBook libraryBook) { // Only audiobook downloads are limited, and only titles the configured scope covers, so the common // case of no limit (or a Plus-only limit against an owned title) costs nothing. if (processable is not DownloadDecryptBook || !DailyDownloadLimit.AppliesTo(libraryBook.IsAudiblePlus, Configuration.Instance)) return false; var now = DateTimeOffset.Now; var allowance = DailyDownloadLimit.Evaluate(Configuration.Instance, DownloadHistoryStore.GetCurrentWindow(now), now); if (!allowance.Blocks(libraryBook.IsAudiblePlus)) return false; if (!announcedDailyLimit) { announcedDailyLimit = true; foreach (var line in DailyDownloadLimitUserMessage.BuildCliSkippedLines(allowance)) { Console.Error.WriteLine(line); Serilog.Log.Logger.Information(line); } } Serilog.Log.Logger.Information( "Daily download limit reached; skipping {libraryBook}. {@DebugInfo}", libraryBook.LogFriendly(), new { allowance.Scope, allowance.Unit, allowance.Quantity, allowance.UsedBooks, allowance.UsedBytes, allowance.NextCapacityAt }); return true; } protected async Task ProcessOneAsync(Processable Processable, LibraryBook libraryBook, bool validate) { try { var statusHandler = await Processable.ProcessSingleAsync(libraryBook, validate); if (statusHandler.IsSuccess) return; foreach (var errorMessage in statusHandler.Errors) { Console.Error.WriteLine(errorMessage); Serilog.Log.Logger.Error(errorMessage); } } catch (ApiErrorException ex) when (WidevineRecommendation.ShouldRecommendWidevine(ex, Configuration.Instance)) { Console.Error.WriteLine(WidevineRecommendation.BuildLogSummary(libraryBook.Book.TitleWithSubtitle)); Serilog.Log.Logger.Error(ex, "ADRM license unavailable (Sable acr:null) {@DebugInfo}", new { Book = libraryBook.LogFriendly() }); ReportNextAttempt(libraryBook); } catch (ContentLicenseDeniedException clEx) { foreach (var line in ContentLicenseDeniedCliSummary.Lines(clEx)) Console.Error.WriteLine(line); Serilog.Log.Logger.Error(clEx, "Content license denied {@DebugInfo}", new { Book = libraryBook.LogFriendly() }); ReportNextAttempt(libraryBook); } catch (Exception ex) { var msg = "Error processing book. Skipping. For options of skipping or marking as error, retry with main Libation app."; Console.Error.WriteLine(msg + ". See log for more details."); Serilog.Log.Logger.Error(ex, $"{msg} {{@DebugInfo}}", new { Book = libraryBook.LogFriendly() }); if (!ReportNextAttempt(libraryBook)) Console.Error.WriteLine("This book will be tried again on next attempt."); } } /// /// Says when a title Libation has decided to wait on will be attempted again, so a scheduled run explains /// its own silence on the next several runs rather than appearing to have forgotten the title. /// /// True when the title is being waited on. private static bool ReportNextAttempt(LibraryBook libraryBook) { var now = DateTimeOffset.Now; if (DownloadAttemptFailureStore.Find(libraryBook, now) is not DeferredDownload deferred) return false; Console.Error.WriteLine( $"Not attempting this title again {DeferredDownloadUserMessage.DescribeWhen(deferred.RetryAfter, now)}. " + "To try it sooner, name it: libationcli liberate " + libraryBook.Book.AudibleProductId); return true; } }