mirror of
https://github.com/rmcrackan/Libation.git
synced 2026-09-13 06:07:30 -04:00
Reported in issue #1973: a scheduled liberate run re-requested a content license for the same 59 titles every 15 minutes, 1397 refused requests in six hours, because nothing about a failed PDF was remembered and the PDF step asked Audible afresh every time. The two download paths have been converging for a while - a PDF is now named and placed by the audiobook path's own logic, and verified like one - and every part of this bug lives where that stopped short. Both steps ask Audible for the same license. The request asks for pdf_url alongside the content reference, and LicenseInfo dropped it, so DownloadPdf turned round and requested an identical license to read the field the first response had already returned. Carry PdfUrl on LicenseInfo and give both steps one ILicensedDownload contract: a license may be supplied to a step, and the one a step ended up using is published for the next step for the same title. The CLI and the GUI queue hand it on, so a title costs one license request per run however many steps want something from it. A carried license is retried once with a fresh one if it does not work, since Audible's links are signed and a long decrypt can run between the two steps. Where the audiobook step obtained no license there is nothing to hand on and the supplement step does not run, which deletes a bug rather than guarding it: Completed fires from a finally, so a refused audio download was followed at once by a PDF request that reproduced the refusal. Error now means the same for a PDF as for a book. The audiobook step has always skipped LiberatedStatus.Error through AudioExists, and NeedsPdfDownload agrees, but DownloadPdf selected on PdfExists and so retried an errored PDF forever. A license that is granted and carries no pdf_url - the 'No PDF URL available' in the report - is Audible saying the title has no PDF, and is written off that same way instead of failing identically on every run. It stays resettable by everything that resets a book: --force, a named title, Set PDF Not Downloaded. Refusals now reach ProcessSingleAsync, which has always recorded them for whichever step throws one; DownloadPdf swallowed everything and recorded nothing. It keeps swallowing what the classifier does not recognise, which is what stopped a missing PDF from taking the app down with it. A bulk CLI run leaves alone the titles the last scan did not find, by the same Downloadable rule every multi-title path in the app already uses, and the PDF back-fill pass waits on a refused title just as the first pass does. --force and a named title still attempt everything. Co-authored-by: rmcrackan <rmcrackan@gmail.com>
190 lines
6.7 KiB
C#
190 lines
6.7 KiB
C#
using ApplicationServices;
|
|
using DataLayer;
|
|
using Dinah.Core;
|
|
using Dinah.Core.ErrorHandling;
|
|
using Dinah.Core.Net.Http;
|
|
using LibationFileManager;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace FileLiberator;
|
|
|
|
public interface IProcessable<T> where T : IProcessable<T>
|
|
{
|
|
/// <summary>
|
|
/// Create a new instance of the Processable which uses a specific Configuration
|
|
/// </summary>
|
|
/// <param name="config">The <see cref="Configuration"/> this <typeparamref name="T"/> will use</param>
|
|
static abstract T Create(Configuration config);
|
|
}
|
|
public abstract class Processable
|
|
{
|
|
public abstract string Name { get; }
|
|
public event EventHandler<LibraryBook>? Begin;
|
|
|
|
/// <summary>General string message to display. DON'T rely on this for success, failure, or control logic</summary>
|
|
public event EventHandler<string>? StatusUpdate;
|
|
/// <summary>Fired when a file is successfully saved to disk</summary>
|
|
public event EventHandler<(string id, string path)>? FileCreated;
|
|
public event EventHandler<DownloadProgress>? StreamingProgressChanged;
|
|
public event EventHandler<TimeSpan>? StreamingTimeRemaining;
|
|
|
|
public event EventHandler<LibraryBook>? Completed;
|
|
|
|
public required Configuration Configuration { get; init; }
|
|
protected Processable() { }
|
|
|
|
/// <returns>True == Valid</returns>
|
|
public abstract bool Validate(LibraryBook libraryBook);
|
|
|
|
/// <returns>True == success</returns>
|
|
public abstract Task<StatusHandler> ProcessAsync(LibraryBook libraryBook);
|
|
|
|
/// <summary>
|
|
/// Whether a refusal from Audible during this step should be remembered, so a scheduled run stops asking
|
|
/// for the same license every time. True for the steps that request a content license - the audiobook
|
|
/// download and the supplement download, which make the same request - because that request is the one
|
|
/// Audible refuses and the one the record gates. A step that only works on files already on disk has
|
|
/// nothing to record.
|
|
/// </summary>
|
|
protected virtual bool RecordsAttemptFailures => false;
|
|
|
|
// when used in foreach: stateful. deferred execution
|
|
public IEnumerable<LibraryBook> GetValidLibraryBooks(IEnumerable<LibraryBook> library)
|
|
=> library.Where(libraryBook =>
|
|
Validate(libraryBook)
|
|
&& (!libraryBook.Book.IsEpisodeChild() || Configuration.DownloadEpisodes)
|
|
);
|
|
|
|
public async Task<StatusHandler> ProcessSingleAsync(LibraryBook libraryBook, bool validate)
|
|
{
|
|
if (validate && !Validate(libraryBook))
|
|
return new StatusHandler { "Validation failed" };
|
|
|
|
Serilog.Log.Logger.Information("Begin " + nameof(ProcessSingleAsync) + " {@DebugInfo}", new
|
|
{
|
|
libraryBook.Book.TitleWithSubtitle,
|
|
libraryBook.Book.AudibleProductId,
|
|
libraryBook.Book.Locale,
|
|
Account = libraryBook.Account?.ToMask() ?? "[empty]"
|
|
});
|
|
|
|
StatusHandler status;
|
|
try
|
|
{
|
|
status
|
|
= (await ProcessAsync(libraryBook))
|
|
?? new StatusHandler { "Processable should never return a null status" };
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
RecordAttemptFailure(libraryBook, ex);
|
|
throw;
|
|
}
|
|
|
|
GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, true, true);
|
|
|
|
if (status.IsSuccess && RecordsAttemptFailures)
|
|
DownloadAttemptFailureStore.Clear(libraryBook);
|
|
|
|
return status;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Recorded here rather than in each host so the CLI, the GUI queue and anything added later all remember
|
|
/// a refusal the same way. Failures Libation cannot attribute to Audible are left unrecorded and keep
|
|
/// being retried on the next run.
|
|
/// </summary>
|
|
private void RecordAttemptFailure(LibraryBook libraryBook, Exception ex)
|
|
{
|
|
if (!RecordsAttemptFailures || !DownloadFailureClassifier.TryClassify(ex, out var diagnosis))
|
|
return;
|
|
|
|
DownloadAttemptFailureStore.Record(libraryBook, diagnosis.Kind, diagnosis.Reason);
|
|
}
|
|
|
|
public async Task<StatusHandler> TryProcessAsync(LibraryBook libraryBook)
|
|
=> Validate(libraryBook)
|
|
? await ProcessAsync(libraryBook)
|
|
: new StatusHandler();
|
|
|
|
protected void OnBegin(LibraryBook libraryBook)
|
|
{
|
|
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(Begin), Book = libraryBook.LogFriendly() });
|
|
Begin?.Invoke(this, libraryBook);
|
|
}
|
|
|
|
protected void OnStatusUpdate(string statusUpdate)
|
|
{
|
|
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(StatusUpdate), Status = statusUpdate });
|
|
StatusUpdate?.Invoke(this, statusUpdate);
|
|
}
|
|
|
|
protected void OnFileCreated(LibraryBook libraryBook, string path)
|
|
{
|
|
Serilog.Log.Logger.Information("File created {@DebugInfo}", new { Name = nameof(FileCreated), libraryBook.Book.AudibleProductId, path });
|
|
FilePathCache.Insert(libraryBook.Book.AudibleProductId, path);
|
|
FileCreated?.Invoke(this, (libraryBook.Book.AudibleProductId, path));
|
|
}
|
|
|
|
protected void OnStreamingProgressChanged(DownloadProgress progress)
|
|
=> OnStreamingProgressChanged(null, progress);
|
|
protected void OnStreamingProgressChanged(object? _, DownloadProgress progress)
|
|
=> StreamingProgressChanged?.Invoke(this, progress);
|
|
|
|
protected void OnStreamingTimeRemaining(TimeSpan timeRemaining)
|
|
=> OnStreamingTimeRemaining(null, timeRemaining);
|
|
protected void OnStreamingTimeRemaining(object? _, TimeSpan timeRemaining)
|
|
=> StreamingTimeRemaining?.Invoke(this, timeRemaining);
|
|
|
|
protected void OnCompleted(LibraryBook libraryBook)
|
|
{
|
|
Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(Completed), Book = libraryBook.LogFriendly() });
|
|
Completed?.Invoke(this, libraryBook);
|
|
}
|
|
|
|
protected void SetFileTime(LibraryBook libraryBook, string file)
|
|
=> setFileSystemTime(libraryBook, new FileInfo(file));
|
|
protected void SetDirectoryTime(LibraryBook libraryBook, string file)
|
|
=> setFileSystemTime(libraryBook, new DirectoryInfo(file));
|
|
|
|
private void setFileSystemTime(LibraryBook libraryBook, FileSystemInfo fileInfo)
|
|
{
|
|
if (!fileInfo.Exists) return;
|
|
|
|
DateTime? getTimeValue(Configuration.DateTimeSource source) => source switch
|
|
{
|
|
Configuration.DateTimeSource.Added => libraryBook.DateAdded,
|
|
Configuration.DateTimeSource.Published => libraryBook.Book.DatePublished,
|
|
_ => null,
|
|
};
|
|
|
|
if (getTimeValue(Configuration.CreationTime) is { } creationUtc)
|
|
{
|
|
try
|
|
{
|
|
fileInfo.CreationTimeUtc = creationUtc;
|
|
}
|
|
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
|
{
|
|
Serilog.Log.Logger.Debug(ex, "Could not set creation time for {Path}; filesystem may not support it.", fileInfo.FullName);
|
|
}
|
|
}
|
|
|
|
if (getTimeValue(Configuration.LastWriteTime) is { } lastWriteUtc)
|
|
{
|
|
try
|
|
{
|
|
fileInfo.LastWriteTimeUtc = lastWriteUtc;
|
|
}
|
|
catch (Exception ex) when (ex is UnauthorizedAccessException or IOException)
|
|
{
|
|
Serilog.Log.Logger.Debug(ex, "Could not set last write time for {Path}; filesystem may not support it.", fileInfo.FullName);
|
|
}
|
|
}
|
|
}
|
|
}
|