diff --git a/.github/ISSUE_TEMPLATE/1-bug.yml b/.github/ISSUE_TEMPLATE/1-bug.yml index b2543db4..200d6e49 100644 --- a/.github/ISSUE_TEMPLATE/1-bug.yml +++ b/.github/ISSUE_TEMPLATE/1-bug.yml @@ -1,6 +1,6 @@ name: Bug report description: File a bug report if something is not working right. -title: "[BUG]: " +title: "[BUG] " labels: ["bug"] body: - type: markdown @@ -40,6 +40,7 @@ body: - Windows - Linux - MacOS + - Unraid validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/2-feature.yml b/.github/ISSUE_TEMPLATE/2-feature.yml index a6332e20..2496f7ff 100644 --- a/.github/ISSUE_TEMPLATE/2-feature.yml +++ b/.github/ISSUE_TEMPLATE/2-feature.yml @@ -1,6 +1,6 @@ name: Feature request description: File a feature request. -title: "[FEATURE]: " +title: "[FEATURE] " labels: ["enhancement"] body: - type: markdown diff --git a/.github/ISSUE_TEMPLATE/3-help.yml b/.github/ISSUE_TEMPLATE/3-help.yml index b90919d6..022f9912 100644 --- a/.github/ISSUE_TEMPLATE/3-help.yml +++ b/.github/ISSUE_TEMPLATE/3-help.yml @@ -1,6 +1,6 @@ name: Help request description: Ask a question to receive help. -title: "[HELP]: " +title: "[HELP] " labels: ["question"] body: - type: markdown diff --git a/README.md b/README.md index c0936ebf..49d5b2d3 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,10 @@ This tool is actively developed and still a work in progress. Join the Discord s 1. Set `QUEUECLEANER__ENABLED` to `true`. 2. Set `QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES` to a desired value. -3. Set `DOWNLOAD_CLIENT` to `none`. +3. Optionally set failed import message patterns to ignore using `QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__`. +4. Set `DOWNLOAD_CLIENT` to `none`. -**No other action involving a download client would work (e.g. content blocking, removing stalled downloads etc.).** +**No other action involving a download client would work (e.g. content blocking, removing stalled downloads, excluding private trackers).** ## Usage @@ -99,7 +100,11 @@ services: - QUEUECLEANER__ENABLED=true - QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 + - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false + # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch + # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required - QUEUECLEANER__STALLED_MAX_STRIKES=5 + - QUEUECLEANER__STALLED_IGNORE_PRIVATE=false - CONTENTBLOCKER__ENABLED=true - CONTENTBLOCKER__BLACKLIST__ENABLED=true @@ -155,7 +160,10 @@ services: | QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner | true | | QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process | true | | QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | After how many strikes should a failed import be removed
0 means never | 0 | +| QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers | false | +| QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | First pattern to look for when an import is failed
If the specified message pattern is found, the item is skipped | empty | | QUEUECLEANER__STALLED_MAX_STRIKES | No | After how many strikes should a stalled download be removed
0 means never | 0 | +| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers | false | ||||| | CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker | false | | CONTENTBLOCKER__BLACKLIST__ENABLED | Yes if content blocker is enabled and whitelist is not enabled | Enable or disable the blacklist | false | @@ -203,6 +211,10 @@ regex: // regex that needs to be marked at the start of the line wi SONARR__INSTANCES____URL SONARR__INSTANCES____APIKEY ``` +6. Multiple failed import patterns can be specified using this format, where `` starts from 0: +``` +QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__ +``` # diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index c3539799..b6fa5bea 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -13,8 +13,17 @@ public sealed record QueueCleanerConfig : IJobConfig [ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")] public ushort ImportFailedMaxStrikes { get; init; } + [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PRIVATE")] + public bool ImportFailedIgnorePrivate { get; init; } + + [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")] + public List? ImportFailedIgnorePatterns { get; init; } + [ConfigurationKeyName("STALLED_MAX_STRIKES")] public ushort StalledMaxStrikes { get; init; } + + [ConfigurationKeyName("STALLED_IGNORE_PRIVATE")] + public bool StalledIgnorePrivate { get; init; } public void Validate() { diff --git a/code/Domain/Models/Arr/Queue/QueueRecord.cs b/code/Domain/Models/Arr/Queue/QueueRecord.cs index 0158cdad..b5a940aa 100644 --- a/code/Domain/Models/Arr/Queue/QueueRecord.cs +++ b/code/Domain/Models/Arr/Queue/QueueRecord.cs @@ -1,6 +1,6 @@ -namespace Domain.Models.Arr.Queue; +namespace Domain.Models.Arr.Queue; -public record QueueRecord +public sealed record QueueRecord { public int SeriesId { get; init; } public int EpisodeId { get; init; } @@ -10,6 +10,7 @@ public record QueueRecord public string Status { get; init; } public string TrackedDownloadStatus { get; init; } public string TrackedDownloadState { get; init; } + public List? StatusMessages { get; init; } public required string DownloadId { get; init; } public required string Protocol { get; init; } public required int Id { get; init; } diff --git a/code/Domain/Models/Arr/Queue/TrackedDownloadStatusMessage.cs b/code/Domain/Models/Arr/Queue/TrackedDownloadStatusMessage.cs new file mode 100644 index 00000000..6d1f8fdf --- /dev/null +++ b/code/Domain/Models/Arr/Queue/TrackedDownloadStatusMessage.cs @@ -0,0 +1,8 @@ +namespace Domain.Models.Arr.Queue; + +public sealed record TrackedDownloadStatusMessage +{ + public string Title { get; set; } + + public List? Messages { get; set; } +} \ No newline at end of file diff --git a/code/Domain/Models/Deluge/Response/TorrentStatus.cs b/code/Domain/Models/Deluge/Response/TorrentStatus.cs index 97d22a43..2aedb0af 100644 --- a/code/Domain/Models/Deluge/Response/TorrentStatus.cs +++ b/code/Domain/Models/Deluge/Response/TorrentStatus.cs @@ -2,11 +2,13 @@ public sealed record TorrentStatus { - public string? Hash { get; set; } + public string? Hash { get; init; } - public string? State { get; set; } + public string? State { get; init; } - public string? Name { get; set; } + public string? Name { get; init; } - public ulong Eta { get; set; } + public ulong Eta { get; init; } + + public bool Private { get; init; } } \ No newline at end of file diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index f0c76faa..c217c5c0 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -26,7 +26,12 @@ "Enabled": true, "RunSequentially": true, "IMPORT_FAILED_MAX_STRIKES": 5, - "STALLED_MAX_STRIKES": 5 + "IMPORT_FAILED_IGNORE_PRIVATE": true, + "IMPORT_FAILED_IGNORE_PATTERNS": [ + "file is a sample" + ], + "STALLED_MAX_STRIKES": 5, + "STALLED_IGNORE_PRIVATE": true }, "DOWNLOAD_CLIENT": "qbittorrent", "qBittorrent": { diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index cd298b77..dd9d597a 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -26,7 +26,10 @@ "Enabled": true, "RunSequentially": true, "IMPORT_FAILED_MAX_STRIKES": 5, - "STALLED_MAX_STRIKES": 5 + "IMPORT_FAILED_IGNORE_PRIVATE": false, + "IMPORT_FAILED_IGNORE_PATTERNS": [], + "STALLED_MAX_STRIKES": 5, + "STALLED_IGNORE_PRIVATE": false }, "DOWNLOAD_CLIENT": "qbittorrent", "qBittorrent": { diff --git a/code/Infrastructure/Verticals/Arr/ArrClient.cs b/code/Infrastructure/Verticals/Arr/ArrClient.cs index ac03ce46..79848dac 100644 --- a/code/Infrastructure/Verticals/Arr/ArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/ArrClient.cs @@ -65,17 +65,30 @@ public abstract class ArrClient return queueResponse; } - public virtual bool ShouldRemoveFromQueue(QueueRecord record) + public virtual bool ShouldRemoveFromQueue(QueueRecord record, bool isPrivateDownload) { + if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload) + { + // ignore private trackers + _logger.LogDebug("skip failed import check | download is private | {name}", record.Title); + return false; + } + bool hasWarn() => record.TrackedDownloadStatus .Equals("warning", StringComparison.InvariantCultureIgnoreCase); bool isImportBlocked() => record.TrackedDownloadState .Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase); bool isImportPending() => record.TrackedDownloadState .Equals("importPending", StringComparison.InvariantCultureIgnoreCase); - + if (hasWarn() && (isImportBlocked() || isImportPending())) { + if (HasIgnoredPatterns(record)) + { + _logger.LogDebug("skip failed import check | contains ignored pattern | {name}", record.Title); + return false; + } + return _striker.StrikeAndCheckLimit( record.DownloadId, record.Title, @@ -134,4 +147,32 @@ public abstract class ArrClient { request.Headers.Add("x-api-key", apiKey); } + + private bool HasIgnoredPatterns(QueueRecord record) + { + if (_queueCleanerConfig.ImportFailedIgnorePatterns?.Count is null or 0) + { + // no patterns are configured + return false; + } + + if (record.StatusMessages?.Count is null or 0) + { + // no status message found + return false; + } + + HashSet messages = record.StatusMessages + .SelectMany(x => x.Messages ?? Enumerable.Empty()) + .ToHashSet(); + record.StatusMessages.Select(x => x.Title) + .ToList() + .ForEach(x => messages.Add(x)); + + return messages.Any( + m => _queueCleanerConfig.ImportFailedIgnorePatterns.Any( + p => !string.IsNullOrWhiteSpace(p.Trim()) && m.Contains(p, StringComparison.InvariantCultureIgnoreCase) + ) + ); + } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index 6e7134a9..0b32e851 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -30,18 +30,19 @@ public sealed class DelugeService : DownloadServiceBase await _client.LoginAsync(); } - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash) { hash = hash.ToLowerInvariant(); DelugeContents? contents = null; + RemoveResult result = new(); TorrentStatus? status = await GetTorrentStatus(hash); if (status?.Hash is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return false; + return result; } try @@ -63,7 +64,10 @@ public sealed class DelugeService : DownloadServiceBase } }); - return shouldRemove || IsItemStuckAndShouldRemove(status); + result.ShouldRemove = shouldRemove || IsItemStuckAndShouldRemove(status); + result.IsPrivate = status.Private; + + return result; } public override async Task BlockUnwantedFilesAsync(string hash) @@ -128,6 +132,18 @@ public sealed class DelugeService : DownloadServiceBase private bool IsItemStuckAndShouldRemove(TorrentStatus status) { + if (_queueCleanerConfig.StalledMaxStrikes is 0) + { + return false; + } + + if (_queueCleanerConfig.StalledIgnorePrivate && status.Private) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", status.Name); + return false; + } + if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) { return false; @@ -146,7 +162,7 @@ public sealed class DelugeService : DownloadServiceBase return await _client.SendRequest( "web.get_torrent_status", hash, - new[] { "hash", "state", "name", "eta" } + new[] { "hash", "state", "name", "eta", "private" } ); } diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs index 6ef71845..d8ec316d 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs @@ -31,7 +31,7 @@ public abstract class DownloadServiceBase : IDownloadService public abstract Task LoginAsync(); - public abstract Task ShouldRemoveFromArrQueueAsync(string hash); + public abstract Task ShouldRemoveFromArrQueueAsync(string hash); public abstract Task BlockUnwantedFilesAsync(string hash); diff --git a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs index bdffec93..eda4f1f1 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs @@ -21,7 +21,7 @@ public sealed class DummyDownloadService : DownloadServiceBase return Task.CompletedTask; } - public override Task ShouldRemoveFromArrQueueAsync(string hash) + public override Task ShouldRemoveFromArrQueueAsync(string hash) { throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs index 3fcdac95..3f42b62f 100644 --- a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs @@ -4,7 +4,7 @@ public interface IDownloadService : IDisposable { public Task LoginAsync(); - public Task ShouldRemoveFromArrQueueAsync(string hash); + public Task ShouldRemoveFromArrQueueAsync(string hash); public Task BlockUnwantedFilesAsync(string hash); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index d6b0de55..deeb571a 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -1,4 +1,4 @@ -using Common.Configuration.DownloadClient; +using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ItemStriker; @@ -36,21 +36,35 @@ public sealed class QBitService : DownloadServiceBase await _client.LoginAsync(_config.Username, _config.Password); } - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash) { + RemoveResult result = new(); TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) .FirstOrDefault(); if (torrent is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return false; + return result; } + TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); + + if (torrentProperties is null) + { + _logger.LogDebug("failed to find torrent properties {hash} in the download client", hash); + return result; + } + + result.IsPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) && + bool.TryParse(dictValue?.ToString(), out bool boolValue) + && boolValue; + // if all files were blocked by qBittorrent if (torrent is { CompletionOn: not null, Downloaded: null or 0 }) { - return true; + result.ShouldRemove = true; + return result; } IReadOnlyList? files = await _client.GetTorrentContentsAsync(hash); @@ -58,10 +72,13 @@ public sealed class QBitService : DownloadServiceBase // if all files are marked as skip if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip)) { - return true; + result.ShouldRemove = true; + return result; } - return IsItemStuckAndShouldRemove(torrent); + result.ShouldRemove = IsItemStuckAndShouldRemove(torrent, result.IsPrivate); + + return result; } public override async Task BlockUnwantedFilesAsync(string hash) @@ -95,8 +112,20 @@ public sealed class QBitService : DownloadServiceBase _client.Dispose(); } - private bool IsItemStuckAndShouldRemove(TorrentInfo torrent) + private bool IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate) { + if (_queueCleanerConfig.StalledMaxStrikes is 0) + { + return false; + } + + if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); + return false; + } + if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata and not TorrentState.ForcedFetchingMetadata) { diff --git a/code/Infrastructure/Verticals/DownloadClient/RemoveResult.cs b/code/Infrastructure/Verticals/DownloadClient/RemoveResult.cs new file mode 100644 index 00000000..75efa446 --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadClient/RemoveResult.cs @@ -0,0 +1,14 @@ +namespace Infrastructure.Verticals.DownloadClient; + +public sealed record RemoveResult +{ + /// + /// True if the download should be removed; otherwise false. + /// + public bool ShouldRemove { get; set; } + + /// + /// True if the download is private; otherwise false. + /// + public bool IsPrivate { get; set; } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 801968eb..48821f6c 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -38,17 +38,19 @@ public sealed class TransmissionService : DownloadServiceBase await _client.GetSessionInformationAsync(); } - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash) { + RemoveResult result = new(); TorrentInfo? torrent = await GetTorrentAsync(hash); if (torrent is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return false; + return result; } bool shouldRemove = torrent.FileStats?.Length > 0; + result.IsPrivate = torrent.IsPrivate ?? false; foreach (TransmissionTorrentFileStats? stats in torrent.FileStats ?? []) { @@ -65,8 +67,10 @@ public sealed class TransmissionService : DownloadServiceBase } } - // remove if all files are unwanted - return shouldRemove || IsItemStuckAndShouldRemove(torrent); + // remove if all files are unwanted or download is stuck + result.ShouldRemove = shouldRemove || IsItemStuckAndShouldRemove(torrent); + + return result; } public override async Task BlockUnwantedFilesAsync(string hash) @@ -116,6 +120,18 @@ public sealed class TransmissionService : DownloadServiceBase private bool IsItemStuckAndShouldRemove(TorrentInfo torrent) { + if (_queueCleanerConfig.StalledMaxStrikes is 0) + { + return false; + } + + if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false)) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); + return false; + } + if (torrent.Status is not 4) { // not in downloading state @@ -144,7 +160,8 @@ public sealed class TransmissionService : DownloadServiceBase TorrentFields.ID, TorrentFields.ETA, TorrentFields.NAME, - TorrentFields.STATUS + TorrentFields.STATUS, + TorrentFields.IS_PRIVATE ]; // refresh cache diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index be9599f1..91e30e86 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -57,11 +57,16 @@ public sealed class QueueCleaner : GenericHandler continue; } - bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(record); - bool shouldRemoveFromDownloadClient = _downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && - await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId); + RemoveResult removeResult = new(); - if (!shouldRemoveFromArr && !shouldRemoveFromDownloadClient) + if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None) + { + removeResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId); + } + + bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(record, removeResult.IsPrivate); + + if (!shouldRemoveFromArr && !removeResult.ShouldRemove) { _logger.LogInformation("skip | {title}", record.Title); continue; diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index 2cf68c68..7b3a2e6b 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -182,7 +182,10 @@ services: - QUEUECLEANER__ENABLED=true - QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 + - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true + - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=file is a sample - QUEUECLEANER__STALLED_MAX_STRIKES=5 + - QUEUECLEANER__STALLED_IGNORE_PRIVATE=true - CONTENTBLOCKER__ENABLED=true - CONTENTBLOCKER__BLACKLIST__ENABLED=true