From 027c4a0f4d8832fbc3598b856fd1dde9b35cc589 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Sun, 9 Mar 2025 23:38:27 +0200 Subject: [PATCH] Add option to ignore specific downloads (#79) --- README.md | 4 + .../ContentBlocker/ContentBlockerConfig.cs | 5 +- .../DownloadCleaner/DownloadCleanerConfig.cs | 7 +- .../Configuration/IIgnoredDownloadsConfig.cs | 6 + .../QueueCleaner/QueueCleanerConfig.cs | 5 +- .../Models/Deluge/Response/TorrentStatus.cs | 7 ++ .../DependencyInjection/ServicesDI.cs | 11 +- code/Executable/appsettings.Development.json | 7 +- code/Executable/appsettings.json | 7 +- .../DownloadClient/TestDownloadService.cs | 9 +- .../Extensions/DelugeExtensions.cs | 29 +++++ .../Extensions/QBitExtensions.cs | 42 +++++++ .../Extensions/TransmissionExtensions.cs | 42 +++++++ code/Infrastructure/Helpers/CacheKeys.cs | 2 + .../Providers/IgnoredDownloadsProvider.cs | 82 +++++++++++++ .../ContentBlocker/ContentBlocker.cs | 18 ++- .../DownloadCleaner/DownloadCleaner.cs | 10 +- .../DownloadClient/Deluge/DelugeClient.cs | 18 ++- .../DownloadClient/Deluge/DelugeService.cs | 53 ++++++--- .../DownloadClient/DownloadService.cs | 13 +-- .../DownloadClient/DummyDownloadService.cs | 8 +- .../DownloadClient/IDownloadService.cs | 16 ++- .../DownloadClient/QBittorrent/QBitService.cs | 62 +++++++--- .../Transmission/TransmissionService.cs | 110 ++++++++++-------- .../Verticals/QueueCleaner/QueueCleaner.cs | 18 ++- code/test/data/cleanuperr/ignored_downloads | 1 + code/test/docker-compose.yml | 4 + variables.md | 77 +++++++++++- 28 files changed, 552 insertions(+), 121 deletions(-) create mode 100644 code/Common/Configuration/IIgnoredDownloadsConfig.cs create mode 100644 code/Infrastructure/Extensions/DelugeExtensions.cs create mode 100644 code/Infrastructure/Extensions/QBitExtensions.cs create mode 100644 code/Infrastructure/Extensions/TransmissionExtensions.cs create mode 100644 code/Infrastructure/Providers/IgnoredDownloadsProvider.cs create mode 100644 code/test/data/cleanuperr/ignored_downloads diff --git a/README.md b/README.md index 2c3ae201..aad24687 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ services: restart: unless-stopped volumes: - ./cleanuperr/logs:/var/logs + - ./cleanuperr/ignored.txt:/ignored.txt environment: - TZ=America/New_York - DRY_RUN=false @@ -153,6 +154,7 @@ services: - TRIGGERS__DOWNLOADCLEANER=0 0 * * * ? - QUEUECLEANER__ENABLED=true + - QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt - QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false @@ -165,10 +167,12 @@ services: - QUEUECLEANER__STALLED_DELETE_PRIVATE=false - CONTENTBLOCKER__ENABLED=true + - CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored.txt - CONTENTBLOCKER__IGNORE_PRIVATE=false - CONTENTBLOCKER__DELETE_PRIVATE=false - DOWNLOADCLEANER__ENABLED=true + - DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt - DOWNLOADCLEANER__DELETE_PRIVATE=false - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 diff --git a/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs b/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs index a0fb4b43..4b071eff 100644 --- a/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs +++ b/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs @@ -2,7 +2,7 @@ namespace Common.Configuration.ContentBlocker; -public sealed record ContentBlockerConfig : IJobConfig +public sealed record ContentBlockerConfig : IJobConfig, IIgnoredDownloadsConfig { public const string SectionName = "ContentBlocker"; @@ -13,6 +13,9 @@ public sealed record ContentBlockerConfig : IJobConfig [ConfigurationKeyName("DELETE_PRIVATE")] public bool DeletePrivate { get; init; } + + [ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")] + public string? IgnoredDownloadsPath { get; init; } public void Validate() { diff --git a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs index 7f08fdbe..b5658fad 100644 --- a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs +++ b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Configuration; namespace Common.Configuration.DownloadCleaner; -public sealed record DownloadCleanerConfig : IJobConfig +public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig { public const string SectionName = "DownloadCleaner"; @@ -12,7 +12,10 @@ public sealed record DownloadCleanerConfig : IJobConfig public List? Categories { get; init; } [ConfigurationKeyName("DELETE_PRIVATE")] - public bool DeletePrivate { get; set; } + public bool DeletePrivate { get; init; } + + [ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")] + public string? IgnoredDownloadsPath { get; init; } public void Validate() { diff --git a/code/Common/Configuration/IIgnoredDownloadsConfig.cs b/code/Common/Configuration/IIgnoredDownloadsConfig.cs new file mode 100644 index 00000000..f08e445a --- /dev/null +++ b/code/Common/Configuration/IIgnoredDownloadsConfig.cs @@ -0,0 +1,6 @@ +namespace Common.Configuration; + +public interface IIgnoredDownloadsConfig +{ + string? IgnoredDownloadsPath { get; } +} \ No newline at end of file diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index 85c29920..f0d7049c 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -2,7 +2,7 @@ namespace Common.Configuration.QueueCleaner; -public sealed record QueueCleanerConfig : IJobConfig +public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig { public const string SectionName = "QueueCleaner"; @@ -10,6 +10,9 @@ public sealed record QueueCleanerConfig : IJobConfig public required bool RunSequentially { get; init; } + [ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")] + public string? IgnoredDownloadsPath { get; init; } + [ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")] public ushort ImportFailedMaxStrikes { get; init; } diff --git a/code/Domain/Models/Deluge/Response/TorrentStatus.cs b/code/Domain/Models/Deluge/Response/TorrentStatus.cs index 5ad65a50..b33dd8f5 100644 --- a/code/Domain/Models/Deluge/Response/TorrentStatus.cs +++ b/code/Domain/Models/Deluge/Response/TorrentStatus.cs @@ -23,4 +23,11 @@ public sealed record TorrentStatus public long SeedingTime { get; init; } public float Ratio { get; init; } + + public required IReadOnlyList Trackers { get; init; } +} + +public sealed record Tracker +{ + public required Uri Url { get; init; } } \ No newline at end of file diff --git a/code/Executable/DependencyInjection/ServicesDI.cs b/code/Executable/DependencyInjection/ServicesDI.cs index e1becadb..395d164c 100644 --- a/code/Executable/DependencyInjection/ServicesDI.cs +++ b/code/Executable/DependencyInjection/ServicesDI.cs @@ -1,4 +1,8 @@ -using Infrastructure.Interceptors; +using Common.Configuration.ContentBlocker; +using Common.Configuration.DownloadCleaner; +using Common.Configuration.QueueCleaner; +using Infrastructure.Interceptors; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadCleaner; @@ -30,5 +34,8 @@ public static class ServicesDI .AddTransient() .AddTransient() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton>() + .AddSingleton>() + .AddSingleton>(); } \ No newline at end of file diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index 27dc69be..26a69d1f 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -18,11 +18,13 @@ "ContentBlocker": { "Enabled": true, "IGNORE_PRIVATE": true, - "DELETE_PRIVATE": false + "DELETE_PRIVATE": false, + "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads" }, "QueueCleaner": { "Enabled": true, "RunSequentially": true, + "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads", "IMPORT_FAILED_MAX_STRIKES": 5, "IMPORT_FAILED_IGNORE_PRIVATE": true, "IMPORT_FAILED_DELETE_PRIVATE": false, @@ -44,7 +46,8 @@ "MIN_SEED_TIME": 0, "MAX_SEED_TIME": -1 } - ] + ], + "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads" }, "DOWNLOAD_CLIENT": "qbittorrent", "qBittorrent": { diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index cf834a4a..3b901e87 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -17,11 +17,13 @@ }, "ContentBlocker": { "Enabled": false, - "IGNORE_PRIVATE": false + "IGNORE_PRIVATE": false, + "IGNORED_DOWNLOADS_PATH": "" }, "QueueCleaner": { "Enabled": true, "RunSequentially": true, + "IGNORED_DOWNLOADS_PATH": "", "IMPORT_FAILED_MAX_STRIKES": 0, "IMPORT_FAILED_IGNORE_PRIVATE": false, "IMPORT_FAILED_DELETE_PRIVATE": false, @@ -34,7 +36,8 @@ "DownloadCleaner": { "Enabled": false, "DELETE_PRIVATE": false, - "CATEGORIES": [] + "CATEGORIES": [], + "IGNORED_DOWNLOADS_PATH": "" }, "DOWNLOAD_CLIENT": "none", "qBittorrent": { diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs index abf0f765..60029be1 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs @@ -35,12 +35,13 @@ public class TestDownloadService : DownloadService public override void Dispose() { } public override Task LoginAsync() => Task.CompletedTask; - public override Task ShouldRemoveFromArrQueueAsync(string hash) => Task.FromResult(new StalledResult()); - public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, - ConcurrentBag patterns, ConcurrentBag regexes) => Task.FromResult(new BlockFilesResult()); + public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) => Task.FromResult(new StalledResult()); + public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, + ConcurrentBag patterns, ConcurrentBag regexes, IReadOnlyList ignoredDownloads) => Task.FromResult(new BlockFilesResult()); public override Task DeleteDownload(string hash) => Task.CompletedTask; public override Task?> GetAllDownloadsToBeCleaned(List categories) => Task.FromResult?>(null); - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) => Task.CompletedTask; + public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) => Task.CompletedTask; // Expose protected methods for testing public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded); diff --git a/code/Infrastructure/Extensions/DelugeExtensions.cs b/code/Infrastructure/Extensions/DelugeExtensions.cs new file mode 100644 index 00000000..0c20a458 --- /dev/null +++ b/code/Infrastructure/Extensions/DelugeExtensions.cs @@ -0,0 +1,29 @@ +using Domain.Models.Deluge.Response; + +namespace Infrastructure.Extensions; + +public static class DelugeExtensions +{ + public static bool ShouldIgnore(this TorrentStatus download, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (download.Hash?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false) + { + return true; + } + + if (download.Label?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false) + { + return true; + } + + if (download.Trackers.Any(x => x.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase))) + { + return true; + } + } + + return false; + } +} \ No newline at end of file diff --git a/code/Infrastructure/Extensions/QBitExtensions.cs b/code/Infrastructure/Extensions/QBitExtensions.cs new file mode 100644 index 00000000..1a6f0200 --- /dev/null +++ b/code/Infrastructure/Extensions/QBitExtensions.cs @@ -0,0 +1,42 @@ +using QBittorrent.Client; + +namespace Infrastructure.Extensions; + +public static class QBitExtensions +{ + public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (download.Hash.Equals(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + + if (download.Category.Equals(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + + if (download.Tags.Contains(value, StringComparer.InvariantCultureIgnoreCase)) + { + return true; + } + } + + return false; + } + + public static bool ShouldIgnore(this TorrentTracker tracker, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (tracker.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + } + + return false; + } +} \ No newline at end of file diff --git a/code/Infrastructure/Extensions/TransmissionExtensions.cs b/code/Infrastructure/Extensions/TransmissionExtensions.cs new file mode 100644 index 00000000..c599945e --- /dev/null +++ b/code/Infrastructure/Extensions/TransmissionExtensions.cs @@ -0,0 +1,42 @@ +using Transmission.API.RPC.Entity; + +namespace Infrastructure.Extensions; + +public static class TransmissionExtensions +{ + public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList ignoredDownloads) + { + foreach (string value in ignoredDownloads) + { + if (download.HashString?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false) + { + return true; + } + + if (download.GetCategory().Equals(value, StringComparison.InvariantCultureIgnoreCase)) + { + return true; + } + + bool? hasIgnoredTracker = download.Trackers? + .Any(x => new Uri(x.Announce!).Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase)); + + if (hasIgnoredTracker is true) + { + return true; + } + } + + return false; + } + + public static string GetCategory(this TorrentInfo download) + { + if (string.IsNullOrEmpty(download.DownloadDir)) + { + return string.Empty; + } + + return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir)); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Helpers/CacheKeys.cs b/code/Infrastructure/Helpers/CacheKeys.cs index 61b1728b..2f0d726a 100644 --- a/code/Infrastructure/Helpers/CacheKeys.cs +++ b/code/Infrastructure/Helpers/CacheKeys.cs @@ -11,4 +11,6 @@ public static class CacheKeys public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes"; public static string Item(string hash) => $"item_{hash}"; + + public static string IgnoredDownloads(string name) => $"{name}_ignored"; } \ No newline at end of file diff --git a/code/Infrastructure/Providers/IgnoredDownloadsProvider.cs b/code/Infrastructure/Providers/IgnoredDownloadsProvider.cs new file mode 100644 index 00000000..fe093b12 --- /dev/null +++ b/code/Infrastructure/Providers/IgnoredDownloadsProvider.cs @@ -0,0 +1,82 @@ +using Common.Configuration; +using Infrastructure.Helpers; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Infrastructure.Providers; + +public sealed class IgnoredDownloadsProvider + where T : IIgnoredDownloadsConfig +{ + private readonly ILogger> _logger; + private IIgnoredDownloadsConfig _config; + private readonly IMemoryCache _cache; + private DateTime _lastModified = DateTime.MinValue; + + public IgnoredDownloadsProvider(ILogger> logger, IOptionsMonitor config, IMemoryCache cache) + { + _config = config.CurrentValue; + config.OnChange((newValue) => _config = newValue); + _logger = logger; + _cache = cache; + + if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath)) + { + return; + } + + if (!File.Exists(_config.IgnoredDownloadsPath)) + { + throw new FileNotFoundException("file not found", _config.IgnoredDownloadsPath); + } + } + + public async Task> GetIgnoredDownloads() + { + if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath)) + { + return Array.Empty(); + } + + FileInfo fileInfo = new(_config.IgnoredDownloadsPath); + + if (fileInfo.LastWriteTime > _lastModified || + !_cache.TryGetValue(CacheKeys.IgnoredDownloads(typeof(T).Name), out IReadOnlyList? ignoredDownloads) || + ignoredDownloads is null) + { + _lastModified = fileInfo.LastWriteTime; + + return await LoadFile(); + } + + return ignoredDownloads; + } + + private async Task> LoadFile() + { + try + { + if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath)) + { + return Array.Empty(); + } + + string[] ignoredDownloads = (await File.ReadAllLinesAsync(_config.IgnoredDownloadsPath)) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .ToArray(); + + _cache.Set(CacheKeys.IgnoredDownloads(typeof(T).Name), ignoredDownloads); + + _logger.LogInformation("ignored downloads reloaded"); + + return ignoredDownloads; + } + catch (Exception exception) + { + _logger.LogError(exception, "error while reading ignored downloads file | {file}", _config.IgnoredDownloadsPath); + } + + return Array.Empty(); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs index 7eff7185..857510f4 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs @@ -6,6 +6,7 @@ using Common.Configuration.DownloadClient; using Domain.Enums; using Domain.Models.Arr; using Domain.Models.Arr.Queue; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Context; @@ -22,7 +23,8 @@ public sealed class ContentBlocker : GenericHandler { private readonly ContentBlockerConfig _config; private readonly BlocklistProvider _blocklistProvider; - + private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; + public ContentBlocker( ILogger logger, IOptions config, @@ -36,7 +38,8 @@ public sealed class ContentBlocker : GenericHandler ArrQueueIterator arrArrQueueIterator, BlocklistProvider blocklistProvider, DownloadServiceFactory downloadServiceFactory, - INotificationPublisher notifier + INotificationPublisher notifier, + IgnoredDownloadsProvider ignoredDownloadsProvider ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, @@ -47,6 +50,7 @@ public sealed class ContentBlocker : GenericHandler { _config = config.Value; _blocklistProvider = blocklistProvider; + _ignoredDownloadsProvider = ignoredDownloadsProvider; } public override async Task ExecuteAsync() @@ -73,6 +77,8 @@ public sealed class ContentBlocker : GenericHandler protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) { + IReadOnlyList ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads(); + using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); HashSet itemsToBeRefreshed = []; @@ -106,13 +112,19 @@ public sealed class ContentBlocker : GenericHandler continue; } + if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase)) + { + _logger.LogInformation("skip | {title} | ignored", record.Title); + continue; + } + // push record to context ContextProvider.Set(nameof(QueueRecord), record); _logger.LogDebug("searching unwanted files for {title}", record.Title); BlockFilesResult result = await _downloadService - .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes); + .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads); if (!result.ShouldRemove) { diff --git a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs index 27326945..e5f767c3 100644 --- a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs +++ b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs @@ -3,6 +3,7 @@ using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Domain.Enums; using Domain.Models.Arr.Queue; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.DownloadClient; @@ -17,6 +18,7 @@ namespace Infrastructure.Verticals.DownloadCleaner; public sealed class DownloadCleaner : GenericHandler { private readonly DownloadCleanerConfig _config; + private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; private readonly HashSet _excludedHashes = []; public DownloadCleaner( @@ -31,7 +33,8 @@ public sealed class DownloadCleaner : GenericHandler LidarrClient lidarrClient, ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, - INotificationPublisher notifier + INotificationPublisher notifier, + IgnoredDownloadsProvider ignoredDownloadsProvider ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, @@ -42,6 +45,7 @@ public sealed class DownloadCleaner : GenericHandler { _config = config.Value; _config.Validate(); + _ignoredDownloadsProvider = ignoredDownloadsProvider; } public override async Task ExecuteAsync() @@ -58,6 +62,8 @@ public sealed class DownloadCleaner : GenericHandler return; } + IReadOnlyList ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads(); + await _downloadService.LoginAsync(); List? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories); @@ -75,7 +81,7 @@ public sealed class DownloadCleaner : GenericHandler await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true); await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true); - await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes); + await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes, ignoredDownloads); } protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs index dd29288e..e7eac073 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs @@ -16,6 +16,20 @@ public sealed class DelugeClient private readonly DelugeConfig _config; private readonly HttpClient _httpClient; + private static readonly IReadOnlyList Fields = + [ + "hash", + "state", + "name", + "eta", + "private", + "total_done", + "label", + "seeding_time", + "ratio", + "trackers" + ]; + public DelugeClient(IOptions config, IHttpClientFactory httpClientFactory) { _config = config.Value; @@ -68,7 +82,7 @@ public sealed class DelugeClient return await SendRequest( "web.get_torrent_status", hash, - new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" } + Fields ); } @@ -77,7 +91,7 @@ public sealed class DelugeClient Dictionary? downloads = await SendRequest?>( "core.get_torrents_status", "", - new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" } + Fields ); return downloads?.Values.ToList(); diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index 47cfc3fe..d104b042 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Domain.Enums; using Domain.Models.Deluge.Response; +using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; @@ -49,20 +50,26 @@ public class DelugeService : DownloadService, IDelugeService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { hash = hash.ToLowerInvariant(); DelugeContents? contents = null; StalledResult result = new(); - TorrentStatus? status = await _client.GetTorrentStatus(hash); + TorrentStatus? download = await _client.GetTorrentStatus(hash); - if (status?.Hash is null) + if (download?.Hash is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } try { @@ -88,8 +95,8 @@ public class DelugeService : DownloadService, IDelugeService result.DeleteReason = DeleteReason.AllFilesBlocked; } - result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(status); - result.IsPrivate = status.Private; + result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download); + result.IsPrivate = download.Private; if (!shouldRemove && result.ShouldRemove) { @@ -100,30 +107,37 @@ public class DelugeService : DownloadService, IDelugeService } /// - public override async Task BlockUnwantedFilesAsync( - string hash, + public override async Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes - ) + ConcurrentBag regexes, IReadOnlyList ignoredDownloads) { hash = hash.ToLowerInvariant(); - TorrentStatus? status = await _client.GetTorrentStatus(hash); + TorrentStatus? download = await _client.GetTorrentStatus(hash); BlockFilesResult result = new(); - if (status?.Hash is null) + if (download?.Hash is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } - - result.IsPrivate = status.Private; - if (_contentBlockerConfig.IgnorePrivate && status.Private) + var ceva = await _client.GetTorrentExtended(hash); + + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + + result.IsPrivate = download.Private; + + if (_contentBlockerConfig.IgnorePrivate && download.Private) { // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", status.Name); + _logger.LogDebug("skip files check | download is private | {name}", download.Name); return result; } @@ -205,7 +219,8 @@ public class DelugeService : DownloadService, IDelugeService } /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { foreach (TorrentStatus download in downloads) { @@ -213,6 +228,12 @@ public class DelugeService : DownloadService, IDelugeService { continue; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } Category? category = categoriesToClean .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs index 3f528d38..587fd53c 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs @@ -60,15 +60,13 @@ public abstract class DownloadService : IDownloadService public abstract Task LoginAsync(); - public abstract Task ShouldRemoveFromArrQueueAsync(string hash); + public abstract Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); /// - public abstract Task BlockUnwantedFilesAsync( - string hash, + public abstract Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes - ); + ConcurrentBag regexes, IReadOnlyList ignoredDownloads); /// public abstract Task DeleteDownload(string hash); @@ -77,7 +75,8 @@ public abstract class DownloadService : IDownloadService public abstract Task?> GetAllDownloadsToBeCleaned(List categories); /// - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads); protected void ResetStrikesOnProgress(string hash, long downloaded) { @@ -131,7 +130,7 @@ public abstract class DownloadService : IDownloadService return new(); } - + private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category) { if (category.MaxRatio < 0) diff --git a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs index 91263283..1d0e87e0 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs @@ -28,12 +28,13 @@ public class DummyDownloadService : DownloadService return Task.CompletedTask; } - public override Task ShouldRemoveFromArrQueueAsync(string hash) + public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { throw new NotImplementedException(); } - public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, ConcurrentBag regexes) + public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, + ConcurrentBag regexes, IReadOnlyList ignoredDownloads) { throw new NotImplementedException(); } @@ -43,7 +44,8 @@ public class DummyDownloadService : DownloadService throw new NotImplementedException(); } - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs index c2d446fc..5ac9db7e 100644 --- a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs @@ -14,7 +14,8 @@ public interface IDownloadService : IDisposable /// Checks whether the download should be removed from the *arr queue. /// /// The download hash. - public Task ShouldRemoveFromArrQueueAsync(string hash); + /// Downloads to ignore from processing. + public Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); /// /// Blocks unwanted files from being fully downloaded. @@ -23,12 +24,13 @@ public interface IDownloadService : IDisposable /// The . /// The patterns to test the files against. /// The regexes to test the files against. + /// Downloads to ignore from processing. /// True if all files have been blocked; otherwise false. - public Task BlockUnwantedFilesAsync( - string hash, + public Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes + ConcurrentBag regexes, + IReadOnlyList ignoredDownloads ); /// @@ -37,14 +39,16 @@ public interface IDownloadService : IDisposable /// The categories by which to filter the downloads. /// A list of downloads for the provided categories. Task?> GetAllDownloadsToBeCleaned(List categories); - + /// /// Cleans the downloads. /// /// /// The categories that should be cleaned. /// The hashes that should not be cleaned. - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes); + /// Downloads to ignore from processing. + public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads); /// /// Deletes a download item. diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 0e14776a..8061fe2b 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Common.Helpers; using Domain.Enums; +using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; @@ -58,18 +59,27 @@ public class QBitService : DownloadService, IQBitService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { StalledResult result = new(); - TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) + TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) .FirstOrDefault(); - if (torrent is null) + if (download is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } + IReadOnlyList trackers = await GetTrackersAsync(hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); if (torrentProperties is null) @@ -83,7 +93,7 @@ public class QBitService : DownloadService, IQBitService && boolValue; // if all files were blocked by qBittorrent - if (torrent is { CompletionOn: not null, Downloaded: null or 0 }) + if (download is { CompletionOn: not null, Downloaded: null or 0 }) { result.ShouldRemove = true; result.DeleteReason = DeleteReason.AllFilesBlocked; @@ -100,7 +110,7 @@ public class QBitService : DownloadService, IQBitService return result; } - result.ShouldRemove = await IsItemStuckAndShouldRemove(torrent, result.IsPrivate); + result.ShouldRemove = await IsItemStuckAndShouldRemove(download, result.IsPrivate); if (result.ShouldRemove) { @@ -111,23 +121,32 @@ public class QBitService : DownloadService, IQBitService } /// - public override async Task BlockUnwantedFilesAsync( - string hash, + public override async Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes + ConcurrentBag regexes, + IReadOnlyList ignoredDownloads ) { - TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) + TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) .FirstOrDefault(); BlockFilesResult result = new(); - if (torrent is null) + if (download is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } + IReadOnlyList trackers = await GetTrackersAsync(hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); if (torrentProperties is null) @@ -145,7 +164,7 @@ public class QBitService : DownloadService, IQBitService if (_contentBlockerConfig.IgnorePrivate && isPrivate) { // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", torrent.Name); + _logger.LogDebug("skip files check | download is private | {name}", download.Name); return result; } @@ -218,7 +237,8 @@ public class QBitService : DownloadService, IQBitService .ToList(); /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { foreach (TorrentInfo download in downloads) { @@ -227,6 +247,15 @@ public class QBitService : DownloadService, IQBitService continue; } + IReadOnlyList trackers = await GetTrackersAsync(download.Hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + Category? category = categoriesToClean .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); @@ -303,7 +332,7 @@ public class QBitService : DownloadService, IQBitService { _client.Dispose(); } - + private async Task IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate) { if (_queueCleanerConfig.StalledMaxStrikes is 0) @@ -329,4 +358,11 @@ public class QBitService : DownloadService, IQBitService return await StrikeAndCheckLimit(torrent.Hash, torrent.Name); } + + private async Task> GetTrackersAsync(string hash) + { + return (await _client.GetTorrentTrackersAsync(hash)) + .Where(x => !x.Url.ToString().Contains("**")) + .ToList(); + } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 80abfa79..51605691 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -7,6 +7,7 @@ using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Common.Helpers; using Domain.Enums; +using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; @@ -27,6 +28,23 @@ public class TransmissionService : DownloadService, ITransmissionService private readonly Client _client; private TorrentInfo[]? _torrentsCache; + private static readonly string[] Fields = + [ + TorrentFields.FILES, + TorrentFields.FILE_STATS, + TorrentFields.HASH_STRING, + TorrentFields.ID, + TorrentFields.ETA, + TorrentFields.NAME, + TorrentFields.STATUS, + TorrentFields.IS_PRIVATE, + TorrentFields.DOWNLOADED_EVER, + TorrentFields.DOWNLOAD_DIR, + TorrentFields.SECONDS_SEEDING, + TorrentFields.UPLOAD_RATIO, + TorrentFields.TRACKERS + ]; + public TransmissionService( IHttpClientFactory httpClientFactory, ILogger logger, @@ -60,21 +78,27 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { StalledResult result = new(); - TorrentInfo? torrent = await GetTorrentAsync(hash); + TorrentInfo? download = await GetTorrentAsync(hash); - if (torrent is null) + if (download is null) { _logger.LogDebug("failed to find torrent {hash} in the download client", hash); return result; } - bool shouldRemove = torrent.FileStats?.Length > 0; - result.IsPrivate = torrent.IsPrivate ?? false; + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + return result; + } + + bool shouldRemove = download.FileStats?.Length > 0; + result.IsPrivate = download.IsPrivate ?? false; - foreach (TransmissionTorrentFileStats? stats in torrent.FileStats ?? []) + foreach (TransmissionTorrentFileStats? stats in download.FileStats ?? []) { if (!stats.Wanted.HasValue) { @@ -95,7 +119,7 @@ public class TransmissionService : DownloadService, ITransmissionService } // remove if all files are unwanted or download is stuck - result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(torrent); + result.ShouldRemove = shouldRemove || await IsItemStuckAndShouldRemove(download); if (!shouldRemove && result.ShouldRemove) { @@ -106,28 +130,32 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task BlockUnwantedFilesAsync( - string hash, + public override async Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, - ConcurrentBag regexes - ) + ConcurrentBag regexes, IReadOnlyList ignoredDownloads) { - TorrentInfo? torrent = await GetTorrentAsync(hash); + TorrentInfo? download = await GetTorrentAsync(hash); BlockFilesResult result = new(); - if (torrent?.FileStats is null || torrent.Files is null) + if (download?.FileStats is null || download.Files is null) { return result; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + return result; + } - bool isPrivate = torrent.IsPrivate ?? false; + bool isPrivate = download.IsPrivate ?? false; result.IsPrivate = isPrivate; if (_contentBlockerConfig.IgnorePrivate && isPrivate) { // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", torrent.Name); + _logger.LogDebug("skip files check | download is private | {name}", download.Name); return result; } @@ -135,27 +163,27 @@ public class TransmissionService : DownloadService, ITransmissionService long totalFiles = 0; long totalUnwantedFiles = 0; - for (int i = 0; i < torrent.Files.Length; i++) + for (int i = 0; i < download.Files.Length; i++) { - if (torrent.FileStats?[i].Wanted == null) + if (download.FileStats?[i].Wanted == null) { continue; } totalFiles++; - if (!torrent.FileStats[i].Wanted.Value) + if (!download.FileStats[i].Wanted.Value) { totalUnwantedFiles++; continue; } - if (_filenameEvaluator.IsValid(torrent.Files[i].Name, blocklistType, patterns, regexes)) + if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes)) { continue; } - _logger.LogInformation("unwanted file found | {file}", torrent.Files[i].Name); + _logger.LogInformation("unwanted file found | {file}", download.Files[i].Name); unwantedFiles.Add(i); totalUnwantedFiles++; } @@ -175,7 +203,7 @@ public class TransmissionService : DownloadService, ITransmissionService _logger.LogDebug("changing priorities | torrent {hash}", hash); - await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, torrent.Id, unwantedFiles.ToArray()); + await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray()); return result; } @@ -183,22 +211,7 @@ public class TransmissionService : DownloadService, ITransmissionService /// public override async Task?> GetAllDownloadsToBeCleaned(List categories) { - string[] fields = [ - TorrentFields.FILES, - TorrentFields.FILE_STATS, - TorrentFields.HASH_STRING, - TorrentFields.ID, - TorrentFields.ETA, - TorrentFields.NAME, - TorrentFields.STATUS, - TorrentFields.IS_PRIVATE, - TorrentFields.DOWNLOADED_EVER, - TorrentFields.DOWNLOAD_DIR, - TorrentFields.SECONDS_SEEDING, - TorrentFields.UPLOAD_RATIO - ]; - - return (await _client.TorrentGetAsync(fields)) + return (await _client.TorrentGetAsync(Fields)) ?.Torrents ?.Where(x => !string.IsNullOrEmpty(x.HashString)) .Where(x => x.Status is 5 or 6) @@ -219,7 +232,8 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes) + public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) { foreach (TorrentInfo download in downloads) { @@ -227,6 +241,12 @@ public class TransmissionService : DownloadService, ITransmissionService { continue; } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + continue; + } Category? category = categoriesToClean .FirstOrDefault(x => @@ -351,20 +371,8 @@ public class TransmissionService : DownloadService, ITransmissionService if (_torrentsCache is null || torrent is null) { - string[] fields = [ - TorrentFields.FILES, - TorrentFields.FILE_STATS, - TorrentFields.HASH_STRING, - TorrentFields.ID, - TorrentFields.ETA, - TorrentFields.NAME, - TorrentFields.STATUS, - TorrentFields.IS_PRIVATE, - TorrentFields.DOWNLOADED_EVER - ]; - // refresh cache - _torrentsCache = (await _client.TorrentGetAsync(fields)) + _torrentsCache = (await _client.TorrentGetAsync(Fields)) ?.Torrents; } diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index 0bba5d1b..561b38c2 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -4,6 +4,7 @@ using Common.Configuration.QueueCleaner; using Domain.Enums; using Domain.Models.Arr; using Domain.Models.Arr.Queue; +using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Context; @@ -19,7 +20,8 @@ namespace Infrastructure.Verticals.QueueCleaner; public sealed class QueueCleaner : GenericHandler { private readonly QueueCleanerConfig _config; - + private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; + public QueueCleaner( ILogger logger, IOptions config, @@ -32,7 +34,8 @@ public sealed class QueueCleaner : GenericHandler LidarrClient lidarrClient, ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, - INotificationPublisher notifier + INotificationPublisher notifier, + IgnoredDownloadsProvider ignoredDownloadsProvider ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, @@ -42,10 +45,13 @@ public sealed class QueueCleaner : GenericHandler ) { _config = config.Value; + _ignoredDownloadsProvider = ignoredDownloadsProvider; } protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) { + IReadOnlyList ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads(); + using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); HashSet itemsToBeRefreshed = []; @@ -75,6 +81,12 @@ public sealed class QueueCleaner : GenericHandler continue; } + if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase)) + { + _logger.LogInformation("skip | {title} | ignored", record.Title); + continue; + } + // push record to context ContextProvider.Set(nameof(QueueRecord), record); @@ -83,7 +95,7 @@ public sealed class QueueCleaner : GenericHandler if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent") { // stalled download check - stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId); + stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads); } // failed import check diff --git a/code/test/data/cleanuperr/ignored_downloads b/code/test/data/cleanuperr/ignored_downloads new file mode 100644 index 00000000..5537770d --- /dev/null +++ b/code/test/data/cleanuperr/ignored_downloads @@ -0,0 +1 @@ +ignored \ No newline at end of file diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index 010531d7..fb8c186b 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -191,6 +191,7 @@ services: - TRIGGERS__DOWNLOADCLEANER=0/30 * * * * ? - QUEUECLEANER__ENABLED=true + - QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored - QUEUECLEANER__RUNSEQUENTIALLY=true - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true @@ -201,10 +202,12 @@ services: - QUEUECLEANER__STALLED_DELETE_PRIVATE=false - CONTENTBLOCKER__ENABLED=true + - CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored - CONTENTBLOCKER__IGNORE_PRIVATE=true - CONTENTBLOCKER__DELETE_PRIVATE=false - DOWNLOADCLEANER__ENABLED=true + - DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH=/ignored - DOWNLOADCLEANER__DELETE_PRIVATE=false - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 @@ -256,6 +259,7 @@ services: # - NOTIFIARR__CHANNEL_ID=discord_channel_id volumes: - ./data/cleanuperr/logs:/var/logs + - ./data/cleanuperr/ignored_downloads:/ignored restart: unless-stopped depends_on: - qbittorrent diff --git a/variables.md b/variables.md index dcd07cef..dbc0aaf9 100644 --- a/variables.md +++ b/variables.md @@ -76,6 +76,31 @@ - Default: `true` - Required: No. +**`QUEUECLEANER__IGNORED_DOWNLOADS_PATH`** +- Local path to the file containing ignored downloads. +- If the contents of the file are changed, they will be reloaded on the next job run. +- Accepted values: + - torrent hash + - qBitTorrent tag or category + - Deluge label + - Transmission category (last directory from the save location) + - torrent tracker domain +- Each value needs to be on a new line. +- Type: String. +- Default: Empty. +- Required: No. +- Example: `/ignored.txt`. +- Example of file contents: + ``` + fa800a7d7c443a2c3561d1f8f393c089036dade1 + tv-sonarr + qbit-tag + mytracker.com + ... + ``` +>[!IMPORTANT] +> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. + **`QUEUECLEANER__RUNSEQUENTIALLY`** - Controls whether queue cleaner runs after content blocker instead of in parallel. - When `true`, streamlines the cleaning process by running immediately after content blocker. @@ -178,6 +203,31 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `false` - Required: No. +**`CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH`** +- Local path to the file containing ignored downloads. +- If the contents of the file are changed, they will be reloaded on the next job run. +- Accepted values: + - torrent hash + - qBitTorrent tag or category + - Deluge label + - Transmission category (last directory from the save location) + - torrent tracker domain +- Each value needs to be on a new line. +- Type: String. +- Default: Empty. +- Required: No. +- Example: `/ignored.txt`. +- Example of file contents: + ``` + fa800a7d7c443a2c3561d1f8f393c089036dade1 + tv-sonarr + qbit-tag + mytracker.com + ... + ``` +>[!IMPORTANT] +> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. + **`CONTENTBLOCKER__IGNORE_PRIVATE`** - Controls whether to ignore downloads from private trackers. - Type: Boolean @@ -217,6 +267,31 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" - Default: `false` - Required: No. +**`DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH`** +- Local path to the file containing ignored downloads. +- If the contents of the file are changed, they will be reloaded on the next job run. +- Accepted values: + - torrent hash + - qBitTorrent tag or category + - Deluge label + - Transmission category (last directory from the save location) + - torrent tracker domain +- Each value needs to be on a new line. +- Type: String. +- Default: Empty. +- Required: No. +- Example: `/ignored.txt`. +- Example of file contents: + ``` + fa800a7d7c443a2c3561d1f8f393c089036dade1 + tv-sonarr + qbit-tag + mytracker.com + ... + ``` +>[!IMPORTANT] +> Some people have experienced problems using Docker where the mounted file would not update inside the container if it was modified on the host. This is a Docker configuration problem and can not be solved by cleanuperr. + **`DOWNLOADCLEANER__DELETE_PRIVATE`** - Controls whether to delete private downloads. - Type: Boolean. @@ -237,7 +312,7 @@ QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1: "manual import required" > The category name must match the category that was set in the *arr. > For qBittorrent, the category name is the name of the download category. > For Deluge, the category name is the name of the label. -> For Transmission, the category name is the name of the download location. +> For Transmission, the category name is the last directory from the save location. **`DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO`** - Maximum ratio to reach before removing a download.