diff --git a/README.md b/README.md index 1a492b6a..e92b9533 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ services: # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required - QUEUECLEANER__STALLED_MAX_STRIKES=5 + - QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS=false - QUEUECLEANER__STALLED_IGNORE_PRIVATE=false - QUEUECLEANER__STALLED_DELETE_PRIVATE=false @@ -184,6 +185,7 @@ services: | QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE | No | - Whether to delete failed imports of private downloads from the download client.
- Does not have any effect if `QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE` is `true`.
- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false | | QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | - First pattern to look for when an import is failed.
- If the specified message pattern is found, the item is skipped. | empty | | QUEUECLEANER__STALLED_MAX_STRIKES | No | - After how many strikes should a stalled download be removed.
- 0 means never. | 0 | +| QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS | No | Removes strikes if any download progress was made since last checked. | false | | QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers. | false | | QUEUECLEANER__STALLED_DELETE_PRIVATE | No | - Whether to delete stalled private downloads from the download client.
- Does not have any effect if `QUEUECLEANER__STALLED_IGNORE_PRIVATE` is `true`.
- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false | ||||| diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index 191b5d98..85c29920 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -25,6 +25,9 @@ public sealed record QueueCleanerConfig : IJobConfig [ConfigurationKeyName("STALLED_MAX_STRIKES")] public ushort StalledMaxStrikes { get; init; } + [ConfigurationKeyName("STALLED_RESET_STRIKES_ON_PROGRESS")] + public bool StalledResetStrikesOnProgress { get; init; } + [ConfigurationKeyName("STALLED_IGNORE_PRIVATE")] public bool StalledIgnorePrivate { get; init; } diff --git a/code/Domain/Models/Cache/CacheItem.cs b/code/Domain/Models/Cache/CacheItem.cs new file mode 100644 index 00000000..8d5b2aef --- /dev/null +++ b/code/Domain/Models/Cache/CacheItem.cs @@ -0,0 +1,9 @@ +namespace Domain.Models.Cache; + +public sealed record CacheItem +{ + /// + /// The amount of bytes that have been downloaded. + /// + public long Downloaded { get; set; } +} \ No newline at end of file diff --git a/code/Domain/Models/Deluge/Response/TorrentStatus.cs b/code/Domain/Models/Deluge/Response/TorrentStatus.cs index 2aedb0af..85cf39f6 100644 --- a/code/Domain/Models/Deluge/Response/TorrentStatus.cs +++ b/code/Domain/Models/Deluge/Response/TorrentStatus.cs @@ -1,4 +1,6 @@ -namespace Domain.Models.Deluge.Response; +using Newtonsoft.Json; + +namespace Domain.Models.Deluge.Response; public sealed record TorrentStatus { @@ -11,4 +13,7 @@ public sealed record TorrentStatus public ulong Eta { get; init; } public bool Private { get; init; } + + [JsonProperty("total_done")] + public long TotalDone { get; init; } } \ No newline at end of file diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index d9800e66..6f1eceb7 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -28,6 +28,7 @@ "file is a sample" ], "STALLED_MAX_STRIKES": 5, + "STALLED_RESET_STRIKES_ON_PROGRESS": true, "STALLED_IGNORE_PRIVATE": true, "STALLED_DELETE_PRIVATE": false }, diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index dfe8d064..cd83efdd 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -25,6 +25,7 @@ "IMPORT_FAILED_DELETE_PRIVATE": false, "IMPORT_FAILED_IGNORE_PATTERNS": [], "STALLED_MAX_STRIKES": 0, + "STALLED_RESET_STRIKES_ON_PROGRESS": false, "STALLED_IGNORE_PRIVATE": false, "STALLED_DELETE_PRIVATE": false }, diff --git a/code/Infrastructure/Helpers/CacheKeys.cs b/code/Infrastructure/Helpers/CacheKeys.cs new file mode 100644 index 00000000..61b1728b --- /dev/null +++ b/code/Infrastructure/Helpers/CacheKeys.cs @@ -0,0 +1,14 @@ +using Domain.Enums; + +namespace Infrastructure.Helpers; + +public static class CacheKeys +{ + public static string Strike(StrikeType strikeType, string hash) => $"{strikeType.ToString()}_{hash}"; + + public static string BlocklistType(InstanceType instanceType) => $"{instanceType.ToString()}_type"; + public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns"; + public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes"; + + public static string Item(string hash) => $"item_{hash}"; +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs index fa368638..90d91e32 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs @@ -5,6 +5,7 @@ using Common.Configuration.Arr; using Common.Configuration.ContentBlocker; using Common.Helpers; using Domain.Enums; +using Infrastructure.Helpers; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -21,10 +22,6 @@ public sealed class BlocklistProvider private readonly IMemoryCache _cache; private bool _initialized; - private const string Type = "type"; - private const string Patterns = "patterns"; - private const string Regexes = "regexes"; - public BlocklistProvider( ILogger logger, IOptions sonarrConfig, @@ -67,21 +64,21 @@ public sealed class BlocklistProvider public BlocklistType GetBlocklistType(InstanceType instanceType) { - _cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType); + _cache.TryGetValue(CacheKeys.BlocklistType(instanceType), out BlocklistType? blocklistType); return blocklistType ?? BlocklistType.Blacklist; } public ConcurrentBag GetPatterns(InstanceType instanceType) { - _cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag? patterns); + _cache.TryGetValue(CacheKeys.BlocklistPatterns(instanceType), out ConcurrentBag? patterns); return patterns ?? []; } public ConcurrentBag GetRegexes(InstanceType instanceType) { - _cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag? regexes); + _cache.TryGetValue(CacheKeys.BlocklistRegexes(instanceType), out ConcurrentBag? regexes); return regexes ?? []; } @@ -124,9 +121,9 @@ public sealed class BlocklistProvider TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime); - _cache.Set($"{instanceType.ToString()}_{Type}", blocklistType); - _cache.Set($"{instanceType.ToString()}_{Patterns}", patterns); - _cache.Set($"{instanceType.ToString()}_{Regexes}", regexes); + _cache.Set(CacheKeys.BlocklistType(instanceType), blocklistType); + _cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns); + _cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes); _logger.LogDebug("loaded {count} patterns", patterns.Count); _logger.LogDebug("loaded {count} regexes", regexes.Count); diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index 7e22740d..b64e467a 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner; using Domain.Models.Deluge.Response; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ItemStriker; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -21,9 +22,10 @@ public sealed class DelugeService : DownloadServiceBase IHttpClientFactory httpClientFactory, IOptions queueCleanerConfig, IOptions contentBlockerConfig, + IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker - ) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker) + ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker) { config.Value.Validate(); _client = new (config, httpClientFactory); @@ -201,6 +203,8 @@ public sealed class DelugeService : DownloadServiceBase { return false; } + + ResetStrikesOnProgress(status.Hash!, status.TotalDone); return StrikeAndCheckLimit(status.Hash!, status.Name!); } @@ -210,7 +214,7 @@ public sealed class DelugeService : DownloadServiceBase return await _client.SendRequest( "web.get_torrent_status", hash, - new[] { "hash", "state", "name", "eta", "private" } + new[] { "hash", "state", "name", "eta", "private", "total_done" } ); } diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs index 87d722db..91f37c2a 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs @@ -2,9 +2,13 @@ using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.QueueCleaner; +using Common.Helpers; using Domain.Enums; +using Domain.Models.Cache; +using Infrastructure.Helpers; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ItemStriker; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -15,13 +19,16 @@ public abstract class DownloadServiceBase : IDownloadService protected readonly ILogger _logger; protected readonly QueueCleanerConfig _queueCleanerConfig; protected readonly ContentBlockerConfig _contentBlockerConfig; + protected readonly IMemoryCache _cache; protected readonly FilenameEvaluator _filenameEvaluator; protected readonly Striker _striker; + protected readonly MemoryCacheEntryOptions _cacheOptions; protected DownloadServiceBase( ILogger logger, IOptions queueCleanerConfig, IOptions contentBlockerConfig, + IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker ) @@ -29,8 +36,11 @@ public abstract class DownloadServiceBase : IDownloadService _logger = logger; _queueCleanerConfig = queueCleanerConfig.Value; _contentBlockerConfig = contentBlockerConfig.Value; + _cache = cache; _filenameEvaluator = filenameEvaluator; _striker = striker; + _cacheOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer); } public abstract void Dispose(); @@ -50,6 +60,23 @@ public abstract class DownloadServiceBase : IDownloadService /// public abstract Task Delete(string hash); + protected void ResetStrikesOnProgress(string hash, long downloaded) + { + if (!_queueCleanerConfig.StalledResetStrikesOnProgress) + { + return; + } + + if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded) + { + // cache item found + _cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash)); + _logger.LogDebug("resetting strikes for {hash} due to progress", hash); + } + + _cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions); + } + /// /// Strikes an item and checks if the limit has been reached. /// diff --git a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs index 19d89b06..41f11bee 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs @@ -4,6 +4,7 @@ using Common.Configuration.ContentBlocker; using Common.Configuration.QueueCleaner; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ItemStriker; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -11,7 +12,7 @@ namespace Infrastructure.Verticals.DownloadClient; public sealed class DummyDownloadService : DownloadServiceBase { - public DummyDownloadService(ILogger logger, IOptions queueCleanerConfig, IOptions contentBlockerConfig, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker) + public DummyDownloadService(ILogger logger, IOptions queueCleanerConfig, IOptions contentBlockerConfig, IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker) { } diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 7b2f4236..19bf99e4 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner; using Common.Helpers; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ItemStriker; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using QBittorrent.Client; @@ -23,9 +24,10 @@ public sealed class QBitService : DownloadServiceBase IOptions config, IOptions queueCleanerConfig, IOptions contentBlockerConfig, + IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker - ) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker) + ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker) { _config = config.Value; _config.Validate(); @@ -216,6 +218,8 @@ public sealed class QBitService : DownloadServiceBase return false; } + ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); + return StrikeAndCheckLimit(torrent.Hash, torrent.Name); } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 5999a643..b53a0ba1 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner; using Common.Helpers; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.ItemStriker; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Transmission.API.RPC; @@ -26,9 +27,10 @@ public sealed class TransmissionService : DownloadServiceBase IOptions config, IOptions queueCleanerConfig, IOptions contentBlockerConfig, + IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker - ) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker) + ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker) { _config = config.Value; _config.Validate(); @@ -200,6 +202,8 @@ public sealed class TransmissionService : DownloadServiceBase { return false; } + + ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0); return StrikeAndCheckLimit(torrent.HashString!, torrent.Name!); } @@ -219,7 +223,8 @@ public sealed class TransmissionService : DownloadServiceBase TorrentFields.ETA, TorrentFields.NAME, TorrentFields.STATUS, - TorrentFields.IS_PRIVATE + TorrentFields.IS_PRIVATE, + TorrentFields.DOWNLOADED_EVER ]; // refresh cache diff --git a/code/Infrastructure/Verticals/ItemStriker/Striker.cs b/code/Infrastructure/Verticals/ItemStriker/Striker.cs index f814260e..dcd7308b 100644 --- a/code/Infrastructure/Verticals/ItemStriker/Striker.cs +++ b/code/Infrastructure/Verticals/ItemStriker/Striker.cs @@ -1,5 +1,6 @@ using Common.Helpers; using Domain.Enums; +using Infrastructure.Helpers; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; @@ -26,7 +27,7 @@ public class Striker return false; } - string key = $"{strikeType.ToString()}_{hash}"; + string key = CacheKeys.Strike(strikeType, hash); if (!_cache.TryGetValue(key, out int? strikeCount)) {