Add more Lidarr checks for failed imports (#48)

README.md

@@ -1,3 +1,5 @@
+_Love this project? Give it a ⭐️ and let others know!_
+
 # <img width="24px" src="./Logo/256.png" alt="cleanuperr"></img> cleanuperr
 
 cleanuperr is a tool for automating the cleanup of unwanted or blocked files in Sonarr, Radarr, and supported download clients like qBittorrent. It removes incomplete or blocked downloads, updates queues, and enforces blacklists or whitelists to manage file selection. After removing blocked content, cleanuperr can also trigger a search to replace the deleted shows/movies.
@@ -110,6 +112,7 @@ services:
       # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
       # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
       - QUEUECLEANER__STALLED_MAX_STRIKES=5
+      - QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS=false
       - QUEUECLEANER__STALLED_IGNORE_PRIVATE=false
       - QUEUECLEANER__STALLED_DELETE_PRIVATE=false
 
@@ -184,6 +187,7 @@ services:
 | QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE | No | - Whether to delete failed imports of private downloads from the download client.<br>- Does not have any effect if `QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
 | QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | - First pattern to look for when an import is failed.<br>- If the specified message pattern is found, the item is skipped. | empty |
 | QUEUECLEANER__STALLED_MAX_STRIKES | No | - After how many strikes should a stalled download be removed.<br>- 0 means never. | 0 |
+| QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS | No | Whether to remove strikes if any download progress was made since last checked. | false |
 | QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers. | false |
 | QUEUECLEANER__STALLED_DELETE_PRIVATE | No | - Whether to delete stalled private downloads from the download client.<br>- Does not have any effect if `QUEUECLEANER__STALLED_IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
 |||||

chart/values.yaml

@@ -11,7 +11,7 @@ deployment:
         tag: latest
       env:
         - name: LOGGING__LOGLEVEL
-          value: Information
+          value: Debug
         - name: LOGGING__FILE__ENABLED
           value: "true"
         - name: LOGGING__FILE__PATH
@@ -22,34 +22,55 @@ deployment:
           value: 0 0/5 * * * ?
         - name: TRIGGERS__CONTENTBLOCKER
           value: 0 0/5 * * * ?
+
         - name: QUEUECLEANER__ENABLED
           value: "true"
         - name: QUEUECLEANER__RUNSEQUENTIALLY
           value: "true"
         - name: QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES
           value: "3"
+        - name: QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE
+          value: "false"
+        - name: QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE
+          value: "false"
         - name: QUEUECLEANER__STALLED_MAX_STRIKES
           value: "3"
+        - name: QUEUECLEANER__STALLED_IGNORE_PRIVATE
+          value: "false"
+        - name: QUEUECLEANER__STALLED_DELETE_PRIVATE
+          value: "false"
+
         - name: CONTENTBLOCKER__ENABLED
           value: "true"
-        - name: CONTENTBLOCKER__BLACKLIST__ENABLED
+        - name: CONTENTBLOCKER__IGNORE_PRIVATE
           value: "true"
-        - name: CONTENTBLOCKER__BLACKLIST__PATH
-          value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
+        - name: CONTENTBLOCKER__DELETE_PRIVATE
+          value: "false"
+
         - name: DOWNLOAD_CLIENT
           value: qbittorrent
         - name: QBITTORRENT__URL
           value: http://service.qbittorrent-videos.svc.cluster.local
+
         - name: SONARR__ENABLED
           value: "true"
         - name: SONARR__SEARCHTYPE
           value: Episode
+        - name: SONARR__BLOCK__TYPE
+          value: blacklist
+        - name: SONARR__BLOCK__PATH
+          value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
         - name: SONARR__INSTANCES__0__URL
           value: http://service.sonarr-low-res.svc.cluster.local
         - name: SONARR__INSTANCES__1__URL
           value: http://service.sonarr-high-res.svc.cluster.local
+
         - name: RADARR__ENABLED
           value: "true"
+        - name: RADARR__BLOCK__TYPE
+          value: blacklist
+        - name: RADARR__BLOCK__PATH
+          value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
         - name: RADARR__INSTANCES__0__URL
           value: http://service.radarr-low-res.svc.cluster.local
         - name: RADARR__INSTANCES__1__URL

code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs

@@ -25,6 +25,9 @@ public sealed record QueueCleanerConfig : IJobConfig
     [ConfigurationKeyName("STALLED_MAX_STRIKES")]
     public ushort StalledMaxStrikes { get; init; }
     
+    [ConfigurationKeyName("STALLED_RESET_STRIKES_ON_PROGRESS")]
+    public bool StalledResetStrikesOnProgress { get; init; }
+    
     [ConfigurationKeyName("STALLED_IGNORE_PRIVATE")]
     public bool StalledIgnorePrivate { get; init; }
     

code/Domain/Models/Cache/CacheItem.cs

@@ -0,0 +1,9 @@
+namespace Domain.Models.Cache;
+
+public sealed record CacheItem
+{
+    /// <summary>
+    /// The amount of bytes that have been downloaded.
+    /// </summary>
+    public long Downloaded { get; set; }
+}

code/Domain/Models/Deluge/Response/TorrentStatus.cs

@@ -1,4 +1,6 @@
-namespace Domain.Models.Deluge.Response;
+using Newtonsoft.Json;
+
+namespace Domain.Models.Deluge.Response;
 
 public sealed record TorrentStatus
 {
@@ -11,4 +13,7 @@ public sealed record TorrentStatus
     public ulong Eta { get; init; }
     
     public bool Private { get; init; }
+    
+    [JsonProperty("total_done")]
+    public long TotalDone { get; init; }
 }

code/Executable/appsettings.Development.json

@@ -28,6 +28,7 @@
       "file is a sample"
     ],
     "STALLED_MAX_STRIKES": 5,
+    "STALLED_RESET_STRIKES_ON_PROGRESS": true,
     "STALLED_IGNORE_PRIVATE": true,
     "STALLED_DELETE_PRIVATE": false
   },

code/Executable/appsettings.json

@@ -20,11 +20,12 @@
   "QueueCleaner": {
     "Enabled": true,
     "RunSequentially": true,
-    "IMPORT_FAILED_MAX_STRIKES": 5,
+    "IMPORT_FAILED_MAX_STRIKES": 0,
     "IMPORT_FAILED_IGNORE_PRIVATE": false,
     "IMPORT_FAILED_DELETE_PRIVATE": false,
     "IMPORT_FAILED_IGNORE_PATTERNS": [],
-    "STALLED_MAX_STRIKES": 5,
+    "STALLED_MAX_STRIKES": 0,
+    "STALLED_RESET_STRIKES_ON_PROGRESS": false,
     "STALLED_IGNORE_PRIVATE": false,
     "STALLED_DELETE_PRIVATE": false
   },

code/Infrastructure/Helpers/CacheKeys.cs

@@ -0,0 +1,14 @@
+using Domain.Enums;
+
+namespace Infrastructure.Helpers;
+
+public static class CacheKeys
+{
+    public static string Strike(StrikeType strikeType, string hash) => $"{strikeType.ToString()}_{hash}";
+
+    public static string BlocklistType(InstanceType instanceType) => $"{instanceType.ToString()}_type";
+    public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns";
+    public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes";
+    
+    public static string Item(string hash) => $"item_{hash}";
+}

code/Infrastructure/Verticals/Arr/ArrClient.cs

@@ -1,4 +1,4 @@
-using Common.Configuration.Arr;
+using Common.Configuration.Arr;
 using Common.Configuration.Logging;
 using Common.Configuration.QueueCleaner;
 using Common.Helpers;
@@ -66,7 +66,7 @@ public abstract class ArrClient
         return queueResponse;
     }
 
-    public virtual bool ShouldRemoveFromQueue(QueueRecord record, bool isPrivateDownload)
+    public virtual bool ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload)
     {
         if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload)
         {
@@ -81,8 +81,14 @@ public abstract class ArrClient
             .Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase);
         bool isImportPending() => record.TrackedDownloadState
             .Equals("importPending", StringComparison.InvariantCultureIgnoreCase);
+        bool isImportFailed() => record.TrackedDownloadState
+            .Equals("importFailed", StringComparison.InvariantCultureIgnoreCase);
+        bool isFailedLidarr() => instanceType is InstanceType.Lidarr &&
+                                 (record.Status.Equals("failed", StringComparison.InvariantCultureIgnoreCase) ||
+                                  record.Status.Equals("completed", StringComparison.InvariantCultureIgnoreCase)) &&
+                                 hasWarn();
         
-        if (hasWarn() && (isImportBlocked() || isImportPending()))
+        if (hasWarn() && (isImportBlocked() || isImportPending() || isImportFailed()) || isFailedLidarr())
         {
             if (HasIgnoredPatterns(record))
             {

code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs

@@ -5,6 +5,7 @@ using Common.Configuration.Arr;
 using Common.Configuration.ContentBlocker;
 using Common.Helpers;
 using Domain.Enums;
+using Infrastructure.Helpers;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -21,10 +22,6 @@ public sealed class BlocklistProvider
     private readonly IMemoryCache _cache;
     private bool _initialized;
 
-    private const string Type = "type";
-    private const string Patterns = "patterns";
-    private const string Regexes = "regexes";
-
     public BlocklistProvider(
         ILogger<BlocklistProvider> logger,
         IOptions<SonarrConfig> sonarrConfig,
@@ -67,21 +64,21 @@ public sealed class BlocklistProvider
 
     public BlocklistType GetBlocklistType(InstanceType instanceType)
     {
-        _cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType);
+        _cache.TryGetValue(CacheKeys.BlocklistType(instanceType), out BlocklistType? blocklistType);
 
         return blocklistType ?? BlocklistType.Blacklist;
     }
     
     public ConcurrentBag<string> GetPatterns(InstanceType instanceType)
     {
-        _cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag<string>? patterns);
+        _cache.TryGetValue(CacheKeys.BlocklistPatterns(instanceType), out ConcurrentBag<string>? patterns);
 
         return patterns ?? [];
     }
 
     public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
     {
-        _cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag<Regex>? regexes);
+        _cache.TryGetValue(CacheKeys.BlocklistRegexes(instanceType), out ConcurrentBag<Regex>? regexes);
         
         return regexes ?? [];
     }
@@ -124,9 +121,9 @@ public sealed class BlocklistProvider
 
         TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
 
-        _cache.Set($"{instanceType.ToString()}_{Type}", blocklistType);
-        _cache.Set($"{instanceType.ToString()}_{Patterns}", patterns);
-        _cache.Set($"{instanceType.ToString()}_{Regexes}", regexes);
+        _cache.Set(CacheKeys.BlocklistType(instanceType), blocklistType);
+        _cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns);
+        _cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes);
         
         _logger.LogDebug("loaded {count} patterns", patterns.Count);
         _logger.LogDebug("loaded {count} regexes", regexes.Count);

code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs

@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
 using Domain.Models.Deluge.Response;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -21,9 +22,10 @@ public sealed class DelugeService : DownloadServiceBase
         IHttpClientFactory httpClientFactory,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
         IOptions<ContentBlockerConfig> contentBlockerConfig,
+        IMemoryCache cache,
         FilenameEvaluator filenameEvaluator,
         Striker striker
-    ) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
+    ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
     {
         config.Value.Validate();
         _client = new (config, httpClientFactory);
@@ -201,6 +203,8 @@ public sealed class DelugeService : DownloadServiceBase
         {
             return false;
         }
+        
+        ResetStrikesOnProgress(status.Hash!, status.TotalDone);
 
         return StrikeAndCheckLimit(status.Hash!, status.Name!);
     }
@@ -210,7 +214,7 @@ public sealed class DelugeService : DownloadServiceBase
         return await _client.SendRequest<TorrentStatus?>(
             "web.get_torrent_status",
             hash,
-            new[] { "hash", "state", "name", "eta", "private" }
+            new[] { "hash", "state", "name", "eta", "private", "total_done" }
         );
     }
     

code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs

@@ -2,9 +2,13 @@
 using System.Text.RegularExpressions;
 using Common.Configuration.ContentBlocker;
 using Common.Configuration.QueueCleaner;
+using Common.Helpers;
 using Domain.Enums;
+using Domain.Models.Cache;
+using Infrastructure.Helpers;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -15,13 +19,16 @@ public abstract class DownloadServiceBase : IDownloadService
     protected readonly ILogger<DownloadServiceBase> _logger;
     protected readonly QueueCleanerConfig _queueCleanerConfig;
     protected readonly ContentBlockerConfig _contentBlockerConfig;
+    protected readonly IMemoryCache _cache;
     protected readonly FilenameEvaluator _filenameEvaluator;
     protected readonly Striker _striker;
+    protected readonly MemoryCacheEntryOptions _cacheOptions;
     
     protected DownloadServiceBase(
         ILogger<DownloadServiceBase> logger,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
         IOptions<ContentBlockerConfig> contentBlockerConfig,
+        IMemoryCache cache,
         FilenameEvaluator filenameEvaluator,
         Striker striker
     )
@@ -29,8 +36,11 @@ public abstract class DownloadServiceBase : IDownloadService
         _logger = logger;
         _queueCleanerConfig = queueCleanerConfig.Value;
         _contentBlockerConfig = contentBlockerConfig.Value;
+        _cache = cache;
         _filenameEvaluator = filenameEvaluator;
         _striker = striker;
+        _cacheOptions = new MemoryCacheEntryOptions()
+            .SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
     }
 
     public abstract void Dispose();
@@ -50,6 +60,23 @@ public abstract class DownloadServiceBase : IDownloadService
     /// <inheritdoc/>
     public abstract Task Delete(string hash);
 
+    protected void ResetStrikesOnProgress(string hash, long downloaded)
+    {
+        if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
+        {
+            return;
+        }
+        
+        if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded)
+        {
+            // cache item found
+            _cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
+            _logger.LogDebug("resetting strikes for {hash} due to progress", hash);
+        }
+        
+        _cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions);
+    }
+
     /// <summary>
     /// Strikes an item and checks if the limit has been reached.
     /// </summary>

code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs

@@ -4,6 +4,7 @@ using Common.Configuration.ContentBlocker;
 using Common.Configuration.QueueCleaner;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -11,7 +12,7 @@ namespace Infrastructure.Verticals.DownloadClient;
 
 public sealed class DummyDownloadService : DownloadServiceBase
 {
-    public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
+    public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
     {
     }
 

code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs

@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
 using Common.Helpers;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using QBittorrent.Client;
@@ -23,9 +24,10 @@ public sealed class QBitService : DownloadServiceBase
         IOptions<QBitConfig> config,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
         IOptions<ContentBlockerConfig> contentBlockerConfig,
+        IMemoryCache cache,
         FilenameEvaluator filenameEvaluator,
         Striker striker
-    ) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
+    ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
     {
         _config = config.Value;
         _config.Validate();
@@ -216,6 +218,8 @@ public sealed class QBitService : DownloadServiceBase
             return false;
         }
 
+        ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
+
         return StrikeAndCheckLimit(torrent.Hash, torrent.Name);
     }
 }

code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs

@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
 using Common.Helpers;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Transmission.API.RPC;
@@ -26,9 +27,10 @@ public sealed class TransmissionService : DownloadServiceBase
         IOptions<TransmissionConfig> config,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
         IOptions<ContentBlockerConfig> contentBlockerConfig,
+        IMemoryCache cache,
         FilenameEvaluator filenameEvaluator,
         Striker striker
-    ) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
+    ) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
     {
         _config = config.Value;
         _config.Validate();
@@ -200,6 +202,8 @@ public sealed class TransmissionService : DownloadServiceBase
         {
             return false;
         }
+        
+        ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0);
 
         return StrikeAndCheckLimit(torrent.HashString!, torrent.Name!);
     }
@@ -219,7 +223,8 @@ public sealed class TransmissionService : DownloadServiceBase
                 TorrentFields.ETA,
                 TorrentFields.NAME,
                 TorrentFields.STATUS,
-                TorrentFields.IS_PRIVATE
+                TorrentFields.IS_PRIVATE,
+                TorrentFields.DOWNLOADED_EVER
             ];
             
             // refresh cache

code/Infrastructure/Verticals/ItemStriker/Striker.cs

@@ -1,5 +1,6 @@
 using Common.Helpers;
 using Domain.Enums;
+using Infrastructure.Helpers;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 
@@ -26,7 +27,7 @@ public class Striker
             return false;
         }
         
-        string key = $"{strikeType.ToString()}_{hash}";
+        string key = CacheKeys.Strike(strikeType, hash);
         
         if (!_cache.TryGetValue(key, out int? strikeCount))
         {

code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs

@@ -75,7 +75,7 @@ public sealed class QueueCleaner : GenericHandler
                 }
                 
                 // failed import check
-                bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(record, stalledCheckResult.IsPrivate);
+                bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(instanceType, record, stalledCheckResult.IsPrivate);
 
                 if (!shouldRemoveFromArr && !stalledCheckResult.ShouldRemove)
                 {