Add configurable number of retries and timeout for http calls (#40)

.github/ISSUE_TEMPLATE/1-bug.yml

@@ -1,6 +1,6 @@
 name: Bug report
 description: File a bug report if something is not working right.
-title: "[BUG]: "
+title: "[BUG] "
 labels: ["bug"]
 body:
   - type: markdown
@@ -40,6 +40,7 @@ body:
         - Windows
         - Linux
         - MacOS
+        - Unraid
     validations:
       required: true
   - type: dropdown

.github/ISSUE_TEMPLATE/2-feature.yml

@@ -1,6 +1,6 @@
 name: Feature request
 description: File a feature request.
-title: "[FEATURE]: "
+title: "[FEATURE] "
 labels: ["enhancement"]
 body:
   - type: markdown

.github/ISSUE_TEMPLATE/3-help.yml

@@ -1,6 +1,6 @@
 name: Help request
 description: Ask a question to receive help.
-title: "[HELP]: "
+title: "[HELP] "
 labels: ["question"]
 body:
   - type: markdown

README.md

@@ -63,12 +63,21 @@ This tool is actively developed and still a work in progress. Join the Discord s
 
 ## Using cleanuperr's blocklist (works with all supported download clients)
 
-1. Set both `QUEUECLEANER_ENABLED` and `CONTENTBLOCKER_ENABLED` to `true` in your environment variables.
+1. Set both `QUEUECLEANER__ENABLED` and `CONTENTBLOCKER_ENABLED` to `true` in your environment variables.
 2. Configure and enable either a **blacklist** or a **whitelist** as described in the [Environment variables](#Environment-variables) section.
 3. Once configured, cleanuperr will perform the following tasks:
    - Execute the **content blocker** job, as explained in the [How it works](#how-it-works) section.
    - Execute the **queue cleaner** job, as explained in the [How it works](#how-it-works) section.
 
+## Using cleanuperr just for failed *arr imports (works for Usenet users as well)
+
+1. Set `QUEUECLEANER__ENABLED` to `true`.
+2. Set `QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES` to a desired value.
+3. Optionally set failed import message patterns to ignore using `QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__<NUMBER>`.
+4. Set `DOWNLOAD_CLIENT` to `none`.
+
+**No other action involving a download client would work (e.g. content blocking, removing stalled downloads, excluding private trackers).**
+
 ## Usage
 
 ### Docker compose yaml
@@ -91,9 +100,14 @@ services:
       - QUEUECLEANER__ENABLED=true
       - QUEUECLEANER__RUNSEQUENTIALLY=true
       - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
+      - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false
+      # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
+      # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
       - QUEUECLEANER__STALLED_MAX_STRIKES=5
+      - QUEUECLEANER__STALLED_IGNORE_PRIVATE=false
 
       - CONTENTBLOCKER__ENABLED=true
+      - CONTENTBLOCKER__IGNORE_PRIVATE=true
       - CONTENTBLOCKER__BLACKLIST__ENABLED=true
       - CONTENTBLOCKER__BLACKLIST__PATH=https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
       # OR
@@ -113,6 +127,8 @@ services:
       # - TRANSMISSION__URL=http://localhost:9091
       # - TRANSMISSION__USERNAME=test
       # - TRANSMISSION__PASSWORD=testing
+      # OR
+      # - DOWNLOAD_CLIENT=none
 
       - SONARR__ENABLED=true
       - SONARR__SEARCHTYPE=Episode
@@ -145,15 +161,19 @@ services:
 | QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner | true |
 | QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process | true |
 | QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | After how many strikes should a failed import be removed<br>0 means never | 0 |
+| QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers | false |
+| QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | First pattern to look for when an import is failed<br>If the specified message pattern is found, the item is skipped | empty |
 | QUEUECLEANER__STALLED_MAX_STRIKES | No | After how many strikes should a stalled download be removed<br>0 means never | 0 |
+| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers | false |
 |||||
 | CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker | false |
+| CONTENTBLOCKER__IGNORE_PRIVATE | No | Whether to ignore downloads from private trackers | false |
 | CONTENTBLOCKER__BLACKLIST__ENABLED | Yes if content blocker is enabled and whitelist is not enabled | Enable or disable the blacklist | false |
 | CONTENTBLOCKER__BLACKLIST__PATH | Yes if blacklist is enabled | Path to the blacklist (local file or url)<br>Needs to be json compatible | empty |
 | CONTENTBLOCKER__WHITELIST__ENABLED | Yes if content blocker is enabled and blacklist is not enabled | Enable or disable the whitelist | false |
 | CONTENTBLOCKER__WHITELIST__PATH | Yes if whitelist is enabled | Path to the whitelist (local file or url)<br>Needs to be json compatible | empty |
 |||||
-| DOWNLOAD_CLIENT | No | Download client that is used by *arrs<br>Can be `qbittorrent`, `deluge` or `transmission` | `qbittorrent` |
+| DOWNLOAD_CLIENT | No | Download client that is used by *arrs<br>Can be `qbittorrent`, `deluge`, `transmission` or `none` | `qbittorrent` |
 | QBITTORRENT__URL | No | qBittorrent instance url | http://localhost:8112 |
 | QBITTORRENT__USERNAME | No | qBittorrent user | empty |
 | QBITTORRENT__PASSWORD | No | qBittorrent password | empty |
@@ -173,7 +193,9 @@ services:
 | RADARR__ENABLED | No | Enable or disable Radarr cleanup  | false |
 | RADARR__INSTANCES__0__URL | No | First Radarr instance url | http://localhost:8989 |
 | RADARR__INSTANCES__0__APIKEY | No | First Radarr instance API key | empty |
-
+|||||
+| HTTP_MAX_RETRIES | No | The number of times to retry a failed HTTP call (to *arrs, download clients etc.) | 0 |
+| HTTP_TIMEOUT | No | The number of seconds to wait before failing an HTTP call (to *arrs, download clients etc.) | 100 |
 #
 ### To be noted
 
@@ -193,6 +215,10 @@ regex:<ANY_REGEX>   // regex that needs to be marked at the start of the line wi
 SONARR__INSTANCES__<NUMBER>__URL
 SONARR__INSTANCES__<NUMBER>__APIKEY
 ```
+6. Multiple failed import patterns can be specified using this format, where `<NUMBER>` starts from 0:
+```
+QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__<NUMBER>
+```
 
 #
 

code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs

@@ -1,4 +1,6 @@
-namespace Common.Configuration.ContentBlocker;
+using Microsoft.Extensions.Configuration;
+
+namespace Common.Configuration.ContentBlocker;
 
 public sealed record ContentBlockerConfig : IJobConfig
 {
@@ -6,6 +8,9 @@ public sealed record ContentBlockerConfig : IJobConfig
     
     public required bool Enabled { get; init; }
     
+    [ConfigurationKeyName("IGNORE_PRIVATE")]
+    public bool IgnorePrivate { get; init; }
+    
     public PatternConfig? Blacklist { get; init; }
     
     public PatternConfig? Whitelist { get; init; }

code/Common/Configuration/DownloadClient/DownloadClientConfig.cs

@@ -0,0 +1,9 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Common.Configuration.DownloadClient;
+
+public sealed record DownloadClientConfig
+{
+    [ConfigurationKeyName("DOWNLOAD_CLIENT")]
+    public Enums.DownloadClient DownloadClient { get; init; } = Enums.DownloadClient.QBittorrent;
+}

code/Common/Configuration/EnvironmentVariables.cs

@@ -1,6 +0,0 @@
-namespace Common.Configuration;
-
-public static class EnvironmentVariables
-{
-    public const string DownloadClient = "DOWNLOAD_CLIENT";
-}

code/Common/Configuration/General/HttpConfig.cs

@@ -0,0 +1,20 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Common.Configuration.General;
+
+public class HttpConfig : IConfig
+{
+    [ConfigurationKeyName("HTTP_MAX_RETRIES")]
+    public ushort MaxRetries { get; init; }
+    
+    [ConfigurationKeyName("HTTP_TIMEOUT")]
+    public ushort Timeout { get; init; } = 100;
+
+    public void Validate()
+    {
+        if (Timeout is 0)
+        {
+            throw new ArgumentException("HTTP_TIMEOUT must be greater than 0");
+        }
+    }
+}

code/Common/Configuration/General/TriggersConfig.cs

@@ -1,4 +1,4 @@
-namespace Common.Configuration;
+namespace Common.Configuration.General;
 
 public sealed class TriggersConfig
 {

code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs

@@ -13,8 +13,17 @@ public sealed record QueueCleanerConfig : IJobConfig
     [ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")]
     public ushort ImportFailedMaxStrikes { get; init; }
     
+    [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PRIVATE")]
+    public bool ImportFailedIgnorePrivate { get; init; }
+    
+    [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")]
+    public List<string>? ImportFailedIgnorePatterns { get; init; }
+    
     [ConfigurationKeyName("STALLED_MAX_STRIKES")]
     public ushort StalledMaxStrikes { get; init; }
+    
+    [ConfigurationKeyName("STALLED_IGNORE_PRIVATE")]
+    public bool StalledIgnorePrivate { get; init; }
 
     public void Validate()
     {

code/Common/Enums/DownloadClient.cs

@@ -1,8 +1,9 @@
-namespace Domain.Enums;
+namespace Common.Enums;
 
 public enum DownloadClient
 {
     QBittorrent,
     Deluge,
-    Transmission
+    Transmission,
+    None
 }

code/Common/Helpers/Constants.cs

@@ -4,4 +4,6 @@ public static class Constants
 {
     public static readonly TimeSpan TriggerMaxLimit  = TimeSpan.FromHours(6);
     public static readonly TimeSpan CacheLimitBuffer = TimeSpan.FromHours(2);
+
+    public const string HttpClientWithRetryName = "retry";
 }

code/Domain/Models/Arr/Queue/QueueRecord.cs

@@ -1,6 +1,6 @@
-namespace Domain.Models.Arr.Queue;
+namespace Domain.Models.Arr.Queue;
 
-public record QueueRecord
+public sealed record QueueRecord
 {
     public int SeriesId { get; init; }
     public int EpisodeId { get; init; }
@@ -10,6 +10,7 @@ public record QueueRecord
     public string Status { get; init; }
     public string TrackedDownloadStatus { get; init; }
     public string TrackedDownloadState { get; init; }
+    public List<TrackedDownloadStatusMessage>? StatusMessages { get; init; }
     public required string DownloadId { get; init; }
     public required string Protocol { get; init; }
     public required int Id { get; init; }

code/Domain/Models/Arr/Queue/TrackedDownloadStatusMessage.cs

@@ -0,0 +1,8 @@
+namespace Domain.Models.Arr.Queue;
+
+public sealed record TrackedDownloadStatusMessage
+{
+    public string Title { get; set; }
+    
+    public List<string>? Messages { get; set; }
+}

code/Domain/Models/Deluge/Response/TorrentStatus.cs

@@ -2,11 +2,13 @@
 
 public sealed record TorrentStatus
 {
-    public string? Hash { get; set; }
+    public string? Hash { get; init; }
     
-    public string? State { get; set; }
+    public string? State { get; init; }
     
-    public string? Name { get; set; }
+    public string? Name { get; init; }
     
-    public ulong Eta { get; set; }
+    public ulong Eta { get; init; }
+    
+    public bool Private { get; init; }
 }

code/Executable/DependencyInjection/ConfigurationDI.cs

@@ -14,6 +14,7 @@ public static class ConfigurationDI
         services
             .Configure<QueueCleanerConfig>(configuration.GetSection(QueueCleanerConfig.SectionName))
             .Configure<ContentBlockerConfig>(configuration.GetSection(ContentBlockerConfig.SectionName))
+            .Configure<DownloadClientConfig>(configuration)
             .Configure<QBitConfig>(configuration.GetSection(QBitConfig.SectionName))
             .Configure<DelugeConfig>(configuration.GetSection(DelugeConfig.SectionName))
             .Configure<TransmissionConfig>(configuration.GetSection(TransmissionConfig.SectionName))

code/Executable/DependencyInjection/MainDI.cs

@@ -1,14 +1,9 @@
 using System.Net;
-using Common.Configuration;
-using Common.Configuration.ContentBlocker;
-using Executable.Jobs;
-using Infrastructure.Verticals.Arr;
-using Infrastructure.Verticals.ContentBlocker;
-using Infrastructure.Verticals.DownloadClient;
+using Common.Configuration.General;
+using Common.Helpers;
 using Infrastructure.Verticals.DownloadClient.Deluge;
-using Infrastructure.Verticals.DownloadClient.QBittorrent;
-using Infrastructure.Verticals.DownloadClient.Transmission;
-using Infrastructure.Verticals.QueueCleaner;
+using Polly;
+using Polly.Extensions.Http;
 
 namespace Executable.DependencyInjection;
 
@@ -17,16 +12,27 @@ public static class MainDI
     public static IServiceCollection AddInfrastructure(this IServiceCollection services, IConfiguration configuration) =>
         services
             .AddLogging(builder => builder.ClearProviders().AddConsole())
-            .AddHttpClients()
+            .AddHttpClients(configuration)
             .AddConfiguration(configuration)
             .AddMemoryCache()
             .AddServices()
             .AddQuartzServices(configuration);
     
-    private static IServiceCollection AddHttpClients(this IServiceCollection services)
+    private static IServiceCollection AddHttpClients(this IServiceCollection services, IConfiguration configuration)
     {
         // add default HttpClient
         services.AddHttpClient();
+        
+        HttpConfig config = configuration.Get<HttpConfig>() ?? new();
+        config.Validate();
+
+        // add retry HttpClient
+        services
+            .AddHttpClient(Constants.HttpClientWithRetryName, x =>
+            {
+                x.Timeout = TimeSpan.FromSeconds(config.Timeout);
+            })
+            .AddRetryPolicyHandler(config);
 
         // add Deluge HttpClient
         services
@@ -44,8 +50,18 @@ public static class MainDI
                     AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
                     ServerCertificateCustomValidationCallback = (_, _, _, _) => true
                 };
-            });
+            })
+            .AddRetryPolicyHandler(config);
 
         return services;
     }
+
+    private static IHttpClientBuilder AddRetryPolicyHandler(this IHttpClientBuilder builder, HttpConfig config) =>
+        builder.AddPolicyHandler(
+            HttpPolicyExtensions
+                .HandleTransientHttpError()
+                // do not retry on Unauthorized
+                .OrResult(response => !response.IsSuccessStatusCode && response.StatusCode != HttpStatusCode.Unauthorized)
+                .WaitAndRetryAsync(config.MaxRetries, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))
+        );
 }

code/Executable/DependencyInjection/QuartzDI.cs

@@ -1,5 +1,6 @@
 using Common.Configuration;
 using Common.Configuration.ContentBlocker;
+using Common.Configuration.General;
 using Common.Configuration.QueueCleaner;
 using Common.Helpers;
 using Executable.Jobs;

code/Executable/DependencyInjection/ServicesDI.cs

@@ -18,6 +18,7 @@ public static class ServicesDI
             .AddTransient<QueueCleaner>()
             .AddTransient<ContentBlocker>()
             .AddTransient<FilenameEvaluator>()
+            .AddTransient<DummyDownloadService>()
             .AddTransient<QBitService>()
             .AddTransient<DelugeService>()
             .AddTransient<TransmissionService>()

code/Executable/Executable.csproj

@@ -12,6 +12,7 @@
     <ItemGroup>
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
         <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="9.0.0" />
         <PackageReference Include="Quartz" Version="3.13.1" />
         <PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.13.1" />
         <PackageReference Include="Quartz.Extensions.Hosting" Version="3.13.1" />

code/Executable/appsettings.Development.json

@@ -1,4 +1,6 @@
 {
+  "HTTP_MAX_RETRIES": 0,
+  "HTTP_TIMEOUT": 10,
   "Logging": {
     "LogLevel": "Debug",
     "Enhanced": true,
@@ -13,6 +15,7 @@
   },
   "ContentBlocker": {
     "Enabled": true,
+    "IGNORE_PRIVATE": true,
     "Blacklist": {
       "Enabled": false,
       "Path": "https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist"
@@ -26,7 +29,12 @@
     "Enabled": true,
     "RunSequentially": true,
     "IMPORT_FAILED_MAX_STRIKES": 5,
-    "STALLED_MAX_STRIKES": 5
+    "IMPORT_FAILED_IGNORE_PRIVATE": true,
+    "IMPORT_FAILED_IGNORE_PATTERNS": [
+      "file is a sample"
+    ],
+    "STALLED_MAX_STRIKES": 5,
+    "STALLED_IGNORE_PRIVATE": true
   },
   "DOWNLOAD_CLIENT": "qbittorrent",
   "qBittorrent": {

code/Executable/appsettings.json

@@ -1,4 +1,6 @@
 {
+  "HTTP_MAX_RETRIES": 0,
+  "HTTP_TIMEOUT": 100,
   "Logging": {
     "LogLevel": "Information",
     "Enhanced": true,
@@ -13,6 +15,7 @@
   },
   "ContentBlocker": {
     "Enabled": false,
+    "IGNORE_PRIVATE": false,
     "Blacklist": {
       "Enabled": false,
       "Path": ""
@@ -26,7 +29,10 @@
     "Enabled": true,
     "RunSequentially": true,
     "IMPORT_FAILED_MAX_STRIKES": 5,
-    "STALLED_MAX_STRIKES": 5
+    "IMPORT_FAILED_IGNORE_PRIVATE": false,
+    "IMPORT_FAILED_IGNORE_PATTERNS": [],
+    "STALLED_MAX_STRIKES": 5,
+    "STALLED_IGNORE_PRIVATE": false
   },
   "DOWNLOAD_CLIENT": "qbittorrent",
   "qBittorrent": {

code/Infrastructure/Infrastructure.csproj

@@ -12,10 +12,10 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="FLM.Transmission" Version="1.0.0" />
+      <PackageReference Include="FLM.QBittorrent" Version="1.0.0" />
+      <PackageReference Include="FLM.Transmission" Version="1.0.2" />
       <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
-      <PackageReference Include="QBittorrent.Client" Version="1.9.24285.1" />
       <PackageReference Include="Quartz" Version="3.13.1" />
     </ItemGroup>
 

code/Infrastructure/Verticals/Arr/ArrClient.cs

@@ -1,6 +1,7 @@
 using Common.Configuration.Arr;
 using Common.Configuration.Logging;
 using Common.Configuration.QueueCleaner;
+using Common.Helpers;
 using Domain.Enums;
 using Domain.Models.Arr;
 using Domain.Models.Arr.Queue;
@@ -29,7 +30,7 @@ public abstract class ArrClient
     {
         _logger = logger;
         _striker = striker;
-        _httpClient = httpClientFactory.CreateClient();
+        _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
         _loggingConfig = loggingConfig.Value;
         _queueCleanerConfig = queueCleanerConfig.Value;
         _striker = striker;
@@ -65,17 +66,30 @@ public abstract class ArrClient
         return queueResponse;
     }
 
-    public virtual bool ShouldRemoveFromQueue(QueueRecord record)
+    public virtual bool ShouldRemoveFromQueue(QueueRecord record, bool isPrivateDownload)
     {
+        if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload)
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip failed import check | download is private | {name}", record.Title);
+            return false;
+        }
+        
         bool hasWarn() => record.TrackedDownloadStatus
             .Equals("warning", StringComparison.InvariantCultureIgnoreCase);
         bool isImportBlocked() => record.TrackedDownloadState
             .Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase);
         bool isImportPending() => record.TrackedDownloadState
             .Equals("importPending", StringComparison.InvariantCultureIgnoreCase);
-
+        
         if (hasWarn() && (isImportBlocked() || isImportPending()))
         {
+            if (HasIgnoredPatterns(record))
+            {
+                _logger.LogDebug("skip failed import check | contains ignored pattern | {name}", record.Title);
+                return false;
+            }
+            
             return _striker.StrikeAndCheckLimit(
                 record.DownloadId,
                 record.Title,
@@ -134,4 +148,32 @@ public abstract class ArrClient
     {
         request.Headers.Add("x-api-key", apiKey);
     }
+
+    private bool HasIgnoredPatterns(QueueRecord record)
+    {
+        if (_queueCleanerConfig.ImportFailedIgnorePatterns?.Count is null or 0)
+        {
+            // no patterns are configured
+            return false;
+        }
+            
+        if (record.StatusMessages?.Count is null or 0)
+        {
+            // no status message found
+            return false;
+        }
+        
+        HashSet<string> messages = record.StatusMessages
+            .SelectMany(x => x.Messages ?? Enumerable.Empty<string>())
+            .ToHashSet();
+        record.StatusMessages.Select(x => x.Title)
+            .ToList()
+            .ForEach(x => messages.Add(x));
+        
+        return messages.Any(
+            m => _queueCleanerConfig.ImportFailedIgnorePatterns.Any(
+                p => !string.IsNullOrWhiteSpace(p.Trim()) && m.Contains(p, StringComparison.InvariantCultureIgnoreCase)
+            )
+        );
+    }
 }

code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs

@@ -2,6 +2,7 @@
 using System.Diagnostics;
 using System.Text.RegularExpressions;
 using Common.Configuration.ContentBlocker;
+using Common.Helpers;
 using Domain.Enums;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -27,7 +28,7 @@ public sealed class BlocklistProvider
     {
         _logger = logger;
         _config = config.Value;
-        _httpClient = httpClientFactory.CreateClient();
+        _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
         
         _config.Validate();
         

code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs

@@ -1,5 +1,5 @@
-using Common.Configuration;
-using Common.Configuration.Arr;
+using Common.Configuration.Arr;
+using Common.Configuration.DownloadClient;
 using Domain.Enums;
 using Domain.Models.Arr.Queue;
 using Infrastructure.Verticals.Arr;
@@ -16,6 +16,7 @@ public sealed class ContentBlocker : GenericHandler
     
     public ContentBlocker(
         ILogger<ContentBlocker> logger,
+        IOptions<DownloadClientConfig> downloadClientConfig,
         IOptions<SonarrConfig> sonarrConfig,
         IOptions<RadarrConfig> radarrConfig,
         SonarrClient sonarrClient,
@@ -23,13 +24,19 @@ public sealed class ContentBlocker : GenericHandler
         ArrQueueIterator arrArrQueueIterator,
         BlocklistProvider blocklistProvider,
         DownloadServiceFactory downloadServiceFactory
-    ) : base(logger, sonarrConfig.Value, radarrConfig.Value, sonarrClient, radarrClient, arrArrQueueIterator, downloadServiceFactory)
+    ) : base(logger, downloadClientConfig, sonarrConfig.Value, radarrConfig.Value, sonarrClient, radarrClient, arrArrQueueIterator, downloadServiceFactory)
     {
         _blocklistProvider = blocklistProvider;
     }
 
     public override async Task ExecuteAsync()
     {
+        if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None)
+        {
+            _logger.LogWarning("download client is set to none");
+            return;
+        }
+        
         await _blocklistProvider.LoadBlocklistAsync();
         await base.ExecuteAsync();
     }

code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs

@@ -30,18 +30,19 @@ public sealed class DelugeService : DownloadServiceBase
         await _client.LoginAsync();
     }
 
-    public override async Task<bool> ShouldRemoveFromArrQueueAsync(string hash)
+    public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
     {
         hash = hash.ToLowerInvariant();
         
         DelugeContents? contents = null;
+        RemoveResult result = new();
 
         TorrentStatus? status = await GetTorrentStatus(hash);
         
         if (status?.Hash is null)
         {
             _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
-            return false;
+            return result;
         }
 
         try
@@ -63,7 +64,10 @@ public sealed class DelugeService : DownloadServiceBase
             }
         });
 
-        return shouldRemove || IsItemStuckAndShouldRemove(status);
+        result.ShouldRemove = shouldRemove || IsItemStuckAndShouldRemove(status);
+        result.IsPrivate = status.Private;
+        
+        return result;
     }
 
     public override async Task BlockUnwantedFilesAsync(string hash)
@@ -78,6 +82,13 @@ public sealed class DelugeService : DownloadServiceBase
             return;
         }
         
+        if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip files check | download is private | {name}", status.Name);
+            return;
+        }
+        
         DelugeContents? contents = null;
 
         try
@@ -128,6 +139,18 @@ public sealed class DelugeService : DownloadServiceBase
     
     private bool IsItemStuckAndShouldRemove(TorrentStatus status)
     {
+        if (_queueCleanerConfig.StalledMaxStrikes is 0)
+        {
+            return false;
+        }
+        
+        if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip stalled check | download is private | {name}", status.Name);
+            return false;
+        }
+        
         if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
         {
             return false;
@@ -146,7 +169,7 @@ public sealed class DelugeService : DownloadServiceBase
         return await _client.SendRequest<TorrentStatus?>(
             "web.get_torrent_status",
             hash,
-            new[] { "hash", "state", "name", "eta" }
+            new[] { "hash", "state", "name", "eta", "private" }
         );
     }
     

code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs

@@ -31,7 +31,7 @@ public abstract class DownloadServiceBase : IDownloadService
 
     public abstract Task LoginAsync();
 
-    public abstract Task<bool> ShouldRemoveFromArrQueueAsync(string hash);
+    public abstract Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
 
     public abstract Task BlockUnwantedFilesAsync(string hash);
 

code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs

@@ -1,9 +1,7 @@
-using Common.Configuration;
-using Common.Configuration.DownloadClient;
+using Common.Configuration.DownloadClient;
 using Infrastructure.Verticals.DownloadClient.Deluge;
 using Infrastructure.Verticals.DownloadClient.QBittorrent;
 using Infrastructure.Verticals.DownloadClient.Transmission;
-using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 
@@ -12,24 +10,21 @@ namespace Infrastructure.Verticals.DownloadClient;
 public sealed class DownloadServiceFactory
 {
     private readonly IServiceProvider _serviceProvider;
-    private readonly Domain.Enums.DownloadClient _downloadClient;
+    private readonly Common.Enums.DownloadClient _downloadClient;
     
-    public DownloadServiceFactory(IServiceProvider serviceProvider, IConfiguration configuration)
+    public DownloadServiceFactory(IServiceProvider serviceProvider, IOptions<DownloadClientConfig> downloadClientConfig)
     {
         _serviceProvider = serviceProvider;
-        _downloadClient = (Domain.Enums.DownloadClient)Enum.Parse(
-            typeof(Domain.Enums.DownloadClient),
-            configuration[EnvironmentVariables.DownloadClient] ?? Domain.Enums.DownloadClient.QBittorrent.ToString(),
-            true
-        );
+        _downloadClient = downloadClientConfig.Value.DownloadClient;
     }
 
     public IDownloadService CreateDownloadClient() =>
         _downloadClient switch
         {
-            Domain.Enums.DownloadClient.QBittorrent => _serviceProvider.GetRequiredService<QBitService>(),
-            Domain.Enums.DownloadClient.Deluge => _serviceProvider.GetRequiredService<DelugeService>(),
-            Domain.Enums.DownloadClient.Transmission => _serviceProvider.GetRequiredService<TransmissionService>(),
+            Common.Enums.DownloadClient.QBittorrent => _serviceProvider.GetRequiredService<QBitService>(),
+            Common.Enums.DownloadClient.Deluge => _serviceProvider.GetRequiredService<DelugeService>(),
+            Common.Enums.DownloadClient.Transmission => _serviceProvider.GetRequiredService<TransmissionService>(),
+            Common.Enums.DownloadClient.None => _serviceProvider.GetRequiredService<DummyDownloadService>(),
             _ => throw new ArgumentOutOfRangeException()
         };
 }

code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs

@@ -0,0 +1,33 @@
+using Common.Configuration.QueueCleaner;
+using Infrastructure.Verticals.ContentBlocker;
+using Infrastructure.Verticals.ItemStriker;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Infrastructure.Verticals.DownloadClient;
+
+public sealed class DummyDownloadService : DownloadServiceBase
+{
+    public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, filenameEvaluator, striker)
+    {
+    }
+
+    public override void Dispose()
+    {
+    }
+
+    public override Task LoginAsync()
+    {
+        return Task.CompletedTask;
+    }
+
+    public override Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
+    {
+        throw new NotImplementedException();
+    }
+
+    public override Task BlockUnwantedFilesAsync(string hash)
+    {
+        throw new NotImplementedException();
+    }
+}

code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs

@@ -4,7 +4,7 @@ public interface IDownloadService : IDisposable
 {
     public Task LoginAsync();
 
-    public Task<bool> ShouldRemoveFromArrQueueAsync(string hash);
+    public Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
 
     public Task BlockUnwantedFilesAsync(string hash);
 }

code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs

@@ -1,5 +1,6 @@
-using Common.Configuration.DownloadClient;
+using Common.Configuration.DownloadClient;
 using Common.Configuration.QueueCleaner;
+using Common.Helpers;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
 using Microsoft.Extensions.Logging;
@@ -15,6 +16,7 @@ public sealed class QBitService : DownloadServiceBase
 
     public QBitService(
         ILogger<QBitService> logger,
+        IHttpClientFactory httpClientFactory,
         IOptions<QBitConfig> config,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
         FilenameEvaluator filenameEvaluator,
@@ -23,7 +25,7 @@ public sealed class QBitService : DownloadServiceBase
     {
         _config = config.Value;
         _config.Validate();
-        _client = new(_config.Url);
+        _client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), _config.Url);
     }
 
     public override async Task LoginAsync()
@@ -36,21 +38,35 @@ public sealed class QBitService : DownloadServiceBase
         await _client.LoginAsync(_config.Username, _config.Password);
     }
 
-    public override async Task<bool> ShouldRemoveFromArrQueueAsync(string hash)
+    public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
     {
+        RemoveResult result = new();
         TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
             .FirstOrDefault();
 
         if (torrent is null)
         {
             _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
-            return false;
+            return result;
         }
 
+        TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
+
+        if (torrentProperties is null)
+        {
+            _logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
+            return result;
+        }
+
+        result.IsPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
+                           bool.TryParse(dictValue?.ToString(), out bool boolValue)
+                           && boolValue;
+
         // if all files were blocked by qBittorrent
         if (torrent is { CompletionOn: not null, Downloaded: null or 0 })
         {
-            return true;
+            result.ShouldRemove = true;
+            return result;
         }
 
         IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
@@ -58,14 +74,45 @@ public sealed class QBitService : DownloadServiceBase
         // if all files are marked as skip
         if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip))
         {
-            return true;
+            result.ShouldRemove = true;
+            return result;
         }
 
-        return IsItemStuckAndShouldRemove(torrent);
+        result.ShouldRemove = IsItemStuckAndShouldRemove(torrent, result.IsPrivate);
+
+        return result;
     }
 
     public override async Task BlockUnwantedFilesAsync(string hash)
     {
+        TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
+            .FirstOrDefault();
+
+        if (torrent is null)
+        {
+            _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
+            return;
+        }
+        
+        TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
+
+        if (torrentProperties is null)
+        {
+            _logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
+            return;
+        }
+
+        bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
+                         bool.TryParse(dictValue?.ToString(), out bool boolValue)
+                         && boolValue;
+
+        if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate)
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
+            return;
+        }
+        
         IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
 
         if (files is null)
@@ -95,8 +142,20 @@ public sealed class QBitService : DownloadServiceBase
         _client.Dispose();
     }
     
-    private bool IsItemStuckAndShouldRemove(TorrentInfo torrent)
+    private bool IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate)
     {
+        if (_queueCleanerConfig.StalledMaxStrikes is 0)
+        {
+            return false;
+        }
+        
+        if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate)
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
+            return false;
+        }
+        
         if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata
             and not TorrentState.ForcedFetchingMetadata)
         {

code/Infrastructure/Verticals/DownloadClient/RemoveResult.cs

@@ -0,0 +1,14 @@
+namespace Infrastructure.Verticals.DownloadClient;
+
+public sealed record RemoveResult
+{
+    /// <summary>
+    /// True if the download should be removed; otherwise false.
+    /// </summary>
+    public bool ShouldRemove { get; set; }
+    
+    /// <summary>
+    /// True if the download is private; otherwise false.
+    /// </summary>
+    public bool IsPrivate { get; set; }
+}

code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs

@@ -1,5 +1,6 @@
 using Common.Configuration.DownloadClient;
 using Common.Configuration.QueueCleaner;
+using Common.Helpers;
 using Infrastructure.Verticals.ContentBlocker;
 using Infrastructure.Verticals.ItemStriker;
 using Microsoft.Extensions.Logging;
@@ -17,6 +18,7 @@ public sealed class TransmissionService : DownloadServiceBase
     private TorrentInfo[]? _torrentsCache;
 
     public TransmissionService(
+        IHttpClientFactory httpClientFactory,
         ILogger<TransmissionService> logger,
         IOptions<TransmissionConfig> config,
         IOptions<QueueCleanerConfig> queueCleanerConfig,
@@ -27,6 +29,7 @@ public sealed class TransmissionService : DownloadServiceBase
         _config = config.Value;
         _config.Validate();
         _client = new(
+            httpClientFactory.CreateClient(Constants.HttpClientWithRetryName),
             new Uri(_config.Url, "/transmission/rpc").ToString(),
             login: _config.Username,
             password: _config.Password
@@ -38,17 +41,19 @@ public sealed class TransmissionService : DownloadServiceBase
         await _client.GetSessionInformationAsync();
     }
 
-    public override async Task<bool> ShouldRemoveFromArrQueueAsync(string hash)
+    public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
     {
+        RemoveResult result = new();
         TorrentInfo? torrent = await GetTorrentAsync(hash);
 
         if (torrent is null)
         {
             _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
-            return false;
+            return result;
         }
         
         bool shouldRemove = torrent.FileStats?.Length > 0;
+        result.IsPrivate = torrent.IsPrivate ?? false;
 
         foreach (TransmissionTorrentFileStats? stats in torrent.FileStats ?? [])
         {
@@ -65,8 +70,10 @@ public sealed class TransmissionService : DownloadServiceBase
             }
         }
 
-        // remove if all files are unwanted
-        return shouldRemove || IsItemStuckAndShouldRemove(torrent);
+        // remove if all files are unwanted or download is stuck
+        result.ShouldRemove = shouldRemove || IsItemStuckAndShouldRemove(torrent);
+        
+        return result;
     }
 
     public override async Task BlockUnwantedFilesAsync(string hash)
@@ -77,6 +84,13 @@ public sealed class TransmissionService : DownloadServiceBase
         {
             return;
         }
+        
+        if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false))
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
+            return;
+        }
 
         List<long> unwantedFiles = [];
         
@@ -116,6 +130,18 @@ public sealed class TransmissionService : DownloadServiceBase
     
     private bool IsItemStuckAndShouldRemove(TorrentInfo torrent)
     {
+        if (_queueCleanerConfig.StalledMaxStrikes is 0)
+        {
+            return false;
+        }
+        
+        if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false))
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
+            return false;
+        }
+        
         if (torrent.Status is not 4)
         {
             // not in downloading state
@@ -144,7 +170,8 @@ public sealed class TransmissionService : DownloadServiceBase
                 TorrentFields.ID,
                 TorrentFields.ETA,
                 TorrentFields.NAME,
-                TorrentFields.STATUS
+                TorrentFields.STATUS,
+                TorrentFields.IS_PRIVATE
             ];
             
             // refresh cache

code/Infrastructure/Verticals/Jobs/GenericHandler.cs

@@ -1,17 +1,19 @@
 using Common.Configuration.Arr;
+using Common.Configuration.DownloadClient;
 using Domain.Enums;
 using Domain.Models.Arr;
 using Domain.Models.Arr.Queue;
 using Infrastructure.Verticals.Arr;
 using Infrastructure.Verticals.DownloadClient;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
+using Microsoft.Extensions.Options;
 
 namespace Infrastructure.Verticals.Jobs;
 
 public abstract class GenericHandler : IDisposable
 {
     protected readonly ILogger<GenericHandler> _logger;
+    protected readonly DownloadClientConfig _downloadClientConfig;
     protected readonly SonarrConfig _sonarrConfig;
     protected readonly RadarrConfig _radarrConfig;
     protected readonly SonarrClient _sonarrClient;
@@ -21,6 +23,7 @@ public abstract class GenericHandler : IDisposable
 
     protected GenericHandler(
         ILogger<GenericHandler> logger,
+        IOptions<DownloadClientConfig> downloadClientConfig,
         SonarrConfig sonarrConfig,
         RadarrConfig radarrConfig,
         SonarrClient sonarrClient,
@@ -30,6 +33,7 @@ public abstract class GenericHandler : IDisposable
     )
     {
         _logger = logger;
+        _downloadClientConfig = downloadClientConfig.Value;
         _sonarrConfig = sonarrConfig;
         _radarrConfig = radarrConfig;
         _sonarrClient = sonarrClient;

code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs

@@ -1,5 +1,5 @@
 using Common.Configuration.Arr;
-using Common.Configuration.QueueCleaner;
+using Common.Configuration.DownloadClient;
 using Domain.Enums;
 using Domain.Models.Arr;
 using Domain.Models.Arr.Queue;
@@ -15,13 +15,14 @@ public sealed class QueueCleaner : GenericHandler
 {
     public QueueCleaner(
         ILogger<QueueCleaner> logger,
+        IOptions<DownloadClientConfig> downloadClientConfig,
         IOptions<SonarrConfig> sonarrConfig,
         IOptions<RadarrConfig> radarrConfig,
         SonarrClient sonarrClient,
         RadarrClient radarrClient,
         ArrQueueIterator arrArrQueueIterator,
         DownloadServiceFactory downloadServiceFactory
-    ) : base(logger, sonarrConfig.Value, radarrConfig.Value, sonarrClient, radarrClient, arrArrQueueIterator, downloadServiceFactory)
+    ) : base(logger, downloadClientConfig, sonarrConfig.Value, radarrConfig.Value, sonarrClient, radarrClient, arrArrQueueIterator, downloadServiceFactory)
     {
     }
     
@@ -56,7 +57,16 @@ public sealed class QueueCleaner : GenericHandler
                     continue;
                 }
 
-                if (!arrClient.ShouldRemoveFromQueue(record) && !await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId))
+                RemoveResult removeResult = new();
+
+                if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None)
+                {
+                    removeResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId);
+                }
+                
+                bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(record, removeResult.IsPrivate);
+
+                if (!shouldRemoveFromArr && !removeResult.ShouldRemove)
                 {
                     _logger.LogInformation("skip | {title}", record.Title);
                     continue;

code/test/docker-compose.yml

@@ -176,15 +176,22 @@ services:
       - LOGGING__FILE__PATH=/var/logs
       - LOGGING__ENHANCED=true
 
+      - HTTP_MAX_RETRIES=0
+      - HTTP_TIMEOUT=20
+
       - TRIGGERS__QUEUECLEANER=0/30 * * * * ?
       - TRIGGERS__CONTENTBLOCKER=0/30 * * * * ?
 
       - QUEUECLEANER__ENABLED=true
       - QUEUECLEANER__RUNSEQUENTIALLY=true
       - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
+      - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true
+      - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=file is a sample
       - QUEUECLEANER__STALLED_MAX_STRIKES=5
+      - QUEUECLEANER__STALLED_IGNORE_PRIVATE=true
 
       - CONTENTBLOCKER__ENABLED=true
+      - CONTENTBLOCKER__IGNORE_PRIVATE=true
       - CONTENTBLOCKER__BLACKLIST__ENABLED=true
       - CONTENTBLOCKER__BLACKLIST__PATH=https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
       # OR