Fix crashing when tracker url is malformed (#121)

README.md

@@ -12,6 +12,7 @@ cleanuperr was created primarily to address malicious files, such as `*.lnk` or
 > **Features:**
 > - Strike system to mark stalled or downloads stuck in metadata downloading.
 > - Remove and block downloads that reached a maximum number of strikes.
+> - Remove and block downloads that have a low download speed or high estimated completion time.
 > - Remove downloads blocked by qBittorrent or by cleanuperr's **content blocker**.
 > - Trigger a search for downloads removed from the *arrs.
 > - Clean up downloads that have been seeding for a certain amount of time.
@@ -91,7 +92,7 @@ I've seen a few discussions on this type of naming and I've decided that I didn'
 #### 2. **Queue cleaner** will:
    - Run every 5 minutes (or configured cron, or right after `content blocker`).
    - Process all items in the *arr queue.
-   - Check each queue item if it is **stalled (download speed is 0)**, **stuck in metadata downloading** or **failed to be imported**.
+   - Check each queue item if it is **stalled (download speed is 0)**, **stuck in metadata downloading**, **failed to be imported** or **slow**.
      - If it is, the item receives a **strike** and will continue to accumulate strikes every time it meets any of these conditions.
    - Check each queue item if it meets one of the following condition in the download client:
      - **Marked as completed, but 0 bytes have been downloaded** (due to files being blocked by qBittorrent or the **content blocker**).
@@ -130,10 +131,14 @@ I've seen a few discussions on this type of naming and I've decided that I didn'
 1. Set `QUEUECLEANER__ENABLED` to `true`.
 2. Set `QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES` to a desired value.
 3. Optionally set failed import message patterns to ignore using `QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__<NUMBER>`.
-4. Set `DOWNLOAD_CLIENT` to `none`.
+4. Set `DOWNLOAD_CLIENT` to `none`(works only for usenet) or `disabled` (works for both usenet and torrent).
 
-> [!WARNING]
-> When `DOWNLOAD_CLIENT=none`, no other action involving a download client would work (e.g. content blocking, removing stalled downloads, excluding private trackers).
+> [!IMPORTANT]
+> When `DOWNLOAD_CLIENT=disabled`, no other action involving a download client would work (e.g. content blocking, removing stalled downloads, excluding private trackers).
+> 
+> When the download client is set to `disabled`, the queue cleaner will be able to remove items that are failed to be imported even if there is no download client configured. This means that all downloads, including private ones, will be completely removed.
+>
+> Setting `DOWNLOAD_CLIENT=disabled` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account.
 
 ## Usage
 
@@ -168,39 +173,62 @@ services:
       - ./cleanuperr/logs:/var/logs
       - ./cleanuperr/ignored.txt:/ignored.txt
     environment:
+      # general settings
       - TZ=America/New_York
       - DRY_RUN=false
+      - HTTP_MAX_RETRIES=0
+      - HTTP_TIMEOUT=100
 
+      # logging
       - LOGGING__LOGLEVEL=Information
       - LOGGING__FILE__ENABLED=false
       - LOGGING__FILE__PATH=/var/logs/
       - LOGGING__ENHANCED=true
 
+      # job triggers
       - TRIGGERS__QUEUECLEANER=0 0/5 * * * ?
       - TRIGGERS__CONTENTBLOCKER=0 0/5 * * * ?
       - TRIGGERS__DOWNLOADCLEANER=0 0 * * * ?
 
+      # queue cleaner
       - QUEUECLEANER__ENABLED=true
       - QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt
       - QUEUECLEANER__RUNSEQUENTIALLY=true
+
+      # failed imports
       - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
       - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false
       - QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=false
-      # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
-      # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
+      - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
+      - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
+
+      # stalled downloads
       - QUEUECLEANER__STALLED_MAX_STRIKES=5
       - QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS=false
       - QUEUECLEANER__STALLED_IGNORE_PRIVATE=false
       - QUEUECLEANER__STALLED_DELETE_PRIVATE=false
 
+      # slow downloads
+      - QUEUECLEANER__SLOW_MAX_STRIKES=5
+      - QUEUECLEANER__SLOW_RESET_STRIKES_ON_PROGRESS=true
+      - QUEUECLEANER__SLOW_IGNORE_PRIVATE=false
+      - QUEUECLEANER__SLOW_DELETE_PRIVATE=false
+      - QUEUECLEANER__SLOW_MIN_SPEED=1MB
+      - QUEUECLEANER__SLOW_MAX_TIME=20
+      - QUEUECLEANER__SLOW_IGNORE_ABOVE_SIZE=60GB
+
+      # content blocker
       - CONTENTBLOCKER__ENABLED=true
       - CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored.txt
       - CONTENTBLOCKER__IGNORE_PRIVATE=false
       - CONTENTBLOCKER__DELETE_PRIVATE=false
 
+      # download cleaner
       - DOWNLOADCLEANER__ENABLED=true
       - DOWNLOADCLEANER__IGNORED_DOWNLOADS_PATH=/ignored.txt
       - DOWNLOADCLEANER__DELETE_PRIVATE=false
+
+      # categories to seed until max ratio or min seed time has been reached
       - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr
       - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1
       - DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME=0
@@ -212,6 +240,8 @@ services:
 
       - DOWNLOAD_CLIENT=none
       # OR
+      # - DOWNLOAD_CLIENT=disabled
+      # OR
       # - DOWNLOAD_CLIENT=qBittorrent
       # - QBITTORRENT__URL=http://localhost:8080
       # - QBITTORRENT__URL_BASE=myCustomPath
@@ -256,6 +286,7 @@ services:
 
       - NOTIFIARR__ON_IMPORT_FAILED_STRIKE=true
       - NOTIFIARR__ON_STALLED_STRIKE=true
+      - NOTIFIARR__ON_SLOW_STRIKE=true
       - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true
       - NOTIFIARR__ON_DOWNLOAD_CLEANED=true
       - NOTIFIARR__API_KEY=notifiarr_secret

code/Common/Configuration/Notification/NotificationConfig.cs

@@ -10,13 +10,16 @@ public abstract record NotificationConfig
     [ConfigurationKeyName("ON_STALLED_STRIKE")]
     public bool OnStalledStrike { get; init; }
     
+    [ConfigurationKeyName("ON_SLOW_STRIKE")]
+    public bool OnSlowStrike { get; init; }
+    
     [ConfigurationKeyName("ON_QUEUE_ITEM_DELETED")]
     public bool OnQueueItemDeleted { get; init; }
     
     [ConfigurationKeyName("ON_DOWNLOAD_CLEANED")]
     public bool OnDownloadCleaned { get; init; }
 
-    public bool IsEnabled => OnImportFailedStrike || OnStalledStrike || OnQueueItemDeleted || OnDownloadCleaned;
+    public bool IsEnabled => OnImportFailedStrike || OnStalledStrike || OnSlowStrike || OnQueueItemDeleted || OnDownloadCleaned;
 
     public abstract bool IsValid();
 }

code/Domain/Models/Deluge/Response/DownloadStatus.cs

@@ -35,5 +35,5 @@ public sealed record DownloadStatus
 
 public sealed record Tracker
 {
-    public required Uri Url { get; init; }
+    public required string Url { get; init; }
 }

code/Executable/DependencyInjection/MainDI.cs

@@ -25,6 +25,7 @@ public static class MainDI
             {
                 config.AddConsumer<NotificationConsumer<FailedImportStrikeNotification>>();
                 config.AddConsumer<NotificationConsumer<StalledStrikeNotification>>();
+                config.AddConsumer<NotificationConsumer<SlowStrikeNotification>>();
                 config.AddConsumer<NotificationConsumer<QueueItemDeletedNotification>>();
                 config.AddConsumer<NotificationConsumer<DownloadCleanedNotification>>();
 
@@ -34,6 +35,7 @@ public static class MainDI
                     {
                         e.ConfigureConsumer<NotificationConsumer<FailedImportStrikeNotification>>(context);
                         e.ConfigureConsumer<NotificationConsumer<StalledStrikeNotification>>(context);
+                        e.ConfigureConsumer<NotificationConsumer<SlowStrikeNotification>>(context);
                         e.ConfigureConsumer<NotificationConsumer<QueueItemDeletedNotification>>(context);
                         e.ConfigureConsumer<NotificationConsumer<DownloadCleanedNotification>>(context);
                         e.ConcurrentMessageLimit = 1;

code/Executable/appsettings.Development.json

@@ -117,6 +117,7 @@
   "Notifiarr": {
     "ON_IMPORT_FAILED_STRIKE": true,
     "ON_STALLED_STRIKE": true,
+    "ON_SLOW_STRIKE": true,
     "ON_QUEUE_ITEM_DELETED": true,
     "ON_DOWNLOAD_CLEANED": true,
     "API_KEY": "",

code/Executable/appsettings.json

@@ -100,6 +100,7 @@
   "Notifiarr": {
     "ON_IMPORT_FAILED_STRIKE": false,
     "ON_STALLED_STRIKE": false,
+    "ON_SLOW_STRIKE": false,
     "ON_QUEUE_ITEM_DELETED": false,
     "ON_DOWNLOAD_CLEANED": false,
     "API_KEY": "",

code/Infrastructure/Extensions/DelugeExtensions.cs

@@ -1,4 +1,5 @@
 using Domain.Models.Deluge.Response;
+using Infrastructure.Helpers;
 
 namespace Infrastructure.Extensions;
 
@@ -18,7 +19,7 @@ public static class DelugeExtensions
                 return true;
             }
             
-            if (download.Trackers.Any(x => x.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase)))
+            if (download.Trackers.Any(x => UriService.GetDomain(x.Url)?.EndsWith(value, StringComparison.InvariantCultureIgnoreCase) ?? false))
             {
                 return true;
             }

code/Infrastructure/Extensions/QBitExtensions.cs

@@ -1,4 +1,5 @@
-using QBittorrent.Client;
+using Infrastructure.Helpers;
+using QBittorrent.Client;
 
 namespace Infrastructure.Extensions;
 
@@ -29,9 +30,16 @@ public static class QBitExtensions
 
     public static bool ShouldIgnore(this TorrentTracker tracker, IReadOnlyList<string> ignoredDownloads)
     {
+        string? trackerUrl = UriService.GetDomain(tracker.Url);
+
+        if (trackerUrl is null)
+        {
+            return false;
+        }
+        
         foreach (string value in ignoredDownloads)
         {
-            if (tracker.Url.Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase))
+            if (trackerUrl.EndsWith(value, StringComparison.InvariantCultureIgnoreCase))
             {
                 return true;
             }

code/Infrastructure/Extensions/TransmissionExtensions.cs

@@ -1,4 +1,5 @@
-using Transmission.API.RPC.Entity;
+using Infrastructure.Helpers;
+using Transmission.API.RPC.Entity;
 
 namespace Infrastructure.Extensions;
 
@@ -19,7 +20,7 @@ public static class TransmissionExtensions
             }
 
             bool? hasIgnoredTracker = download.Trackers?
-                .Any(x => new Uri(x.Announce!).Host.EndsWith(value, StringComparison.InvariantCultureIgnoreCase));
+                .Any(x => UriService.GetDomain(x.Announce)?.EndsWith(value, StringComparison.InvariantCultureIgnoreCase) ?? false);
             
             if (hasIgnoredTracker is true)
             {

code/Infrastructure/Helpers/UriService.cs

@@ -0,0 +1,37 @@
+using System.Text.RegularExpressions;
+
+namespace Infrastructure.Helpers;
+
+public static class UriService
+{
+    public static string? GetDomain(string? input)
+    {
+        if (string.IsNullOrWhiteSpace(input))
+        {
+            return null;
+        }
+
+        // add "http://" if scheme is missing to help Uri.TryCreate
+        if (!input.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+        {
+            input = "http://" + input;
+        }
+
+        if (Uri.TryCreate(input, UriKind.Absolute, out var uri))
+        {
+            return uri.Host;
+        }
+
+        // url might be malformed
+        var regex = new Regex(@"^(?:https?:\/\/)?([^\/\?:]+)", RegexOptions.IgnoreCase);
+        var match = regex.Match(input);
+        
+        if (match.Success)
+        {
+            return match.Groups[1].Value;
+        }
+
+        // could not extract
+        return null;
+    }
+}

code/Infrastructure/Infrastructure.csproj

@@ -12,7 +12,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="FLM.QBittorrent" Version="1.0.0" />
+      <PackageReference Include="FLM.QBittorrent" Version="1.0.1" />
       <PackageReference Include="FLM.Transmission" Version="1.0.3" />
       <PackageReference Include="Mapster" Version="7.4.0" />
       <PackageReference Include="MassTransit" Version="8.3.6" />

code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs

@@ -428,7 +428,7 @@ public class QBitService : DownloadService, IQBitService
     private async Task<IReadOnlyList<TorrentTracker>> GetTrackersAsync(string hash)
     {
         return (await _client.GetTorrentTrackersAsync(hash))
-            .Where(x => !x.Url.ToString().Contains("**"))
+            .Where(x => x.Url.Contains("**"))
             .ToList();
     }
 }

code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs

@@ -27,6 +27,9 @@ public sealed class NotificationConsumer<T> : IConsumer<T> where T : Notificatio
                 case StalledStrikeNotification stalledMessage:
                     await _notificationService.Notify(stalledMessage);
                     break;
+                case SlowStrikeNotification slowMessage:
+                    await _notificationService.Notify(slowMessage);
+                    break;
                 case QueueItemDeletedNotification queueItemDeleteMessage:
                     await _notificationService.Notify(queueItemDeleteMessage);
                     break;

code/Infrastructure/Verticals/Notifications/INotificationFactory.cs

@@ -6,6 +6,8 @@ public interface INotificationFactory
     
     List<INotificationProvider> OnStalledStrikeEnabled();
     
+    List<INotificationProvider> OnSlowStrikeEnabled();
+    
     List<INotificationProvider> OnQueueItemDeletedEnabled();
 
     List<INotificationProvider> OnDownloadCleanedEnabled();

code/Infrastructure/Verticals/Notifications/INotificationProvider.cs

@@ -12,6 +12,8 @@ public interface INotificationProvider
     Task OnFailedImportStrike(FailedImportStrikeNotification notification);
         
     Task OnStalledStrike(StalledStrikeNotification notification);
+    
+    Task OnSlowStrike(SlowStrikeNotification notification);
 
     Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
 

code/Infrastructure/Verticals/Notifications/Models/SlowStrikeNotification.cs

@@ -0,0 +1,5 @@
+namespace Infrastructure.Verticals.Notifications.Models;
+
+public sealed record SlowStrikeNotification : ArrNotification
+{
+}

code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs

@@ -32,6 +32,11 @@ public class NotifiarrProvider : NotificationProvider
         await _proxy.SendNotification(BuildPayload(notification, WarningColor), _config);
     }
     
+    public override async Task OnSlowStrike(SlowStrikeNotification notification)
+    {
+        await _proxy.SendNotification(BuildPayload(notification, WarningColor), _config);
+    }
+    
     public override async Task OnQueueItemDeleted(QueueItemDeletedNotification notification)
     {
         await _proxy.SendNotification(BuildPayload(notification, ImportantColor), _config);

code/Infrastructure/Verticals/Notifications/NotificationFactory.cs

@@ -24,6 +24,11 @@ public class NotificationFactory : INotificationFactory
         ActiveProviders()
             .Where(n => n.Config.OnStalledStrike)
             .ToList();
+    
+    public List<INotificationProvider> OnSlowStrikeEnabled() =>
+        ActiveProviders()
+            .Where(n => n.Config.OnSlowStrike)
+            .ToList();
 
     public List<INotificationProvider> OnQueueItemDeletedEnabled() =>
         ActiveProviders()

code/Infrastructure/Verticals/Notifications/NotificationProvider.cs

@@ -18,6 +18,8 @@ public abstract class NotificationProvider : INotificationProvider
     public abstract Task OnFailedImportStrike(FailedImportStrikeNotification notification);
 
     public abstract Task OnStalledStrike(StalledStrikeNotification notification);
+    
+    public abstract Task OnSlowStrike(SlowStrikeNotification notification);
 
     public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
 

code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs

@@ -48,11 +48,16 @@ public class NotificationPublisher : INotificationPublisher
             switch (strikeType)
             {
                 case StrikeType.Stalled:
+                case StrikeType.DownloadingMetadata:
                     await _dryRunInterceptor.InterceptAsync(Notify<StalledStrikeNotification>, notification.Adapt<StalledStrikeNotification>());
                     break;
                 case StrikeType.ImportFailed:
                     await _dryRunInterceptor.InterceptAsync(Notify<FailedImportStrikeNotification>, notification.Adapt<FailedImportStrikeNotification>());
                     break;
+                case StrikeType.SlowSpeed:
+                case StrikeType.SlowTime:
+                    await _dryRunInterceptor.InterceptAsync(Notify<SlowStrikeNotification>, notification.Adapt<SlowStrikeNotification>());
+                    break;
             }
         }
         catch (Exception ex)

code/Infrastructure/Verticals/Notifications/NotificationService.cs

@@ -44,6 +44,21 @@ public class NotificationService
         }
     }
     
+    public async Task Notify(SlowStrikeNotification notification)
+    {
+        foreach (INotificationProvider provider in _notificationFactory.OnSlowStrikeEnabled())
+        {
+            try
+            {
+                await provider.OnSlowStrike(notification);
+            }
+            catch (Exception exception)
+            {
+                _logger.LogWarning(exception, "failed to send notification | provider {provider}", provider.Name);
+            }
+        }
+    }
+    
     public async Task Notify(QueueItemDeletedNotification notification)
     {
         foreach (INotificationProvider provider in _notificationFactory.OnQueueItemDeletedEnabled())

code/test/docker-compose.yml

@@ -262,12 +262,13 @@ services:
       - LIDARR__INSTANCES__0__URL=http://lidarr:8686
       - LIDARR__INSTANCES__0__APIKEY=7f677cfdc074414397af53dd633860c5
 
-      # - NOTIFIARR__ON_IMPORT_FAILED_STRIKE=true
-      # - NOTIFIARR__ON_STALLED_STRIKE=true
-      # - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true
-      # - NOTIFIARR__ON_DOWNLOAD_CLEANED=true
-      # - NOTIFIARR__API_KEY=notifiarr_secret
-      # - NOTIFIARR__CHANNEL_ID=discord_channel_id
+      - NOTIFIARR__ON_IMPORT_FAILED_STRIKE=true
+      - NOTIFIARR__ON_STALLED_STRIKE=true
+      - NOTIFIARR__ON_SLOW_STRIKE=true
+      - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true
+      - NOTIFIARR__ON_DOWNLOAD_CLEANED=true
+      - NOTIFIARR__API_KEY=notifiarr_secret
+      - NOTIFIARR__CHANNEL_ID=discord_channel_id
     volumes:
       - ./data/cleanuperr/logs:/var/logs
       - ./data/cleanuperr/ignored_downloads:/ignored

variables.md

@@ -675,7 +675,14 @@
 - Required: No.
 
 #### **`NOTIFIARR__ON_STALLED_STRIKE`**
-- Controls whether to notify when an item receives a stalled download strike.
+- Controls whether to notify when an item receives a stalled download strike. This includes strikes for being stuck while downloading metadata.
+- Type: Boolean
+- Possible values: `true`, `false`
+- Default: `false`
+- Required: No.
+
+#### **`NOTIFIARR__ON_SLOW_STRIKE`**
+- Controls whether to notify when an item receives a slow download strike. This includes strikes for having a low download speed or slow estimated finish time.
 - Type: Boolean
 - Possible values: `true`, `false`
 - Default: `false`