Compare commits

...

9 Commits

Author SHA1 Message Date
Marius Nechifor
c813215f3e Add more Lidarr checks for failed imports (#48) 2025-01-28 19:10:07 +02:00
Flaminel
0f63a2d271 updated README 2025-01-26 01:36:08 +02:00
Marius Nechifor
133c34de53 Add option to reset stalled strikes on download progress (#50) 2025-01-25 03:27:40 +02:00
Flaminel
a3ca735b12 updated deployment 2025-01-25 01:18:03 +02:00
Marius Nechifor
519ab6a0cd Fix strike defaults (#49) 2025-01-22 22:18:31 +02:00
Marius Nechifor
0c691a540a Add missing failed import status (#47) 2025-01-21 00:14:55 +02:00
Marius Nechifor
209f78717f Fix usenet usage (#46) 2025-01-18 19:12:28 +02:00
Flaminel
a02be80ac1 updated README 2025-01-18 17:25:15 +02:00
Marius Nechifor
8a8b906b6f Add option to not remove private downloads from the download client (#45) 2025-01-18 17:20:23 +02:00
26 changed files with 312 additions and 106 deletions

View File

@@ -1,3 +1,5 @@
_Love this project? Give it a ⭐️ and let others know!_
# <img width="24px" src="./Logo/256.png" alt="cleanuperr"></img> cleanuperr
cleanuperr is a tool for automating the cleanup of unwanted or blocked files in Sonarr, Radarr, and supported download clients like qBittorrent. It removes incomplete or blocked downloads, updates queues, and enforces blacklists or whitelists to manage file selection. After removing blocked content, cleanuperr can also trigger a search to replace the deleted shows/movies.
@@ -106,13 +108,17 @@ services:
- QUEUECLEANER__RUNSEQUENTIALLY=true
- QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false
- QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=false
# - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
# - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
- QUEUECLEANER__STALLED_MAX_STRIKES=5
- QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS=false
- QUEUECLEANER__STALLED_IGNORE_PRIVATE=false
- QUEUECLEANER__STALLED_DELETE_PRIVATE=false
- CONTENTBLOCKER__ENABLED=true
- CONTENTBLOCKER__IGNORE_PRIVATE=true
- CONTENTBLOCKER__IGNORE_PRIVATE=false
- CONTENTBLOCKER__DELETE_PRIVATE=false
- DOWNLOAD_CLIENT=none
# OR
@@ -166,24 +172,28 @@ services:
| Variable | Required | Description | Default value |
|---|---|---|---|
| LOGGING__LOGLEVEL | No | Can be `Verbose`, `Debug`, `Information`, `Warning`, `Error` or `Fatal` | `Information` |
| LOGGING__FILE__ENABLED | No | Enable or disable logging to file | false |
| LOGGING__FILE__PATH | No | Directory where to save the log files | empty |
| LOGGING__ENHANCED | No | Enhance logs whenever possible<br>A more detailed description is provided [here](variables.md#LOGGING__ENHANCED) | true |
| LOGGING__LOGLEVEL | No | Can be `Verbose`, `Debug`, `Information`, `Warning`, `Error` or `Fatal`. | `Information` |
| LOGGING__FILE__ENABLED | No | Enable or disable logging to file. | false |
| LOGGING__FILE__PATH | No | Directory where to save the log files. | empty |
| LOGGING__ENHANCED | No | Enhance logs whenever possible.<br>A more detailed description is provided. [here](variables.md#LOGGING__ENHANCED) | true |
|||||
| TRIGGERS__QUEUECLEANER | Yes if queue cleaner is enabled | [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html)<br>Can be a max of 6h interval<br>**Is ignored if `QUEUECLEANER__RUNSEQUENTIALLY=true` and `CONTENTBLOCKER__ENABLED=true`** | 0 0/5 * * * ? |
| TRIGGERS__CONTENTBLOCKER | Yes if content blocker is enabled | [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html)<br>Can be a max of 6h interval | 0 0/5 * * * ? |
| TRIGGERS__QUEUECLEANER | Yes if queue cleaner is enabled | - [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).<br>- Can be a max of 6h interval.<br>- **Is ignored if `QUEUECLEANER__RUNSEQUENTIALLY=true` and `CONTENTBLOCKER__ENABLED=true`**. | 0 0/5 * * * ? |
| TRIGGERS__CONTENTBLOCKER | Yes if content blocker is enabled | - [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).<br>- Can be a max of 6h interval. | 0 0/5 * * * ? |
|||||
| QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner | true |
| QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process | true |
| QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | After how many strikes should a failed import be removed<br>0 means never | 0 |
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers | false |
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | First pattern to look for when an import is failed<br>If the specified message pattern is found, the item is skipped | empty |
| QUEUECLEANER__STALLED_MAX_STRIKES | No | After how many strikes should a stalled download be removed<br>0 means never | 0 |
| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers | false |
| QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner. | true |
| QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process. | true |
| QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | - After how many strikes should a failed import be removed.<br>- 0 means never. | 0 |
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers. | false |
| QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE | No | - Whether to delete failed imports of private downloads from the download client.<br>- Does not have any effect if `QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | - First pattern to look for when an import is failed.<br>- If the specified message pattern is found, the item is skipped. | empty |
| QUEUECLEANER__STALLED_MAX_STRIKES | No | - After how many strikes should a stalled download be removed.<br>- 0 means never. | 0 |
| QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS | No | Whether to remove strikes if any download progress was made since last checked. | false |
| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers. | false |
| QUEUECLEANER__STALLED_DELETE_PRIVATE | No | - Whether to delete stalled private downloads from the download client.<br>- Does not have any effect if `QUEUECLEANER__STALLED_IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
|||||
| CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker | false |
| CONTENTBLOCKER__IGNORE_PRIVATE | No | Whether to ignore downloads from private trackers | false |
| CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker. | false |
| CONTENTBLOCKER__IGNORE_PRIVATE | No | Whether to ignore downloads from private trackers. | false |
| CONTENTBLOCKER__DELETE_PRIVATE | No | - Whether to delete private downloads that have all files blocked from the download client.<br>- Does not have any effect if `CONTENTBLOCKER__IGNORE_PRIVATE` is `true`.<br>- **Set this to `true` if you don't care about seeding, ratio, H&R and potentially losing your tracker account.** | false |
</details>
### Download client variables

View File

@@ -11,7 +11,7 @@ deployment:
tag: latest
env:
- name: LOGGING__LOGLEVEL
value: Information
value: Debug
- name: LOGGING__FILE__ENABLED
value: "true"
- name: LOGGING__FILE__PATH
@@ -22,34 +22,55 @@ deployment:
value: 0 0/5 * * * ?
- name: TRIGGERS__CONTENTBLOCKER
value: 0 0/5 * * * ?
- name: QUEUECLEANER__ENABLED
value: "true"
- name: QUEUECLEANER__RUNSEQUENTIALLY
value: "true"
- name: QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES
value: "3"
- name: QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE
value: "false"
- name: QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE
value: "false"
- name: QUEUECLEANER__STALLED_MAX_STRIKES
value: "3"
- name: QUEUECLEANER__STALLED_IGNORE_PRIVATE
value: "false"
- name: QUEUECLEANER__STALLED_DELETE_PRIVATE
value: "false"
- name: CONTENTBLOCKER__ENABLED
value: "true"
- name: CONTENTBLOCKER__BLACKLIST__ENABLED
- name: CONTENTBLOCKER__IGNORE_PRIVATE
value: "true"
- name: CONTENTBLOCKER__BLACKLIST__PATH
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
- name: CONTENTBLOCKER__DELETE_PRIVATE
value: "false"
- name: DOWNLOAD_CLIENT
value: qbittorrent
- name: QBITTORRENT__URL
value: http://service.qbittorrent-videos.svc.cluster.local
- name: SONARR__ENABLED
value: "true"
- name: SONARR__SEARCHTYPE
value: Episode
- name: SONARR__BLOCK__TYPE
value: blacklist
- name: SONARR__BLOCK__PATH
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
- name: SONARR__INSTANCES__0__URL
value: http://service.sonarr-low-res.svc.cluster.local
- name: SONARR__INSTANCES__1__URL
value: http://service.sonarr-high-res.svc.cluster.local
- name: RADARR__ENABLED
value: "true"
- name: RADARR__BLOCK__TYPE
value: blacklist
- name: RADARR__BLOCK__PATH
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
- name: RADARR__INSTANCES__0__URL
value: http://service.radarr-low-res.svc.cluster.local
- name: RADARR__INSTANCES__1__URL

View File

@@ -1,4 +1,4 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Configuration;
namespace Common.Configuration.ContentBlocker;
@@ -11,6 +11,9 @@ public sealed record ContentBlockerConfig : IJobConfig
[ConfigurationKeyName("IGNORE_PRIVATE")]
public bool IgnorePrivate { get; init; }
[ConfigurationKeyName("DELETE_PRIVATE")]
public bool DeletePrivate { get; init; }
public void Validate()
{
}

View File

@@ -16,14 +16,23 @@ public sealed record QueueCleanerConfig : IJobConfig
[ConfigurationKeyName("IMPORT_FAILED_IGNORE_PRIVATE")]
public bool ImportFailedIgnorePrivate { get; init; }
[ConfigurationKeyName("IMPORT_FAILED_DELETE_PRIVATE")]
public bool ImportFailedDeletePrivate { get; init; }
[ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")]
public List<string>? ImportFailedIgnorePatterns { get; init; }
[ConfigurationKeyName("STALLED_MAX_STRIKES")]
public ushort StalledMaxStrikes { get; init; }
[ConfigurationKeyName("STALLED_RESET_STRIKES_ON_PROGRESS")]
public bool StalledResetStrikesOnProgress { get; init; }
[ConfigurationKeyName("STALLED_IGNORE_PRIVATE")]
public bool StalledIgnorePrivate { get; init; }
[ConfigurationKeyName("STALLED_DELETE_PRIVATE")]
public bool StalledDeletePrivate { get; init; }
public void Validate()
{

View File

@@ -0,0 +1,9 @@
namespace Domain.Models.Cache;
public sealed record CacheItem
{
/// <summary>
/// The amount of bytes that have been downloaded.
/// </summary>
public long Downloaded { get; set; }
}

View File

@@ -1,4 +1,6 @@
namespace Domain.Models.Deluge.Response;
using Newtonsoft.Json;
namespace Domain.Models.Deluge.Response;
public sealed record TorrentStatus
{
@@ -11,4 +13,7 @@ public sealed record TorrentStatus
public ulong Eta { get; init; }
public bool Private { get; init; }
[JsonProperty("total_done")]
public long TotalDone { get; init; }
}

View File

@@ -15,18 +15,22 @@
},
"ContentBlocker": {
"Enabled": true,
"IGNORE_PRIVATE": true
"IGNORE_PRIVATE": true,
"DELETE_PRIVATE": false
},
"QueueCleaner": {
"Enabled": true,
"RunSequentially": true,
"IMPORT_FAILED_MAX_STRIKES": 5,
"IMPORT_FAILED_IGNORE_PRIVATE": true,
"IMPORT_FAILED_DELETE_PRIVATE": false,
"IMPORT_FAILED_IGNORE_PATTERNS": [
"file is a sample"
],
"STALLED_MAX_STRIKES": 5,
"STALLED_IGNORE_PRIVATE": true
"STALLED_RESET_STRIKES_ON_PROGRESS": true,
"STALLED_IGNORE_PRIVATE": true,
"STALLED_DELETE_PRIVATE": false
},
"DOWNLOAD_CLIENT": "qbittorrent",
"qBittorrent": {

View File

@@ -20,11 +20,14 @@
"QueueCleaner": {
"Enabled": true,
"RunSequentially": true,
"IMPORT_FAILED_MAX_STRIKES": 5,
"IMPORT_FAILED_MAX_STRIKES": 0,
"IMPORT_FAILED_IGNORE_PRIVATE": false,
"IMPORT_FAILED_DELETE_PRIVATE": false,
"IMPORT_FAILED_IGNORE_PATTERNS": [],
"STALLED_MAX_STRIKES": 5,
"STALLED_IGNORE_PRIVATE": false
"STALLED_MAX_STRIKES": 0,
"STALLED_RESET_STRIKES_ON_PROGRESS": false,
"STALLED_IGNORE_PRIVATE": false,
"STALLED_DELETE_PRIVATE": false
},
"DOWNLOAD_CLIENT": "none",
"qBittorrent": {

View File

@@ -0,0 +1,14 @@
using Domain.Enums;
namespace Infrastructure.Helpers;
public static class CacheKeys
{
public static string Strike(StrikeType strikeType, string hash) => $"{strikeType.ToString()}_{hash}";
public static string BlocklistType(InstanceType instanceType) => $"{instanceType.ToString()}_type";
public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns";
public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes";
public static string Item(string hash) => $"item_{hash}";
}

View File

@@ -1,4 +1,4 @@
using Common.Configuration.Arr;
using Common.Configuration.Arr;
using Common.Configuration.Logging;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
@@ -66,7 +66,7 @@ public abstract class ArrClient
return queueResponse;
}
public virtual bool ShouldRemoveFromQueue(QueueRecord record, bool isPrivateDownload)
public virtual bool ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload)
{
if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload)
{
@@ -81,8 +81,14 @@ public abstract class ArrClient
.Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase);
bool isImportPending() => record.TrackedDownloadState
.Equals("importPending", StringComparison.InvariantCultureIgnoreCase);
bool isImportFailed() => record.TrackedDownloadState
.Equals("importFailed", StringComparison.InvariantCultureIgnoreCase);
bool isFailedLidarr() => instanceType is InstanceType.Lidarr &&
(record.Status.Equals("failed", StringComparison.InvariantCultureIgnoreCase) ||
record.Status.Equals("completed", StringComparison.InvariantCultureIgnoreCase)) &&
hasWarn();
if (hasWarn() && (isImportBlocked() || isImportPending()))
if (hasWarn() && (isImportBlocked() || isImportPending() || isImportFailed()) || isFailedLidarr())
{
if (HasIgnoredPatterns(record))
{
@@ -101,9 +107,9 @@ public abstract class ArrClient
return false;
}
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record)
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient)
{
Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id));
Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id, removeFromClient));
using HttpRequestMessage request = new(HttpMethod.Delete, uri);
SetApiKey(request, arrInstance.ApiKey);
@@ -113,8 +119,14 @@ public abstract class ArrClient
try
{
response.EnsureSuccessStatusCode();
_logger.LogInformation("queue item deleted | {url} | {title}", arrInstance.Url, record.Title);
_logger.LogInformation(
removeFromClient
? "queue item deleted | {url} | {title}"
: "queue item removed from arr | {url} | {title}",
arrInstance.Url,
record.Title
);
}
catch
{
@@ -144,7 +156,7 @@ public abstract class ArrClient
protected abstract string GetQueueUrlPath(int page);
protected abstract string GetQueueDeleteUrlPath(long recordId);
protected abstract string GetQueueDeleteUrlPath(long recordId, bool removeFromClient);
protected virtual void SetApiKey(HttpRequestMessage request, string apiKey)
{

View File

@@ -29,9 +29,13 @@ public sealed class LidarrClient : ArrClient
return $"/api/v1/queue?page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true";
}
protected override string GetQueueDeleteUrlPath(long recordId)
protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
{
return $"/api/v1/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
string path = $"/api/v1/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return path;
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)

View File

@@ -29,9 +29,13 @@ public sealed class RadarrClient : ArrClient
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true";
}
protected override string GetQueueDeleteUrlPath(long recordId)
protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
{
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return path;
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)

View File

@@ -29,9 +29,13 @@ public sealed class SonarrClient : ArrClient
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true";
}
protected override string GetQueueDeleteUrlPath(long recordId)
protected override string GetQueueDeleteUrlPath(long recordId, bool removeFromClient)
{
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
string path = $"/api/v3/queue/{recordId}?blocklist=true&skipRedownload=true&changeCategory=false";
path += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
return path;
}
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)

View File

@@ -5,6 +5,7 @@ using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Helpers;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -21,10 +22,6 @@ public sealed class BlocklistProvider
private readonly IMemoryCache _cache;
private bool _initialized;
private const string Type = "type";
private const string Patterns = "patterns";
private const string Regexes = "regexes";
public BlocklistProvider(
ILogger<BlocklistProvider> logger,
IOptions<SonarrConfig> sonarrConfig,
@@ -67,21 +64,21 @@ public sealed class BlocklistProvider
public BlocklistType GetBlocklistType(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType);
_cache.TryGetValue(CacheKeys.BlocklistType(instanceType), out BlocklistType? blocklistType);
return blocklistType ?? BlocklistType.Blacklist;
}
public ConcurrentBag<string> GetPatterns(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag<string>? patterns);
_cache.TryGetValue(CacheKeys.BlocklistPatterns(instanceType), out ConcurrentBag<string>? patterns);
return patterns ?? [];
}
public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
{
_cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag<Regex>? regexes);
_cache.TryGetValue(CacheKeys.BlocklistRegexes(instanceType), out ConcurrentBag<Regex>? regexes);
return regexes ?? [];
}
@@ -124,9 +121,9 @@ public sealed class BlocklistProvider
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
_cache.Set($"{instanceType.ToString()}_{Type}", blocklistType);
_cache.Set($"{instanceType.ToString()}_{Patterns}", patterns);
_cache.Set($"{instanceType.ToString()}_{Regexes}", regexes);
_cache.Set(CacheKeys.BlocklistType(instanceType), blocklistType);
_cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns);
_cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes);
_logger.LogDebug("loaded {count} patterns", patterns.Count);
_logger.LogDebug("loaded {count} regexes", regexes.Count);

View File

@@ -17,10 +17,12 @@ namespace Infrastructure.Verticals.ContentBlocker;
public sealed class ContentBlocker : GenericHandler
{
private readonly ContentBlockerConfig _config;
private readonly BlocklistProvider _blocklistProvider;
public ContentBlocker(
ILogger<ContentBlocker> logger,
IOptions<ContentBlockerConfig> config,
IOptions<DownloadClientConfig> downloadClientConfig,
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
@@ -38,6 +40,7 @@ public sealed class ContentBlocker : GenericHandler
arrArrQueueIterator, downloadServiceFactory
)
{
_config = config.Value;
_blocklistProvider = blocklistProvider;
}
@@ -96,7 +99,10 @@ public sealed class ContentBlocker : GenericHandler
_logger.LogDebug("searching unwanted files for {title}", record.Title);
if (!await _downloadService.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes))
BlockFilesResult result = await _downloadService
.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes);
if (!result.ShouldRemove)
{
continue;
}
@@ -104,7 +110,15 @@ public sealed class ContentBlocker : GenericHandler
_logger.LogDebug("all files are marked as unwanted | {hash}", record.Title);
itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
await arrClient.DeleteQueueItemAsync(instance, record);
bool removeFromClient = true;
if (result.IsPrivate && !_config.DeletePrivate)
{
removeFromClient = false;
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
}
});

View File

@@ -0,0 +1,14 @@
namespace Infrastructure.Verticals.DownloadClient;
public sealed record BlockFilesResult
{
/// <summary>
/// True if the download should be removed; otherwise false.
/// </summary>
public bool ShouldRemove { get; set; }
/// <summary>
/// True if the download is private; otherwise false.
/// </summary>
public bool IsPrivate { get; set; }
}

View File

@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Models.Deluge.Response;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -21,9 +22,10 @@ public sealed class DelugeService : DownloadServiceBase
IHttpClientFactory httpClientFactory,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
config.Value.Validate();
_client = new (config, httpClientFactory);
@@ -35,12 +37,12 @@ public sealed class DelugeService : DownloadServiceBase
}
/// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{
hash = hash.ToLowerInvariant();
DelugeContents? contents = null;
RemoveResult result = new();
StalledResult result = new();
TorrentStatus? status = await GetTorrentStatus(hash);
@@ -76,7 +78,7 @@ public sealed class DelugeService : DownloadServiceBase
}
/// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync(
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
@@ -86,18 +88,21 @@ public sealed class DelugeService : DownloadServiceBase
hash = hash.ToLowerInvariant();
TorrentStatus? status = await GetTorrentStatus(hash);
BlockFilesResult result = new();
if (status?.Hash is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return false;
return result;
}
result.IsPrivate = status.Private;
if (_contentBlockerConfig.IgnorePrivate && status.Private)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", status.Name);
return false;
return result;
}
DelugeContents? contents = null;
@@ -113,7 +118,7 @@ public sealed class DelugeService : DownloadServiceBase
if (contents is null)
{
return false;
return result;
}
Dictionary<int, int> priorities = [];
@@ -144,7 +149,7 @@ public sealed class DelugeService : DownloadServiceBase
if (!hasPriorityUpdates)
{
return false;
return result;
}
_logger.LogDebug("changing priorities | torrent {hash}", hash);
@@ -157,12 +162,14 @@ public sealed class DelugeService : DownloadServiceBase
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return true;
result.ShouldRemove = true;
return result;
}
await _client.ChangeFilesPriority(hash, sortedPriorities);
return false;
return result;
}
/// <inheritdoc/>
@@ -196,6 +203,8 @@ public sealed class DelugeService : DownloadServiceBase
{
return false;
}
ResetStrikesOnProgress(status.Hash!, status.TotalDone);
return StrikeAndCheckLimit(status.Hash!, status.Name!);
}
@@ -205,7 +214,7 @@ public sealed class DelugeService : DownloadServiceBase
return await _client.SendRequest<TorrentStatus?>(
"web.get_torrent_status",
hash,
new[] { "hash", "state", "name", "eta", "private" }
new[] { "hash", "state", "name", "eta", "private", "total_done" }
);
}

View File

@@ -2,9 +2,13 @@
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Domain.Enums;
using Domain.Models.Cache;
using Infrastructure.Helpers;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -15,13 +19,16 @@ public abstract class DownloadServiceBase : IDownloadService
protected readonly ILogger<DownloadServiceBase> _logger;
protected readonly QueueCleanerConfig _queueCleanerConfig;
protected readonly ContentBlockerConfig _contentBlockerConfig;
protected readonly IMemoryCache _cache;
protected readonly FilenameEvaluator _filenameEvaluator;
protected readonly Striker _striker;
protected readonly MemoryCacheEntryOptions _cacheOptions;
protected DownloadServiceBase(
ILogger<DownloadServiceBase> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
)
@@ -29,18 +36,21 @@ public abstract class DownloadServiceBase : IDownloadService
_logger = logger;
_queueCleanerConfig = queueCleanerConfig.Value;
_contentBlockerConfig = contentBlockerConfig.Value;
_cache = cache;
_filenameEvaluator = filenameEvaluator;
_striker = striker;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
}
public abstract void Dispose();
public abstract Task LoginAsync();
public abstract Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
/// <inheritdoc/>
public abstract Task<bool> BlockUnwantedFilesAsync(
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
@@ -50,6 +60,23 @@ public abstract class DownloadServiceBase : IDownloadService
/// <inheritdoc/>
public abstract Task Delete(string hash);
protected void ResetStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
{
return;
}
if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded)
{
// cache item found
_cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
_logger.LogDebug("resetting strikes for {hash} due to progress", hash);
}
_cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions);
}
/// <summary>
/// Strikes an item and checks if the limit has been reached.
/// </summary>

View File

@@ -4,6 +4,7 @@ using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -11,7 +12,7 @@ namespace Infrastructure.Verticals.DownloadClient;
public sealed class DummyDownloadService : DownloadServiceBase
{
public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
}
@@ -24,12 +25,12 @@ public sealed class DummyDownloadService : DownloadServiceBase
return Task.CompletedTask;
}
public override Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{
throw new NotImplementedException();
}
public override Task<bool> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
{
throw new NotImplementedException();
}

View File

@@ -12,7 +12,7 @@ public interface IDownloadService : IDisposable
/// Checks whether the download should be removed from the *arr queue.
/// </summary>
/// <param name="hash">The download hash.</param>
public Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
public Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
/// <summary>
/// Blocks unwanted files from being fully downloaded.
@@ -22,7 +22,7 @@ public interface IDownloadService : IDisposable
/// <param name="patterns">The patterns to test the files against.</param>
/// <param name="regexes">The regexes to test the files against.</param>
/// <returns>True if all files have been blocked; otherwise false.</returns>
public Task<bool> BlockUnwantedFilesAsync(
public Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,

View File

@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using QBittorrent.Client;
@@ -23,9 +24,10 @@ public sealed class QBitService : DownloadServiceBase
IOptions<QBitConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
_config = config.Value;
_config.Validate();
@@ -43,9 +45,9 @@ public sealed class QBitService : DownloadServiceBase
}
/// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{
RemoveResult result = new();
StalledResult result = new();
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
@@ -89,7 +91,7 @@ public sealed class QBitService : DownloadServiceBase
}
/// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync(
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
@@ -98,11 +100,12 @@ public sealed class QBitService : DownloadServiceBase
{
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
BlockFilesResult result = new();
if (torrent is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return false;
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
@@ -110,25 +113,27 @@ public sealed class QBitService : DownloadServiceBase
if (torrentProperties is null)
{
_logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
return false;
return result;
}
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
result.IsPrivate = isPrivate;
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
return false;
return result;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
if (files is null)
{
return false;
return result;
}
List<int> unwantedFiles = [];
@@ -162,13 +167,15 @@ public sealed class QBitService : DownloadServiceBase
if (unwantedFiles.Count is 0)
{
return false;
return result;
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return true;
result.ShouldRemove = true;
return result;
}
foreach (int fileIndex in unwantedFiles)
@@ -176,7 +183,7 @@ public sealed class QBitService : DownloadServiceBase
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
}
return false;
return result;
}
/// <inheritdoc/>
@@ -211,6 +218,8 @@ public sealed class QBitService : DownloadServiceBase
return false;
}
ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
return StrikeAndCheckLimit(torrent.Hash, torrent.Name);
}
}

View File

@@ -1,6 +1,6 @@
namespace Infrastructure.Verticals.DownloadClient;
public sealed record RemoveResult
public sealed record StalledResult
{
/// <summary>
/// True if the download should be removed; otherwise false.

View File

@@ -6,6 +6,7 @@ using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Transmission.API.RPC;
@@ -26,9 +27,10 @@ public sealed class TransmissionService : DownloadServiceBase
IOptions<TransmissionConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
{
_config = config.Value;
_config.Validate();
@@ -46,9 +48,9 @@ public sealed class TransmissionService : DownloadServiceBase
}
/// <inheritdoc/>
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
public override async Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash)
{
RemoveResult result = new();
StalledResult result = new();
TorrentInfo? torrent = await GetTorrentAsync(hash);
if (torrent is null)
@@ -82,7 +84,7 @@ public sealed class TransmissionService : DownloadServiceBase
}
/// <inheritdoc/>
public override async Task<bool> BlockUnwantedFilesAsync(
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
@@ -90,17 +92,21 @@ public sealed class TransmissionService : DownloadServiceBase
)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
BlockFilesResult result = new();
if (torrent?.FileStats is null || torrent.Files is null)
{
return false;
return result;
}
bool isPrivate = torrent.IsPrivate ?? false;
result.IsPrivate = isPrivate;
if (_contentBlockerConfig.IgnorePrivate && (torrent.IsPrivate ?? false))
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
return false;
return result;
}
List<long> unwantedFiles = [];
@@ -134,13 +140,15 @@ public sealed class TransmissionService : DownloadServiceBase
if (unwantedFiles.Count is 0)
{
return false;
return result;
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return true;
result.ShouldRemove = true;
return result;
}
_logger.LogDebug("changing priorities | torrent {hash}", hash);
@@ -151,7 +159,7 @@ public sealed class TransmissionService : DownloadServiceBase
FilesUnwanted = unwantedFiles.ToArray(),
});
return false;
return result;
}
public override async Task Delete(string hash)
@@ -194,6 +202,8 @@ public sealed class TransmissionService : DownloadServiceBase
{
return false;
}
ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0);
return StrikeAndCheckLimit(torrent.HashString!, torrent.Name!);
}
@@ -213,7 +223,8 @@ public sealed class TransmissionService : DownloadServiceBase
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER
];
// refresh cache

View File

@@ -1,5 +1,6 @@
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Helpers;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
@@ -26,7 +27,7 @@ public class Striker
return false;
}
string key = $"{strikeType.ToString()}_{hash}";
string key = CacheKeys.Strike(strikeType, hash);
if (!_cache.TryGetValue(key, out int? strikeCount))
{

View File

@@ -1,5 +1,6 @@
using Common.Configuration.Arr;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
@@ -14,8 +15,11 @@ namespace Infrastructure.Verticals.QueueCleaner;
public sealed class QueueCleaner : GenericHandler
{
private readonly QueueCleanerConfig _config;
public QueueCleaner(
ILogger<QueueCleaner> logger,
IOptions<QueueCleanerConfig> config,
IOptions<DownloadClientConfig> downloadClientConfig,
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
@@ -32,6 +36,7 @@ public sealed class QueueCleaner : GenericHandler
arrArrQueueIterator, downloadServiceFactory
)
{
_config = config.Value;
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
@@ -56,33 +61,46 @@ public sealed class QueueCleaner : GenericHandler
QueueRecord record = group.First();
if (record.Protocol is not "torrent")
{
continue;
}
if (!arrClient.IsRecordValid(record))
{
continue;
}
RemoveResult removeResult = new();
StalledResult stalledCheckResult = new();
if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None)
if (_downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.None && record.Protocol is "torrent")
{
removeResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId);
// stalled download check
stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId);
}
bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(record, removeResult.IsPrivate);
// failed import check
bool shouldRemoveFromArr = arrClient.ShouldRemoveFromQueue(instanceType, record, stalledCheckResult.IsPrivate);
if (!shouldRemoveFromArr && !removeResult.ShouldRemove)
if (!shouldRemoveFromArr && !stalledCheckResult.ShouldRemove)
{
_logger.LogInformation("skip | {title}", record.Title);
continue;
}
itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
await arrClient.DeleteQueueItemAsync(instance, record);
bool removeFromClient = true;
if (stalledCheckResult.IsPrivate)
{
if (stalledCheckResult.ShouldRemove && !_config.StalledDeletePrivate)
{
removeFromClient = false;
}
if (shouldRemoveFromArr && !_config.ImportFailedDeletePrivate)
{
removeFromClient = false;
}
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
}
});

View File

@@ -186,12 +186,15 @@ services:
- QUEUECLEANER__RUNSEQUENTIALLY=true
- QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true
- QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=false
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=file is a sample
- QUEUECLEANER__STALLED_MAX_STRIKES=5
- QUEUECLEANER__STALLED_IGNORE_PRIVATE=true
- QUEUECLEANER__STALLED_DELETE_PRIVATE=false
- CONTENTBLOCKER__ENABLED=true
- CONTENTBLOCKER__IGNORE_PRIVATE=true
- CONTENTBLOCKER__DELETE_PRIVATE=false
- DOWNLOAD_CLIENT=qbittorrent
- QBITTORRENT__URL=http://qbittorrent:8080