mirror/Cleanuparr
1
0
Fork 0
mirror of https://github.com/Cleanuparr/Cleanuparr.git synced 2026-03-04 06:18:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

try remove content blocker

Browse Source
This commit is contained in:
Flaminel
2025-06-06 20:46:38 +03:00
parent f6b0014ec6
commit cae4e323a5
35 changed files with 2472 additions and 2219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
code/Infrastructure/Configuration/ConfigInitializer.cs
View File

@@ -1,4 +1,3 @@
using Common.Configuration.ContentBlocker;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Configuration;

1
code/Infrastructure/Configuration/ConfigManager.cs
View File

@@ -1,7 +1,6 @@
using System.Reflection;
using Common.Configuration;
using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.General;

1
code/Infrastructure/Configuration/IConfigManager.cs
View File

@@ -1,5 +1,4 @@
using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.General;

10
code/Infrastructure/Verticals/Arr/ArrClient.cs
View File

@@ -72,9 +72,9 @@ public abstract class ArrClient : IArrClient
return queueResponse;
}
public virtual async Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, short arrMaxStrikes)
public virtual async Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, ushort arrMaxStrikes)
{
if (_queueCleanerConfig.FailedImportIgnorePrivate && isPrivateDownload)
if (_queueCleanerConfig.FailedImport.IgnorePrivate && isPrivateDownload)
{
// ignore private trackers
_logger.LogDebug("skip failed import check | download is private | {name}", record.Title);
@@ -108,7 +108,7 @@ public abstract class ArrClient : IArrClient
return false;
}
ushort maxStrikes = arrMaxStrikes > 0 ? (ushort)arrMaxStrikes : _queueCleanerConfig.FailedImportMaxStrikes;
ushort maxStrikes = arrMaxStrikes > 0 ? (ushort)arrMaxStrikes : _queueCleanerConfig.FailedImport.MaxStrikes;
return await _striker.StrikeAndCheckLimit(
record.DownloadId,
@@ -214,7 +214,7 @@ public abstract class ArrClient : IArrClient
private bool HasIgnoredPatterns(QueueRecord record)
{
if (_queueCleanerConfig.FailedImportIgnorePatterns.Count is 0)
if (_queueCleanerConfig.FailedImport.IgnoredPatterns.Count is 0)
{
// no patterns are configured
return false;
@@ -234,7 +234,7 @@ public abstract class ArrClient : IArrClient
.ForEach(x => messages.Add(x));
return messages.Any(
m => _queueCleanerConfig.FailedImportIgnorePatterns.Any(
m => _queueCleanerConfig.FailedImport.IgnoredPatterns.Any(
p => !string.IsNullOrWhiteSpace(p.Trim()) && m.Contains(p, StringComparison.InvariantCultureIgnoreCase)
)
);

2
code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs
View File

@@ -9,7 +9,7 @@ public interface IArrClient
{
Task<QueueListResponse> GetQueueItemsAsync(ArrInstance arrInstance, int page);
Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, short arrMaxStrikes);
Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, ushort arrMaxStrikes);
Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient, DeleteReason deleteReason);

27
code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs
View File

@@ -1,7 +1,7 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Data.Enums;
using Infrastructure.Configuration;
@@ -14,8 +14,7 @@ namespace Infrastructure.Verticals.ContentBlocker;
public sealed class BlocklistProvider
{
private readonly ILogger<BlocklistProvider> _logger;
private readonly IConfigManager _configManager;
private ContentBlockerConfig _contentBlockerConfig;
private readonly QueueCleanerConfig _queueCleanerConfig;
private readonly HttpClient _httpClient;
private readonly IMemoryCache _cache;
private bool _initialized;
@@ -28,11 +27,10 @@ public sealed class BlocklistProvider
)
{
_logger = logger;
_configManager = configManager;
_cache = cache;
_httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
_contentBlockerConfig = _configManager.GetConfiguration<ContentBlockerConfig>();
_queueCleanerConfig = configManager.GetConfiguration<QueueCleanerConfig>();
}
public async Task LoadBlocklistsAsync()
@@ -45,9 +43,9 @@ public sealed class BlocklistProvider
try
{
await LoadPatternsAndRegexesAsync(_contentBlockerConfig.Sonarr, InstanceType.Sonarr);
await LoadPatternsAndRegexesAsync(_contentBlockerConfig.Radarr, InstanceType.Radarr);
await LoadPatternsAndRegexesAsync(_contentBlockerConfig.Lidarr, InstanceType.Lidarr);
await LoadPatternsAndRegexesAsync(_queueCleanerConfig.ContentBlocker.Sonarr, InstanceType.Sonarr);
await LoadPatternsAndRegexesAsync(_queueCleanerConfig.ContentBlocker.Radarr, InstanceType.Radarr);
await LoadPatternsAndRegexesAsync(_queueCleanerConfig.ContentBlocker.Lidarr, InstanceType.Lidarr);
_initialized = true;
}
@@ -81,17 +79,12 @@ public sealed class BlocklistProvider
private async Task LoadPatternsAndRegexesAsync(BlocklistSettings blocklistSettings, InstanceType instanceType)
{
if (!blocklistSettings.Enabled)
if (string.IsNullOrEmpty(blocklistSettings.BlocklistPath))
{
return;
}
if (string.IsNullOrEmpty(blocklistSettings.Path))
{
return;
}
string[] filePatterns = await ReadContentAsync(blocklistSettings.Path);
string[] filePatterns = await ReadContentAsync(blocklistSettings.BlocklistPath);
long startTime = Stopwatch.GetTimestamp();
ParallelOptions options = new() { MaxDegreeOfParallelism = 5 };
@@ -122,13 +115,13 @@ public sealed class BlocklistProvider
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
_cache.Set(CacheKeys.BlocklistType(instanceType), blocklistSettings.Type);
_cache.Set(CacheKeys.BlocklistType(instanceType), blocklistSettings.BlocklistType);
_cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns);
_cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes);
_logger.LogDebug("loaded {count} patterns", patterns.Count);
_logger.LogDebug("loaded {count} regexes", regexes.Count);
_logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistSettings.Path);
_logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistSettings.BlocklistPath);
}
private async Task<string[]> ReadContentAsync(string path)

374
code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs
View File

@@ -1,187 +1,187 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.Arr;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadClient;
using Data.Enums;
using Data.Models.Arr.Queue;
using Infrastructure.Configuration;
using Infrastructure.Helpers;
using Infrastructure.Services;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Jobs;
using MassTransit;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using LogContext = Serilog.Context.LogContext;
namespace Infrastructure.Verticals.ContentBlocker;
public sealed class ContentBlocker : GenericHandler
{
private readonly ContentBlockerConfig _config;
private readonly BlocklistProvider _blocklistProvider;
private readonly IIgnoredDownloadsService _ignoredDownloadsService;
public ContentBlocker(
ILogger<ContentBlocker> logger,
IConfigManager configManager,
IMemoryCache cache,
IBus messageBus,
ArrClientFactory arrClientFactory,
ArrQueueIterator arrArrQueueIterator,
BlocklistProvider blocklistProvider,
IIgnoredDownloadsService ignoredDownloadsService,
DownloadServiceFactory downloadServiceFactory
) : base(
logger, cache, messageBus,
arrClientFactory, arrArrQueueIterator, downloadServiceFactory
)
{
_blocklistProvider = blocklistProvider;
_ignoredDownloadsService = ignoredDownloadsService;
_config = configManager.GetConfiguration<ContentBlockerConfig>();
_downloadClientConfig = configManager.GetConfiguration<DownloadClientConfig>();
_sonarrConfig = configManager.GetConfiguration<SonarrConfig>();
_radarrConfig = configManager.GetConfiguration<RadarrConfig>();
_lidarrConfig = configManager.GetConfiguration<LidarrConfig>();
}
public override async Task ExecuteAsync()
{
if (_downloadClientConfig.Clients.Count is 0)
{
_logger.LogWarning("No download clients configured");
return;
}
bool blocklistIsConfigured = _config.Sonarr.Enabled && !string.IsNullOrEmpty(_config.Sonarr.Path) ||
_config.Radarr.Enabled && !string.IsNullOrEmpty(_config.Radarr.Path) ||
_config.Lidarr.Enabled && !string.IsNullOrEmpty(_config.Lidarr.Path);
if (!blocklistIsConfigured)
{
_logger.LogWarning("no blocklist is configured");
return;
}
await _blocklistProvider.LoadBlocklistsAsync();
await base.ExecuteAsync();
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType, ArrConfig config)
{
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsService.GetIgnoredDownloadsAsync();
using var _ = LogContext.PushProperty(LogProperties.Category, instanceType.ToString());
IArrClient arrClient = _arrClientFactory.GetClient(instanceType);
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
{
var groups = items
.GroupBy(x => x.DownloadId)
.ToList();
foreach (var group in groups)
{
if (group.Any(x => !arrClient.IsRecordValid(x)))
{
continue;
}
QueueRecord record = group.First();
if (record.Protocol is not "torrent")
{
continue;
}
if (string.IsNullOrEmpty(record.DownloadId))
{
_logger.LogDebug("skip | download id is null for {title}", record.Title);
continue;
}
if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase))
{
_logger.LogInformation("skip | {title} | ignored", record.Title);
continue;
}
_logger.LogTrace("processing | {name}", record.Title);
string downloadRemovalKey = CacheKeys.DownloadMarkedForRemoval(record.DownloadId, instance.Url);
if (_cache.TryGetValue(downloadRemovalKey, out bool _))
{
_logger.LogDebug("skip | already marked for removal | {title}", record.Title);
continue;
}
_logger.LogDebug("searching unwanted files for {title}", record.Title);
bool found = false;
foreach (var downloadService in _downloadServices)
{
try
{
BlockFilesResult result = await downloadService
.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads);
if (!result.Found)
{
continue;
}
found = true;
if (!result.ShouldRemove)
{
break;
}
_logger.LogDebug("all files are marked as unwanted | {hash}", record.Title);
bool removeFromClient = true;
if (result.IsPrivate && !_config.DeletePrivate)
{
removeFromClient = false;
}
await PublishQueueItemRemoveRequest(
downloadRemovalKey,
instanceType,
instance,
record,
group.Count() > 1,
removeFromClient,
DeleteReason.AllFilesBlocked
);
break;
}
catch (Exception ex)
{
_logger.LogError(
ex,
"Error blocking unwanted files for {hash} with download client",
record.DownloadId);
}
}
if (!found)
{
_logger.LogWarning("skip | download not found {title}", record.Title);
}
}
});
}
}
// using System.Collections.Concurrent;
// using System.Text.RegularExpressions;
// using Common.Configuration.Arr;
// using Common.Configuration.DownloadClient;
// using Common.Configuration.QueueCleaner;
// using Data.Enums;
// using Data.Models.Arr.Queue;
// using Infrastructure.Configuration;
// using Infrastructure.Helpers;
// using Infrastructure.Services;
// using Infrastructure.Verticals.Arr;
// using Infrastructure.Verticals.Arr.Interfaces;
// using Infrastructure.Verticals.DownloadClient;
// using Infrastructure.Verticals.Jobs;
// using MassTransit;
// using Microsoft.Extensions.Caching.Memory;
// using Microsoft.Extensions.Logging;
// using LogContext = Serilog.Context.LogContext;
//
// namespace Infrastructure.Verticals.ContentBlocker;
//
// public sealed class ContentBlocker : GenericHandler
// {
// private readonly ContentBlockerConfig _config;
// private readonly BlocklistProvider _blocklistProvider;
// private readonly IIgnoredDownloadsService _ignoredDownloadsService;
//
// public ContentBlocker(
// ILogger<ContentBlocker> logger,
// IConfigManager configManager,
// IMemoryCache cache,
// IBus messageBus,
// ArrClientFactory arrClientFactory,
// ArrQueueIterator arrArrQueueIterator,
// BlocklistProvider blocklistProvider,
// IIgnoredDownloadsService ignoredDownloadsService,
// DownloadServiceFactory downloadServiceFactory
// ) : base(
// logger, cache, messageBus,
// arrClientFactory, arrArrQueueIterator, downloadServiceFactory
// )
// {
// _blocklistProvider = blocklistProvider;
// _ignoredDownloadsService = ignoredDownloadsService;
//
// _config = configManager.GetConfiguration<ContentBlockerConfig>();
// _downloadClientConfig = configManager.GetConfiguration<DownloadClientConfig>();
// _sonarrConfig = configManager.GetConfiguration<SonarrConfig>();
// _radarrConfig = configManager.GetConfiguration<RadarrConfig>();
// _lidarrConfig = configManager.GetConfiguration<LidarrConfig>();
// }
//
// public override async Task ExecuteAsync()
// {
// if (_downloadClientConfig.Clients.Count is 0)
// {
// _logger.LogWarning("No download clients configured");
// return;
// }
//
// bool blocklistIsConfigured = _config.Sonarr.Enabled && !string.IsNullOrEmpty(_config.Sonarr.Path) ||
// _config.Radarr.Enabled && !string.IsNullOrEmpty(_config.Radarr.Path) ||
// _config.Lidarr.Enabled && !string.IsNullOrEmpty(_config.Lidarr.Path);
//
// if (!blocklistIsConfigured)
// {
// _logger.LogWarning("no blocklist is configured");
// return;
// }
//
// await _blocklistProvider.LoadBlocklistsAsync();
// await base.ExecuteAsync();
// }
//
// protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType, ArrConfig config)
// {
// IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsService.GetIgnoredDownloadsAsync();
//
// using var _ = LogContext.PushProperty(LogProperties.Category, instanceType.ToString());
//
// IArrClient arrClient = _arrClientFactory.GetClient(instanceType);
// BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
// ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
// ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
//
// await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
// {
// var groups = items
// .GroupBy(x => x.DownloadId)
// .ToList();
//
// foreach (var group in groups)
// {
// if (group.Any(x => !arrClient.IsRecordValid(x)))
// {
// continue;
// }
//
// QueueRecord record = group.First();
//
// if (record.Protocol is not "torrent")
// {
// continue;
// }
//
// if (string.IsNullOrEmpty(record.DownloadId))
// {
// _logger.LogDebug("skip | download id is null for {title}", record.Title);
// continue;
// }
//
// if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase))
// {
// _logger.LogInformation("skip | {title} | ignored", record.Title);
// continue;
// }
//
// _logger.LogTrace("processing | {name}", record.Title);
//
// string downloadRemovalKey = CacheKeys.DownloadMarkedForRemoval(record.DownloadId, instance.Url);
//
// if (_cache.TryGetValue(downloadRemovalKey, out bool _))
// {
// _logger.LogDebug("skip | already marked for removal | {title}", record.Title);
// continue;
// }
//
// _logger.LogDebug("searching unwanted files for {title}", record.Title);
// bool found = false;
//
// foreach (var downloadService in _downloadServices)
// {
// try
// {
// BlockFilesResult result = await downloadService
// .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads);
//
// if (!result.Found)
// {
// continue;
// }
//
// found = true;
//
// if (!result.ShouldRemove)
// {
// break;
// }
//
// _logger.LogDebug("all files are marked as unwanted | {hash}", record.Title);
//
// bool removeFromClient = true;
//
// if (result.IsPrivate && !_config.DeletePrivate)
// {
// removeFromClient = false;
// }
//
// await PublishQueueItemRemoveRequest(
// downloadRemovalKey,
// instanceType,
// instance,
// record,
// group.Count() > 1,
// removeFromClient,
// DeleteReason.AllFilesBlocked
// );
//
// break;
// }
// catch (Exception ex)
// {
// _logger.LogError(
// ex,
// "Error blocking unwanted files for {hash} with download client",
// record.DownloadId);
// }
// }
//
// if (!found)
// {
// _logger.LogWarning("skip | download not found {title}", record.Title);
// }
// }
// });
// }
// }

2
code/Infrastructure/Verticals/ContentBlocker/FilenameEvaluator.cs
View File

@@ -1,6 +1,6 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.ContentBlocker;

2
code/Infrastructure/Verticals/ContentBlocker/IFilenameEvaluator.cs
View File

@@ -1,6 +1,6 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
namespace Infrastructure.Verticals.ContentBlocker;

487
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs
View File

@@ -1,21 +1,11 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.CustomDataTypes;
using Common.Exceptions;
using Data.Enums;
using Data.Models.Deluge.Response;
using Infrastructure.Events;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Infrastructure.Configuration;
@@ -23,7 +13,7 @@ using Infrastructure.Http;
namespace Infrastructure.Verticals.DownloadClient.Deluge;
public class DelugeService : DownloadService, IDelugeService
public partial class DelugeService : DownloadService, IDelugeService
{
private DelugeClient? _client;
@@ -36,11 +26,12 @@ public class DelugeService : DownloadService, IDelugeService
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService,
IDynamicHttpClientProvider httpClientProvider,
EventPublisher eventPublisher
EventPublisher eventPublisher,
BlocklistProvider blocklistProvider
) : base(
logger, configManager, cache,
filenameEvaluator, striker, dryRunInterceptor, hardLinkFileService,
httpClientProvider, eventPublisher
httpClientProvider, eventPublisher, blocklistProvider
)
{
// Client will be initialized when Initialize() is called with a specific client configuration
@@ -95,476 +86,6 @@ public class DelugeService : DownloadService, IDelugeService
throw;
}
}
/// <inheritdoc/>
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("Deluge client is not initialized");
}
hash = hash.ToLowerInvariant();
DelugeContents? contents = null;
DownloadCheckResult result = new();
DownloadStatus? download = await _client.GetTorrentStatus(hash);
if (download?.Hash is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.Found = true;
result.IsPrivate = download.Private;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
try
{
contents = await _client.GetTorrentFiles(hash);
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash);
}
bool shouldRemove = contents?.Contents?.Count > 0;
ProcessFiles(contents.Contents, (_, file) =>
{
if (file.Priority > 0)
{
shouldRemove = false;
}
});
if (shouldRemove)
{
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download);
return result;
}
/// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
{
hash = hash.ToLowerInvariant();
DownloadStatus? download = await _client.GetTorrentStatus(hash);
BlockFilesResult result = new();
if (download?.Hash is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
// Mark as processed since we found the download
result.Found = true;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
result.IsPrivate = download.Private;
if (_contentBlockerConfig.IgnorePrivate && download.Private)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result;
}
DelugeContents? contents = null;
try
{
contents = await _client.GetTorrentFiles(hash);
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash);
}
if (contents is null)
{
return result;
}
Dictionary<int, int> priorities = [];
bool hasPriorityUpdates = false;
long totalFiles = 0;
long totalUnwantedFiles = 0;
ProcessFiles(contents.Contents, (name, file) =>
{
totalFiles++;
int priority = file.Priority;
if (file.Priority is 0)
{
totalUnwantedFiles++;
}
if (file.Priority is not 0 && !_filenameEvaluator.IsValid(name, blocklistType, patterns, regexes))
{
totalUnwantedFiles++;
priority = 0;
hasPriorityUpdates = true;
_logger.LogInformation("unwanted file found | {file}", file.Path);
}
priorities.Add(file.Index, priority);
});
if (!hasPriorityUpdates)
{
return result;
}
_logger.LogDebug("changing priorities | torrent {hash}", hash);
List<int> sortedPriorities = priorities
.OrderBy(x => x.Key)
.Select(x => x.Value)
.ToList();
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
result.ShouldRemove = true;
return result;
}
await _dryRunInterceptor.InterceptAsync(ChangeFilesPriority, hash, sortedPriorities);
return result;
}
public override async Task<List<object>?> GetSeedingDownloads()
{
return (await _client.GetStatusForAllTorrents())
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true)
.Cast<object>()
.ToList();
}
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<DownloadStatus>()
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<DownloadStatus>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (DownloadStatus download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.Private)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SeedingTime);
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, seedingTime, category);
if (!result.ShouldClean)
{
continue;
}
await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _eventPublisher.PublishDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
IReadOnlyList<string> existingLabels = await _client.GetLabels();
if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateLabel, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (DownloadStatus download in downloads.Cast<DownloadStatus>())
{
if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
DelugeContents? contents = null;
try
{
contents = await _client.GetTorrentFiles(download.Hash);
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name);
continue;
}
bool hasHardlinks = false;
ProcessFiles(contents?.Contents, (_, file) =>
{
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/']));
if (file.Priority <= 0)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
return;
}
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
return;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
}
});
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
_logger.LogInformation("category changed for {name}", download.Name);
await _eventPublisher.PublishCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory);
download.Label = _downloadCleanerConfig.UnlinkedTargetCategory;
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
hash = hash.ToLowerInvariant();
await _client.DeleteTorrents([hash]);
}
[DryRunSafeguard]
protected async Task CreateLabel(string name)
{
await _client.CreateLabel(name);
}
[DryRunSafeguard]
protected virtual async Task ChangeFilesPriority(string hash, List<int> sortedPriorities)
{
await _client.ChangeFilesPriority(hash, sortedPriorities);
}
[DryRunSafeguard]
protected virtual async Task ChangeLabel(string hash, string newLabel)
{
await _client.SetTorrentLabel(hash, newLabel);
}
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(DownloadStatus status)
{
(bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(status);
if (result.ShouldRemove)
{
return result;
}
return await CheckIfStuck(status);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(DownloadStatus download)
{
if (_queueCleanerConfig.SlowMaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.State is null || !download.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
return (false, DeleteReason.None);
}
if (download.DownloadSpeed <= 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.SlowIgnorePrivate && download.Private)
{
// ignore private trackers
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
{
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
return (false, DeleteReason.None);
}
ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize;
ByteSize currentSpeed = new ByteSize(download.DownloadSpeed);
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime);
SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta);
return await CheckIfSlow(
download.Hash!,
download.Name!,
minSpeed,
currentSpeed,
maxTime,
currentTime
);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(DownloadStatus status)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", status.Name);
return (false, DeleteReason.None);
}
if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
return (false, DeleteReason.None);
}
if (status.Eta > 0)
{
return (false, DeleteReason.None);
}
ResetStalledStrikesOnProgress(status.Hash!, status.TotalDone);
return (await _striker.StrikeAndCheckLimit(status.Hash!, status.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
}
private static void ProcessFiles(Dictionary<string, DelugeFileOrDirectory>? contents, Action<string, DelugeFileOrDirectory> processFile)
{

225
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceDC.cs Normal file
View File

@@ -0,0 +1,225 @@
using Common.Attributes;
using Common.Configuration.DownloadCleaner;
using Data.Enums;
using Data.Models.Deluge.Response;
using Infrastructure.Extensions;
using Infrastructure.Verticals.Context;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.DownloadClient.Deluge;
public partial class DelugeService
{
public override async Task<List<object>?> GetSeedingDownloads()
{
return (await _client.GetStatusForAllTorrents())
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true)
.Cast<object>()
.ToList();
}
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<DownloadStatus>()
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<DownloadStatus>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (DownloadStatus download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.Private)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SeedingTime);
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, seedingTime, category);
if (!result.ShouldClean)
{
continue;
}
await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _eventPublisher.PublishDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
IReadOnlyList<string> existingLabels = await _client.GetLabels();
if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateLabel, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (DownloadStatus download in downloads.Cast<DownloadStatus>())
{
if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
DelugeContents? contents = null;
try
{
contents = await _client.GetTorrentFiles(download.Hash);
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name);
continue;
}
bool hasHardlinks = false;
ProcessFiles(contents?.Contents, (_, file) =>
{
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/']));
if (file.Priority <= 0)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
return;
}
long hardlinkCount = _hardLinkFileService
.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
return;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
}
});
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
_logger.LogInformation("category changed for {name}", download.Name);
await _eventPublisher.PublishCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory);
download.Label = _downloadCleanerConfig.UnlinkedTargetCategory;
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
hash = hash.ToLowerInvariant();
await _client.DeleteTorrents([hash]);
}
[DryRunSafeguard]
protected async Task CreateLabel(string name)
{
await _client.CreateLabel(name);
}
[DryRunSafeguard]
protected virtual async Task ChangeLabel(string hash, string newLabel)
{
await _client.SetTorrentLabel(hash, newLabel);
}
}

261
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceQC.cs Normal file
View File

@@ -0,0 +1,261 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.QueueCleaner;
using Common.CustomDataTypes;
using Data.Enums;
using Data.Models.Deluge.Response;
using Infrastructure.Extensions;
using Infrastructure.Verticals.Context;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.DownloadClient.Deluge;
public partial class DelugeService
{
/// <inheritdoc/>
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash,
IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("Deluge client is not initialized");
}
hash = hash.ToLowerInvariant();
DelugeContents? contents = null;
DownloadCheckResult result = new();
DownloadStatus? download = await _client.GetTorrentStatus(hash);
if (download?.Hash is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.Found = true;
result.IsPrivate = download.Private;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
try
{
contents = await _client.GetTorrentFiles(hash);
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash);
}
bool shouldRemove = contents?.Contents?.Count > 0;
ProcessFiles(contents.Contents, (_, file) =>
{
if (file.Priority > 0)
{
shouldRemove = false;
}
});
if (shouldRemove)
{
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, download.Private, contents.Contents);
return result;
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> BlockUnwantedFilesAsync(
DownloadStatus download,
bool isPrivate,
Dictionary<string, DelugeFileOrDirectory>? files
)
{
if (!_queueCleanerConfig.ContentBlocker.Enabled)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.ContentBlocker.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip unwanted files check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (files is null)
{
_logger.LogDebug("failed to find files for {name}", download.Name);
return (false, DeleteReason.None);
}
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
Dictionary<int, int> priorities = [];
bool hasPriorityUpdates = false;
long totalFiles = 0;
long totalUnwantedFiles = 0;
ProcessFiles(files, (name, file) =>
{
totalFiles++;
int priority = file.Priority;
if (file.Priority is 0)
{
totalUnwantedFiles++;
}
if (file.Priority is not 0 && !_filenameEvaluator.IsValid(name, blocklistType, patterns, regexes))
{
totalUnwantedFiles++;
priority = 0;
hasPriorityUpdates = true;
_logger.LogInformation("unwanted file found | {file}", file.Path);
}
priorities.Add(file.Index, priority);
});
if (!hasPriorityUpdates)
{
return (false, DeleteReason.None);
}
_logger.LogTrace("marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name);
List<int> sortedPriorities = priorities
.OrderBy(x => x.Key)
.Select(x => x.Value)
.ToList();
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return (true, DeleteReason.AllFilesBlocked);
}
await _dryRunInterceptor.InterceptAsync(ChangeFilesPriority, download.Hash, sortedPriorities);
return (false, DeleteReason.None);
}
[DryRunSafeguard]
protected virtual async Task ChangeFilesPriority(string hash, List<int> sortedPriorities)
{
await _client.ChangeFilesPriority(hash, sortedPriorities);
}
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(
DownloadStatus status,
bool isPrivate,
Dictionary<string, DelugeFileOrDirectory>? files
)
{
(bool ShouldRemove, DeleteReason Reason) result = await BlockUnwantedFilesAsync(status, isPrivate, files);
if (result.ShouldRemove)
{
return result;
}
result = await CheckIfSlow(status);
if (result.ShouldRemove)
{
return result;
}
return await CheckIfStuck(status);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(DownloadStatus download)
{
if (_queueCleanerConfig.Slow.MaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.State is null || !download.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
return (false, DeleteReason.None);
}
if (download.DownloadSpeed <= 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.Slow.IgnorePrivate && download.Private)
{
// ignore private trackers
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.Size > (_queueCleanerConfig.Slow.IgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
{
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
return (false, DeleteReason.None);
}
ByteSize minSpeed = _queueCleanerConfig.Slow.MinSpeedByteSize;
ByteSize currentSpeed = new ByteSize(download.DownloadSpeed);
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.Slow.MaxTime);
SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta);
return await CheckIfSlow(
download.Hash!,
download.Name!,
minSpeed,
currentSpeed,
maxTime,
currentTime
);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(DownloadStatus status)
{
if (_queueCleanerConfig.Stalled.MaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.Stalled.IgnorePrivate && status.Private)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", status.Name);
return (false, DeleteReason.None);
}
if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
return (false, DeleteReason.None);
}
if (status.Eta > 0)
{
return (false, DeleteReason.None);
}
ResetStalledStrikesOnProgress(status.Hash!, status.TotalDone);
return (await _striker.StrikeAndCheckLimit(status.Hash!, status.Name!, _queueCleanerConfig.Stalled.MaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
}
}

32
code/Infrastructure/Verticals/DownloadClient/DownloadService.cs
View File

@@ -1,6 +1,5 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
@@ -17,9 +16,9 @@ using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using QBittorrent.Client;
namespace Infrastructure.Verticals.DownloadClient;
@@ -35,15 +34,16 @@ public abstract class DownloadService : IDownloadService
protected readonly IHardLinkFileService _hardLinkFileService;
protected readonly IDynamicHttpClientProvider _httpClientProvider;
protected readonly EventPublisher _eventPublisher;
protected readonly BlocklistProvider _blocklistProvider;
protected readonly QueueCleanerConfig _queueCleanerConfig;
protected readonly ContentBlockerConfig _contentBlockerConfig;
protected readonly DownloadCleanerConfig _downloadCleanerConfig;
protected HttpClient? _httpClient;
// Client-specific configuration
protected ClientConfig _clientConfig;
// HTTP client for this service
protected HttpClient? _httpClient;
protected DownloadService(
ILogger<DownloadService> logger,
@@ -54,7 +54,8 @@ public abstract class DownloadService : IDownloadService
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService,
IDynamicHttpClientProvider httpClientProvider,
EventPublisher eventPublisher
EventPublisher eventPublisher,
BlocklistProvider blocklistProvider
)
{
_logger = logger;
@@ -66,6 +67,7 @@ public abstract class DownloadService : IDownloadService
_hardLinkFileService = hardLinkFileService;
_httpClientProvider = httpClientProvider;
_eventPublisher = eventPublisher;
_blocklistProvider = blocklistProvider;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
@@ -73,7 +75,6 @@ public abstract class DownloadService : IDownloadService
_clientConfig = new ClientConfig();
_queueCleanerConfig = _configManager.GetConfiguration<QueueCleanerConfig>();
_contentBlockerConfig = _configManager.GetConfiguration<ContentBlockerConfig>();
_downloadCleanerConfig = _configManager.GetConfiguration<DownloadCleanerConfig>();
}
@@ -99,13 +100,8 @@ public abstract class DownloadService : IDownloadService
public abstract Task LoginAsync();
public abstract Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads);
public abstract Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash,
IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task DeleteDownload(string hash);
@@ -130,7 +126,7 @@ public abstract class DownloadService : IDownloadService
protected void ResetStalledStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
if (!_queueCleanerConfig.Stalled.ResetStrikesOnProgress)
{
return;
}
@@ -148,7 +144,7 @@ public abstract class DownloadService : IDownloadService
protected void ResetSlowSpeedStrikesOnProgress(string downloadName, string hash)
{
if (_queueCleanerConfig.SlowResetStrikesOnProgress)
if (_queueCleanerConfig.Slow.ResetStrikesOnProgress)
{
return;
}
@@ -166,7 +162,7 @@ public abstract class DownloadService : IDownloadService
protected void ResetSlowTimeStrikesOnProgress(string downloadName, string hash)
{
if (_queueCleanerConfig.SlowResetStrikesOnProgress)
if (_queueCleanerConfig.Slow.ResetStrikesOnProgress)
{
return;
}
@@ -196,7 +192,7 @@ public abstract class DownloadService : IDownloadService
_logger.LogTrace("slow speed | {speed}/s | {name}", currentSpeed.ToString(), downloadName);
bool shouldRemove = await _striker
.StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowSpeed);
.StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.Slow.MaxStrikes, StrikeType.SlowSpeed);
if (shouldRemove)
{
@@ -213,7 +209,7 @@ public abstract class DownloadService : IDownloadService
_logger.LogTrace("slow estimated time | {time} | {name}", currentTime.ToString(), downloadName);
bool shouldRemove = await _striker
.StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowTime);
.StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.Slow.MaxStrikes, StrikeType.SlowTime);
if (shouldRemove)
{

49
code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs
View File

@@ -28,30 +28,30 @@ public sealed class DownloadServiceFactory
_logger = logger;
}
/// <summary>
/// Creates a download service using the specified client ID
/// </summary>
/// <param name="clientId">The client ID to create a service for</param>
/// <returns>An implementation of IDownloadService or null if the client is not available</returns>
public IDownloadService? GetDownloadService(Guid clientId)
{
var config = _configManager.GetConfiguration<DownloadClientConfig>();
var clientConfig = config.GetClientConfig(clientId);
if (clientConfig == null)
{
_logger.LogWarning("No download client configuration found for ID {clientId}", clientId);
return null;
}
if (!clientConfig.Enabled)
{
_logger.LogWarning("Download client {clientId} is disabled", clientId);
return null;
}
return GetDownloadService(clientConfig);
}
// /// <summary>
// /// Creates a download service using the specified client ID
// /// </summary>
// /// <param name="clientId">The client ID to create a service for</param>
// /// <returns>An implementation of IDownloadService or null if the client is not available</returns>
// public IDownloadService? GetDownloadService(Guid clientId)
// {
// var config = _configManager.GetConfiguration<DownloadClientConfig>();
// var clientConfig = config.GetClientConfig(clientId);
//
// if (clientConfig == null)
// {
// _logger.LogWarning("No download client configuration found for ID {clientId}", clientId);
// return null;
// }
//
// if (!clientConfig.Enabled)
// {
// _logger.LogWarning("Download client {clientId} is disabled", clientId);
// return null;
// }
//
// return GetDownloadService(clientConfig);
// }
/// <summary>
/// Creates a download service using the specified client configuration
@@ -83,7 +83,6 @@ public sealed class DownloadServiceFactory
/// <returns>An implementation of IDownloadService</returns>
private T CreateClientService<T>(ClientConfig clientConfig) where T : IDownloadService
{
// TODO
var service = _serviceProvider.GetRequiredService<T>();
service.Initialize(clientConfig);
return service;

23
code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs
View File

@@ -1,9 +1,11 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Data.Enums;
using Infrastructure.Interceptors;
using QBittorrent.Client;
namespace Infrastructure.Verticals.DownloadClient;
@@ -28,23 +30,8 @@ public interface IDownloadService : IDisposable
/// </summary>
/// <param name="hash">The download hash.</param>
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
public Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Blocks unwanted files from being fully downloaded.
/// </summary>
/// <param name="hash">The torrent hash.</param>
/// <param name="blocklistType">The <see cref="BlocklistType"/>.</param>
/// <param name="patterns">The patterns to test the files against.</param>
/// <param name="regexes">The regexes to test the files against.</param>
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
/// <returns>True if all files have been blocked; otherwise false.</returns>
public Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes,
IReadOnlyList<string> ignoredDownloads
);
public Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash,
IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Fetches all seeding downloads.

604
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs
View File

@@ -1,7 +1,6 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
@@ -24,7 +23,7 @@ using Infrastructure.Events;
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
public class QBitService : DownloadService, IQBitService
public partial class QBitService : DownloadService, IQBitService
{
protected QBittorrentClient? _client;
@@ -38,10 +37,11 @@ public class QBitService : DownloadService, IQBitService
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService,
IDynamicHttpClientProvider httpClientProvider,
EventPublisher eventPublisher
EventPublisher eventPublisher,
BlocklistProvider blocklistProvider
) : base(
logger, configManager, cache, filenameEvaluator, striker, dryRunInterceptor, hardLinkFileService,
httpClientProvider, eventPublisher
httpClientProvider, eventPublisher, blocklistProvider
)
{
// Client will be initialized when Initialize() is called with a specific client configuration
@@ -53,12 +53,6 @@ public class QBitService : DownloadService, IQBitService
// Initialize base service first
base.Initialize(clientConfig);
// Ensure client type is correct
if (clientConfig.Type != Common.Enums.DownloadClientType.QBittorrent)
{
throw new InvalidOperationException($"Cannot initialize QBitService with client type {clientConfig.Type}");
}
// Create QBittorrent client
_client = new QBittorrentClient(_httpClient, clientConfig.Url);
@@ -90,589 +84,7 @@ public class QBitService : DownloadService, IQBitService
throw;
}
}
/// <inheritdoc/>
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
DownloadCheckResult result = new();
TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.Found = true;
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
if (torrentProperties is null)
{
_logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
return result;
}
result.IsPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip))
{
result.ShouldRemove = true;
// if all files were blocked by qBittorrent
if (download is { CompletionOn: not null, Downloaded: null or 0 })
{
result.DeleteReason = DeleteReason.AllFilesSkippedByQBit;
return result;
}
// remove if all files are unwanted
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, result.IsPrivate);
return result;
}
/// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes,
IReadOnlyList<string> ignoredDownloads
)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
BlockFilesResult result = new();
TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
// Mark as processed since we found the download
result.Found = true;
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
if (torrentProperties is null)
{
_logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
return result;
}
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
result.IsPrivate = isPrivate;
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
if (files is null)
{
return result;
}
List<int> unwantedFiles = [];
long totalFiles = 0;
long totalUnwantedFiles = 0;
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
{
continue;
}
totalFiles++;
if (file.Priority is TorrentContentPriority.Skip)
{
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(file.Name, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogInformation("unwanted file found | {file}", file.Name);
unwantedFiles.Add(file.Index.Value);
totalUnwantedFiles++;
}
if (unwantedFiles.Count is 0)
{
return result;
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
result.ShouldRemove = true;
return result;
}
foreach (int fileIndex in unwantedFiles)
{
await _dryRunInterceptor.InterceptAsync(SkipFile, hash, fileIndex);
}
return result;
}
/// <inheritdoc/>
public override async Task<List<object>?> GetSeedingDownloads()
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
var torrentList = await _client.GetTorrentListAsync(new TorrentListQuery { Filter = TorrentListFilter.Seeding });
return torrentList?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Where(x =>
{
if (_downloadCleanerConfig.UnlinkedUseTag)
{
return !x.Tags.Any(tag => tag.Equals(_downloadCleanerConfig.UnlinkedTargetCategory, StringComparison.InvariantCultureIgnoreCase));
}
return true;
})
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (!_downloadCleanerConfig.DeletePrivate)
{
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash);
if (torrentProperties is null)
{
_logger.LogDebug("failed to find torrent properties in the download client | {name}", download.Name);
return;
}
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
if (isPrivate)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category);
if (!result.ShouldClean)
{
continue;
}
await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _eventPublisher.PublishDownloadCleaned(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
IReadOnlyDictionary<string, Category>? existingCategories = await _client.GetCategoriesAsync();
if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateCategory, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(download.Hash);
if (files is null)
{
_logger.LogDebug("failed to find files for {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
bool hasHardlinks = false;
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
{
_logger.LogDebug("skip | file index is null for {name}", download.Name);
hasHardlinks = true;
break;
}
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/']));
if (file.Priority is TorrentContentPriority.Skip)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
continue;
}
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
break;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
break;
}
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
if (_downloadCleanerConfig.UnlinkedUseTag)
{
_logger.LogInformation("tag added for {name}", download.Name);
}
else
{
_logger.LogInformation("category changed for {name}", download.Name);
download.Category = _downloadCleanerConfig.UnlinkedTargetCategory;
}
await _eventPublisher.PublishCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory, _downloadCleanerConfig.UnlinkedUseTag);
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
await _client.DeleteAsync([hash], deleteDownloadedData: true);
}
[DryRunSafeguard]
protected async Task CreateCategory(string name)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
await _client.AddCategoryAsync(name);
}
[DryRunSafeguard]
protected virtual async Task SkipFile(string hash, int fileIndex)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
}
[DryRunSafeguard]
protected virtual async Task ChangeCategory(string hash, string newCategory)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
if (_downloadCleanerConfig.UnlinkedUseTag)
{
await _client.AddTorrentTagAsync([hash], newCategory);
return;
}
await _client.SetTorrentCategoryAsync([hash], newCategory);
}
public override void Dispose()
{
_client?.Dispose();
_httpClient?.Dispose();
}
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent, bool isPrivate)
{
(bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent, isPrivate);
if (result.ShouldRemove)
{
return result;
}
return await CheckIfStuck(torrent, isPrivate);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download, bool isPrivate)
{
if (_queueCleanerConfig.SlowMaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.State is not (TorrentState.Downloading or TorrentState.ForcedDownload))
{
return (false, DeleteReason.None);
}
if (download.DownloadSpeed <= 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.SlowIgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
{
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
return (false, DeleteReason.None);
}
ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize;
ByteSize currentSpeed = new ByteSize(download.DownloadSpeed);
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime);
SmartTimeSpan currentTime = new SmartTimeSpan(download.EstimatedTime ?? TimeSpan.Zero);
return await CheckIfSlow(
download.Hash,
download.Name,
minSpeed,
currentSpeed,
maxTime,
currentTime
);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo torrent, bool isPrivate)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0 && _queueCleanerConfig.DownloadingMetadataMaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata
and not TorrentState.ForcedFetchingMetadata)
{
// ignore other states
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.StalledMaxStrikes > 0 && torrent.State is TorrentState.StalledDownload)
{
if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
}
else
{
ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
}
}
if (_queueCleanerConfig.DownloadingMetadataMaxStrikes > 0 && torrent.State is not TorrentState.StalledDownload)
{
return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.DownloadingMetadataMaxStrikes, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata);
}
return (false, DeleteReason.None);
}
private async Task<IReadOnlyList<TorrentTracker>> GetTrackersAsync(string hash)
{
if (_client == null)
@@ -684,4 +96,10 @@ public class QBitService : DownloadService, IQBitService
.Where(x => x.Url.Contains("**"))
.ToList();
}
public override void Dispose()
{
_client?.Dispose();
_httpClient?.Dispose();
}
}

305
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceDC.cs Normal file
View File

@@ -0,0 +1,305 @@
using Common.Attributes;
using Common.Configuration.DownloadCleaner;
using Data.Enums;
using Infrastructure.Extensions;
using Infrastructure.Verticals.Context;
using Microsoft.Extensions.Logging;
using QBittorrent.Client;
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
public partial class QBitService
{
/// <inheritdoc/>
public override async Task<List<object>?> GetSeedingDownloads()
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
var torrentList = await _client.GetTorrentListAsync(new TorrentListQuery { Filter = TorrentListFilter.Seeding });
return torrentList?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Where(x =>
{
if (_downloadCleanerConfig.UnlinkedUseTag)
{
return !x.Tags.Any(tag => tag.Equals(_downloadCleanerConfig.UnlinkedTargetCategory, StringComparison.InvariantCultureIgnoreCase));
}
return true;
})
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (!_downloadCleanerConfig.DeletePrivate)
{
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash);
if (torrentProperties is null)
{
_logger.LogDebug("failed to find torrent properties in the download client | {name}", download.Name);
return;
}
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
if (isPrivate)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category);
if (!result.ShouldClean)
{
continue;
}
await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _eventPublisher.PublishDownloadCleaned(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
IReadOnlyDictionary<string, Category>? existingCategories = await _client.GetCategoriesAsync();
if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateCategory, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(download.Hash);
if (files is null)
{
_logger.LogDebug("failed to find files for {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
bool hasHardlinks = false;
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
{
_logger.LogDebug("skip | file index is null for {name}", download.Name);
hasHardlinks = true;
break;
}
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/']));
if (file.Priority is TorrentContentPriority.Skip)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
continue;
}
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
break;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
break;
}
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
if (_downloadCleanerConfig.UnlinkedUseTag)
{
_logger.LogInformation("tag added for {name}", download.Name);
}
else
{
_logger.LogInformation("category changed for {name}", download.Name);
download.Category = _downloadCleanerConfig.UnlinkedTargetCategory;
}
await _eventPublisher.PublishCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory, _downloadCleanerConfig.UnlinkedUseTag);
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
await _client.DeleteAsync([hash], deleteDownloadedData: true);
}
[DryRunSafeguard]
protected async Task CreateCategory(string name)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
await _client.AddCategoryAsync(name);
}
[DryRunSafeguard]
protected virtual async Task ChangeCategory(string hash, string newCategory)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
if (_downloadCleanerConfig.UnlinkedUseTag)
{
await _client.AddTorrentTagAsync([hash], newCategory);
return;
}
await _client.SetTorrentCategoryAsync([hash], newCategory);
}
}

273
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceQC.cs Normal file
View File

@@ -0,0 +1,273 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.QueueCleaner;
using Common.CustomDataTypes;
using Data.Enums;
using Infrastructure.Extensions;
using Infrastructure.Verticals.Context;
using Microsoft.Extensions.Logging;
using QBittorrent.Client;
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
public partial class QBitService
{
/// <inheritdoc/>
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("QBittorrent client is not initialized");
}
DownloadCheckResult result = new();
TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
.FirstOrDefault();
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.Found = true;
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
return result;
}
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
if (torrentProperties is null)
{
_logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
return result;
}
result.IsPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip))
{
result.ShouldRemove = true;
// if all files were blocked by qBittorrent
if (download is { CompletionOn: not null, Downloaded: null or 0 })
{
result.DeleteReason = DeleteReason.AllFilesSkippedByQBit;
return result;
}
// remove if all files are unwanted
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, result.IsPrivate, files);
return result;
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> BlockUnwantedFilesAsync(
TorrentInfo torrent,
bool isPrivate,
IReadOnlyList<TorrentContent>? files
)
{
if (!_queueCleanerConfig.ContentBlocker.Enabled)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.ContentBlocker.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip unwanted files check | download is private | {name}", torrent.Name);
return (false, DeleteReason.None);
}
if (files is null)
{
_logger.LogDebug("failed to find files for {name}", torrent.Name);
return (false, DeleteReason.None);
}
List<int> unwantedFiles = [];
long totalFiles = 0;
long totalUnwantedFiles = 0;
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
{
continue;
}
totalFiles++;
if (file.Priority is TorrentContentPriority.Skip)
{
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(file.Name, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogInformation("unwanted file found | {file}", file.Name);
unwantedFiles.Add(file.Index.Value);
totalUnwantedFiles++;
}
if (unwantedFiles.Count is 0)
{
return (false, DeleteReason.None);
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return (true, DeleteReason.AllFilesBlocked);
}
_logger.LogTrace("marking {count} unwanted files as skipped for {name}", unwantedFiles.Count, torrent.Name);
foreach (int fileIndex in unwantedFiles)
{
await _dryRunInterceptor.InterceptAsync(MarkFileAsSkipped, torrent.Hash, fileIndex);
}
return (false, DeleteReason.None);
}
[DryRunSafeguard]
protected virtual async Task MarkFileAsSkipped(string hash, int fileIndex)
{
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
}
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(
TorrentInfo torrent,
bool isPrivate,
IReadOnlyList<TorrentContent>? files
)
{
(bool ShouldRemove, DeleteReason Reason) result = await BlockUnwantedFilesAsync(torrent, isPrivate, files);
if (result.ShouldRemove)
{
return result;
}
result = await CheckIfSlow(torrent, isPrivate);
if (result.ShouldRemove)
{
return result;
}
return await CheckIfStuck(torrent, isPrivate);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download, bool isPrivate)
{
if (_queueCleanerConfig.Slow.MaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.State is not (TorrentState.Downloading or TorrentState.ForcedDownload))
{
return (false, DeleteReason.None);
}
if (download.DownloadSpeed <= 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.Slow.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.Size > (_queueCleanerConfig.Slow.IgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
{
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
return (false, DeleteReason.None);
}
ByteSize minSpeed = _queueCleanerConfig.Slow.MinSpeedByteSize;
ByteSize currentSpeed = new ByteSize(download.DownloadSpeed);
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.Slow.MaxTime);
SmartTimeSpan currentTime = new SmartTimeSpan(download.EstimatedTime ?? TimeSpan.Zero);
return await CheckIfSlow(
download.Hash,
download.Name,
minSpeed,
currentSpeed,
maxTime,
currentTime
);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo torrent, bool isPrivate)
{
if (_queueCleanerConfig.Stalled.MaxStrikes is 0 && _queueCleanerConfig.Stalled.DownloadingMetadataMaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata
and not TorrentState.ForcedFetchingMetadata)
{
// ignore other states
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.Stalled.MaxStrikes > 0 && torrent.State is TorrentState.StalledDownload)
{
if (_queueCleanerConfig.Stalled.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
}
else
{
ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
return (
await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.Stalled.MaxStrikes,
StrikeType.Stalled), DeleteReason.Stalled);
}
}
if (_queueCleanerConfig.Stalled.DownloadingMetadataMaxStrikes > 0 && torrent.State is not TorrentState.StalledDownload)
{
return (
await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.Stalled.DownloadingMetadataMaxStrikes,
StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata);
}
return (false, DeleteReason.None);
}
}

475
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs
View File

@@ -1,7 +1,6 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
@@ -26,7 +25,7 @@ using Transmission.API.RPC.Entity;
namespace Infrastructure.Verticals.DownloadClient.Transmission;
public class TransmissionService : DownloadService, ITransmissionService
public partial class TransmissionService : DownloadService, ITransmissionService
{
private Client? _client;
@@ -58,11 +57,12 @@ public class TransmissionService : DownloadService, ITransmissionService
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService,
IDynamicHttpClientProvider httpClientProvider,
EventPublisher eventPublisher
EventPublisher eventPublisher,
BlocklistProvider blocklistProvider
) : base(
logger, configManager, cache,
filenameEvaluator, striker, dryRunInterceptor, hardLinkFileService,
httpClientProvider, eventPublisher
httpClientProvider, eventPublisher, blocklistProvider
)
{
// Client will be initialized when Initialize() is called with a specific client configuration
@@ -118,482 +118,15 @@ public class TransmissionService : DownloadService, ITransmissionService
throw;
}
}
/// <inheritdoc/>
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("Transmission client is not initialized");
}
DownloadCheckResult result = new();
TorrentInfo? download = await GetTorrentAsync(hash);
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.Found = true;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
return result;
}
bool shouldRemove = download.FileStats?.Length > 0;
result.IsPrivate = download.IsPrivate ?? false;
foreach (TransmissionTorrentFileStats? stats in download.FileStats ?? [])
{
if (!stats.Wanted.HasValue)
{
// if any files stats are missing, do not remove
shouldRemove = false;
}
if (stats.Wanted.HasValue && stats.Wanted.Value)
{
// if any files are wanted, do not remove
shouldRemove = false;
}
}
if (shouldRemove)
{
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
return result;
}
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download);
return result;
}
/// <inheritdoc/>
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
{
TorrentInfo? download = await GetTorrentAsync(hash);
BlockFilesResult result = new();
if (download?.FileStats is null || download.Files is null)
{
return result;
}
// Mark as processed since we found the download
result.Found = true;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
return result;
}
bool isPrivate = download.IsPrivate ?? false;
result.IsPrivate = isPrivate;
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
return result;
}
List<long> unwantedFiles = [];
long totalFiles = 0;
long totalUnwantedFiles = 0;
for (int i = 0; i < download.Files.Length; i++)
{
if (download.FileStats?[i].Wanted == null)
{
continue;
}
totalFiles++;
if (!download.FileStats[i].Wanted.Value)
{
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogInformation("unwanted file found | {file}", download.Files[i].Name);
unwantedFiles.Add(i);
totalUnwantedFiles++;
}
if (unwantedFiles.Count is 0)
{
return result;
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
result.ShouldRemove = true;
return result;
}
_logger.LogDebug("changing priorities | torrent {hash}", hash);
await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray());
return result;
}
public override async Task<List<object>?> GetSeedingDownloads() =>
(await _client.TorrentGetAsync(Fields))
?.Torrents
?.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => x.Status is 5 or 6)
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
{
return downloads
?
.Cast<TorrentInfo>()
.Where(x => categories
.Any(cat => cat.Name.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase))
)
.Cast<object>()
.ToList();
}
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
{
return downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => categories.Any(cat => cat.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.HashString))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x =>
{
if (download.DownloadDir is null)
{
return false;
}
return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir))
.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase);
});
if (category is null)
{
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.HashString);
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SecondsSeeding ?? 0);
SeedingCheckResult result = ShouldCleanDownload(download.uploadRatio ?? 0, seedingTime, category);
if (!result.ShouldClean)
{
continue;
}
await _dryRunInterceptor.InterceptAsync(RemoveDownloadAsync, download.Id);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _eventPublisher.PublishDownloadCleaned(download.uploadRatio ?? 0, seedingTime, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
await Task.CompletedTask;
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentInfo download in downloads.Cast<TorrentInfo>())
{
if (string.IsNullOrEmpty(download.HashString) || string.IsNullOrEmpty(download.Name) || download.DownloadDir == null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.HashString);
bool hasHardlinks = false;
if (download.Files is null || download.FileStats is null)
{
_logger.LogDebug("skip | download has no files | {name}", download.Name);
continue;
}
for (int i = 0; i < download.Files.Length; i++)
{
TransmissionTorrentFiles file = download.Files[i];
TransmissionTorrentFileStats stats = download.FileStats[i];
if (stats.Wanted is null or false || string.IsNullOrEmpty(file.Name))
{
continue;
}
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, file.Name).Split(['\\', '/']));
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
break;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
break;
}
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
string currentCategory = download.GetCategory();
string newLocation = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, _downloadCleanerConfig.UnlinkedTargetCategory).Split(['\\', '/']));
await _dryRunInterceptor.InterceptAsync(ChangeDownloadLocation, download.Id, newLocation);
_logger.LogInformation("category changed for {name}", download.Name);
await _eventPublisher.PublishCategoryChanged(currentCategory, _downloadCleanerConfig.UnlinkedTargetCategory);
download.DownloadDir = newLocation;
}
}
[DryRunSafeguard]
protected virtual async Task ChangeDownloadLocation(long downloadId, string newLocation)
{
await _client.TorrentSetLocationAsync([downloadId], newLocation, true);
}
public override async Task DeleteDownload(string hash)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
if (torrent is null)
{
return;
}
await _client.TorrentRemoveAsync([torrent.Id], true);
}
public override void Dispose()
{
_client = null;
_httpClient?.Dispose();
}
[DryRunSafeguard]
protected virtual async Task RemoveDownloadAsync(long downloadId)
{
await _client.TorrentRemoveAsync([downloadId], true);
}
[DryRunSafeguard]
protected virtual async Task SetUnwantedFiles(long downloadId, long[] unwantedFiles)
{
await _client.TorrentSetAsync(new TorrentSettings
{
Ids = [downloadId],
FilesUnwanted = unwantedFiles,
});
}
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent)
{
(bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent);
if (result.ShouldRemove)
{
return result;
}
return await CheckIfStuck(torrent);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download)
{
if (_queueCleanerConfig.SlowMaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.Status is not 4)
{
// not in downloading state
return (false, DeleteReason.None);
}
if (download.RateDownload <= 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.SlowIgnorePrivate && download.IsPrivate is true)
{
// ignore private trackers
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.TotalSize > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
{
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
return (false, DeleteReason.None);
}
ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize;
ByteSize currentSpeed = new ByteSize(download.RateDownload ?? long.MaxValue);
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime);
SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta ?? 0);
return await CheckIfSlow(
download.HashString!,
download.Name!,
minSpeed,
currentSpeed,
maxTime,
currentTime
);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo download)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.Status is not 4)
{
// not in downloading state
return (false, DeleteReason.None);
}
if (download.RateDownload > 0 || download.Eta > 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.StalledIgnorePrivate && (download.IsPrivate ?? false))
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
ResetStalledStrikesOnProgress(download.HashString!, download.DownloadedEver ?? 0);
return (await _striker.StrikeAndCheckLimit(download.HashString!, download.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
}
private async Task<TorrentInfo?> GetTorrentAsync(string hash)
{
if (_client == null)
{
throw new InvalidOperationException("Transmission client is not initialized");
}
return (await _client.TorrentGetAsync(Fields, hash))
?.Torrents
?.FirstOrDefault();

237
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceDC.cs Normal file
View File

@@ -0,0 +1,237 @@
using Common.Attributes;
using Common.Configuration.DownloadCleaner;
using Data.Enums;
using Infrastructure.Extensions;
using Infrastructure.Verticals.Context;
using Microsoft.Extensions.Logging;
using Transmission.API.RPC.Entity;
namespace Infrastructure.Verticals.DownloadClient.Transmission;
public partial class TransmissionService
{
public override async Task<List<object>?> GetSeedingDownloads() =>
(await _client.TorrentGetAsync(Fields))
?.Torrents
?.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => x.Status is 5 or 6)
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
{
return downloads
?
.Cast<TorrentInfo>()
.Where(x => categories
.Any(cat => cat.Name.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase))
)
.Cast<object>()
.ToList();
}
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
{
return downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => categories.Any(cat => cat.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.HashString))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
continue;
}
CleanCategory? category = categoriesToClean
.FirstOrDefault(x =>
{
if (download.DownloadDir is null)
{
return false;
}
return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir))
.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase);
});
if (category is null)
{
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.HashString);
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SecondsSeeding ?? 0);
SeedingCheckResult result = ShouldCleanDownload(download.uploadRatio ?? 0, seedingTime, category);
if (!result.ShouldClean)
{
continue;
}
await _dryRunInterceptor.InterceptAsync(RemoveDownloadAsync, download.Id);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _eventPublisher.PublishDownloadCleaned(download.uploadRatio ?? 0, seedingTime, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
await Task.CompletedTask;
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentInfo download in downloads.Cast<TorrentInfo>())
{
if (string.IsNullOrEmpty(download.HashString) || string.IsNullOrEmpty(download.Name) || download.DownloadDir == null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.HashString);
bool hasHardlinks = false;
if (download.Files is null || download.FileStats is null)
{
_logger.LogDebug("skip | download has no files | {name}", download.Name);
continue;
}
for (int i = 0; i < download.Files.Length; i++)
{
TransmissionTorrentFiles file = download.Files[i];
TransmissionTorrentFileStats stats = download.FileStats[i];
if (stats.Wanted is null or false || string.IsNullOrEmpty(file.Name))
{
continue;
}
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, file.Name).Split(['\\', '/']));
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
break;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
break;
}
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
string currentCategory = download.GetCategory();
string newLocation = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, _downloadCleanerConfig.UnlinkedTargetCategory).Split(['\\', '/']));
await _dryRunInterceptor.InterceptAsync(ChangeDownloadLocation, download.Id, newLocation);
_logger.LogInformation("category changed for {name}", download.Name);
await _eventPublisher.PublishCategoryChanged(currentCategory, _downloadCleanerConfig.UnlinkedTargetCategory);
download.DownloadDir = newLocation;
}
}
[DryRunSafeguard]
protected virtual async Task ChangeDownloadLocation(long downloadId, string newLocation)
{
await _client.TorrentSetLocationAsync([downloadId], newLocation, true);
}
public override async Task DeleteDownload(string hash)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
if (torrent is null)
{
return;
}
await _client.TorrentRemoveAsync([torrent.Id], true);
}
[DryRunSafeguard]
protected virtual async Task RemoveDownloadAsync(long downloadId)
{
await _client.TorrentRemoveAsync([downloadId], true);
}
}

255
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceQC.cs Normal file
View File

@@ -0,0 +1,255 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.QueueCleaner;
using Common.CustomDataTypes;
using Data.Enums;
using Infrastructure.Extensions;
using Infrastructure.Verticals.Context;
using Microsoft.Extensions.Logging;
using Transmission.API.RPC.Arguments;
using Transmission.API.RPC.Entity;
namespace Infrastructure.Verticals.DownloadClient.Transmission;
public partial class TransmissionService
{
/// <inheritdoc/>
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash,
IReadOnlyList<string> ignoredDownloads)
{
if (_client == null)
{
throw new InvalidOperationException("Transmission client is not initialized");
}
DownloadCheckResult result = new();
TorrentInfo? download = await GetTorrentAsync(hash);
if (download is null)
{
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
return result;
}
result.Found = true;
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
return result;
}
bool shouldRemove = download.FileStats?.Length > 0;
bool isPrivate = download.IsPrivate ?? false;
result.IsPrivate = isPrivate;
foreach (TransmissionTorrentFileStats stats in download.FileStats ?? [])
{
if (!stats.Wanted.HasValue)
{
// if any files stats are missing, do not remove
shouldRemove = false;
}
if (stats.Wanted.HasValue && stats.Wanted.Value)
{
// if any files are wanted, do not remove
shouldRemove = false;
}
}
if (shouldRemove)
{
// remove if all files are unwanted
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
// remove if download is stuck
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, isPrivate);
return result;
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> BlockUnwantedFilesAsync(
TorrentInfo download,
bool isPrivate
)
{
if (!_queueCleanerConfig.ContentBlocker.Enabled)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.ContentBlocker.IgnorePrivate && isPrivate)
{
// ignore private trackers
_logger.LogDebug("skip unwanted files check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.Files is null)
{
_logger.LogDebug("failed to find files for {name}", download.Name);
return (false, DeleteReason.None);
}
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
List<long> unwantedFiles = [];
long totalFiles = 0;
long totalUnwantedFiles = 0;
for (int i = 0; i < download.Files.Length; i++)
{
if (download.FileStats?[i].Wanted == null)
{
continue;
}
totalFiles++;
if (!download.FileStats[i].Wanted.Value)
{
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes))
{
continue;
}
_logger.LogInformation("unwanted file found | {file}", download.Files[i].Name);
unwantedFiles.Add(i);
totalUnwantedFiles++;
}
if (unwantedFiles.Count is 0)
{
return (false, DeleteReason.None);
}
if (totalUnwantedFiles == totalFiles)
{
// Skip marking files as unwanted. The download will be removed completely.
return (true, DeleteReason.AllFilesBlocked);
}
_logger.LogTrace("marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name);
await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray());
return (false, DeleteReason.None);
}
[DryRunSafeguard]
protected virtual async Task SetUnwantedFiles(long downloadId, long[] unwantedFiles)
{
await _client.TorrentSetAsync(new TorrentSettings
{
Ids = [downloadId],
FilesUnwanted = unwantedFiles,
});
}
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo download, bool isPrivate)
{
(bool ShouldRemove, DeleteReason Reason) result = await BlockUnwantedFilesAsync(download, isPrivate);
if (result.ShouldRemove)
{
return result;
}
result = await CheckIfSlow(download);
if (result.ShouldRemove)
{
return result;
}
return await CheckIfStuck(download);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download)
{
if (_queueCleanerConfig.Slow.MaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.Status is not 4)
{
// not in downloading state
return (false, DeleteReason.None);
}
if (download.RateDownload <= 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.Slow.IgnorePrivate && download.IsPrivate is true)
{
// ignore private trackers
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.TotalSize > (_queueCleanerConfig.Slow.IgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
{
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
return (false, DeleteReason.None);
}
ByteSize minSpeed = _queueCleanerConfig.Slow.MinSpeedByteSize;
ByteSize currentSpeed = new ByteSize(download.RateDownload ?? long.MaxValue);
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.Slow.MaxTime);
SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta ?? 0);
return await CheckIfSlow(
download.HashString!,
download.Name!,
minSpeed,
currentSpeed,
maxTime,
currentTime
);
}
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo download)
{
if (_queueCleanerConfig.Stalled.MaxStrikes is 0)
{
return (false, DeleteReason.None);
}
if (download.Status is not 4)
{
// not in downloading state
return (false, DeleteReason.None);
}
if (download.RateDownload > 0 || download.Eta > 0)
{
return (false, DeleteReason.None);
}
if (_queueCleanerConfig.Stalled.IgnorePrivate && (download.IsPrivate ?? false))
{
// ignore private trackers
_logger.LogDebug("skip stalled check | download is private | {name}", download.Name);
return (false, DeleteReason.None);
}
ResetStalledStrikesOnProgress(download.HashString!, download.DownloadedEver ?? 0);
return (await _striker.StrikeAndCheckLimit(download.HashString!, download.Name!, _queueCleanerConfig.Stalled.MaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
}
}

92
code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs
View File

@@ -20,6 +20,10 @@ using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using LogContext = Serilog.Context.LogContext;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Infrastructure.Verticals.ContentBlocker;
using BlocklistSettings = Common.Configuration.QueueCleaner.BlocklistSettings;
namespace Infrastructure.Verticals.QueueCleaner;
@@ -28,6 +32,7 @@ public sealed class QueueCleaner : GenericHandler
private readonly QueueCleanerConfig _config;
private readonly IIgnoredDownloadsService _ignoredDownloadsService;
private readonly IDownloadClientFactory _downloadClientFactory;
private readonly BlocklistProvider _blocklistProvider;
public QueueCleaner(
ILogger<QueueCleaner> logger,
@@ -38,7 +43,8 @@ public sealed class QueueCleaner : GenericHandler
ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory,
IIgnoredDownloadsService ignoredDownloadsService,
IDownloadClientFactory downloadClientFactory
IDownloadClientFactory downloadClientFactory,
BlocklistProvider blocklistProvider
) : base(
logger, cache, messageBus,
arrClientFactory, arrArrQueueIterator, downloadServiceFactory
@@ -46,7 +52,8 @@ public sealed class QueueCleaner : GenericHandler
{
_ignoredDownloadsService = ignoredDownloadsService;
_downloadClientFactory = downloadClientFactory;
_blocklistProvider = blocklistProvider;
_config = configManager.GetConfiguration<QueueCleanerConfig>();
_downloadClientConfig = configManager.GetConfiguration<DownloadClientConfig>();
_sonarrConfig = configManager.GetConfiguration<SonarrConfig>();
@@ -54,6 +61,26 @@ public sealed class QueueCleaner : GenericHandler
_lidarrConfig = configManager.GetConfiguration<LidarrConfig>();
}
public override async Task ExecuteAsync()
{
if (_downloadClientConfig.Clients.Count is 0)
{
_logger.LogWarning("No download clients configured");
return;
}
bool blocklistIsConfigured = _sonarrConfig.Enabled && !string.IsNullOrEmpty(_config.ContentBlocker.Sonarr.BlocklistPath) ||
_radarrConfig.Enabled && !string.IsNullOrEmpty(_config.ContentBlocker.Radarr.BlocklistPath) ||
_lidarrConfig.Enabled && !string.IsNullOrEmpty(_config.ContentBlocker.Lidarr.BlocklistPath);
if (_config.ContentBlocker.Enabled && blocklistIsConfigured)
{
await _blocklistProvider.LoadBlocklistsAsync();
}
await base.ExecuteAsync();
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType, ArrConfig config)
{
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsService.GetIgnoredDownloadsAsync();
@@ -112,41 +139,42 @@ public sealed class QueueCleaner : GenericHandler
if (_downloadClientConfig.Clients.Count == 0)
{
_logger.LogWarning("skip | no download clients configured | {title}", record.Title);
continue;
}
// Check each download client for the download item
foreach (var downloadService in _downloadClientFactory.GetAllEnabledClients())
else
{
try
// Check each download client for the download item
foreach (var downloadService in _downloadClientFactory.GetAllEnabledClients())
{
// stalled download check
var result = await downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads);
if (result.Found)
try
{
downloadCheckResult = result;
// Add client ID to context for tracking
ContextProvider.Set("ClientId", downloadService.GetClientId());
break;
// stalled download check
DownloadCheckResult result = await downloadService
.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads);
if (result.Found)
{
downloadCheckResult = result;
break;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking download {id} with download client {clientId}",
record.DownloadId, downloadService.GetClientId());
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Error checking download {id} with download client {clientId}",
record.DownloadId, downloadService.GetClientId());
}
}
if (!downloadCheckResult.Found)
{
_logger.LogWarning("skip | download not found {title}", record.Title);
if (!downloadCheckResult.Found)
{
_logger.LogWarning("skip | download not found {title}", record.Title);
}
}
}
// failed import check
bool shouldRemoveFromArr = await arrClient.ShouldRemoveFromQueue(instanceType, record, downloadCheckResult.IsPrivate, config.FailedImportMaxStrikes);
bool shouldRemoveFromArr = await arrClient.ShouldRemoveFromQueue(instanceType, record, downloadCheckResult.IsPrivate, _config.FailedImport.MaxStrikes);
DeleteReason deleteReason = downloadCheckResult.ShouldRemove ? downloadCheckResult.DeleteReason : DeleteReason.FailedImport;
if (!shouldRemoveFromArr && !downloadCheckResult.ShouldRemove)
{
_logger.LogInformation("skip | {title}", record.Title);
@@ -159,14 +187,20 @@ public sealed class QueueCleaner : GenericHandler
{
bool isStalledWithoutPruneFlag =
downloadCheckResult.DeleteReason is DeleteReason.Stalled &&
!_config.StalledDeletePrivate;
!_config.Stalled.DeletePrivate;
bool isSlowWithoutPruneFlag =
downloadCheckResult.DeleteReason is DeleteReason.SlowSpeed or DeleteReason.SlowTime &&
!_config.SlowDeletePrivate;
!_config.Slow.DeletePrivate;
bool isContentBlockerWithoutPruneFlag =
deleteReason is DeleteReason.AllFilesBlocked &&
!_config.ContentBlocker.DeletePrivate;
bool shouldKeepDueToDeleteRules = downloadCheckResult.ShouldRemove && (isStalledWithoutPruneFlag || isSlowWithoutPruneFlag);
bool shouldKeepDueToImportRules = shouldRemoveFromArr && !_config.FailedImportDeletePrivate;
bool shouldKeepDueToDeleteRules = downloadCheckResult.ShouldRemove &&
(isStalledWithoutPruneFlag || isSlowWithoutPruneFlag || isContentBlockerWithoutPruneFlag);
bool shouldKeepDueToImportRules = shouldRemoveFromArr && !_config.FailedImport.DeletePrivate;
if (shouldKeepDueToDeleteRules || shouldKeepDueToImportRules)
{