diff --git a/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs b/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs deleted file mode 100644 index 15ea46ee..00000000 --- a/code/Common/Configuration/ContentBlocker/ContentBlockerConfig.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace Common.Configuration.ContentBlocker; - -public sealed record ContentBlockerConfig : IJobConfig -{ - public bool Enabled { get; init; } - - public string CronExpression { get; init; } = "0 0/5 * * * ?"; - - public bool IgnorePrivate { get; init; } - - public bool DeletePrivate { get; init; } - - // TODO - public string IgnoredDownloadsPath { get; init; } = string.Empty; - - public BlocklistSettings Sonarr { get; init; } = new(); - - public BlocklistSettings Radarr { get; init; } = new(); - - public BlocklistSettings Lidarr { get; init; } = new(); - - public void Validate() - { - } -} - -public record BlocklistSettings -{ - public bool Enabled { get; init; } - - public BlocklistType Type { get; init; } - - public string? Path { get; init; } -} \ No newline at end of file diff --git a/code/Common/Configuration/DTOs/ContentBlocker/ContentBlockerConfigDto.cs b/code/Common/Configuration/DTOs/ContentBlocker/ContentBlockerConfigDto.cs index 8e285c5e..632a1d33 100644 --- a/code/Common/Configuration/DTOs/ContentBlocker/ContentBlockerConfigDto.cs +++ b/code/Common/Configuration/DTOs/ContentBlocker/ContentBlockerConfigDto.cs @@ -1,4 +1,4 @@ -using Common.Configuration.ContentBlocker; +using Common.Configuration.QueueCleaner; namespace Common.Configuration.DTOs.ContentBlocker; diff --git a/code/Common/Configuration/QueueCleaner/BlocklistSettings.cs b/code/Common/Configuration/QueueCleaner/BlocklistSettings.cs new file mode 100644 index 00000000..536da6b9 --- /dev/null +++ b/code/Common/Configuration/QueueCleaner/BlocklistSettings.cs @@ -0,0 +1,11 @@ +namespace Common.Configuration.QueueCleaner; + +/// +/// Settings for a blocklist +/// +public sealed record BlocklistSettings +{ + public BlocklistType BlocklistType { get; init; } + + public string? BlocklistPath { get; init; } +} \ No newline at end of file diff --git a/code/Common/Configuration/ContentBlocker/BlocklistType.cs b/code/Common/Configuration/QueueCleaner/BlocklistType.cs similarity index 53% rename from code/Common/Configuration/ContentBlocker/BlocklistType.cs rename to code/Common/Configuration/QueueCleaner/BlocklistType.cs index fa0cee07..4d371089 100644 --- a/code/Common/Configuration/ContentBlocker/BlocklistType.cs +++ b/code/Common/Configuration/QueueCleaner/BlocklistType.cs @@ -1,4 +1,4 @@ -namespace Common.Configuration.ContentBlocker; +namespace Common.Configuration.QueueCleaner; public enum BlocklistType { diff --git a/code/Common/Configuration/QueueCleaner/ContentBlockerConfig.cs b/code/Common/Configuration/QueueCleaner/ContentBlockerConfig.cs new file mode 100644 index 00000000..6571efb0 --- /dev/null +++ b/code/Common/Configuration/QueueCleaner/ContentBlockerConfig.cs @@ -0,0 +1,20 @@ +namespace Common.Configuration.QueueCleaner; + +public sealed record ContentBlockerConfig +{ + public bool Enabled { get; init; } + + public bool IgnorePrivate { get; init; } + + public bool DeletePrivate { get; init; } + + public BlocklistSettings Sonarr { get; init; } = new(); + + public BlocklistSettings Radarr { get; init; } = new(); + + public BlocklistSettings Lidarr { get; init; } = new(); + + public void Validate() + { + } +} diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index 7dbee54c..bd1b2ad5 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -12,94 +12,123 @@ public sealed record QueueCleanerConfig : IJobConfig public string CronExpression { get; init; } = "0 0/5 * * * ?"; - public bool RunSequentially { get; init; } - public string IgnoredDownloadsPath { get; init; } = string.Empty; - - public ushort FailedImportMaxStrikes { get; init; } - - public bool FailedImportIgnorePrivate { get; init; } - - public bool FailedImportDeletePrivate { get; init; } - public IReadOnlyList FailedImportIgnorePatterns { get; init; } = []; + public FailedImportConfig FailedImport { get; init; } = new(); - public ushort StalledMaxStrikes { get; init; } + public StalledConfig Stalled { get; init; } = new(); - public bool StalledResetStrikesOnProgress { get; init; } + public SlowConfig Slow { get; init; } = new(); - public bool StalledIgnorePrivate { get; init; } - - public bool StalledDeletePrivate { get; init; } - - public ushort DownloadingMetadataMaxStrikes { get; init; } - - public ushort SlowMaxStrikes { get; init; } - - public bool SlowResetStrikesOnProgress { get; init; } - - public bool SlowIgnorePrivate { get; init; } - - public bool SlowDeletePrivate { get; init; } - - public string SlowMinSpeed { get; init; } = string.Empty; - - [JsonIgnore] - public ByteSize SlowMinSpeedByteSize => string.IsNullOrEmpty(SlowMinSpeed) ? new ByteSize(0) : ByteSize.Parse(SlowMinSpeed); - - public double SlowMaxTime { get; init; } - - public string SlowIgnoreAboveSize { get; init; } = string.Empty; - - [JsonIgnore] - public ByteSize? SlowIgnoreAboveSizeByteSize => string.IsNullOrEmpty(SlowIgnoreAboveSize) ? null : ByteSize.Parse(SlowIgnoreAboveSize); + public ContentBlockerConfig ContentBlocker { get; init; } = new(); public void Validate() { - if (FailedImportMaxStrikes is > 0 and < 3) - { - throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__IMPORT_FAILED_MAX_STRIKES must be 3"); - } + FailedImport.Validate(SectionName); + Stalled.Validate(SectionName); + Slow.Validate(SectionName); + ContentBlocker.Validate(); + } +} - if (StalledMaxStrikes is > 0 and < 3) +public sealed record FailedImportConfig +{ + public ushort MaxStrikes { get; init; } + + public bool IgnorePrivate { get; init; } + + public bool DeletePrivate { get; init; } + + public IReadOnlyList IgnoredPatterns { get; init; } = []; + + public void Validate(string sectionName) + { + if (MaxStrikes is > 0 and < 3) { - throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__STALLED_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {sectionName.ToUpperInvariant()}__FAILED_IMPORT__MAX_STRIKES must be 3"); + } + } +} + +public sealed record StalledConfig +{ + public ushort MaxStrikes { get; init; } + + public bool ResetStrikesOnProgress { get; init; } + + public bool IgnorePrivate { get; init; } + + public bool DeletePrivate { get; init; } + + public ushort DownloadingMetadataMaxStrikes { get; init; } + + public void Validate(string sectionName) + { + if (MaxStrikes is > 0 and < 3) + { + throw new ValidationException($"the minimum value for {sectionName.ToUpperInvariant()}__STALLED__MAX_STRIKES must be 3"); } if (DownloadingMetadataMaxStrikes is > 0 and < 3) { - throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__DOWNLOADING_METADATA_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {sectionName.ToUpperInvariant()}__STALLED__DOWNLOADING_METADATA_MAX_STRIKES must be 3"); } - - if (SlowMaxStrikes is > 0 and < 3) + } +} + +public sealed record SlowConfig +{ + public ushort MaxStrikes { get; init; } + + public bool ResetStrikesOnProgress { get; init; } + + public bool IgnorePrivate { get; init; } + + public bool DeletePrivate { get; init; } + + public string MinSpeed { get; init; } = string.Empty; + + [JsonIgnore] + public ByteSize MinSpeedByteSize => string.IsNullOrEmpty(MinSpeed) ? new ByteSize(0) : ByteSize.Parse(MinSpeed); + + public double MaxTime { get; init; } + + public string IgnoreAboveSize { get; init; } = string.Empty; + + [JsonIgnore] + public ByteSize? IgnoreAboveSizeByteSize => string.IsNullOrEmpty(IgnoreAboveSize) ? null : ByteSize.Parse(IgnoreAboveSize); + + public void Validate(string sectionName) + { + if (MaxStrikes is > 0 and < 3) { - throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__SLOW_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {sectionName.ToUpperInvariant()}__SLOW__MAX_STRIKES must be 3"); } - if (SlowMaxStrikes > 0) + if (MaxStrikes > 0) { - bool isSlowSpeedSet = !string.IsNullOrEmpty(SlowMinSpeed); + bool isSpeedSet = !string.IsNullOrEmpty(MinSpeed); - if (isSlowSpeedSet && ByteSize.TryParse(SlowMinSpeed, out _) is false) + if (isSpeedSet && ByteSize.TryParse(MinSpeed, out _) is false) { - throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_MIN_SPEED"); + throw new ValidationException($"invalid value for {sectionName.ToUpperInvariant()}__SLOW__MIN_SPEED"); } - if (SlowMaxTime < 0) + if (MaxTime < 0) { - throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_MAX_TIME"); + throw new ValidationException($"invalid value for {sectionName.ToUpperInvariant()}__SLOW__MAX_TIME"); } - if (!isSlowSpeedSet && SlowMaxTime is 0) + if (!isSpeedSet && MaxTime is 0) { - throw new ValidationException($"either {SectionName.ToUpperInvariant()}__SLOW_MIN_SPEED or {SectionName.ToUpperInvariant()}__SLOW_MAX_STRIKES must be set"); + throw new ValidationException($"either {sectionName.ToUpperInvariant()}__SLOW__MIN_SPEED or {sectionName.ToUpperInvariant()}__SLOW__MAX_TIME must be set"); } - bool isSlowIgnoreAboveSizeSet = !string.IsNullOrEmpty(SlowIgnoreAboveSize); + bool isIgnoreAboveSizeSet = !string.IsNullOrEmpty(IgnoreAboveSize); - if (isSlowIgnoreAboveSizeSet && ByteSize.TryParse(SlowIgnoreAboveSize, out _) is false) + if (isIgnoreAboveSizeSet && ByteSize.TryParse(IgnoreAboveSize, out _) is false) { - throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_IGNORE_ABOVE_SIZE"); + throw new ValidationException($"invalid value for {sectionName.ToUpperInvariant()}__SLOW__IGNORE_ABOVE_SIZE"); } } } diff --git a/code/Executable/Controllers/ConfigurationController.cs b/code/Executable/Controllers/ConfigurationController.cs index 79a6068e..46d42bb3 100644 --- a/code/Executable/Controllers/ConfigurationController.cs +++ b/code/Executable/Controllers/ConfigurationController.cs @@ -1,6 +1,5 @@ using Common.Configuration; using Common.Configuration.Arr; -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.DTOs.Arr; diff --git a/code/Executable/DependencyInjection/LoggingDI.cs b/code/Executable/DependencyInjection/LoggingDI.cs index 470965d2..7de518af 100644 --- a/code/Executable/DependencyInjection/LoggingDI.cs +++ b/code/Executable/DependencyInjection/LoggingDI.cs @@ -29,12 +29,8 @@ public static class LoggingDI const string consoleOutputTemplate = $"[{{@t:yyyy-MM-dd HH:mm:ss.fff}} {{@l:u3}}]{jobNameTemplate}{categoryTemplate} {{@m}}\n{{@x}}"; const string fileOutputTemplate = $"{{@t:yyyy-MM-dd HH:mm:ss.fff zzz}} [{{@l:u3}}]{jobNameTemplate}{categoryTemplate} {{@m:lj}}\n{{@x}}"; - // Determine categories and padding sizes - List categories = ["SYSTEM", "API", "JOBS", "NOTIFICATIONS"]; - int catPadding = categories.Max(x => x.Length) + 2; - // Determine job name padding - List jobNames = [nameof(ContentBlocker), nameof(QueueCleaner), nameof(DownloadCleaner)]; + List jobNames = [nameof(QueueCleaner), nameof(DownloadCleaner)]; int jobPadding = jobNames.Max(x => x.Length) + 2; // Determine instance name padding @@ -46,7 +42,7 @@ public static class LoggingDI InstanceType.Whisparr.ToString(), "SYSTEM" ]; - int arrPadding = categoryNames.Max(x => x.Length) + 2; + int catPadding = categoryNames.Max(x => x.Length) + 2; // Apply padding values to templates string consoleTemplate = consoleOutputTemplate diff --git a/code/Executable/DependencyInjection/ServicesDI.cs b/code/Executable/DependencyInjection/ServicesDI.cs index 0d6f532c..12d703ef 100644 --- a/code/Executable/DependencyInjection/ServicesDI.cs +++ b/code/Executable/DependencyInjection/ServicesDI.cs @@ -38,7 +38,6 @@ public static class ServicesDI .AddTransient() .AddTransient() .AddTransient() - .AddTransient() .AddTransient() .AddTransient() .AddTransient() diff --git a/code/Executable/Jobs/BackgroundJobManager.cs b/code/Executable/Jobs/BackgroundJobManager.cs index ad00e751..93e5c437 100644 --- a/code/Executable/Jobs/BackgroundJobManager.cs +++ b/code/Executable/Jobs/BackgroundJobManager.cs @@ -1,15 +1,11 @@ -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.QueueCleaner; using Common.Helpers; using Infrastructure.Configuration; -using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadCleaner; using Infrastructure.Verticals.Jobs; using Infrastructure.Verticals.QueueCleaner; -using Microsoft.Extensions.Hosting; using Quartz; -using Quartz.Impl.Matchers; using Quartz.Spi; namespace Executable.Jobs; @@ -28,8 +24,8 @@ public class BackgroundJobManager : IHostedService public BackgroundJobManager( ISchedulerFactory schedulerFactory, IConfigManager configManager, - IServiceProvider serviceProvider, - ILogger logger) + ILogger logger + ) { _schedulerFactory = schedulerFactory; _configManager = configManager; @@ -78,48 +74,11 @@ public class BackgroundJobManager : IHostedService } // Get configurations from JSON files - ContentBlockerConfig? contentBlockerConfig = await _configManager.GetConfigurationAsync(); - QueueCleanerConfig? queueCleanerConfig = await _configManager.GetConfigurationAsync(); - DownloadCleanerConfig? downloadCleanerConfig = await _configManager.GetConfigurationAsync(); + QueueCleanerConfig queueCleanerConfig = await _configManager.GetConfigurationAsync(); + DownloadCleanerConfig downloadCleanerConfig = await _configManager.GetConfigurationAsync(); - // Add ContentBlocker job if enabled - if (contentBlockerConfig?.Enabled == true) - { - await AddContentBlockerJob(contentBlockerConfig, cancellationToken); - } - - // Add QueueCleaner job if enabled - if (queueCleanerConfig?.Enabled == true) - { - // Check if we need to chain it after ContentBlocker - bool shouldChainAfterContentBlocker = - contentBlockerConfig?.Enabled == true && - queueCleanerConfig.RunSequentially; - - await AddQueueCleanerJob(queueCleanerConfig, shouldChainAfterContentBlocker, cancellationToken); - } - - // Add DownloadCleaner job if enabled - if (downloadCleanerConfig?.Enabled == true) - { - await AddDownloadCleanerJob(downloadCleanerConfig, cancellationToken); - } - } - - /// - /// Adds the ContentBlocker job to the scheduler. - /// - public async Task AddContentBlockerJob(ContentBlockerConfig config, CancellationToken cancellationToken = default) - { - if (!config.Enabled) - { - return; - } - - await AddJobWithTrigger( - config, - config.CronExpression, - cancellationToken); + await AddQueueCleanerJob(queueCleanerConfig, cancellationToken); + await AddDownloadCleanerJob(downloadCleanerConfig, cancellationToken); } /// @@ -127,7 +86,6 @@ public class BackgroundJobManager : IHostedService /// public async Task AddQueueCleanerJob( QueueCleanerConfig config, - bool chainAfterContentBlocker = false, CancellationToken cancellationToken = default) { if (!config.Enabled) @@ -135,28 +93,10 @@ public class BackgroundJobManager : IHostedService return; } - var jobKey = new JobKey(nameof(QueueCleaner)); - - // If the job should be chained after ContentBlocker, add it without a cron trigger - if (chainAfterContentBlocker) - { - await AddJobWithoutTrigger(cancellationToken); - - // Add job listener to chain QueueCleaner after ContentBlocker - if (_scheduler != null) - { - var chainListener = new JobChainingListener(nameof(ContentBlocker), nameof(QueueCleaner)); - _scheduler.ListenerManager.AddJobListener(chainListener, KeyMatcher.KeyEquals(new JobKey(nameof(ContentBlocker)))); - } - } - else - { - // Add job with normal cron trigger - await AddJobWithTrigger( - config, - config.CronExpression, - cancellationToken); - } + await AddJobWithTrigger( + config, + config.CronExpression, + cancellationToken); } /// diff --git a/code/Infrastructure.Tests/Verticals/ContentBlocker/FilenameEvaluatorTests.cs b/code/Infrastructure.Tests/Verticals/ContentBlocker/FilenameEvaluatorTests.cs index f295eb5c..ba434556 100644 --- a/code/Infrastructure.Tests/Verticals/ContentBlocker/FilenameEvaluatorTests.cs +++ b/code/Infrastructure.Tests/Verticals/ContentBlocker/FilenameEvaluatorTests.cs @@ -1,6 +1,6 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; -using Common.Configuration.ContentBlocker; +using Common.Configuration.QueueCleaner; using Shouldly; namespace Infrastructure.Tests.Verticals.ContentBlocker; diff --git a/code/Infrastructure/Configuration/ConfigInitializer.cs b/code/Infrastructure/Configuration/ConfigInitializer.cs index c089798f..98b7da63 100644 --- a/code/Infrastructure/Configuration/ConfigInitializer.cs +++ b/code/Infrastructure/Configuration/ConfigInitializer.cs @@ -1,4 +1,3 @@ -using Common.Configuration.ContentBlocker; using Microsoft.Extensions.Logging; namespace Infrastructure.Configuration; diff --git a/code/Infrastructure/Configuration/ConfigManager.cs b/code/Infrastructure/Configuration/ConfigManager.cs index fa250673..a317c3e9 100644 --- a/code/Infrastructure/Configuration/ConfigManager.cs +++ b/code/Infrastructure/Configuration/ConfigManager.cs @@ -1,7 +1,6 @@ using System.Reflection; using Common.Configuration; using Common.Configuration.Arr; -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.General; diff --git a/code/Infrastructure/Configuration/IConfigManager.cs b/code/Infrastructure/Configuration/IConfigManager.cs index dccbce95..8c1b313b 100644 --- a/code/Infrastructure/Configuration/IConfigManager.cs +++ b/code/Infrastructure/Configuration/IConfigManager.cs @@ -1,5 +1,4 @@ using Common.Configuration.Arr; -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.General; diff --git a/code/Infrastructure/Verticals/Arr/ArrClient.cs b/code/Infrastructure/Verticals/Arr/ArrClient.cs index 4dbcc359..c842c678 100644 --- a/code/Infrastructure/Verticals/Arr/ArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/ArrClient.cs @@ -72,9 +72,9 @@ public abstract class ArrClient : IArrClient return queueResponse; } - public virtual async Task ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, short arrMaxStrikes) + public virtual async Task ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, ushort arrMaxStrikes) { - if (_queueCleanerConfig.FailedImportIgnorePrivate && isPrivateDownload) + if (_queueCleanerConfig.FailedImport.IgnorePrivate && isPrivateDownload) { // ignore private trackers _logger.LogDebug("skip failed import check | download is private | {name}", record.Title); @@ -108,7 +108,7 @@ public abstract class ArrClient : IArrClient return false; } - ushort maxStrikes = arrMaxStrikes > 0 ? (ushort)arrMaxStrikes : _queueCleanerConfig.FailedImportMaxStrikes; + ushort maxStrikes = arrMaxStrikes > 0 ? (ushort)arrMaxStrikes : _queueCleanerConfig.FailedImport.MaxStrikes; return await _striker.StrikeAndCheckLimit( record.DownloadId, @@ -214,7 +214,7 @@ public abstract class ArrClient : IArrClient private bool HasIgnoredPatterns(QueueRecord record) { - if (_queueCleanerConfig.FailedImportIgnorePatterns.Count is 0) + if (_queueCleanerConfig.FailedImport.IgnoredPatterns.Count is 0) { // no patterns are configured return false; @@ -234,7 +234,7 @@ public abstract class ArrClient : IArrClient .ForEach(x => messages.Add(x)); return messages.Any( - m => _queueCleanerConfig.FailedImportIgnorePatterns.Any( + m => _queueCleanerConfig.FailedImport.IgnoredPatterns.Any( p => !string.IsNullOrWhiteSpace(p.Trim()) && m.Contains(p, StringComparison.InvariantCultureIgnoreCase) ) ); diff --git a/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs b/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs index d9966d94..07ff5e40 100644 --- a/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs @@ -9,7 +9,7 @@ public interface IArrClient { Task GetQueueItemsAsync(ArrInstance arrInstance, int page); - Task ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, short arrMaxStrikes); + Task ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, ushort arrMaxStrikes); Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient, DeleteReason deleteReason); diff --git a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs index 74c453cc..23f1bad3 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs @@ -1,7 +1,7 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Text.RegularExpressions; -using Common.Configuration.ContentBlocker; +using Common.Configuration.QueueCleaner; using Common.Helpers; using Data.Enums; using Infrastructure.Configuration; @@ -14,8 +14,7 @@ namespace Infrastructure.Verticals.ContentBlocker; public sealed class BlocklistProvider { private readonly ILogger _logger; - private readonly IConfigManager _configManager; - private ContentBlockerConfig _contentBlockerConfig; + private readonly QueueCleanerConfig _queueCleanerConfig; private readonly HttpClient _httpClient; private readonly IMemoryCache _cache; private bool _initialized; @@ -28,11 +27,10 @@ public sealed class BlocklistProvider ) { _logger = logger; - _configManager = configManager; _cache = cache; _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName); - _contentBlockerConfig = _configManager.GetConfiguration(); + _queueCleanerConfig = configManager.GetConfiguration(); } public async Task LoadBlocklistsAsync() @@ -45,9 +43,9 @@ public sealed class BlocklistProvider try { - await LoadPatternsAndRegexesAsync(_contentBlockerConfig.Sonarr, InstanceType.Sonarr); - await LoadPatternsAndRegexesAsync(_contentBlockerConfig.Radarr, InstanceType.Radarr); - await LoadPatternsAndRegexesAsync(_contentBlockerConfig.Lidarr, InstanceType.Lidarr); + await LoadPatternsAndRegexesAsync(_queueCleanerConfig.ContentBlocker.Sonarr, InstanceType.Sonarr); + await LoadPatternsAndRegexesAsync(_queueCleanerConfig.ContentBlocker.Radarr, InstanceType.Radarr); + await LoadPatternsAndRegexesAsync(_queueCleanerConfig.ContentBlocker.Lidarr, InstanceType.Lidarr); _initialized = true; } @@ -81,17 +79,12 @@ public sealed class BlocklistProvider private async Task LoadPatternsAndRegexesAsync(BlocklistSettings blocklistSettings, InstanceType instanceType) { - if (!blocklistSettings.Enabled) + if (string.IsNullOrEmpty(blocklistSettings.BlocklistPath)) { return; } - if (string.IsNullOrEmpty(blocklistSettings.Path)) - { - return; - } - - string[] filePatterns = await ReadContentAsync(blocklistSettings.Path); + string[] filePatterns = await ReadContentAsync(blocklistSettings.BlocklistPath); long startTime = Stopwatch.GetTimestamp(); ParallelOptions options = new() { MaxDegreeOfParallelism = 5 }; @@ -122,13 +115,13 @@ public sealed class BlocklistProvider TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime); - _cache.Set(CacheKeys.BlocklistType(instanceType), blocklistSettings.Type); + _cache.Set(CacheKeys.BlocklistType(instanceType), blocklistSettings.BlocklistType); _cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns); _cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes); _logger.LogDebug("loaded {count} patterns", patterns.Count); _logger.LogDebug("loaded {count} regexes", regexes.Count); - _logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistSettings.Path); + _logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistSettings.BlocklistPath); } private async Task ReadContentAsync(string path) diff --git a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs index 722597e3..8f0d35f7 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs @@ -1,187 +1,187 @@ -using System.Collections.Concurrent; -using System.Text.RegularExpressions; -using Common.Configuration.Arr; -using Common.Configuration.ContentBlocker; -using Common.Configuration.DownloadClient; -using Data.Enums; -using Data.Models.Arr.Queue; -using Infrastructure.Configuration; -using Infrastructure.Helpers; -using Infrastructure.Services; -using Infrastructure.Verticals.Arr; -using Infrastructure.Verticals.Arr.Interfaces; -using Infrastructure.Verticals.DownloadClient; -using Infrastructure.Verticals.Jobs; -using MassTransit; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using LogContext = Serilog.Context.LogContext; - -namespace Infrastructure.Verticals.ContentBlocker; - -public sealed class ContentBlocker : GenericHandler -{ - private readonly ContentBlockerConfig _config; - private readonly BlocklistProvider _blocklistProvider; - private readonly IIgnoredDownloadsService _ignoredDownloadsService; - - public ContentBlocker( - ILogger logger, - IConfigManager configManager, - IMemoryCache cache, - IBus messageBus, - ArrClientFactory arrClientFactory, - ArrQueueIterator arrArrQueueIterator, - BlocklistProvider blocklistProvider, - IIgnoredDownloadsService ignoredDownloadsService, - DownloadServiceFactory downloadServiceFactory - ) : base( - logger, cache, messageBus, - arrClientFactory, arrArrQueueIterator, downloadServiceFactory - ) - { - _blocklistProvider = blocklistProvider; - _ignoredDownloadsService = ignoredDownloadsService; - - _config = configManager.GetConfiguration(); - _downloadClientConfig = configManager.GetConfiguration(); - _sonarrConfig = configManager.GetConfiguration(); - _radarrConfig = configManager.GetConfiguration(); - _lidarrConfig = configManager.GetConfiguration(); - } - - public override async Task ExecuteAsync() - { - if (_downloadClientConfig.Clients.Count is 0) - { - _logger.LogWarning("No download clients configured"); - return; - } - - bool blocklistIsConfigured = _config.Sonarr.Enabled && !string.IsNullOrEmpty(_config.Sonarr.Path) || - _config.Radarr.Enabled && !string.IsNullOrEmpty(_config.Radarr.Path) || - _config.Lidarr.Enabled && !string.IsNullOrEmpty(_config.Lidarr.Path); - - if (!blocklistIsConfigured) - { - _logger.LogWarning("no blocklist is configured"); - return; - } - - await _blocklistProvider.LoadBlocklistsAsync(); - await base.ExecuteAsync(); - } - - protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType, ArrConfig config) - { - IReadOnlyList ignoredDownloads = await _ignoredDownloadsService.GetIgnoredDownloadsAsync(); - - using var _ = LogContext.PushProperty(LogProperties.Category, instanceType.ToString()); - - IArrClient arrClient = _arrClientFactory.GetClient(instanceType); - BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType); - ConcurrentBag patterns = _blocklistProvider.GetPatterns(instanceType); - ConcurrentBag regexes = _blocklistProvider.GetRegexes(instanceType); - - await _arrArrQueueIterator.Iterate(arrClient, instance, async items => - { - var groups = items - .GroupBy(x => x.DownloadId) - .ToList(); - - foreach (var group in groups) - { - if (group.Any(x => !arrClient.IsRecordValid(x))) - { - continue; - } - - QueueRecord record = group.First(); - - if (record.Protocol is not "torrent") - { - continue; - } - - if (string.IsNullOrEmpty(record.DownloadId)) - { - _logger.LogDebug("skip | download id is null for {title}", record.Title); - continue; - } - - if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase)) - { - _logger.LogInformation("skip | {title} | ignored", record.Title); - continue; - } - - _logger.LogTrace("processing | {name}", record.Title); - - string downloadRemovalKey = CacheKeys.DownloadMarkedForRemoval(record.DownloadId, instance.Url); - - if (_cache.TryGetValue(downloadRemovalKey, out bool _)) - { - _logger.LogDebug("skip | already marked for removal | {title}", record.Title); - continue; - } - - _logger.LogDebug("searching unwanted files for {title}", record.Title); - bool found = false; - - foreach (var downloadService in _downloadServices) - { - try - { - BlockFilesResult result = await downloadService - .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads); - - if (!result.Found) - { - continue; - } - - found = true; - - if (!result.ShouldRemove) - { - break; - } - - _logger.LogDebug("all files are marked as unwanted | {hash}", record.Title); - - bool removeFromClient = true; - - if (result.IsPrivate && !_config.DeletePrivate) - { - removeFromClient = false; - } - - await PublishQueueItemRemoveRequest( - downloadRemovalKey, - instanceType, - instance, - record, - group.Count() > 1, - removeFromClient, - DeleteReason.AllFilesBlocked - ); - - break; - } - catch (Exception ex) - { - _logger.LogError( - ex, - "Error blocking unwanted files for {hash} with download client", - record.DownloadId); - } - } - - if (!found) - { - _logger.LogWarning("skip | download not found {title}", record.Title); - } - } - }); - } -} \ No newline at end of file +// using System.Collections.Concurrent; +// using System.Text.RegularExpressions; +// using Common.Configuration.Arr; +// using Common.Configuration.DownloadClient; +// using Common.Configuration.QueueCleaner; +// using Data.Enums; +// using Data.Models.Arr.Queue; +// using Infrastructure.Configuration; +// using Infrastructure.Helpers; +// using Infrastructure.Services; +// using Infrastructure.Verticals.Arr; +// using Infrastructure.Verticals.Arr.Interfaces; +// using Infrastructure.Verticals.DownloadClient; +// using Infrastructure.Verticals.Jobs; +// using MassTransit; +// using Microsoft.Extensions.Caching.Memory; +// using Microsoft.Extensions.Logging; +// using LogContext = Serilog.Context.LogContext; +// +// namespace Infrastructure.Verticals.ContentBlocker; +// +// public sealed class ContentBlocker : GenericHandler +// { +// private readonly ContentBlockerConfig _config; +// private readonly BlocklistProvider _blocklistProvider; +// private readonly IIgnoredDownloadsService _ignoredDownloadsService; +// +// public ContentBlocker( +// ILogger logger, +// IConfigManager configManager, +// IMemoryCache cache, +// IBus messageBus, +// ArrClientFactory arrClientFactory, +// ArrQueueIterator arrArrQueueIterator, +// BlocklistProvider blocklistProvider, +// IIgnoredDownloadsService ignoredDownloadsService, +// DownloadServiceFactory downloadServiceFactory +// ) : base( +// logger, cache, messageBus, +// arrClientFactory, arrArrQueueIterator, downloadServiceFactory +// ) +// { +// _blocklistProvider = blocklistProvider; +// _ignoredDownloadsService = ignoredDownloadsService; +// +// _config = configManager.GetConfiguration(); +// _downloadClientConfig = configManager.GetConfiguration(); +// _sonarrConfig = configManager.GetConfiguration(); +// _radarrConfig = configManager.GetConfiguration(); +// _lidarrConfig = configManager.GetConfiguration(); +// } +// +// public override async Task ExecuteAsync() +// { +// if (_downloadClientConfig.Clients.Count is 0) +// { +// _logger.LogWarning("No download clients configured"); +// return; +// } +// +// bool blocklistIsConfigured = _config.Sonarr.Enabled && !string.IsNullOrEmpty(_config.Sonarr.Path) || +// _config.Radarr.Enabled && !string.IsNullOrEmpty(_config.Radarr.Path) || +// _config.Lidarr.Enabled && !string.IsNullOrEmpty(_config.Lidarr.Path); +// +// if (!blocklistIsConfigured) +// { +// _logger.LogWarning("no blocklist is configured"); +// return; +// } +// +// await _blocklistProvider.LoadBlocklistsAsync(); +// await base.ExecuteAsync(); +// } +// +// protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType, ArrConfig config) +// { +// IReadOnlyList ignoredDownloads = await _ignoredDownloadsService.GetIgnoredDownloadsAsync(); +// +// using var _ = LogContext.PushProperty(LogProperties.Category, instanceType.ToString()); +// +// IArrClient arrClient = _arrClientFactory.GetClient(instanceType); +// BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType); +// ConcurrentBag patterns = _blocklistProvider.GetPatterns(instanceType); +// ConcurrentBag regexes = _blocklistProvider.GetRegexes(instanceType); +// +// await _arrArrQueueIterator.Iterate(arrClient, instance, async items => +// { +// var groups = items +// .GroupBy(x => x.DownloadId) +// .ToList(); +// +// foreach (var group in groups) +// { +// if (group.Any(x => !arrClient.IsRecordValid(x))) +// { +// continue; +// } +// +// QueueRecord record = group.First(); +// +// if (record.Protocol is not "torrent") +// { +// continue; +// } +// +// if (string.IsNullOrEmpty(record.DownloadId)) +// { +// _logger.LogDebug("skip | download id is null for {title}", record.Title); +// continue; +// } +// +// if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase)) +// { +// _logger.LogInformation("skip | {title} | ignored", record.Title); +// continue; +// } +// +// _logger.LogTrace("processing | {name}", record.Title); +// +// string downloadRemovalKey = CacheKeys.DownloadMarkedForRemoval(record.DownloadId, instance.Url); +// +// if (_cache.TryGetValue(downloadRemovalKey, out bool _)) +// { +// _logger.LogDebug("skip | already marked for removal | {title}", record.Title); +// continue; +// } +// +// _logger.LogDebug("searching unwanted files for {title}", record.Title); +// bool found = false; +// +// foreach (var downloadService in _downloadServices) +// { +// try +// { +// BlockFilesResult result = await downloadService +// .BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads); +// +// if (!result.Found) +// { +// continue; +// } +// +// found = true; +// +// if (!result.ShouldRemove) +// { +// break; +// } +// +// _logger.LogDebug("all files are marked as unwanted | {hash}", record.Title); +// +// bool removeFromClient = true; +// +// if (result.IsPrivate && !_config.DeletePrivate) +// { +// removeFromClient = false; +// } +// +// await PublishQueueItemRemoveRequest( +// downloadRemovalKey, +// instanceType, +// instance, +// record, +// group.Count() > 1, +// removeFromClient, +// DeleteReason.AllFilesBlocked +// ); +// +// break; +// } +// catch (Exception ex) +// { +// _logger.LogError( +// ex, +// "Error blocking unwanted files for {hash} with download client", +// record.DownloadId); +// } +// } +// +// if (!found) +// { +// _logger.LogWarning("skip | download not found {title}", record.Title); +// } +// } +// }); +// } +// } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/ContentBlocker/FilenameEvaluator.cs b/code/Infrastructure/Verticals/ContentBlocker/FilenameEvaluator.cs index be7738e1..c8cefc87 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/FilenameEvaluator.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/FilenameEvaluator.cs @@ -1,6 +1,6 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; -using Common.Configuration.ContentBlocker; +using Common.Configuration.QueueCleaner; using Microsoft.Extensions.Logging; namespace Infrastructure.Verticals.ContentBlocker; diff --git a/code/Infrastructure/Verticals/ContentBlocker/IFilenameEvaluator.cs b/code/Infrastructure/Verticals/ContentBlocker/IFilenameEvaluator.cs index 91e69c00..6119f5c6 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/IFilenameEvaluator.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/IFilenameEvaluator.cs @@ -1,6 +1,6 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; -using Common.Configuration.ContentBlocker; +using Common.Configuration.QueueCleaner; namespace Infrastructure.Verticals.ContentBlocker; diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index ee2d3ed3..63f175f1 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -1,21 +1,11 @@ -using System.Collections.Concurrent; -using System.Text.RegularExpressions; -using Common.Attributes; -using Common.Configuration.ContentBlocker; -using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; -using Common.CustomDataTypes; using Common.Exceptions; -using Data.Enums; using Data.Models.Deluge.Response; using Infrastructure.Events; -using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; -using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; -using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Infrastructure.Configuration; @@ -23,7 +13,7 @@ using Infrastructure.Http; namespace Infrastructure.Verticals.DownloadClient.Deluge; -public class DelugeService : DownloadService, IDelugeService +public partial class DelugeService : DownloadService, IDelugeService { private DelugeClient? _client; @@ -36,11 +26,12 @@ public class DelugeService : DownloadService, IDelugeService IDryRunInterceptor dryRunInterceptor, IHardLinkFileService hardLinkFileService, IDynamicHttpClientProvider httpClientProvider, - EventPublisher eventPublisher + EventPublisher eventPublisher, + BlocklistProvider blocklistProvider ) : base( logger, configManager, cache, filenameEvaluator, striker, dryRunInterceptor, hardLinkFileService, - httpClientProvider, eventPublisher + httpClientProvider, eventPublisher, blocklistProvider ) { // Client will be initialized when Initialize() is called with a specific client configuration @@ -95,476 +86,6 @@ public class DelugeService : DownloadService, IDelugeService throw; } } - - /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) - { - if (_client == null) - { - throw new InvalidOperationException("Deluge client is not initialized"); - } - - hash = hash.ToLowerInvariant(); - - DelugeContents? contents = null; - DownloadCheckResult result = new(); - - DownloadStatus? download = await _client.GetTorrentStatus(hash); - - if (download?.Hash is null) - { - _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return result; - } - - result.Found = true; - result.IsPrivate = download.Private; - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - return result; - } - - try - { - contents = await _client.GetTorrentFiles(hash); - } - catch (Exception exception) - { - _logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash); - } - - - bool shouldRemove = contents?.Contents?.Count > 0; - - ProcessFiles(contents.Contents, (_, file) => - { - if (file.Priority > 0) - { - shouldRemove = false; - } - }); - - if (shouldRemove) - { - // remove if all files are unwanted - result.ShouldRemove = true; - result.DeleteReason = DeleteReason.AllFilesSkipped; - return result; - } - - // remove if download is stuck - (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download); - - return result; - } - - /// - public override async Task BlockUnwantedFilesAsync(string hash, - BlocklistType blocklistType, - ConcurrentBag patterns, - ConcurrentBag regexes, IReadOnlyList ignoredDownloads) - { - hash = hash.ToLowerInvariant(); - - DownloadStatus? download = await _client.GetTorrentStatus(hash); - BlockFilesResult result = new(); - - if (download?.Hash is null) - { - _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return result; - } - - // Mark as processed since we found the download - result.Found = true; - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - return result; - } - - result.IsPrivate = download.Private; - - if (_contentBlockerConfig.IgnorePrivate && download.Private) - { - // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", download.Name); - return result; - } - - DelugeContents? contents = null; - - try - { - contents = await _client.GetTorrentFiles(hash); - } - catch (Exception exception) - { - _logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash); - } - - if (contents is null) - { - return result; - } - - Dictionary priorities = []; - bool hasPriorityUpdates = false; - long totalFiles = 0; - long totalUnwantedFiles = 0; - - ProcessFiles(contents.Contents, (name, file) => - { - totalFiles++; - int priority = file.Priority; - - if (file.Priority is 0) - { - totalUnwantedFiles++; - } - - if (file.Priority is not 0 && !_filenameEvaluator.IsValid(name, blocklistType, patterns, regexes)) - { - totalUnwantedFiles++; - priority = 0; - hasPriorityUpdates = true; - _logger.LogInformation("unwanted file found | {file}", file.Path); - } - - priorities.Add(file.Index, priority); - }); - - if (!hasPriorityUpdates) - { - return result; - } - - _logger.LogDebug("changing priorities | torrent {hash}", hash); - - List sortedPriorities = priorities - .OrderBy(x => x.Key) - .Select(x => x.Value) - .ToList(); - - if (totalUnwantedFiles == totalFiles) - { - // Skip marking files as unwanted. The download will be removed completely. - result.ShouldRemove = true; - - return result; - } - - await _dryRunInterceptor.InterceptAsync(ChangeFilesPriority, hash, sortedPriorities); - - return result; - } - - public override async Task?> GetSeedingDownloads() - { - return (await _client.GetStatusForAllTorrents()) - ?.Where(x => !string.IsNullOrEmpty(x.Hash)) - .Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true) - .Cast() - .ToList(); - } - - public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) => - downloads - ?.Cast() - .Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase))) - .Cast() - .ToList(); - - public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) => - downloads - ?.Cast() - .Where(x => !string.IsNullOrEmpty(x.Hash)) - .Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase))) - .Cast() - .ToList(); - - /// - public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, HashSet excludedHashes, - IReadOnlyList ignoredDownloads) - { - if (downloads?.Count is null or 0) - { - return; - } - - foreach (DownloadStatus download in downloads) - { - if (string.IsNullOrEmpty(download.Hash)) - { - continue; - } - - if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - continue; - } - - CleanCategory? category = categoriesToClean - .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); - - if (category is null) - { - continue; - } - - if (!_downloadCleanerConfig.DeletePrivate && download.Private) - { - _logger.LogDebug("skip | download is private | {name}", download.Name); - continue; - } - - ContextProvider.Set("downloadName", download.Name); - ContextProvider.Set("hash", download.Hash); - - TimeSpan seedingTime = TimeSpan.FromSeconds(download.SeedingTime); - SeedingCheckResult result = ShouldCleanDownload(download.Ratio, seedingTime, category); - - if (!result.ShouldClean) - { - continue; - } - - await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash); - - _logger.LogInformation( - "download cleaned | {reason} reached | {name}", - result.Reason is CleanReason.MaxRatioReached - ? "MAX_RATIO & MIN_SEED_TIME" - : "MAX_SEED_TIME", - download.Name - ); - - await _eventPublisher.PublishDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason); - } - } - - public override async Task CreateCategoryAsync(string name) - { - IReadOnlyList existingLabels = await _client.GetLabels(); - - if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase)) - { - return; - } - - await _dryRunInterceptor.InterceptAsync(CreateLabel, name); - } - - public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) - { - if (downloads?.Count is null or 0) - { - return; - } - - if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) - { - _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); - } - - foreach (DownloadStatus download in downloads.Cast()) - { - if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label)) - { - continue; - } - - if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - continue; - } - - ContextProvider.Set("downloadName", download.Name); - ContextProvider.Set("hash", download.Hash); - - DelugeContents? contents = null; - try - { - contents = await _client.GetTorrentFiles(download.Hash); - } - catch (Exception exception) - { - _logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name); - continue; - } - - bool hasHardlinks = false; - - ProcessFiles(contents?.Contents, (_, file) => - { - string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/'])); - - if (file.Priority <= 0) - { - _logger.LogDebug("skip | file is not downloaded | {file}", filePath); - return; - } - - long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); - - if (hardlinkCount < 0) - { - _logger.LogDebug("skip | could not get file properties | {file}", filePath); - hasHardlinks = true; - return; - } - - if (hardlinkCount > 0) - { - hasHardlinks = true; - } - }); - - if (hasHardlinks) - { - _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); - continue; - } - - await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory); - - _logger.LogInformation("category changed for {name}", download.Name); - - await _eventPublisher.PublishCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory); - - download.Label = _downloadCleanerConfig.UnlinkedTargetCategory; - } - } - - /// - [DryRunSafeguard] - public override async Task DeleteDownload(string hash) - { - hash = hash.ToLowerInvariant(); - - await _client.DeleteTorrents([hash]); - } - - [DryRunSafeguard] - protected async Task CreateLabel(string name) - { - await _client.CreateLabel(name); - } - - [DryRunSafeguard] - protected virtual async Task ChangeFilesPriority(string hash, List sortedPriorities) - { - await _client.ChangeFilesPriority(hash, sortedPriorities); - } - - [DryRunSafeguard] - protected virtual async Task ChangeLabel(string hash, string newLabel) - { - await _client.SetTorrentLabel(hash, newLabel); - } - - private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(DownloadStatus status) - { - (bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(status); - - if (result.ShouldRemove) - { - return result; - } - - return await CheckIfStuck(status); - } - - private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(DownloadStatus download) - { - if (_queueCleanerConfig.SlowMaxStrikes is 0) - { - return (false, DeleteReason.None); - } - - if (download.State is null || !download.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) - { - return (false, DeleteReason.None); - } - - if (download.DownloadSpeed <= 0) - { - return (false, DeleteReason.None); - } - - if (_queueCleanerConfig.SlowIgnorePrivate && download.Private) - { - // ignore private trackers - _logger.LogDebug("skip slow check | download is private | {name}", download.Name); - return (false, DeleteReason.None); - } - - if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) - { - _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); - return (false, DeleteReason.None); - } - - ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize; - ByteSize currentSpeed = new ByteSize(download.DownloadSpeed); - SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime); - SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta); - - return await CheckIfSlow( - download.Hash!, - download.Name!, - minSpeed, - currentSpeed, - maxTime, - currentTime - ); - } - - private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(DownloadStatus status) - { - if (_queueCleanerConfig.StalledMaxStrikes is 0) - { - return (false, DeleteReason.None); - } - - if (_queueCleanerConfig.StalledIgnorePrivate && status.Private) - { - // ignore private trackers - _logger.LogDebug("skip stalled check | download is private | {name}", status.Name); - return (false, DeleteReason.None); - } - - if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) - { - return (false, DeleteReason.None); - } - - if (status.Eta > 0) - { - return (false, DeleteReason.None); - } - - ResetStalledStrikesOnProgress(status.Hash!, status.TotalDone); - - return (await _striker.StrikeAndCheckLimit(status.Hash!, status.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); - } private static void ProcessFiles(Dictionary? contents, Action processFile) { diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceDC.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceDC.cs new file mode 100644 index 00000000..78e01f26 --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceDC.cs @@ -0,0 +1,225 @@ +using Common.Attributes; +using Common.Configuration.DownloadCleaner; +using Data.Enums; +using Data.Models.Deluge.Response; +using Infrastructure.Extensions; +using Infrastructure.Verticals.Context; +using Microsoft.Extensions.Logging; + +namespace Infrastructure.Verticals.DownloadClient.Deluge; + +public partial class DelugeService +{ + public override async Task?> GetSeedingDownloads() + { + return (await _client.GetStatusForAllTorrents()) + ?.Where(x => !string.IsNullOrEmpty(x.Hash)) + .Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true) + .Cast() + .ToList(); + } + + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.Hash)) + .Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + + /// + public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, HashSet excludedHashes, + IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + + foreach (DownloadStatus download in downloads) + { + if (string.IsNullOrEmpty(download.Hash)) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + + CleanCategory? category = categoriesToClean + .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); + + if (category is null) + { + continue; + } + + if (!_downloadCleanerConfig.DeletePrivate && download.Private) + { + _logger.LogDebug("skip | download is private | {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.Hash); + + TimeSpan seedingTime = TimeSpan.FromSeconds(download.SeedingTime); + SeedingCheckResult result = ShouldCleanDownload(download.Ratio, seedingTime, category); + + if (!result.ShouldClean) + { + continue; + } + + await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash); + + _logger.LogInformation( + "download cleaned | {reason} reached | {name}", + result.Reason is CleanReason.MaxRatioReached + ? "MAX_RATIO & MIN_SEED_TIME" + : "MAX_SEED_TIME", + download.Name + ); + + await _eventPublisher.PublishDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason); + } + } + + public override async Task CreateCategoryAsync(string name) + { + IReadOnlyList existingLabels = await _client.GetLabels(); + + if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase)) + { + return; + } + + await _dryRunInterceptor.InterceptAsync(CreateLabel, name); + } + + public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + + if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) + { + _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); + } + + foreach (DownloadStatus download in downloads.Cast()) + { + if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label)) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.Hash); + + DelugeContents? contents = null; + try + { + contents = await _client.GetTorrentFiles(download.Hash); + } + catch (Exception exception) + { + _logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name); + continue; + } + + bool hasHardlinks = false; + + ProcessFiles(contents?.Contents, (_, file) => + { + string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/'])); + + if (file.Priority <= 0) + { + _logger.LogDebug("skip | file is not downloaded | {file}", filePath); + return; + } + + long hardlinkCount = _hardLinkFileService + .GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); + + if (hardlinkCount < 0) + { + _logger.LogDebug("skip | could not get file properties | {file}", filePath); + hasHardlinks = true; + return; + } + + if (hardlinkCount > 0) + { + hasHardlinks = true; + } + }); + + if (hasHardlinks) + { + _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); + continue; + } + + await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory); + + _logger.LogInformation("category changed for {name}", download.Name); + + await _eventPublisher.PublishCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory); + + download.Label = _downloadCleanerConfig.UnlinkedTargetCategory; + } + } + + /// + [DryRunSafeguard] + public override async Task DeleteDownload(string hash) + { + hash = hash.ToLowerInvariant(); + + await _client.DeleteTorrents([hash]); + } + + [DryRunSafeguard] + protected async Task CreateLabel(string name) + { + await _client.CreateLabel(name); + } + + [DryRunSafeguard] + protected virtual async Task ChangeLabel(string hash, string newLabel) + { + await _client.SetTorrentLabel(hash, newLabel); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceQC.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceQC.cs new file mode 100644 index 00000000..7a124f3d --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeServiceQC.cs @@ -0,0 +1,261 @@ +using System.Collections.Concurrent; +using System.Text.RegularExpressions; +using Common.Attributes; +using Common.Configuration.QueueCleaner; +using Common.CustomDataTypes; +using Data.Enums; +using Data.Models.Deluge.Response; +using Infrastructure.Extensions; +using Infrastructure.Verticals.Context; +using Microsoft.Extensions.Logging; + +namespace Infrastructure.Verticals.DownloadClient.Deluge; + +public partial class DelugeService +{ + /// + public override async Task ShouldRemoveFromArrQueueAsync(string hash, + IReadOnlyList ignoredDownloads) + { + if (_client == null) + { + throw new InvalidOperationException("Deluge client is not initialized"); + } + + hash = hash.ToLowerInvariant(); + + DelugeContents? contents = null; + DownloadCheckResult result = new(); + + DownloadStatus? download = await _client.GetTorrentStatus(hash); + + if (download?.Hash is null) + { + _logger.LogDebug("failed to find torrent {hash} in the download client", hash); + return result; + } + + result.Found = true; + result.IsPrivate = download.Private; + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + + try + { + contents = await _client.GetTorrentFiles(hash); + } + catch (Exception exception) + { + _logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash); + } + + + bool shouldRemove = contents?.Contents?.Count > 0; + + ProcessFiles(contents.Contents, (_, file) => + { + if (file.Priority > 0) + { + shouldRemove = false; + } + }); + + if (shouldRemove) + { + // remove if all files are unwanted + result.ShouldRemove = true; + result.DeleteReason = DeleteReason.AllFilesSkipped; + return result; + } + + // remove if download is stuck + (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, download.Private, contents.Contents); + + return result; + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> BlockUnwantedFilesAsync( + DownloadStatus download, + bool isPrivate, + Dictionary? files + ) + { + if (!_queueCleanerConfig.ContentBlocker.Enabled) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.ContentBlocker.IgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip unwanted files check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (files is null) + { + _logger.LogDebug("failed to find files for {name}", download.Name); + return (false, DeleteReason.None); + } + + InstanceType instanceType = (InstanceType)ContextProvider.Get(nameof(InstanceType)); + BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType); + ConcurrentBag patterns = _blocklistProvider.GetPatterns(instanceType); + ConcurrentBag regexes = _blocklistProvider.GetRegexes(instanceType); + + Dictionary priorities = []; + bool hasPriorityUpdates = false; + long totalFiles = 0; + long totalUnwantedFiles = 0; + + ProcessFiles(files, (name, file) => + { + totalFiles++; + int priority = file.Priority; + + if (file.Priority is 0) + { + totalUnwantedFiles++; + } + + if (file.Priority is not 0 && !_filenameEvaluator.IsValid(name, blocklistType, patterns, regexes)) + { + totalUnwantedFiles++; + priority = 0; + hasPriorityUpdates = true; + _logger.LogInformation("unwanted file found | {file}", file.Path); + } + + priorities.Add(file.Index, priority); + }); + + if (!hasPriorityUpdates) + { + return (false, DeleteReason.None); + } + + _logger.LogTrace("marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name); + + List sortedPriorities = priorities + .OrderBy(x => x.Key) + .Select(x => x.Value) + .ToList(); + + if (totalUnwantedFiles == totalFiles) + { + // Skip marking files as unwanted. The download will be removed completely. + return (true, DeleteReason.AllFilesBlocked); + } + + await _dryRunInterceptor.InterceptAsync(ChangeFilesPriority, download.Hash, sortedPriorities); + + return (false, DeleteReason.None); + } + + [DryRunSafeguard] + protected virtual async Task ChangeFilesPriority(string hash, List sortedPriorities) + { + await _client.ChangeFilesPriority(hash, sortedPriorities); + } + + private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval( + DownloadStatus status, + bool isPrivate, + Dictionary? files + ) + { + (bool ShouldRemove, DeleteReason Reason) result = await BlockUnwantedFilesAsync(status, isPrivate, files); + + if (result.ShouldRemove) + { + return result; + } + + result = await CheckIfSlow(status); + + if (result.ShouldRemove) + { + return result; + } + + return await CheckIfStuck(status); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(DownloadStatus download) + { + if (_queueCleanerConfig.Slow.MaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (download.State is null || !download.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) + { + return (false, DeleteReason.None); + } + + if (download.DownloadSpeed <= 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.Slow.IgnorePrivate && download.Private) + { + // ignore private trackers + _logger.LogDebug("skip slow check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (download.Size > (_queueCleanerConfig.Slow.IgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) + { + _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); + return (false, DeleteReason.None); + } + + ByteSize minSpeed = _queueCleanerConfig.Slow.MinSpeedByteSize; + ByteSize currentSpeed = new ByteSize(download.DownloadSpeed); + SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.Slow.MaxTime); + SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta); + + return await CheckIfSlow( + download.Hash!, + download.Name!, + minSpeed, + currentSpeed, + maxTime, + currentTime + ); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(DownloadStatus status) + { + if (_queueCleanerConfig.Stalled.MaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.Stalled.IgnorePrivate && status.Private) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", status.Name); + return (false, DeleteReason.None); + } + + if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) + { + return (false, DeleteReason.None); + } + + if (status.Eta > 0) + { + return (false, DeleteReason.None); + } + + ResetStalledStrikesOnProgress(status.Hash!, status.TotalDone); + + return (await _striker.StrikeAndCheckLimit(status.Hash!, status.Name!, _queueCleanerConfig.Stalled.MaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs index bd5f23f2..9335ae90 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; @@ -17,9 +16,9 @@ using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; -using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; +using QBittorrent.Client; namespace Infrastructure.Verticals.DownloadClient; @@ -35,15 +34,16 @@ public abstract class DownloadService : IDownloadService protected readonly IHardLinkFileService _hardLinkFileService; protected readonly IDynamicHttpClientProvider _httpClientProvider; protected readonly EventPublisher _eventPublisher; + protected readonly BlocklistProvider _blocklistProvider; protected readonly QueueCleanerConfig _queueCleanerConfig; - protected readonly ContentBlockerConfig _contentBlockerConfig; protected readonly DownloadCleanerConfig _downloadCleanerConfig; + protected HttpClient? _httpClient; + // Client-specific configuration protected ClientConfig _clientConfig; // HTTP client for this service - protected HttpClient? _httpClient; protected DownloadService( ILogger logger, @@ -54,7 +54,8 @@ public abstract class DownloadService : IDownloadService IDryRunInterceptor dryRunInterceptor, IHardLinkFileService hardLinkFileService, IDynamicHttpClientProvider httpClientProvider, - EventPublisher eventPublisher + EventPublisher eventPublisher, + BlocklistProvider blocklistProvider ) { _logger = logger; @@ -66,6 +67,7 @@ public abstract class DownloadService : IDownloadService _hardLinkFileService = hardLinkFileService; _httpClientProvider = httpClientProvider; _eventPublisher = eventPublisher; + _blocklistProvider = blocklistProvider; _cacheOptions = new MemoryCacheEntryOptions() .SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer); @@ -73,7 +75,6 @@ public abstract class DownloadService : IDownloadService _clientConfig = new ClientConfig(); _queueCleanerConfig = _configManager.GetConfiguration(); - _contentBlockerConfig = _configManager.GetConfiguration(); _downloadCleanerConfig = _configManager.GetConfiguration(); } @@ -99,13 +100,8 @@ public abstract class DownloadService : IDownloadService public abstract Task LoginAsync(); - public abstract Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); - - /// - public abstract Task BlockUnwantedFilesAsync(string hash, - BlocklistType blocklistType, - ConcurrentBag patterns, - ConcurrentBag regexes, IReadOnlyList ignoredDownloads); + public abstract Task ShouldRemoveFromArrQueueAsync(string hash, + IReadOnlyList ignoredDownloads); /// public abstract Task DeleteDownload(string hash); @@ -130,7 +126,7 @@ public abstract class DownloadService : IDownloadService protected void ResetStalledStrikesOnProgress(string hash, long downloaded) { - if (!_queueCleanerConfig.StalledResetStrikesOnProgress) + if (!_queueCleanerConfig.Stalled.ResetStrikesOnProgress) { return; } @@ -148,7 +144,7 @@ public abstract class DownloadService : IDownloadService protected void ResetSlowSpeedStrikesOnProgress(string downloadName, string hash) { - if (_queueCleanerConfig.SlowResetStrikesOnProgress) + if (_queueCleanerConfig.Slow.ResetStrikesOnProgress) { return; } @@ -166,7 +162,7 @@ public abstract class DownloadService : IDownloadService protected void ResetSlowTimeStrikesOnProgress(string downloadName, string hash) { - if (_queueCleanerConfig.SlowResetStrikesOnProgress) + if (_queueCleanerConfig.Slow.ResetStrikesOnProgress) { return; } @@ -196,7 +192,7 @@ public abstract class DownloadService : IDownloadService _logger.LogTrace("slow speed | {speed}/s | {name}", currentSpeed.ToString(), downloadName); bool shouldRemove = await _striker - .StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowSpeed); + .StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.Slow.MaxStrikes, StrikeType.SlowSpeed); if (shouldRemove) { @@ -213,7 +209,7 @@ public abstract class DownloadService : IDownloadService _logger.LogTrace("slow estimated time | {time} | {name}", currentTime.ToString(), downloadName); bool shouldRemove = await _striker - .StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowTime); + .StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.Slow.MaxStrikes, StrikeType.SlowTime); if (shouldRemove) { diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs index e0dd01b9..34710f75 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadServiceFactory.cs @@ -28,30 +28,30 @@ public sealed class DownloadServiceFactory _logger = logger; } - /// - /// Creates a download service using the specified client ID - /// - /// The client ID to create a service for - /// An implementation of IDownloadService or null if the client is not available - public IDownloadService? GetDownloadService(Guid clientId) - { - var config = _configManager.GetConfiguration(); - var clientConfig = config.GetClientConfig(clientId); - - if (clientConfig == null) - { - _logger.LogWarning("No download client configuration found for ID {clientId}", clientId); - return null; - } - - if (!clientConfig.Enabled) - { - _logger.LogWarning("Download client {clientId} is disabled", clientId); - return null; - } - - return GetDownloadService(clientConfig); - } + // /// + // /// Creates a download service using the specified client ID + // /// + // /// The client ID to create a service for + // /// An implementation of IDownloadService or null if the client is not available + // public IDownloadService? GetDownloadService(Guid clientId) + // { + // var config = _configManager.GetConfiguration(); + // var clientConfig = config.GetClientConfig(clientId); + // + // if (clientConfig == null) + // { + // _logger.LogWarning("No download client configuration found for ID {clientId}", clientId); + // return null; + // } + // + // if (!clientConfig.Enabled) + // { + // _logger.LogWarning("Download client {clientId} is disabled", clientId); + // return null; + // } + // + // return GetDownloadService(clientConfig); + // } /// /// Creates a download service using the specified client configuration @@ -83,7 +83,6 @@ public sealed class DownloadServiceFactory /// An implementation of IDownloadService private T CreateClientService(ClientConfig clientConfig) where T : IDownloadService { - // TODO var service = _serviceProvider.GetRequiredService(); service.Initialize(clientConfig); return service; diff --git a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs index 36fa9af2..0b00d014 100644 --- a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs @@ -1,9 +1,11 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; +using Common.Configuration.QueueCleaner; +using Data.Enums; using Infrastructure.Interceptors; +using QBittorrent.Client; namespace Infrastructure.Verticals.DownloadClient; @@ -28,23 +30,8 @@ public interface IDownloadService : IDisposable /// /// The download hash. /// Downloads to ignore from processing. - public Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); - - /// - /// Blocks unwanted files from being fully downloaded. - /// - /// The torrent hash. - /// The . - /// The patterns to test the files against. - /// The regexes to test the files against. - /// Downloads to ignore from processing. - /// True if all files have been blocked; otherwise false. - public Task BlockUnwantedFilesAsync(string hash, - BlocklistType blocklistType, - ConcurrentBag patterns, - ConcurrentBag regexes, - IReadOnlyList ignoredDownloads - ); + public Task ShouldRemoveFromArrQueueAsync(string hash, + IReadOnlyList ignoredDownloads); /// /// Fetches all seeding downloads. diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 60c4d29b..772dc563 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Attributes; -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; @@ -24,7 +23,7 @@ using Infrastructure.Events; namespace Infrastructure.Verticals.DownloadClient.QBittorrent; -public class QBitService : DownloadService, IQBitService +public partial class QBitService : DownloadService, IQBitService { protected QBittorrentClient? _client; @@ -38,10 +37,11 @@ public class QBitService : DownloadService, IQBitService IDryRunInterceptor dryRunInterceptor, IHardLinkFileService hardLinkFileService, IDynamicHttpClientProvider httpClientProvider, - EventPublisher eventPublisher + EventPublisher eventPublisher, + BlocklistProvider blocklistProvider ) : base( logger, configManager, cache, filenameEvaluator, striker, dryRunInterceptor, hardLinkFileService, - httpClientProvider, eventPublisher + httpClientProvider, eventPublisher, blocklistProvider ) { // Client will be initialized when Initialize() is called with a specific client configuration @@ -53,12 +53,6 @@ public class QBitService : DownloadService, IQBitService // Initialize base service first base.Initialize(clientConfig); - // Ensure client type is correct - if (clientConfig.Type != Common.Enums.DownloadClientType.QBittorrent) - { - throw new InvalidOperationException($"Cannot initialize QBitService with client type {clientConfig.Type}"); - } - // Create QBittorrent client _client = new QBittorrentClient(_httpClient, clientConfig.Url); @@ -90,589 +84,7 @@ public class QBitService : DownloadService, IQBitService throw; } } - - /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - DownloadCheckResult result = new(); - TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) - .FirstOrDefault(); - - if (download is null) - { - _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return result; - } - - result.Found = true; - - IReadOnlyList trackers = await GetTrackersAsync(hash); - - if (ignoredDownloads.Count > 0 && - (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - return result; - } - - TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); - - if (torrentProperties is null) - { - _logger.LogDebug("failed to find torrent properties {hash} in the download client", hash); - return result; - } - - result.IsPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) && - bool.TryParse(dictValue?.ToString(), out bool boolValue) - && boolValue; - - IReadOnlyList? files = await _client.GetTorrentContentsAsync(hash); - - if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip)) - { - result.ShouldRemove = true; - - // if all files were blocked by qBittorrent - if (download is { CompletionOn: not null, Downloaded: null or 0 }) - { - result.DeleteReason = DeleteReason.AllFilesSkippedByQBit; - return result; - } - - // remove if all files are unwanted - result.DeleteReason = DeleteReason.AllFilesSkipped; - return result; - } - - (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, result.IsPrivate); - - return result; - } - - /// - public override async Task BlockUnwantedFilesAsync( - string hash, - BlocklistType blocklistType, - ConcurrentBag patterns, - ConcurrentBag regexes, - IReadOnlyList ignoredDownloads - ) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - BlockFilesResult result = new(); - TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) - .FirstOrDefault(); - - if (download is null) - { - _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return result; - } - - // Mark as processed since we found the download - result.Found = true; - - IReadOnlyList trackers = await GetTrackersAsync(hash); - - if (ignoredDownloads.Count > 0 && - (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - return result; - } - - TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); - - if (torrentProperties is null) - { - _logger.LogDebug("failed to find torrent properties {hash} in the download client", hash); - return result; - } - - bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) && - bool.TryParse(dictValue?.ToString(), out bool boolValue) - && boolValue; - - result.IsPrivate = isPrivate; - - if (_contentBlockerConfig.IgnorePrivate && isPrivate) - { - // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", download.Name); - return result; - } - - IReadOnlyList? files = await _client.GetTorrentContentsAsync(hash); - - if (files is null) - { - return result; - } - - List unwantedFiles = []; - long totalFiles = 0; - long totalUnwantedFiles = 0; - - foreach (TorrentContent file in files) - { - if (!file.Index.HasValue) - { - continue; - } - - totalFiles++; - - if (file.Priority is TorrentContentPriority.Skip) - { - totalUnwantedFiles++; - continue; - } - - if (_filenameEvaluator.IsValid(file.Name, blocklistType, patterns, regexes)) - { - continue; - } - - _logger.LogInformation("unwanted file found | {file}", file.Name); - unwantedFiles.Add(file.Index.Value); - totalUnwantedFiles++; - } - - if (unwantedFiles.Count is 0) - { - return result; - } - - if (totalUnwantedFiles == totalFiles) - { - // Skip marking files as unwanted. The download will be removed completely. - result.ShouldRemove = true; - - return result; - } - - foreach (int fileIndex in unwantedFiles) - { - await _dryRunInterceptor.InterceptAsync(SkipFile, hash, fileIndex); - } - - return result; - } - - /// - public override async Task?> GetSeedingDownloads() - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - var torrentList = await _client.GetTorrentListAsync(new TorrentListQuery { Filter = TorrentListFilter.Seeding }); - return torrentList?.Where(x => !string.IsNullOrEmpty(x.Hash)) - .Cast() - .ToList(); - } - - /// - public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) => - downloads - ?.Cast() - .Where(x => !string.IsNullOrEmpty(x.Hash)) - .Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase))) - .Cast() - .ToList(); - - /// - public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) => - downloads - ?.Cast() - .Where(x => !string.IsNullOrEmpty(x.Hash)) - .Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase))) - .Where(x => - { - if (_downloadCleanerConfig.UnlinkedUseTag) - { - return !x.Tags.Any(tag => tag.Equals(_downloadCleanerConfig.UnlinkedTargetCategory, StringComparison.InvariantCultureIgnoreCase)); - } - - return true; - }) - .Cast() - .ToList(); - - /// - public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, - HashSet excludedHashes, IReadOnlyList ignoredDownloads) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - if (downloads?.Count is null or 0) - { - return; - } - - foreach (TorrentInfo download in downloads) - { - if (string.IsNullOrEmpty(download.Hash)) - { - continue; - } - - if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - - IReadOnlyList trackers = await GetTrackersAsync(download.Hash); - - if (ignoredDownloads.Count > 0 && - (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)))) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - continue; - } - - CleanCategory? category = categoriesToClean - .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); - - if (category is null) - { - continue; - } - - if (!_downloadCleanerConfig.DeletePrivate) - { - TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash); - - if (torrentProperties is null) - { - _logger.LogDebug("failed to find torrent properties in the download client | {name}", download.Name); - return; - } - - bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) && - bool.TryParse(dictValue?.ToString(), out bool boolValue) - && boolValue; - - if (isPrivate) - { - _logger.LogDebug("skip | download is private | {name}", download.Name); - continue; - } - } - - ContextProvider.Set("downloadName", download.Name); - ContextProvider.Set("hash", download.Hash); - - SeedingCheckResult result = ShouldCleanDownload(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category); - - if (!result.ShouldClean) - { - continue; - } - - await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash); - - _logger.LogInformation( - "download cleaned | {reason} reached | {name}", - result.Reason is CleanReason.MaxRatioReached - ? "MAX_RATIO & MIN_SEED_TIME" - : "MAX_SEED_TIME", - download.Name - ); - - await _eventPublisher.PublishDownloadCleaned(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category.Name, result.Reason); - } - } - - public override async Task CreateCategoryAsync(string name) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - IReadOnlyDictionary? existingCategories = await _client.GetCategoriesAsync(); - - if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase))) - { - return; - } - - await _dryRunInterceptor.InterceptAsync(CreateCategory, name); - } - - public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - if (downloads?.Count is null or 0) - { - return; - } - - if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) - { - _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); - } - - foreach (TorrentInfo download in downloads) - { - if (string.IsNullOrEmpty(download.Hash)) - { - continue; - } - - if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - - IReadOnlyList trackers = await GetTrackersAsync(download.Hash); - - if (ignoredDownloads.Count > 0 && - (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)))) - { - _logger.LogInformation("skip | download is ignored | {name}", download.Name); - continue; - } - - IReadOnlyList? files = await _client.GetTorrentContentsAsync(download.Hash); - - if (files is null) - { - _logger.LogDebug("failed to find files for {name}", download.Name); - continue; - } - - ContextProvider.Set("downloadName", download.Name); - ContextProvider.Set("hash", download.Hash); - bool hasHardlinks = false; - - foreach (TorrentContent file in files) - { - if (!file.Index.HasValue) - { - _logger.LogDebug("skip | file index is null for {name}", download.Name); - hasHardlinks = true; - break; - } - - string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/'])); - - if (file.Priority is TorrentContentPriority.Skip) - { - _logger.LogDebug("skip | file is not downloaded | {file}", filePath); - continue; - } - - long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); - - if (hardlinkCount < 0) - { - _logger.LogDebug("skip | could not get file properties | {file}", filePath); - hasHardlinks = true; - break; - } - - if (hardlinkCount > 0) - { - hasHardlinks = true; - break; - } - } - - if (hasHardlinks) - { - _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); - continue; - } - - await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory); - - if (_downloadCleanerConfig.UnlinkedUseTag) - { - _logger.LogInformation("tag added for {name}", download.Name); - } - else - { - _logger.LogInformation("category changed for {name}", download.Name); - download.Category = _downloadCleanerConfig.UnlinkedTargetCategory; - } - - await _eventPublisher.PublishCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory, _downloadCleanerConfig.UnlinkedUseTag); - } - } - - /// - [DryRunSafeguard] - public override async Task DeleteDownload(string hash) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - await _client.DeleteAsync([hash], deleteDownloadedData: true); - } - - [DryRunSafeguard] - protected async Task CreateCategory(string name) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - await _client.AddCategoryAsync(name); - } - - [DryRunSafeguard] - protected virtual async Task SkipFile(string hash, int fileIndex) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip); - } - - [DryRunSafeguard] - protected virtual async Task ChangeCategory(string hash, string newCategory) - { - if (_client == null) - { - throw new InvalidOperationException("QBittorrent client is not initialized"); - } - - if (_downloadCleanerConfig.UnlinkedUseTag) - { - await _client.AddTorrentTagAsync([hash], newCategory); - return; - } - - await _client.SetTorrentCategoryAsync([hash], newCategory); - } - - public override void Dispose() - { - _client?.Dispose(); - _httpClient?.Dispose(); - } - - private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent, bool isPrivate) - { - (bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent, isPrivate); - - if (result.ShouldRemove) - { - return result; - } - - return await CheckIfStuck(torrent, isPrivate); - } - - private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download, bool isPrivate) - { - if (_queueCleanerConfig.SlowMaxStrikes is 0) - { - return (false, DeleteReason.None); - } - - if (download.State is not (TorrentState.Downloading or TorrentState.ForcedDownload)) - { - return (false, DeleteReason.None); - } - - if (download.DownloadSpeed <= 0) - { - return (false, DeleteReason.None); - } - - if (_queueCleanerConfig.SlowIgnorePrivate && isPrivate) - { - // ignore private trackers - _logger.LogDebug("skip slow check | download is private | {name}", download.Name); - return (false, DeleteReason.None); - } - - if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) - { - _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); - return (false, DeleteReason.None); - } - - ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize; - ByteSize currentSpeed = new ByteSize(download.DownloadSpeed); - SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime); - SmartTimeSpan currentTime = new SmartTimeSpan(download.EstimatedTime ?? TimeSpan.Zero); - - return await CheckIfSlow( - download.Hash, - download.Name, - minSpeed, - currentSpeed, - maxTime, - currentTime - ); - } - - private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo torrent, bool isPrivate) - { - if (_queueCleanerConfig.StalledMaxStrikes is 0 && _queueCleanerConfig.DownloadingMetadataMaxStrikes is 0) - { - return (false, DeleteReason.None); - } - - if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata - and not TorrentState.ForcedFetchingMetadata) - { - // ignore other states - return (false, DeleteReason.None); - } - - if (_queueCleanerConfig.StalledMaxStrikes > 0 && torrent.State is TorrentState.StalledDownload) - { - if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) - { - // ignore private trackers - _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); - } - else - { - ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); - - return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); - } - } - - if (_queueCleanerConfig.DownloadingMetadataMaxStrikes > 0 && torrent.State is not TorrentState.StalledDownload) - { - return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.DownloadingMetadataMaxStrikes, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata); - } - - return (false, DeleteReason.None); - } - + private async Task> GetTrackersAsync(string hash) { if (_client == null) @@ -684,4 +96,10 @@ public class QBitService : DownloadService, IQBitService .Where(x => x.Url.Contains("**")) .ToList(); } + + public override void Dispose() + { + _client?.Dispose(); + _httpClient?.Dispose(); + } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceDC.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceDC.cs new file mode 100644 index 00000000..ae51fb6a --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceDC.cs @@ -0,0 +1,305 @@ +using Common.Attributes; +using Common.Configuration.DownloadCleaner; +using Data.Enums; +using Infrastructure.Extensions; +using Infrastructure.Verticals.Context; +using Microsoft.Extensions.Logging; +using QBittorrent.Client; + +namespace Infrastructure.Verticals.DownloadClient.QBittorrent; + +public partial class QBitService +{ + /// + public override async Task?> GetSeedingDownloads() + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + var torrentList = await _client.GetTorrentListAsync(new TorrentListQuery { Filter = TorrentListFilter.Seeding }); + return torrentList?.Where(x => !string.IsNullOrEmpty(x.Hash)) + .Cast() + .ToList(); + } + + /// + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.Hash)) + .Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + + /// + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.Hash)) + .Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase))) + .Where(x => + { + if (_downloadCleanerConfig.UnlinkedUseTag) + { + return !x.Tags.Any(tag => tag.Equals(_downloadCleanerConfig.UnlinkedTargetCategory, StringComparison.InvariantCultureIgnoreCase)); + } + + return true; + }) + .Cast() + .ToList(); + + /// + public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, + HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + if (downloads?.Count is null or 0) + { + return; + } + + foreach (TorrentInfo download in downloads) + { + if (string.IsNullOrEmpty(download.Hash)) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + IReadOnlyList trackers = await GetTrackersAsync(download.Hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)))) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + + CleanCategory? category = categoriesToClean + .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); + + if (category is null) + { + continue; + } + + if (!_downloadCleanerConfig.DeletePrivate) + { + TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash); + + if (torrentProperties is null) + { + _logger.LogDebug("failed to find torrent properties in the download client | {name}", download.Name); + return; + } + + bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) && + bool.TryParse(dictValue?.ToString(), out bool boolValue) + && boolValue; + + if (isPrivate) + { + _logger.LogDebug("skip | download is private | {name}", download.Name); + continue; + } + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.Hash); + + SeedingCheckResult result = ShouldCleanDownload(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category); + + if (!result.ShouldClean) + { + continue; + } + + await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash); + + _logger.LogInformation( + "download cleaned | {reason} reached | {name}", + result.Reason is CleanReason.MaxRatioReached + ? "MAX_RATIO & MIN_SEED_TIME" + : "MAX_SEED_TIME", + download.Name + ); + + await _eventPublisher.PublishDownloadCleaned(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category.Name, result.Reason); + } + } + + public override async Task CreateCategoryAsync(string name) + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + IReadOnlyDictionary? existingCategories = await _client.GetCategoriesAsync(); + + if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase))) + { + return; + } + + await _dryRunInterceptor.InterceptAsync(CreateCategory, name); + } + + public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + if (downloads?.Count is null or 0) + { + return; + } + + if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) + { + _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); + } + + foreach (TorrentInfo download in downloads) + { + if (string.IsNullOrEmpty(download.Hash)) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + IReadOnlyList trackers = await GetTrackersAsync(download.Hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)))) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + + IReadOnlyList? files = await _client.GetTorrentContentsAsync(download.Hash); + + if (files is null) + { + _logger.LogDebug("failed to find files for {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.Hash); + bool hasHardlinks = false; + + foreach (TorrentContent file in files) + { + if (!file.Index.HasValue) + { + _logger.LogDebug("skip | file index is null for {name}", download.Name); + hasHardlinks = true; + break; + } + + string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/'])); + + if (file.Priority is TorrentContentPriority.Skip) + { + _logger.LogDebug("skip | file is not downloaded | {file}", filePath); + continue; + } + + long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); + + if (hardlinkCount < 0) + { + _logger.LogDebug("skip | could not get file properties | {file}", filePath); + hasHardlinks = true; + break; + } + + if (hardlinkCount > 0) + { + hasHardlinks = true; + break; + } + } + + if (hasHardlinks) + { + _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); + continue; + } + + await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory); + + if (_downloadCleanerConfig.UnlinkedUseTag) + { + _logger.LogInformation("tag added for {name}", download.Name); + } + else + { + _logger.LogInformation("category changed for {name}", download.Name); + download.Category = _downloadCleanerConfig.UnlinkedTargetCategory; + } + + await _eventPublisher.PublishCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory, _downloadCleanerConfig.UnlinkedUseTag); + } + } + + /// + [DryRunSafeguard] + public override async Task DeleteDownload(string hash) + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + await _client.DeleteAsync([hash], deleteDownloadedData: true); + } + + [DryRunSafeguard] + protected async Task CreateCategory(string name) + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + await _client.AddCategoryAsync(name); + } + + [DryRunSafeguard] + protected virtual async Task ChangeCategory(string hash, string newCategory) + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + if (_downloadCleanerConfig.UnlinkedUseTag) + { + await _client.AddTorrentTagAsync([hash], newCategory); + return; + } + + await _client.SetTorrentCategoryAsync([hash], newCategory); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceQC.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceQC.cs new file mode 100644 index 00000000..ad218b68 --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitServiceQC.cs @@ -0,0 +1,273 @@ +using System.Collections.Concurrent; +using System.Text.RegularExpressions; +using Common.Attributes; +using Common.Configuration.QueueCleaner; +using Common.CustomDataTypes; +using Data.Enums; +using Infrastructure.Extensions; +using Infrastructure.Verticals.Context; +using Microsoft.Extensions.Logging; +using QBittorrent.Client; + +namespace Infrastructure.Verticals.DownloadClient.QBittorrent; + +public partial class QBitService +{ + /// + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) + { + if (_client == null) + { + throw new InvalidOperationException("QBittorrent client is not initialized"); + } + + DownloadCheckResult result = new(); + TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) + .FirstOrDefault(); + + if (download is null) + { + _logger.LogDebug("failed to find torrent {hash} in the download client", hash); + return result; + } + + result.Found = true; + + IReadOnlyList trackers = await GetTrackersAsync(hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)))) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + return result; + } + + TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash); + + if (torrentProperties is null) + { + _logger.LogDebug("failed to find torrent properties {hash} in the download client", hash); + return result; + } + + result.IsPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) && + bool.TryParse(dictValue?.ToString(), out bool boolValue) + && boolValue; + + IReadOnlyList? files = await _client.GetTorrentContentsAsync(hash); + + if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip)) + { + result.ShouldRemove = true; + + // if all files were blocked by qBittorrent + if (download is { CompletionOn: not null, Downloaded: null or 0 }) + { + result.DeleteReason = DeleteReason.AllFilesSkippedByQBit; + return result; + } + + // remove if all files are unwanted + result.DeleteReason = DeleteReason.AllFilesSkipped; + return result; + } + + (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, result.IsPrivate, files); + + return result; + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> BlockUnwantedFilesAsync( + TorrentInfo torrent, + bool isPrivate, + IReadOnlyList? files + ) + { + if (!_queueCleanerConfig.ContentBlocker.Enabled) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.ContentBlocker.IgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip unwanted files check | download is private | {name}", torrent.Name); + return (false, DeleteReason.None); + } + + if (files is null) + { + _logger.LogDebug("failed to find files for {name}", torrent.Name); + return (false, DeleteReason.None); + } + + List unwantedFiles = []; + long totalFiles = 0; + long totalUnwantedFiles = 0; + + InstanceType instanceType = (InstanceType)ContextProvider.Get(nameof(InstanceType)); + BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType); + ConcurrentBag patterns = _blocklistProvider.GetPatterns(instanceType); + ConcurrentBag regexes = _blocklistProvider.GetRegexes(instanceType); + + foreach (TorrentContent file in files) + { + if (!file.Index.HasValue) + { + continue; + } + + totalFiles++; + + if (file.Priority is TorrentContentPriority.Skip) + { + totalUnwantedFiles++; + continue; + } + + if (_filenameEvaluator.IsValid(file.Name, blocklistType, patterns, regexes)) + { + continue; + } + + _logger.LogInformation("unwanted file found | {file}", file.Name); + unwantedFiles.Add(file.Index.Value); + totalUnwantedFiles++; + } + + if (unwantedFiles.Count is 0) + { + return (false, DeleteReason.None); + } + + if (totalUnwantedFiles == totalFiles) + { + // Skip marking files as unwanted. The download will be removed completely. + return (true, DeleteReason.AllFilesBlocked); + } + + _logger.LogTrace("marking {count} unwanted files as skipped for {name}", unwantedFiles.Count, torrent.Name); + + foreach (int fileIndex in unwantedFiles) + { + await _dryRunInterceptor.InterceptAsync(MarkFileAsSkipped, torrent.Hash, fileIndex); + } + + return (false, DeleteReason.None); + } + + [DryRunSafeguard] + protected virtual async Task MarkFileAsSkipped(string hash, int fileIndex) + { + await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip); + } + + private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval( + TorrentInfo torrent, + bool isPrivate, + IReadOnlyList? files + ) + { + (bool ShouldRemove, DeleteReason Reason) result = await BlockUnwantedFilesAsync(torrent, isPrivate, files); + + if (result.ShouldRemove) + { + return result; + } + + result = await CheckIfSlow(torrent, isPrivate); + + if (result.ShouldRemove) + { + return result; + } + + return await CheckIfStuck(torrent, isPrivate); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download, bool isPrivate) + { + if (_queueCleanerConfig.Slow.MaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (download.State is not (TorrentState.Downloading or TorrentState.ForcedDownload)) + { + return (false, DeleteReason.None); + } + + if (download.DownloadSpeed <= 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.Slow.IgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip slow check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (download.Size > (_queueCleanerConfig.Slow.IgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) + { + _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); + return (false, DeleteReason.None); + } + + ByteSize minSpeed = _queueCleanerConfig.Slow.MinSpeedByteSize; + ByteSize currentSpeed = new ByteSize(download.DownloadSpeed); + SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.Slow.MaxTime); + SmartTimeSpan currentTime = new SmartTimeSpan(download.EstimatedTime ?? TimeSpan.Zero); + + return await CheckIfSlow( + download.Hash, + download.Name, + minSpeed, + currentSpeed, + maxTime, + currentTime + ); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo torrent, bool isPrivate) + { + if (_queueCleanerConfig.Stalled.MaxStrikes is 0 && _queueCleanerConfig.Stalled.DownloadingMetadataMaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata + and not TorrentState.ForcedFetchingMetadata) + { + // ignore other states + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.Stalled.MaxStrikes > 0 && torrent.State is TorrentState.StalledDownload) + { + if (_queueCleanerConfig.Stalled.IgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); + } + else + { + ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); + + return ( + await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.Stalled.MaxStrikes, + StrikeType.Stalled), DeleteReason.Stalled); + } + } + + if (_queueCleanerConfig.Stalled.DownloadingMetadataMaxStrikes > 0 && torrent.State is not TorrentState.StalledDownload) + { + return ( + await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.Stalled.DownloadingMetadataMaxStrikes, + StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata); + } + + return (false, DeleteReason.None); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 17e42c11..2474f9f2 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -1,7 +1,6 @@ using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Attributes; -using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; @@ -26,7 +25,7 @@ using Transmission.API.RPC.Entity; namespace Infrastructure.Verticals.DownloadClient.Transmission; -public class TransmissionService : DownloadService, ITransmissionService +public partial class TransmissionService : DownloadService, ITransmissionService { private Client? _client; @@ -58,11 +57,12 @@ public class TransmissionService : DownloadService, ITransmissionService IDryRunInterceptor dryRunInterceptor, IHardLinkFileService hardLinkFileService, IDynamicHttpClientProvider httpClientProvider, - EventPublisher eventPublisher + EventPublisher eventPublisher, + BlocklistProvider blocklistProvider ) : base( logger, configManager, cache, filenameEvaluator, striker, dryRunInterceptor, hardLinkFileService, - httpClientProvider, eventPublisher + httpClientProvider, eventPublisher, blocklistProvider ) { // Client will be initialized when Initialize() is called with a specific client configuration @@ -118,482 +118,15 @@ public class TransmissionService : DownloadService, ITransmissionService throw; } } - - /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) - { - if (_client == null) - { - throw new InvalidOperationException("Transmission client is not initialized"); - } - - DownloadCheckResult result = new(); - TorrentInfo? download = await GetTorrentAsync(hash); - - if (download is null) - { - _logger.LogDebug("failed to find torrent {hash} in the download client", hash); - return result; - } - - result.Found = true; - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogDebug("skip | download is ignored | {name}", download.Name); - return result; - } - - bool shouldRemove = download.FileStats?.Length > 0; - result.IsPrivate = download.IsPrivate ?? false; - - foreach (TransmissionTorrentFileStats? stats in download.FileStats ?? []) - { - if (!stats.Wanted.HasValue) - { - // if any files stats are missing, do not remove - shouldRemove = false; - } - - if (stats.Wanted.HasValue && stats.Wanted.Value) - { - // if any files are wanted, do not remove - shouldRemove = false; - } - } - - if (shouldRemove) - { - // remove if all files are unwanted - result.ShouldRemove = true; - result.DeleteReason = DeleteReason.AllFilesBlocked; - return result; - } - - // remove if download is stuck - (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download); - - return result; - } - - /// - public override async Task BlockUnwantedFilesAsync(string hash, - BlocklistType blocklistType, - ConcurrentBag patterns, - ConcurrentBag regexes, IReadOnlyList ignoredDownloads) - { - TorrentInfo? download = await GetTorrentAsync(hash); - BlockFilesResult result = new(); - - if (download?.FileStats is null || download.Files is null) - { - return result; - } - - // Mark as processed since we found the download - result.Found = true; - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogDebug("skip | download is ignored | {name}", download.Name); - return result; - } - - bool isPrivate = download.IsPrivate ?? false; - result.IsPrivate = isPrivate; - - if (_contentBlockerConfig.IgnorePrivate && isPrivate) - { - // ignore private trackers - _logger.LogDebug("skip files check | download is private | {name}", download.Name); - return result; - } - - List unwantedFiles = []; - long totalFiles = 0; - long totalUnwantedFiles = 0; - - for (int i = 0; i < download.Files.Length; i++) - { - if (download.FileStats?[i].Wanted == null) - { - continue; - } - - totalFiles++; - - if (!download.FileStats[i].Wanted.Value) - { - totalUnwantedFiles++; - continue; - } - - if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes)) - { - continue; - } - - _logger.LogInformation("unwanted file found | {file}", download.Files[i].Name); - unwantedFiles.Add(i); - totalUnwantedFiles++; - } - - if (unwantedFiles.Count is 0) - { - return result; - } - - if (totalUnwantedFiles == totalFiles) - { - // Skip marking files as unwanted. The download will be removed completely. - result.ShouldRemove = true; - - return result; - } - - _logger.LogDebug("changing priorities | torrent {hash}", hash); - - await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray()); - - return result; - } - public override async Task?> GetSeedingDownloads() => - (await _client.TorrentGetAsync(Fields)) - ?.Torrents - ?.Where(x => !string.IsNullOrEmpty(x.HashString)) - .Where(x => x.Status is 5 or 6) - .Cast() - .ToList(); - - /// - public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) - { - return downloads - ? - .Cast() - .Where(x => categories - .Any(cat => cat.Name.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase)) - ) - .Cast() - .ToList(); - } - - public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) - { - return downloads - ?.Cast() - .Where(x => !string.IsNullOrEmpty(x.HashString)) - .Where(x => categories.Any(cat => cat.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase))) - .Cast() - .ToList(); - } - - /// - public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, - HashSet excludedHashes, IReadOnlyList ignoredDownloads) - { - if (downloads?.Count is null or 0) - { - return; - } - - foreach (TorrentInfo download in downloads) - { - if (string.IsNullOrEmpty(download.HashString)) - { - continue; - } - - if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogDebug("skip | download is ignored | {name}", download.Name); - continue; - } - - CleanCategory? category = categoriesToClean - .FirstOrDefault(x => - { - if (download.DownloadDir is null) - { - return false; - } - - return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir)) - .Equals(x.Name, StringComparison.InvariantCultureIgnoreCase); - }); - - if (category is null) - { - continue; - } - - if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true) - { - _logger.LogDebug("skip | download is private | {name}", download.Name); - continue; - } - - ContextProvider.Set("downloadName", download.Name); - ContextProvider.Set("hash", download.HashString); - - TimeSpan seedingTime = TimeSpan.FromSeconds(download.SecondsSeeding ?? 0); - SeedingCheckResult result = ShouldCleanDownload(download.uploadRatio ?? 0, seedingTime, category); - - if (!result.ShouldClean) - { - continue; - } - - await _dryRunInterceptor.InterceptAsync(RemoveDownloadAsync, download.Id); - - _logger.LogInformation( - "download cleaned | {reason} reached | {name}", - result.Reason is CleanReason.MaxRatioReached - ? "MAX_RATIO & MIN_SEED_TIME" - : "MAX_SEED_TIME", - download.Name - ); - - await _eventPublisher.PublishDownloadCleaned(download.uploadRatio ?? 0, seedingTime, category.Name, result.Reason); - } - } - - public override async Task CreateCategoryAsync(string name) - { - await Task.CompletedTask; - } - - public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) - { - if (downloads?.Count is null or 0) - { - return; - } - - if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) - { - _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); - } - - foreach (TorrentInfo download in downloads.Cast()) - { - if (string.IsNullOrEmpty(download.HashString) || string.IsNullOrEmpty(download.Name) || download.DownloadDir == null) - { - continue; - } - - if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) - { - _logger.LogDebug("skip | download is ignored | {name}", download.Name); - continue; - } - - ContextProvider.Set("downloadName", download.Name); - ContextProvider.Set("hash", download.HashString); - - bool hasHardlinks = false; - - if (download.Files is null || download.FileStats is null) - { - _logger.LogDebug("skip | download has no files | {name}", download.Name); - continue; - } - - for (int i = 0; i < download.Files.Length; i++) - { - TransmissionTorrentFiles file = download.Files[i]; - TransmissionTorrentFileStats stats = download.FileStats[i]; - - if (stats.Wanted is null or false || string.IsNullOrEmpty(file.Name)) - { - continue; - } - - string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, file.Name).Split(['\\', '/'])); - - long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); - - if (hardlinkCount < 0) - { - _logger.LogDebug("skip | could not get file properties | {file}", filePath); - hasHardlinks = true; - break; - } - - if (hardlinkCount > 0) - { - hasHardlinks = true; - break; - } - } - - if (hasHardlinks) - { - _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); - continue; - } - - string currentCategory = download.GetCategory(); - string newLocation = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, _downloadCleanerConfig.UnlinkedTargetCategory).Split(['\\', '/'])); - - await _dryRunInterceptor.InterceptAsync(ChangeDownloadLocation, download.Id, newLocation); - - _logger.LogInformation("category changed for {name}", download.Name); - - await _eventPublisher.PublishCategoryChanged(currentCategory, _downloadCleanerConfig.UnlinkedTargetCategory); - - download.DownloadDir = newLocation; - } - } - - [DryRunSafeguard] - protected virtual async Task ChangeDownloadLocation(long downloadId, string newLocation) - { - await _client.TorrentSetLocationAsync([downloadId], newLocation, true); - } - - public override async Task DeleteDownload(string hash) - { - TorrentInfo? torrent = await GetTorrentAsync(hash); - - if (torrent is null) - { - return; - } - - await _client.TorrentRemoveAsync([torrent.Id], true); - } - public override void Dispose() { _client = null; _httpClient?.Dispose(); } - - [DryRunSafeguard] - protected virtual async Task RemoveDownloadAsync(long downloadId) - { - await _client.TorrentRemoveAsync([downloadId], true); - } - - [DryRunSafeguard] - protected virtual async Task SetUnwantedFiles(long downloadId, long[] unwantedFiles) - { - await _client.TorrentSetAsync(new TorrentSettings - { - Ids = [downloadId], - FilesUnwanted = unwantedFiles, - }); - } - - private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent) - { - (bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent); - - if (result.ShouldRemove) - { - return result; - } - - return await CheckIfStuck(torrent); - } - - private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download) - { - if (_queueCleanerConfig.SlowMaxStrikes is 0) - { - return (false, DeleteReason.None); - } - - if (download.Status is not 4) - { - // not in downloading state - return (false, DeleteReason.None); - } - - if (download.RateDownload <= 0) - { - return (false, DeleteReason.None); - } - - if (_queueCleanerConfig.SlowIgnorePrivate && download.IsPrivate is true) - { - // ignore private trackers - _logger.LogDebug("skip slow check | download is private | {name}", download.Name); - return (false, DeleteReason.None); - } - - if (download.TotalSize > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) - { - _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); - return (false, DeleteReason.None); - } - - ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize; - ByteSize currentSpeed = new ByteSize(download.RateDownload ?? long.MaxValue); - SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime); - SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta ?? 0); - - return await CheckIfSlow( - download.HashString!, - download.Name!, - minSpeed, - currentSpeed, - maxTime, - currentTime - ); - } - - private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo download) - { - if (_queueCleanerConfig.StalledMaxStrikes is 0) - { - return (false, DeleteReason.None); - } - - if (download.Status is not 4) - { - // not in downloading state - return (false, DeleteReason.None); - } - - if (download.RateDownload > 0 || download.Eta > 0) - { - return (false, DeleteReason.None); - } - - if (_queueCleanerConfig.StalledIgnorePrivate && (download.IsPrivate ?? false)) - { - // ignore private trackers - _logger.LogDebug("skip stalled check | download is private | {name}", download.Name); - return (false, DeleteReason.None); - } - - ResetStalledStrikesOnProgress(download.HashString!, download.DownloadedEver ?? 0); - - return (await _striker.StrikeAndCheckLimit(download.HashString!, download.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); - } private async Task GetTorrentAsync(string hash) { - if (_client == null) - { - throw new InvalidOperationException("Transmission client is not initialized"); - } - return (await _client.TorrentGetAsync(Fields, hash)) ?.Torrents ?.FirstOrDefault(); diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceDC.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceDC.cs new file mode 100644 index 00000000..cd9a09e7 --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceDC.cs @@ -0,0 +1,237 @@ +using Common.Attributes; +using Common.Configuration.DownloadCleaner; +using Data.Enums; +using Infrastructure.Extensions; +using Infrastructure.Verticals.Context; +using Microsoft.Extensions.Logging; +using Transmission.API.RPC.Entity; + +namespace Infrastructure.Verticals.DownloadClient.Transmission; + +public partial class TransmissionService +{ + public override async Task?> GetSeedingDownloads() => + (await _client.TorrentGetAsync(Fields)) + ?.Torrents + ?.Where(x => !string.IsNullOrEmpty(x.HashString)) + .Where(x => x.Status is 5 or 6) + .Cast() + .ToList(); + + /// + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) + { + return downloads + ? + .Cast() + .Where(x => categories + .Any(cat => cat.Name.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase)) + ) + .Cast() + .ToList(); + } + + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) + { + return downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.HashString)) + .Where(x => categories.Any(cat => cat.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + } + + /// + public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, + HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + + foreach (TorrentInfo download in downloads) + { + if (string.IsNullOrEmpty(download.HashString)) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + continue; + } + + CleanCategory? category = categoriesToClean + .FirstOrDefault(x => + { + if (download.DownloadDir is null) + { + return false; + } + + return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir)) + .Equals(x.Name, StringComparison.InvariantCultureIgnoreCase); + }); + + if (category is null) + { + continue; + } + + if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true) + { + _logger.LogDebug("skip | download is private | {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.HashString); + + TimeSpan seedingTime = TimeSpan.FromSeconds(download.SecondsSeeding ?? 0); + SeedingCheckResult result = ShouldCleanDownload(download.uploadRatio ?? 0, seedingTime, category); + + if (!result.ShouldClean) + { + continue; + } + + await _dryRunInterceptor.InterceptAsync(RemoveDownloadAsync, download.Id); + + _logger.LogInformation( + "download cleaned | {reason} reached | {name}", + result.Reason is CleanReason.MaxRatioReached + ? "MAX_RATIO & MIN_SEED_TIME" + : "MAX_SEED_TIME", + download.Name + ); + + await _eventPublisher.PublishDownloadCleaned(download.uploadRatio ?? 0, seedingTime, category.Name, result.Reason); + } + } + + public override async Task CreateCategoryAsync(string name) + { + await Task.CompletedTask; + } + + public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + + if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) + { + _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); + } + + foreach (TorrentInfo download in downloads.Cast()) + { + if (string.IsNullOrEmpty(download.HashString) || string.IsNullOrEmpty(download.Name) || download.DownloadDir == null) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.HashString); + + bool hasHardlinks = false; + + if (download.Files is null || download.FileStats is null) + { + _logger.LogDebug("skip | download has no files | {name}", download.Name); + continue; + } + + for (int i = 0; i < download.Files.Length; i++) + { + TransmissionTorrentFiles file = download.Files[i]; + TransmissionTorrentFileStats stats = download.FileStats[i]; + + if (stats.Wanted is null or false || string.IsNullOrEmpty(file.Name)) + { + continue; + } + + string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, file.Name).Split(['\\', '/'])); + + long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); + + if (hardlinkCount < 0) + { + _logger.LogDebug("skip | could not get file properties | {file}", filePath); + hasHardlinks = true; + break; + } + + if (hardlinkCount > 0) + { + hasHardlinks = true; + break; + } + } + + if (hasHardlinks) + { + _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); + continue; + } + + string currentCategory = download.GetCategory(); + string newLocation = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, _downloadCleanerConfig.UnlinkedTargetCategory).Split(['\\', '/'])); + + await _dryRunInterceptor.InterceptAsync(ChangeDownloadLocation, download.Id, newLocation); + + _logger.LogInformation("category changed for {name}", download.Name); + + await _eventPublisher.PublishCategoryChanged(currentCategory, _downloadCleanerConfig.UnlinkedTargetCategory); + + download.DownloadDir = newLocation; + } + } + + [DryRunSafeguard] + protected virtual async Task ChangeDownloadLocation(long downloadId, string newLocation) + { + await _client.TorrentSetLocationAsync([downloadId], newLocation, true); + } + + public override async Task DeleteDownload(string hash) + { + TorrentInfo? torrent = await GetTorrentAsync(hash); + + if (torrent is null) + { + return; + } + + await _client.TorrentRemoveAsync([torrent.Id], true); + } + + [DryRunSafeguard] + protected virtual async Task RemoveDownloadAsync(long downloadId) + { + await _client.TorrentRemoveAsync([downloadId], true); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceQC.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceQC.cs new file mode 100644 index 00000000..2991ad9b --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionServiceQC.cs @@ -0,0 +1,255 @@ +using System.Collections.Concurrent; +using System.Text.RegularExpressions; +using Common.Attributes; +using Common.Configuration.QueueCleaner; +using Common.CustomDataTypes; +using Data.Enums; +using Infrastructure.Extensions; +using Infrastructure.Verticals.Context; +using Microsoft.Extensions.Logging; +using Transmission.API.RPC.Arguments; +using Transmission.API.RPC.Entity; + +namespace Infrastructure.Verticals.DownloadClient.Transmission; + +public partial class TransmissionService +{ + /// + public override async Task ShouldRemoveFromArrQueueAsync(string hash, + IReadOnlyList ignoredDownloads) + { + if (_client == null) + { + throw new InvalidOperationException("Transmission client is not initialized"); + } + + DownloadCheckResult result = new(); + TorrentInfo? download = await GetTorrentAsync(hash); + + if (download is null) + { + _logger.LogDebug("failed to find torrent {hash} in the download client", hash); + return result; + } + + result.Found = true; + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + return result; + } + + bool shouldRemove = download.FileStats?.Length > 0; + bool isPrivate = download.IsPrivate ?? false; + result.IsPrivate = isPrivate; + + foreach (TransmissionTorrentFileStats stats in download.FileStats ?? []) + { + if (!stats.Wanted.HasValue) + { + // if any files stats are missing, do not remove + shouldRemove = false; + } + + if (stats.Wanted.HasValue && stats.Wanted.Value) + { + // if any files are wanted, do not remove + shouldRemove = false; + } + } + + if (shouldRemove) + { + // remove if all files are unwanted + result.ShouldRemove = true; + result.DeleteReason = DeleteReason.AllFilesSkipped; + return result; + } + + // remove if download is stuck + (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, isPrivate); + + return result; + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> BlockUnwantedFilesAsync( + TorrentInfo download, + bool isPrivate + ) + { + if (!_queueCleanerConfig.ContentBlocker.Enabled) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.ContentBlocker.IgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip unwanted files check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (download.Files is null) + { + _logger.LogDebug("failed to find files for {name}", download.Name); + return (false, DeleteReason.None); + } + + InstanceType instanceType = (InstanceType)ContextProvider.Get(nameof(InstanceType)); + BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType); + ConcurrentBag patterns = _blocklistProvider.GetPatterns(instanceType); + ConcurrentBag regexes = _blocklistProvider.GetRegexes(instanceType); + + List unwantedFiles = []; + long totalFiles = 0; + long totalUnwantedFiles = 0; + + for (int i = 0; i < download.Files.Length; i++) + { + if (download.FileStats?[i].Wanted == null) + { + continue; + } + + totalFiles++; + + if (!download.FileStats[i].Wanted.Value) + { + totalUnwantedFiles++; + continue; + } + + if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes)) + { + continue; + } + + _logger.LogInformation("unwanted file found | {file}", download.Files[i].Name); + unwantedFiles.Add(i); + totalUnwantedFiles++; + } + + if (unwantedFiles.Count is 0) + { + return (false, DeleteReason.None); + } + + if (totalUnwantedFiles == totalFiles) + { + // Skip marking files as unwanted. The download will be removed completely. + return (true, DeleteReason.AllFilesBlocked); + } + + _logger.LogTrace("marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name); + + await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray()); + + return (false, DeleteReason.None); + } + + [DryRunSafeguard] + protected virtual async Task SetUnwantedFiles(long downloadId, long[] unwantedFiles) + { + await _client.TorrentSetAsync(new TorrentSettings + { + Ids = [downloadId], + FilesUnwanted = unwantedFiles, + }); + } + + private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo download, bool isPrivate) + { + (bool ShouldRemove, DeleteReason Reason) result = await BlockUnwantedFilesAsync(download, isPrivate); + + if (result.ShouldRemove) + { + return result; + } + + result = await CheckIfSlow(download); + + if (result.ShouldRemove) + { + return result; + } + + return await CheckIfStuck(download); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download) + { + if (_queueCleanerConfig.Slow.MaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (download.Status is not 4) + { + // not in downloading state + return (false, DeleteReason.None); + } + + if (download.RateDownload <= 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.Slow.IgnorePrivate && download.IsPrivate is true) + { + // ignore private trackers + _logger.LogDebug("skip slow check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (download.TotalSize > (_queueCleanerConfig.Slow.IgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) + { + _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); + return (false, DeleteReason.None); + } + + ByteSize minSpeed = _queueCleanerConfig.Slow.MinSpeedByteSize; + ByteSize currentSpeed = new ByteSize(download.RateDownload ?? long.MaxValue); + SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.Slow.MaxTime); + SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta ?? 0); + + return await CheckIfSlow( + download.HashString!, + download.Name!, + minSpeed, + currentSpeed, + maxTime, + currentTime + ); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo download) + { + if (_queueCleanerConfig.Stalled.MaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (download.Status is not 4) + { + // not in downloading state + return (false, DeleteReason.None); + } + + if (download.RateDownload > 0 || download.Eta > 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.Stalled.IgnorePrivate && (download.IsPrivate ?? false)) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + ResetStalledStrikesOnProgress(download.HashString!, download.DownloadedEver ?? 0); + + return (await _striker.StrikeAndCheckLimit(download.HashString!, download.Name!, _queueCleanerConfig.Stalled.MaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index 9e91aa10..ee04dd96 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -20,6 +20,10 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using LogContext = Serilog.Context.LogContext; using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Text.RegularExpressions; +using Infrastructure.Verticals.ContentBlocker; +using BlocklistSettings = Common.Configuration.QueueCleaner.BlocklistSettings; namespace Infrastructure.Verticals.QueueCleaner; @@ -28,6 +32,7 @@ public sealed class QueueCleaner : GenericHandler private readonly QueueCleanerConfig _config; private readonly IIgnoredDownloadsService _ignoredDownloadsService; private readonly IDownloadClientFactory _downloadClientFactory; + private readonly BlocklistProvider _blocklistProvider; public QueueCleaner( ILogger logger, @@ -38,7 +43,8 @@ public sealed class QueueCleaner : GenericHandler ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, IIgnoredDownloadsService ignoredDownloadsService, - IDownloadClientFactory downloadClientFactory + IDownloadClientFactory downloadClientFactory, + BlocklistProvider blocklistProvider ) : base( logger, cache, messageBus, arrClientFactory, arrArrQueueIterator, downloadServiceFactory @@ -46,7 +52,8 @@ public sealed class QueueCleaner : GenericHandler { _ignoredDownloadsService = ignoredDownloadsService; _downloadClientFactory = downloadClientFactory; - + _blocklistProvider = blocklistProvider; + _config = configManager.GetConfiguration(); _downloadClientConfig = configManager.GetConfiguration(); _sonarrConfig = configManager.GetConfiguration(); @@ -54,6 +61,26 @@ public sealed class QueueCleaner : GenericHandler _lidarrConfig = configManager.GetConfiguration(); } + public override async Task ExecuteAsync() + { + if (_downloadClientConfig.Clients.Count is 0) + { + _logger.LogWarning("No download clients configured"); + return; + } + + bool blocklistIsConfigured = _sonarrConfig.Enabled && !string.IsNullOrEmpty(_config.ContentBlocker.Sonarr.BlocklistPath) || + _radarrConfig.Enabled && !string.IsNullOrEmpty(_config.ContentBlocker.Radarr.BlocklistPath) || + _lidarrConfig.Enabled && !string.IsNullOrEmpty(_config.ContentBlocker.Lidarr.BlocklistPath); + + if (_config.ContentBlocker.Enabled && blocklistIsConfigured) + { + await _blocklistProvider.LoadBlocklistsAsync(); + } + + await base.ExecuteAsync(); + } + protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType, ArrConfig config) { IReadOnlyList ignoredDownloads = await _ignoredDownloadsService.GetIgnoredDownloadsAsync(); @@ -112,41 +139,42 @@ public sealed class QueueCleaner : GenericHandler if (_downloadClientConfig.Clients.Count == 0) { _logger.LogWarning("skip | no download clients configured | {title}", record.Title); - continue; } - - // Check each download client for the download item - foreach (var downloadService in _downloadClientFactory.GetAllEnabledClients()) + else { - try + // Check each download client for the download item + foreach (var downloadService in _downloadClientFactory.GetAllEnabledClients()) { - // stalled download check - var result = await downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads); - if (result.Found) + try { - downloadCheckResult = result; - // Add client ID to context for tracking - ContextProvider.Set("ClientId", downloadService.GetClientId()); - break; + // stalled download check + DownloadCheckResult result = await downloadService + .ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads); + + if (result.Found) + { + downloadCheckResult = result; + break; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error checking download {id} with download client {clientId}", + record.DownloadId, downloadService.GetClientId()); } } - catch (Exception ex) - { - _logger.LogError(ex, "Error checking download {id} with download client {clientId}", - record.DownloadId, downloadService.GetClientId()); - } - } - if (!downloadCheckResult.Found) - { - _logger.LogWarning("skip | download not found {title}", record.Title); + if (!downloadCheckResult.Found) + { + _logger.LogWarning("skip | download not found {title}", record.Title); + } } } // failed import check - bool shouldRemoveFromArr = await arrClient.ShouldRemoveFromQueue(instanceType, record, downloadCheckResult.IsPrivate, config.FailedImportMaxStrikes); + bool shouldRemoveFromArr = await arrClient.ShouldRemoveFromQueue(instanceType, record, downloadCheckResult.IsPrivate, _config.FailedImport.MaxStrikes); DeleteReason deleteReason = downloadCheckResult.ShouldRemove ? downloadCheckResult.DeleteReason : DeleteReason.FailedImport; - + if (!shouldRemoveFromArr && !downloadCheckResult.ShouldRemove) { _logger.LogInformation("skip | {title}", record.Title); @@ -159,14 +187,20 @@ public sealed class QueueCleaner : GenericHandler { bool isStalledWithoutPruneFlag = downloadCheckResult.DeleteReason is DeleteReason.Stalled && - !_config.StalledDeletePrivate; + !_config.Stalled.DeletePrivate; bool isSlowWithoutPruneFlag = downloadCheckResult.DeleteReason is DeleteReason.SlowSpeed or DeleteReason.SlowTime && - !_config.SlowDeletePrivate; + !_config.Slow.DeletePrivate; + + bool isContentBlockerWithoutPruneFlag = + deleteReason is DeleteReason.AllFilesBlocked && + !_config.ContentBlocker.DeletePrivate; - bool shouldKeepDueToDeleteRules = downloadCheckResult.ShouldRemove && (isStalledWithoutPruneFlag || isSlowWithoutPruneFlag); - bool shouldKeepDueToImportRules = shouldRemoveFromArr && !_config.FailedImportDeletePrivate; + bool shouldKeepDueToDeleteRules = downloadCheckResult.ShouldRemove && + (isStalledWithoutPruneFlag || isSlowWithoutPruneFlag || isContentBlockerWithoutPruneFlag); + + bool shouldKeepDueToImportRules = shouldRemoveFromArr && !_config.FailedImport.DeletePrivate; if (shouldKeepDueToDeleteRules || shouldKeepDueToImportRules) { diff --git a/code/UI/src/app/settings/queue-cleaner/queue-cleaner-settings.component.ts b/code/UI/src/app/settings/queue-cleaner/queue-cleaner-settings.component.ts index ca6c842a..8e8c61ed 100644 --- a/code/UI/src/app/settings/queue-cleaner/queue-cleaner-settings.component.ts +++ b/code/UI/src/app/settings/queue-cleaner/queue-cleaner-settings.component.ts @@ -3,7 +3,7 @@ import { CommonModule } from "@angular/common"; import { FormBuilder, FormGroup, ReactiveFormsModule, Validators } from "@angular/forms"; import { Subject, takeUntil } from "rxjs"; import { QueueCleanerConfigStore } from "./queue-cleaner-config.store"; -import { QueueCleanerConfig, ScheduleUnit } from "../../shared/models/queue-cleaner-config.model"; +import { QueueCleanerConfig, ScheduleUnit, BlocklistType, FailedImportConfig, StalledConfig, SlowConfig, ContentBlockerConfig } from "../../shared/models/queue-cleaner-config.model"; import { SettingsCardComponent } from "../components/settings-card/settings-card.component"; import { ByteSizeInputComponent } from "../../shared/components/byte-size-input/byte-size-input.component"; @@ -18,6 +18,7 @@ import { SelectButtonModule } from "primeng/selectbutton"; import { ChipsModule } from "primeng/chips"; import { ToastModule } from "primeng/toast"; import { MessageService } from "primeng/api"; +import { DropdownModule } from "primeng/dropdown"; @Component({ selector: "app-queue-cleaner-settings", @@ -25,7 +26,6 @@ import { MessageService } from "primeng/api"; imports: [ CommonModule, ReactiveFormsModule, - SettingsCardComponent, CardModule, InputTextModule, CheckboxModule, @@ -36,6 +36,7 @@ import { MessageService } from "primeng/api"; ChipsModule, ToastModule, ByteSizeInputComponent, + DropdownModule, ], providers: [QueueCleanerConfigStore, MessageService], templateUrl: "./queue-cleaner-settings.component.html", @@ -80,29 +81,52 @@ export class QueueCleanerSettingsComponent implements OnDestroy { runSequentially: [{value: false, disabled: true}], ignoredDownloadsPath: [{value: "", disabled: true}], - // Failed Import settings - failedImportMaxStrikes: [0, [Validators.min(0)]], - failedImportIgnorePrivate: [{value: false, disabled: true}], - failedImportDeletePrivate: [{value: false, disabled: true}], - failedImportIgnorePatterns: [{value: [], disabled: true}], + // Failed Import settings - nested group + failedImport: this.formBuilder.group({ + maxStrikes: [0, [Validators.min(0)]], + ignorePrivate: [{value: false, disabled: true}], + deletePrivate: [{value: false, disabled: true}], + ignorePatterns: [{value: [], disabled: true}], + }), - // Stalled settings - stalledMaxStrikes: [0, [Validators.min(0)]], - stalledResetStrikesOnProgress: [{value: false, disabled: true}], - stalledIgnorePrivate: [{value: false, disabled: true}], - stalledDeletePrivate: [{value: false, disabled: true}], + // Stalled settings - nested group + stalled: this.formBuilder.group({ + maxStrikes: [0, [Validators.min(0)]], + resetStrikesOnProgress: [{value: false, disabled: true}], + ignorePrivate: [{value: false, disabled: true}], + deletePrivate: [{value: false, disabled: true}], + downloadingMetadataMaxStrikes: [0, [Validators.min(0)]], + }), - // Downloading Metadata settings - downloadingMetadataMaxStrikes: [0, [Validators.min(0)]], - - // Slow Download settings - slowMaxStrikes: [0, [Validators.min(0)]], - slowResetStrikesOnProgress: [{value: false, disabled: true}], - slowIgnorePrivate: [{value: false, disabled: true}], - slowDeletePrivate: [{value: false, disabled: true}], - slowMinSpeed: [{value: "", disabled: true}], - slowMaxTime: [{value: 0, disabled: true}], - slowIgnoreAboveSize: [{value: "", disabled: true}], + // Slow Download settings - nested group + slow: this.formBuilder.group({ + maxStrikes: [0, [Validators.min(0)]], + resetStrikesOnProgress: [{value: false, disabled: true}], + ignorePrivate: [{value: false, disabled: true}], + deletePrivate: [{value: false, disabled: true}], + minSpeed: [{value: "", disabled: true}], + maxTime: [{value: 0, disabled: true}], + ignoreAboveSize: [{value: "", disabled: true}], + }), + + // Content Blocker settings - nested group + contentBlocker: this.formBuilder.group({ + enabled: [{value: false, disabled: true}], + ignorePrivate: [{value: false, disabled: true}], + deletePrivate: [{value: false, disabled: true}], + sonarrBlocklist: this.formBuilder.group({ + path: [{value: "", disabled: true}], + type: [{value: BlocklistType.Blacklist, disabled: true}], + }), + radarrBlocklist: this.formBuilder.group({ + path: [{value: "", disabled: true}], + type: [{value: BlocklistType.Blacklist, disabled: true}], + }), + lidarrBlocklist: this.formBuilder.group({ + path: [{value: "", disabled: true}], + type: [{value: BlocklistType.Blacklist, disabled: true}], + }), + }), }); // Set up form control value change subscriptions to manage dependent control states @@ -112,44 +136,97 @@ export class QueueCleanerSettingsComponent implements OnDestroy { effect(() => { const config = this.queueCleanerConfig(); if (config) { - // Update the form with the current configuration - this.queueCleanerForm.patchValue({ + // Build form values for the nested configuration structure + const formValues: any = { enabled: config.enabled, runSequentially: config.runSequentially, ignoredDownloadsPath: config.ignoredDownloadsPath, - - // Failed Import settings - failedImportMaxStrikes: config.failedImportMaxStrikes, - failedImportIgnorePrivate: config.failedImportIgnorePrivate, - failedImportDeletePrivate: config.failedImportDeletePrivate, - failedImportIgnorePatterns: config.failedImportIgnorePatterns, - - // Stalled settings - stalledMaxStrikes: config.stalledMaxStrikes, - stalledResetStrikesOnProgress: config.stalledResetStrikesOnProgress, - stalledIgnorePrivate: config.stalledIgnorePrivate, - stalledDeletePrivate: config.stalledDeletePrivate, - - // Downloading Metadata settings - downloadingMetadataMaxStrikes: config.downloadingMetadataMaxStrikes, - - // Slow Download settings - slowMaxStrikes: config.slowMaxStrikes, - slowResetStrikesOnProgress: config.slowResetStrikesOnProgress, - slowIgnorePrivate: config.slowIgnorePrivate, - slowDeletePrivate: config.slowDeletePrivate, - slowMinSpeed: config.slowMinSpeed, - slowMaxTime: config.slowMaxTime, - slowIgnoreAboveSize: config.slowIgnoreAboveSize, - }); - - // Update job schedule if it exists + }; + + // Add jobSchedule if it exists if (config.jobSchedule) { - this.queueCleanerForm.get("jobSchedule")?.patchValue({ + formValues.jobSchedule = { every: config.jobSchedule.every, type: config.jobSchedule.type, - }); + }; } + + // Add Failed Import settings + if (config.failedImport) { + formValues.failedImport = { + maxStrikes: config.failedImport.maxStrikes, + ignorePrivate: config.failedImport.ignorePrivate, + deletePrivate: config.failedImport.deletePrivate, + ignorePatterns: config.failedImport.ignorePatterns, + }; + } else if (config.failedImportMaxStrikes !== undefined) { + // Fall back to legacy flat properties if nested ones don't exist + formValues.failedImport = { + maxStrikes: config.failedImportMaxStrikes, + ignorePrivate: config.failedImportIgnorePrivate, + deletePrivate: config.failedImportDeletePrivate, + ignorePatterns: config.failedImportIgnorePatterns || [], + }; + } + + // Add Stalled settings + if (config.stalled) { + formValues.stalled = { + maxStrikes: config.stalled.maxStrikes, + resetStrikesOnProgress: config.stalled.resetStrikesOnProgress, + ignorePrivate: config.stalled.ignorePrivate, + deletePrivate: config.stalled.deletePrivate, + downloadingMetadataMaxStrikes: config.stalled.downloadingMetadataMaxStrikes, + }; + } else if (config.stalledMaxStrikes !== undefined) { + // Fall back to legacy flat properties if nested ones don't exist + formValues.stalled = { + maxStrikes: config.stalledMaxStrikes, + resetStrikesOnProgress: config.stalledResetStrikesOnProgress, + ignorePrivate: config.stalledIgnorePrivate, + deletePrivate: config.stalledDeletePrivate, + downloadingMetadataMaxStrikes: config.downloadingMetadataMaxStrikes || 0, + }; + } + + // Add Slow Download settings + if (config.slow) { + formValues.slow = { + maxStrikes: config.slow.maxStrikes, + resetStrikesOnProgress: config.slow.resetStrikesOnProgress, + ignorePrivate: config.slow.ignorePrivate, + deletePrivate: config.slow.deletePrivate, + minSpeed: config.slow.minSpeed, + maxTime: config.slow.maxTime, + ignoreAboveSize: config.slow.ignoreAboveSize, + }; + } else if (config.slowMaxStrikes !== undefined) { + // Fall back to legacy flat properties if nested ones don't exist + formValues.slow = { + maxStrikes: config.slowMaxStrikes, + resetStrikesOnProgress: config.slowResetStrikesOnProgress, + ignorePrivate: config.slowIgnorePrivate, + deletePrivate: config.slowDeletePrivate, + minSpeed: config.slowMinSpeed || "", + maxTime: config.slowMaxTime || 0, + ignoreAboveSize: config.slowIgnoreAboveSize || "", + }; + } + + // Add Content Blocker settings + if (config.contentBlocker) { + formValues.contentBlocker = { + enabled: config.contentBlocker.enabled, + ignorePrivate: config.contentBlocker.ignorePrivate, + deletePrivate: config.contentBlocker.deletePrivate, + sonarrBlocklist: config.contentBlocker.sonarrBlocklist, + radarrBlocklist: config.contentBlocker.radarrBlocklist, + lidarrBlocklist: config.contentBlocker.lidarrBlocklist, + }; + } + + // Update the form with the current configuration + this.queueCleanerForm.patchValue(formValues); // Update form control disabled states based on the configuration this.updateFormControlDisabledStates(config); @@ -188,93 +265,84 @@ export class QueueCleanerSettingsComponent implements OnDestroy { * Set up listeners for form control value changes to manage dependent control states */ private setupFormValueChangeListeners(): void { - // Listen for changes on the 'enabled' control - this.queueCleanerForm.get('enabled')?.valueChanges - .pipe(takeUntil(this.destroy$)) - .subscribe((enabled: boolean) => { - this.updateMainControlsState(enabled); - }); + // Listen for changes to the enabled control + this.queueCleanerForm.get('enabled')?.valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(enabled => { + this.updateMainControlsState(enabled); + }); - // Listen for changes on 'failedImportMaxStrikes' control - this.queueCleanerForm.get('failedImportMaxStrikes')?.valueChanges - .pipe(takeUntil(this.destroy$)) - .subscribe((strikes: number) => { - this.updateFailedImportDependentControls(strikes); - }); + // Failed import settings + this.queueCleanerForm.get('failedImport.maxStrikes')?.valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(strikes => { + this.updateFailedImportDependentControls(strikes); + }); - // Listen for changes on 'stalledMaxStrikes' control - this.queueCleanerForm.get('stalledMaxStrikes')?.valueChanges - .pipe(takeUntil(this.destroy$)) - .subscribe((strikes: number) => { - this.updateStalledDependentControls(strikes); - }); + // Stalled settings + this.queueCleanerForm.get('stalled.maxStrikes')?.valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(strikes => { + this.updateStalledDependentControls(strikes); + }); - // Listen for changes on 'slowMaxStrikes' control - this.queueCleanerForm.get('slowMaxStrikes')?.valueChanges - .pipe(takeUntil(this.destroy$)) - .subscribe((strikes: number) => { - this.updateSlowDependentControls(strikes); - }); + // Slow downloads settings + this.queueCleanerForm.get('slow.maxStrikes')?.valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(strikes => { + this.updateSlowDependentControls(strikes); + }); + + // Content blocker settings + this.queueCleanerForm.get('enabled')?.valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(enabled => { + if (enabled) { + this.queueCleanerForm.get('contentBlocker.enabled')?.enable(); + } else { + this.queueCleanerForm.get('contentBlocker.enabled')?.disable(); + } + }); + + // Update content blocker dependent controls when enabled changes + this.queueCleanerForm.get('contentBlocker.enabled')?.valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe(enabled => { + this.updateContentBlockerDependentControls(enabled); + }); } /** * Update form control disabled states based on the configuration */ private updateFormControlDisabledStates(config: QueueCleanerConfig): void { - const enabled = config.enabled; - const options = { onlySelf: true }; + // Update main form controls based on the 'enabled' state + this.updateMainControlsState(config.enabled); - // Job schedule - if (enabled) { - this.queueCleanerForm.get("jobSchedule")?.enable(options); - this.queueCleanerForm.get("runSequentially")?.enable(options); - this.queueCleanerForm.get("ignoredDownloadsPath")?.enable(options); - } else { - this.queueCleanerForm.get("jobSchedule")?.disable(options); - this.queueCleanerForm.get("runSequentially")?.disable(options); - this.queueCleanerForm.get("ignoredDownloadsPath")?.disable(options); + // Check if failed import strikes are set and update dependent controls + if (config.failedImport?.maxStrikes !== undefined) { + this.updateFailedImportDependentControls(config.failedImport.maxStrikes); + } else if (config.failedImportMaxStrikes !== undefined) { // Fall back to legacy property if needed + this.updateFailedImportDependentControls(config.failedImportMaxStrikes); } - // Failed Import settings - const failedImportEnabled = enabled && config.failedImportMaxStrikes >= 3; - if (failedImportEnabled) { - this.queueCleanerForm.get("failedImportIgnorePrivate")?.enable(options); - this.queueCleanerForm.get("failedImportDeletePrivate")?.enable(options); - this.queueCleanerForm.get("failedImportIgnorePatterns")?.enable(options); - } else { - this.queueCleanerForm.get("failedImportIgnorePrivate")?.disable(options); - this.queueCleanerForm.get("failedImportDeletePrivate")?.disable(options); - this.queueCleanerForm.get("failedImportIgnorePatterns")?.disable(options); + // Check if stalled strikes are set and update dependent controls + if (config.stalled?.maxStrikes !== undefined) { + this.updateStalledDependentControls(config.stalled.maxStrikes); + } else if (config.stalledMaxStrikes !== undefined) { // Fall back to legacy property if needed + this.updateStalledDependentControls(config.stalledMaxStrikes); } - // Stalled settings - const stalledEnabled = enabled && config.stalledMaxStrikes >= 3; - if (stalledEnabled) { - this.queueCleanerForm.get("stalledResetStrikesOnProgress")?.enable(options); - this.queueCleanerForm.get("stalledIgnorePrivate")?.enable(options); - this.queueCleanerForm.get("stalledDeletePrivate")?.enable(options); - } else { - this.queueCleanerForm.get("stalledResetStrikesOnProgress")?.disable(options); - this.queueCleanerForm.get("stalledIgnorePrivate")?.disable(options); - this.queueCleanerForm.get("stalledDeletePrivate")?.disable(options); + // Check if slow download strikes are set and update dependent controls + if (config.slow?.maxStrikes !== undefined) { + this.updateSlowDependentControls(config.slow.maxStrikes); + } else if (config.slowMaxStrikes !== undefined) { // Fall back to legacy property if needed + this.updateSlowDependentControls(config.slowMaxStrikes); } - - // Slow Download settings - const slowEnabled = enabled && config.slowMaxStrikes >= 3; - if (slowEnabled) { - this.queueCleanerForm.get("slowResetStrikesOnProgress")?.enable(options); - this.queueCleanerForm.get("slowIgnorePrivate")?.enable(options); - this.queueCleanerForm.get("slowDeletePrivate")?.enable(options); - this.queueCleanerForm.get("slowMinSpeed")?.enable(options); - this.queueCleanerForm.get("slowMaxTime")?.enable(options); - this.queueCleanerForm.get("slowIgnoreAboveSize")?.enable(options); - } else { - this.queueCleanerForm.get("slowResetStrikesOnProgress")?.disable(options); - this.queueCleanerForm.get("slowIgnorePrivate")?.disable(options); - this.queueCleanerForm.get("slowDeletePrivate")?.disable(options); - this.queueCleanerForm.get("slowMinSpeed")?.disable(options); - this.queueCleanerForm.get("slowMaxTime")?.disable(options); - this.queueCleanerForm.get("slowIgnoreAboveSize")?.disable(options); + + // Check if content blocker is enabled and update dependent controls + if (config.contentBlocker?.enabled !== undefined) { + this.updateContentBlockerDependentControls(config.contentBlocker.enabled); } } @@ -283,30 +351,43 @@ export class QueueCleanerSettingsComponent implements OnDestroy { */ private updateMainControlsState(enabled: boolean): void { const options = { onlySelf: true }; - + if (enabled) { this.queueCleanerForm.get('jobSchedule')?.enable(options); this.queueCleanerForm.get('runSequentially')?.enable(options); this.queueCleanerForm.get('ignoredDownloadsPath')?.enable(options); + this.queueCleanerForm.get('contentBlocker')?.get('enabled')?.enable(options); } else { this.queueCleanerForm.get('jobSchedule')?.disable(options); this.queueCleanerForm.get('runSequentially')?.disable(options); this.queueCleanerForm.get('ignoredDownloadsPath')?.disable(options); + this.queueCleanerForm.get('contentBlocker')?.get('enabled')?.disable(options); + this.updateContentBlockerDependentControls(false); } } /** - * Update the state of Failed Import dependent controls based on the 'failedImportMaxStrikes' value + * Update the state of Failed Import dependent controls based on the 'maxStrikes' value */ private updateFailedImportDependentControls(strikes: number): void { const enable = strikes >= 3; const options = { onlySelf: true }; if (enable) { + this.queueCleanerForm.get('failedImport')?.get('ignorePrivate')?.enable(options); + this.queueCleanerForm.get('failedImport')?.get('deletePrivate')?.enable(options); + this.queueCleanerForm.get('failedImport')?.get('ignorePatterns')?.enable(options); + + // Also enable legacy controls if they exist for backward compatibility this.queueCleanerForm.get('failedImportIgnorePrivate')?.enable(options); this.queueCleanerForm.get('failedImportDeletePrivate')?.enable(options); this.queueCleanerForm.get('failedImportIgnorePatterns')?.enable(options); } else { + this.queueCleanerForm.get('failedImport')?.get('ignorePrivate')?.disable(options); + this.queueCleanerForm.get('failedImport')?.get('deletePrivate')?.disable(options); + this.queueCleanerForm.get('failedImport')?.get('ignorePatterns')?.disable(options); + + // Also disable legacy controls if they exist this.queueCleanerForm.get('failedImportIgnorePrivate')?.disable(options); this.queueCleanerForm.get('failedImportDeletePrivate')?.disable(options); this.queueCleanerForm.get('failedImportIgnorePatterns')?.disable(options); @@ -314,17 +395,27 @@ export class QueueCleanerSettingsComponent implements OnDestroy { } /** - * Update the state of Stalled dependent controls based on the 'stalledMaxStrikes' value + * Update the state of Stalled dependent controls based on the 'maxStrikes' value */ private updateStalledDependentControls(strikes: number): void { const enable = strikes >= 3; const options = { onlySelf: true }; if (enable) { + this.queueCleanerForm.get('stalled')?.get('resetStrikesOnProgress')?.enable(options); + this.queueCleanerForm.get('stalled')?.get('ignorePrivate')?.enable(options); + this.queueCleanerForm.get('stalled')?.get('deletePrivate')?.enable(options); + + // Also enable legacy controls if they exist for backward compatibility this.queueCleanerForm.get('stalledResetStrikesOnProgress')?.enable(options); this.queueCleanerForm.get('stalledIgnorePrivate')?.enable(options); this.queueCleanerForm.get('stalledDeletePrivate')?.enable(options); } else { + this.queueCleanerForm.get('stalled')?.get('resetStrikesOnProgress')?.disable(options); + this.queueCleanerForm.get('stalled')?.get('ignorePrivate')?.disable(options); + this.queueCleanerForm.get('stalled')?.get('deletePrivate')?.disable(options); + + // Also disable legacy controls if they exist this.queueCleanerForm.get('stalledResetStrikesOnProgress')?.disable(options); this.queueCleanerForm.get('stalledIgnorePrivate')?.disable(options); this.queueCleanerForm.get('stalledDeletePrivate')?.disable(options); @@ -332,13 +423,21 @@ export class QueueCleanerSettingsComponent implements OnDestroy { } /** - * Update the state of Slow Download dependent controls based on the 'slowMaxStrikes' value + * Update the state of Slow Download dependent controls based on the 'maxStrikes' value */ private updateSlowDependentControls(strikes: number): void { const enable = strikes >= 3; const options = { onlySelf: true }; if (enable) { + this.queueCleanerForm.get('slow')?.get('resetStrikesOnProgress')?.enable(options); + this.queueCleanerForm.get('slow')?.get('ignorePrivate')?.enable(options); + this.queueCleanerForm.get('slow')?.get('deletePrivate')?.enable(options); + this.queueCleanerForm.get('slow')?.get('minSpeed')?.enable(options); + this.queueCleanerForm.get('slow')?.get('maxTime')?.enable(options); + this.queueCleanerForm.get('slow')?.get('ignoreAboveSize')?.enable(options); + + // Also enable legacy controls if they exist for backward compatibility this.queueCleanerForm.get('slowResetStrikesOnProgress')?.enable(options); this.queueCleanerForm.get('slowIgnorePrivate')?.enable(options); this.queueCleanerForm.get('slowDeletePrivate')?.enable(options); @@ -346,6 +445,14 @@ export class QueueCleanerSettingsComponent implements OnDestroy { this.queueCleanerForm.get('slowMaxTime')?.enable(options); this.queueCleanerForm.get('slowIgnoreAboveSize')?.enable(options); } else { + this.queueCleanerForm.get('slow')?.get('resetStrikesOnProgress')?.disable(options); + this.queueCleanerForm.get('slow')?.get('ignorePrivate')?.disable(options); + this.queueCleanerForm.get('slow')?.get('deletePrivate')?.disable(options); + this.queueCleanerForm.get('slow')?.get('minSpeed')?.disable(options); + this.queueCleanerForm.get('slow')?.get('maxTime')?.disable(options); + this.queueCleanerForm.get('slow')?.get('ignoreAboveSize')?.disable(options); + + // Also disable legacy controls if they exist this.queueCleanerForm.get('slowResetStrikesOnProgress')?.disable(options); this.queueCleanerForm.get('slowIgnorePrivate')?.disable(options); this.queueCleanerForm.get('slowDeletePrivate')?.disable(options); @@ -354,6 +461,47 @@ export class QueueCleanerSettingsComponent implements OnDestroy { this.queueCleanerForm.get('slowIgnoreAboveSize')?.disable(options); } } + + /** + * Update the state of Content Blocker dependent controls based on the 'enabled' value + */ + private updateContentBlockerDependentControls(enabled: boolean): void { + const options = { onlySelf: true }; + + if (enabled) { + // Enable blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('ignorePrivate')?.enable(options); + this.queueCleanerForm.get('contentBlocker')?.get('deletePrivate')?.enable(options); + + // Enable Sonarr blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('sonarrBlocklist')?.get('path')?.enable(options); + this.queueCleanerForm.get('contentBlocker')?.get('sonarrBlocklist')?.get('type')?.enable(options); + + // Enable Radarr blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('radarrBlocklist')?.get('path')?.enable(options); + this.queueCleanerForm.get('contentBlocker')?.get('radarrBlocklist')?.get('type')?.enable(options); + + // Enable Lidarr blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('lidarrBlocklist')?.get('path')?.enable(options); + this.queueCleanerForm.get('contentBlocker')?.get('lidarrBlocklist')?.get('type')?.enable(options); + } else { + // Disable blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('ignorePrivate')?.disable(options); + this.queueCleanerForm.get('contentBlocker')?.get('deletePrivate')?.disable(options); + + // Disable Sonarr blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('sonarrBlocklist')?.get('path')?.disable(options); + this.queueCleanerForm.get('contentBlocker')?.get('sonarrBlocklist')?.get('type')?.disable(options); + + // Disable Radarr blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('radarrBlocklist')?.get('path')?.disable(options); + this.queueCleanerForm.get('contentBlocker')?.get('radarrBlocklist')?.get('type')?.disable(options); + + // Disable Lidarr blocklist settings + this.queueCleanerForm.get('contentBlocker')?.get('lidarrBlocklist')?.get('path')?.disable(options); + this.queueCleanerForm.get('contentBlocker')?.get('lidarrBlocklist')?.get('type')?.disable(options); + } + } /** * Save the queue cleaner configuration @@ -374,7 +522,7 @@ export class QueueCleanerSettingsComponent implements OnDestroy { // Get the form values const formValues = this.queueCleanerForm.getRawValue(); // Get values including disabled fields - // Build the configuration object + // Build the configuration object with nested structure const config: QueueCleanerConfig = { enabled: formValues.enabled, // The cronExpression will be generated from the jobSchedule when saving @@ -383,29 +531,69 @@ export class QueueCleanerSettingsComponent implements OnDestroy { runSequentially: formValues.runSequentially, ignoredDownloadsPath: formValues.ignoredDownloadsPath || "", - // Failed Import settings - failedImportMaxStrikes: formValues.failedImportMaxStrikes, - failedImportIgnorePrivate: formValues.failedImportIgnorePrivate, - failedImportDeletePrivate: formValues.failedImportDeletePrivate, - failedImportIgnorePatterns: formValues.failedImportIgnorePatterns || [], + // Legacy flat properties for backward compatibility + failedImportMaxStrikes: formValues.failedImport?.maxStrikes, + failedImportIgnorePrivate: formValues.failedImport?.ignorePrivate, + failedImportDeletePrivate: formValues.failedImport?.deletePrivate, + failedImportIgnorePatterns: formValues.failedImport?.ignorePatterns || [], + + stalledMaxStrikes: formValues.stalled?.maxStrikes, + stalledResetStrikesOnProgress: formValues.stalled?.resetStrikesOnProgress, + stalledIgnorePrivate: formValues.stalled?.ignorePrivate, + stalledDeletePrivate: formValues.stalled?.deletePrivate, + downloadingMetadataMaxStrikes: formValues.stalled?.downloadingMetadataMaxStrikes, + + slowMaxStrikes: formValues.slow?.maxStrikes, + slowResetStrikesOnProgress: formValues.slow?.resetStrikesOnProgress, + slowIgnorePrivate: formValues.slow?.ignorePrivate, + slowDeletePrivate: formValues.slow?.deletePrivate, + slowMinSpeed: formValues.slow?.minSpeed || "", + slowMaxTime: formValues.slow?.maxTime, + slowIgnoreAboveSize: formValues.slow?.ignoreAboveSize || "", - // Stalled settings - stalledMaxStrikes: formValues.stalledMaxStrikes, - stalledResetStrikesOnProgress: formValues.stalledResetStrikesOnProgress, - stalledIgnorePrivate: formValues.stalledIgnorePrivate, - stalledDeletePrivate: formValues.stalledDeletePrivate, - - // Downloading Metadata settings - downloadingMetadataMaxStrikes: formValues.downloadingMetadataMaxStrikes, - - // Slow Download settings - slowMaxStrikes: formValues.slowMaxStrikes, - slowResetStrikesOnProgress: formValues.slowResetStrikesOnProgress, - slowIgnorePrivate: formValues.slowIgnorePrivate, - slowDeletePrivate: formValues.slowDeletePrivate, - slowMinSpeed: formValues.slowMinSpeed || "", - slowMaxTime: formValues.slowMaxTime, - slowIgnoreAboveSize: formValues.slowIgnoreAboveSize || "", + // Nested configuration objects + failedImport: { + maxStrikes: formValues.failedImport?.maxStrikes || 0, + ignorePrivate: formValues.failedImport?.ignorePrivate || false, + deletePrivate: formValues.failedImport?.deletePrivate || false, + ignorePatterns: formValues.failedImport?.ignorePatterns || [], + }, + + stalled: { + maxStrikes: formValues.stalled?.maxStrikes || 0, + resetStrikesOnProgress: formValues.stalled?.resetStrikesOnProgress || false, + ignorePrivate: formValues.stalled?.ignorePrivate || false, + deletePrivate: formValues.stalled?.deletePrivate || false, + downloadingMetadataMaxStrikes: formValues.stalled?.downloadingMetadataMaxStrikes || 0, + }, + + slow: { + maxStrikes: formValues.slow?.maxStrikes || 0, + resetStrikesOnProgress: formValues.slow?.resetStrikesOnProgress || false, + ignorePrivate: formValues.slow?.ignorePrivate || false, + deletePrivate: formValues.slow?.deletePrivate || false, + minSpeed: formValues.slow?.minSpeed || "", + maxTime: formValues.slow?.maxTime || 0, + ignoreAboveSize: formValues.slow?.ignoreAboveSize || "", + }, + + contentBlocker: { + enabled: formValues.contentBlocker?.enabled || false, + ignorePrivate: formValues.contentBlocker?.ignorePrivate || false, + deletePrivate: formValues.contentBlocker?.deletePrivate || false, + sonarrBlocklist: formValues.contentBlocker?.sonarrBlocklist || { + path: "", + type: BlocklistType.Blacklist + }, + radarrBlocklist: formValues.contentBlocker?.radarrBlocklist || { + path: "", + type: BlocklistType.Blacklist + }, + lidarrBlocklist: formValues.contentBlocker?.lidarrBlocklist || { + path: "", + type: BlocklistType.Blacklist + }, + }, }; // Save the configuration @@ -425,29 +613,52 @@ export class QueueCleanerSettingsComponent implements OnDestroy { runSequentially: false, ignoredDownloadsPath: "", - // Failed Import settings - failedImportMaxStrikes: 0, - failedImportIgnorePrivate: false, - failedImportDeletePrivate: false, - failedImportIgnorePatterns: [], + // Failed Import settings (nested) + failedImport: { + maxStrikes: 0, + ignorePrivate: false, + deletePrivate: false, + ignorePatterns: [], + }, - // Stalled settings - stalledMaxStrikes: 0, - stalledResetStrikesOnProgress: false, - stalledIgnorePrivate: false, - stalledDeletePrivate: false, + // Stalled settings (nested) + stalled: { + maxStrikes: 0, + resetStrikesOnProgress: false, + ignorePrivate: false, + deletePrivate: false, + downloadingMetadataMaxStrikes: 0, + }, - // Downloading Metadata settings - downloadingMetadataMaxStrikes: 0, + // Slow Download settings (nested) + slow: { + maxStrikes: 0, + resetStrikesOnProgress: false, + ignorePrivate: false, + deletePrivate: false, + minSpeed: "", + maxTime: 0, + ignoreAboveSize: "", + }, - // Slow Download settings - slowMaxStrikes: 0, - slowResetStrikesOnProgress: false, - slowIgnorePrivate: false, - slowDeletePrivate: false, - slowMinSpeed: "", - slowMaxTime: 0, - slowIgnoreAboveSize: "", + // Content Blocker settings (nested) + contentBlocker: { + enabled: false, + ignorePrivate: false, + deletePrivate: false, + sonarrBlocklist: { + path: "", + type: BlocklistType.Blacklist, + }, + radarrBlocklist: { + path: "", + type: BlocklistType.Blacklist, + }, + lidarrBlocklist: { + path: "", + type: BlocklistType.Blacklist, + }, + }, }); // Manually update control states after reset @@ -455,6 +666,7 @@ export class QueueCleanerSettingsComponent implements OnDestroy { this.updateFailedImportDependentControls(0); this.updateStalledDependentControls(0); this.updateSlowDependentControls(0); + this.updateContentBlockerDependentControls(false); } /** diff --git a/code/UI/src/app/shared/models/queue-cleaner-config.model.ts b/code/UI/src/app/shared/models/queue-cleaner-config.model.ts index 114fb250..11593261 100644 --- a/code/UI/src/app/shared/models/queue-cleaner-config.model.ts +++ b/code/UI/src/app/shared/models/queue-cleaner-config.model.ts @@ -9,6 +9,52 @@ export interface JobSchedule { type: ScheduleUnit; } +export enum BlocklistType { + Blacklist = 'Blacklist', + Whitelist = 'Whitelist' +} + +export interface BlocklistSettings { + path: string; + type: BlocklistType; +} + +// Nested configuration interfaces +export interface FailedImportConfig { + maxStrikes: number; + ignorePrivate: boolean; + deletePrivate: boolean; + ignorePatterns: string[]; +} + +export interface StalledConfig { + maxStrikes: number; + resetStrikesOnProgress: boolean; + ignorePrivate: boolean; + deletePrivate: boolean; + downloadingMetadataMaxStrikes: number; +} + +export interface SlowConfig { + maxStrikes: number; + resetStrikesOnProgress: boolean; + ignorePrivate: boolean; + deletePrivate: boolean; + minSpeed: string; + maxTime: number; + ignoreAboveSize: string; +} + +export interface ContentBlockerConfig { + enabled: boolean; + ignorePrivate: boolean; + deletePrivate: boolean; + sonarrBlocklist?: BlocklistSettings; + radarrBlocklist?: BlocklistSettings; + lidarrBlocklist?: BlocklistSettings; + customBlocklists?: BlocklistSettings[]; +} + export interface QueueCleanerConfig { enabled: boolean; cronExpression: string; @@ -16,27 +62,28 @@ export interface QueueCleanerConfig { runSequentially: boolean; ignoredDownloadsPath: string; - // Failed Import settings - failedImportMaxStrikes: number; - failedImportIgnorePrivate: boolean; - failedImportDeletePrivate: boolean; - failedImportIgnorePatterns: string[]; + // Nested configurations + failedImport: FailedImportConfig; + stalled: StalledConfig; + slow: SlowConfig; + contentBlocker: ContentBlockerConfig; - // Stalled settings - stalledMaxStrikes: number; - stalledResetStrikesOnProgress: boolean; - stalledIgnorePrivate: boolean; - stalledDeletePrivate: boolean; - - // Downloading Metadata settings - downloadingMetadataMaxStrikes: number; - - // Slow Download settings - slowMaxStrikes: number; - slowResetStrikesOnProgress: boolean; - slowIgnorePrivate: boolean; - slowDeletePrivate: boolean; - slowMinSpeed: string; - slowMaxTime: number; - slowIgnoreAboveSize: string; + // Legacy flat properties for backward compatibility + // These will be mapped to/from the nested structure + failedImportMaxStrikes?: number; + failedImportIgnorePrivate?: boolean; + failedImportDeletePrivate?: boolean; + failedImportIgnorePatterns?: string[]; + stalledMaxStrikes?: number; + stalledResetStrikesOnProgress?: boolean; + stalledIgnorePrivate?: boolean; + stalledDeletePrivate?: boolean; + downloadingMetadataMaxStrikes?: number; + slowMaxStrikes?: number; + slowResetStrikesOnProgress?: boolean; + slowIgnorePrivate?: boolean; + slowDeletePrivate?: boolean; + slowMinSpeed?: string; + slowMaxTime?: number; + slowIgnoreAboveSize?: string; }