From c82b5e11b1139841b2f71d32d027cbc3ea80fb92 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Sun, 11 May 2025 13:27:51 +0300 Subject: [PATCH] Add rate limiting for download removal (#141) --- .../Configuration/General/SearchConfig.cs | 12 +++ .../DependencyInjection/ConfigurationDI.cs | 1 + code/Executable/DependencyInjection/MainDI.cs | 14 ++++ .../DependencyInjection/ServicesDI.cs | 4 + code/Executable/appsettings.Development.json | 2 + code/Executable/appsettings.json | 2 + .../Extensions/DelugeExtensions.cs | 1 + .../Extensions/QBitExtensions.cs | 1 + .../Extensions/TransmissionExtensions.cs | 1 + code/Infrastructure/Helpers/CacheKeys.cs | 2 + .../{Helpers => Services}/UriService.cs | 2 +- .../Infrastructure/Verticals/Arr/ArrClient.cs | 2 +- .../Verticals/Arr/ArrClientFactory.cs | 31 ++++++++ .../Verticals/Arr/Interfaces/IArrClient.cs | 2 +- .../Verticals/Arr/LidarrClient.cs | 2 +- .../Verticals/Arr/RadarrClient.cs | 2 +- .../Verticals/Arr/SonarrClient.cs | 2 +- .../ContentBlocker/ContentBlocker.cs | 46 +++++++----- .../DownloadCleaner/DownloadCleaner.cs | 15 ++-- .../Consumers/DownloadRemoverConsumer.cs | 39 ++++++++++ .../Interfaces/IQueueItemRemover.cs | 9 +++ .../Models/QueueItemRemoveRequest.cs | 22 ++++++ .../DownloadRemover/QueueItemRemover.cs | 66 +++++++++++++++++ .../Verticals/Jobs/GenericHandler.cs | 74 ++++++++++++++----- .../Verticals/QueueCleaner/QueueCleaner.cs | 43 +++++++---- code/test/docker-compose.yml | 3 + docs/docs/configuration/1_general.mdx | 2 +- docs/docs/configuration/2_search.mdx | 11 +++ docs/docs/configuration/examples/1_docker.mdx | 7 ++ .../configuration/examples/2_config-file.mdx | 2 + .../configuration/SearchSettings.tsx | 35 +++++++++ 31 files changed, 390 insertions(+), 67 deletions(-) create mode 100644 code/Common/Configuration/General/SearchConfig.cs rename code/Infrastructure/{Helpers => Services}/UriService.cs (95%) create mode 100644 code/Infrastructure/Verticals/Arr/ArrClientFactory.cs create mode 100644 code/Infrastructure/Verticals/DownloadRemover/Consumers/DownloadRemoverConsumer.cs create mode 100644 code/Infrastructure/Verticals/DownloadRemover/Interfaces/IQueueItemRemover.cs create mode 100644 code/Infrastructure/Verticals/DownloadRemover/Models/QueueItemRemoveRequest.cs create mode 100644 code/Infrastructure/Verticals/DownloadRemover/QueueItemRemover.cs create mode 100644 docs/docs/configuration/2_search.mdx create mode 100644 docs/src/components/configuration/SearchSettings.tsx diff --git a/code/Common/Configuration/General/SearchConfig.cs b/code/Common/Configuration/General/SearchConfig.cs new file mode 100644 index 00000000..4439b187 --- /dev/null +++ b/code/Common/Configuration/General/SearchConfig.cs @@ -0,0 +1,12 @@ +using Microsoft.Extensions.Configuration; + +namespace Common.Configuration.General; + +public sealed record SearchConfig +{ + [ConfigurationKeyName("SEARCH_ENABLED")] + public bool SearchEnabled { get; init; } = true; + + [ConfigurationKeyName("SEARCH_DELAY")] + public ushort SearchDelay { get; init; } = 30; +} \ No newline at end of file diff --git a/code/Executable/DependencyInjection/ConfigurationDI.cs b/code/Executable/DependencyInjection/ConfigurationDI.cs index 6bccee3d..ee5cacb4 100644 --- a/code/Executable/DependencyInjection/ConfigurationDI.cs +++ b/code/Executable/DependencyInjection/ConfigurationDI.cs @@ -13,6 +13,7 @@ public static class ConfigurationDI public static IServiceCollection AddConfiguration(this IServiceCollection services, IConfiguration configuration) => services .Configure(configuration) + .Configure(configuration) .Configure(configuration.GetSection(QueueCleanerConfig.SectionName)) .Configure(configuration.GetSection(ContentBlockerConfig.SectionName)) .Configure(configuration.GetSection(DownloadCleanerConfig.SectionName)) diff --git a/code/Executable/DependencyInjection/MainDI.cs b/code/Executable/DependencyInjection/MainDI.cs index f542b792..1c498537 100644 --- a/code/Executable/DependencyInjection/MainDI.cs +++ b/code/Executable/DependencyInjection/MainDI.cs @@ -1,11 +1,14 @@ using System.Net; using Common.Configuration.General; using Common.Helpers; +using Domain.Models.Arr; using Infrastructure.Services; using Infrastructure.Verticals.DownloadClient.Deluge; +using Infrastructure.Verticals.DownloadRemover.Consumers; using Infrastructure.Verticals.Notifications.Consumers; using Infrastructure.Verticals.Notifications.Models; using MassTransit; +using MassTransit.Configuration; using Microsoft.Extensions.Options; using Polly; using Polly.Extensions.Http; @@ -27,6 +30,9 @@ public static class MainDI .AddNotifications(configuration) .AddMassTransit(config => { + config.AddConsumer>(); + config.AddConsumer>(); + config.AddConsumer>(); config.AddConsumer>(); config.AddConsumer>(); @@ -36,6 +42,14 @@ public static class MainDI config.UsingInMemory((context, cfg) => { + cfg.ReceiveEndpoint("download-remover-queue", e => + { + e.ConfigureConsumer>(context); + e.ConfigureConsumer>(context); + e.ConcurrentMessageLimit = 1; + e.PrefetchCount = 1; + }); + cfg.ReceiveEndpoint("notification-queue", e => { e.ConfigureConsumer>(context); diff --git a/code/Executable/DependencyInjection/ServicesDI.cs b/code/Executable/DependencyInjection/ServicesDI.cs index 498e6c48..e2e557de 100644 --- a/code/Executable/DependencyInjection/ServicesDI.cs +++ b/code/Executable/DependencyInjection/ServicesDI.cs @@ -11,6 +11,8 @@ using Infrastructure.Verticals.DownloadClient; using Infrastructure.Verticals.DownloadClient.Deluge; using Infrastructure.Verticals.DownloadClient.QBittorrent; using Infrastructure.Verticals.DownloadClient.Transmission; +using Infrastructure.Verticals.DownloadRemover; +using Infrastructure.Verticals.DownloadRemover.Interfaces; using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.QueueCleaner; @@ -26,9 +28,11 @@ public static class ServicesDI .AddTransient() .AddTransient() .AddTransient() + .AddTransient() .AddTransient() .AddTransient() .AddTransient() + .AddTransient() .AddTransient() .AddTransient() .AddTransient() diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index 460153f7..536b1ced 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -11,6 +11,8 @@ "Path": "" } }, + "SEARCH_ENABLED": true, + "SEARCH_DELAY": 5, "Triggers": { "QueueCleaner": "0/10 * * * * ?", "ContentBlocker": "0/10 * * * * ?", diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index e9690a47..35bb7679 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -11,6 +11,8 @@ "Path": "" } }, + "SEARCH_ENABLED": true, + "SEARCH_DELAY": 30, "Triggers": { "QueueCleaner": "0 0/5 * * * ?", "ContentBlocker": "0 0/5 * * * ?", diff --git a/code/Infrastructure/Extensions/DelugeExtensions.cs b/code/Infrastructure/Extensions/DelugeExtensions.cs index 77497bd6..93656906 100644 --- a/code/Infrastructure/Extensions/DelugeExtensions.cs +++ b/code/Infrastructure/Extensions/DelugeExtensions.cs @@ -1,5 +1,6 @@ using Domain.Models.Deluge.Response; using Infrastructure.Helpers; +using Infrastructure.Services; namespace Infrastructure.Extensions; diff --git a/code/Infrastructure/Extensions/QBitExtensions.cs b/code/Infrastructure/Extensions/QBitExtensions.cs index 5e30742a..ca26ff8a 100644 --- a/code/Infrastructure/Extensions/QBitExtensions.cs +++ b/code/Infrastructure/Extensions/QBitExtensions.cs @@ -1,4 +1,5 @@ using Infrastructure.Helpers; +using Infrastructure.Services; using QBittorrent.Client; namespace Infrastructure.Extensions; diff --git a/code/Infrastructure/Extensions/TransmissionExtensions.cs b/code/Infrastructure/Extensions/TransmissionExtensions.cs index 8997048a..47c29e70 100644 --- a/code/Infrastructure/Extensions/TransmissionExtensions.cs +++ b/code/Infrastructure/Extensions/TransmissionExtensions.cs @@ -1,4 +1,5 @@ using Infrastructure.Helpers; +using Infrastructure.Services; using Transmission.API.RPC.Entity; namespace Infrastructure.Extensions; diff --git a/code/Infrastructure/Helpers/CacheKeys.cs b/code/Infrastructure/Helpers/CacheKeys.cs index 61314fca..84cdffc3 100644 --- a/code/Infrastructure/Helpers/CacheKeys.cs +++ b/code/Infrastructure/Helpers/CacheKeys.cs @@ -13,4 +13,6 @@ public static class CacheKeys public static string StrikeItem(string hash, StrikeType strikeType) => $"item_{hash}_{strikeType.ToString()}"; public static string IgnoredDownloads(string name) => $"{name}_ignored"; + + public static string DownloadMarkedForRemoval(string hash, Uri url) => $"remove_{hash.ToLowerInvariant()}_{url}"; } \ No newline at end of file diff --git a/code/Infrastructure/Helpers/UriService.cs b/code/Infrastructure/Services/UriService.cs similarity index 95% rename from code/Infrastructure/Helpers/UriService.cs rename to code/Infrastructure/Services/UriService.cs index 2a24828b..aa8ed7fd 100644 --- a/code/Infrastructure/Helpers/UriService.cs +++ b/code/Infrastructure/Services/UriService.cs @@ -1,6 +1,6 @@ using System.Text.RegularExpressions; -namespace Infrastructure.Helpers; +namespace Infrastructure.Services; public static class UriService { diff --git a/code/Infrastructure/Verticals/Arr/ArrClient.cs b/code/Infrastructure/Verticals/Arr/ArrClient.cs index 38af9b1f..421aa90a 100644 --- a/code/Infrastructure/Verticals/Arr/ArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/ArrClient.cs @@ -157,7 +157,7 @@ public abstract class ArrClient : IArrClient } } - public abstract Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items); + public abstract Task SearchItemsAsync(ArrInstance arrInstance, HashSet? items); public virtual bool IsRecordValid(QueueRecord record) { diff --git a/code/Infrastructure/Verticals/Arr/ArrClientFactory.cs b/code/Infrastructure/Verticals/Arr/ArrClientFactory.cs new file mode 100644 index 00000000..a270a206 --- /dev/null +++ b/code/Infrastructure/Verticals/Arr/ArrClientFactory.cs @@ -0,0 +1,31 @@ +using Domain.Enums; +using Infrastructure.Verticals.Arr.Interfaces; + +namespace Infrastructure.Verticals.Arr; + +public sealed class ArrClientFactory +{ + private readonly ISonarrClient _sonarrClient; + private readonly IRadarrClient _radarrClient; + private readonly ILidarrClient _lidarrClient; + + public ArrClientFactory( + SonarrClient sonarrClient, + RadarrClient radarrClient, + LidarrClient lidarrClient + ) + { + _sonarrClient = sonarrClient; + _radarrClient = radarrClient; + _lidarrClient = lidarrClient; + } + + public IArrClient GetClient(InstanceType type) => + type switch + { + InstanceType.Sonarr => _sonarrClient, + InstanceType.Radarr => _radarrClient, + InstanceType.Lidarr => _lidarrClient, + _ => throw new NotImplementedException($"instance type {type} is not yet supported") + }; +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs b/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs index e74f112b..e2cd654d 100644 --- a/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs +++ b/code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs @@ -13,7 +13,7 @@ public interface IArrClient Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient, DeleteReason deleteReason); - Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items); + Task SearchItemsAsync(ArrInstance arrInstance, HashSet? items); bool IsRecordValid(QueueRecord record); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Arr/LidarrClient.cs b/code/Infrastructure/Verticals/Arr/LidarrClient.cs index 730615cf..af180eec 100644 --- a/code/Infrastructure/Verticals/Arr/LidarrClient.cs +++ b/code/Infrastructure/Verticals/Arr/LidarrClient.cs @@ -50,7 +50,7 @@ public class LidarrClient : ArrClient, ILidarrClient return query; } - public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items) + public override async Task SearchItemsAsync(ArrInstance arrInstance, HashSet? items) { if (items?.Count is null or 0) { diff --git a/code/Infrastructure/Verticals/Arr/RadarrClient.cs b/code/Infrastructure/Verticals/Arr/RadarrClient.cs index 7b9e696e..d7cd4782 100644 --- a/code/Infrastructure/Verticals/Arr/RadarrClient.cs +++ b/code/Infrastructure/Verticals/Arr/RadarrClient.cs @@ -50,7 +50,7 @@ public class RadarrClient : ArrClient, IRadarrClient return query; } - public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items) + public override async Task SearchItemsAsync(ArrInstance arrInstance, HashSet? items) { if (items?.Count is null or 0) { diff --git a/code/Infrastructure/Verticals/Arr/SonarrClient.cs b/code/Infrastructure/Verticals/Arr/SonarrClient.cs index 2f5aa448..0144a154 100644 --- a/code/Infrastructure/Verticals/Arr/SonarrClient.cs +++ b/code/Infrastructure/Verticals/Arr/SonarrClient.cs @@ -51,7 +51,7 @@ public class SonarrClient : ArrClient, ISonarrClient return query; } - public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet? items) + public override async Task SearchItemsAsync(ArrInstance arrInstance, HashSet? items) { if (items?.Count is null or 0) { diff --git a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs index aca7dee3..56ce043e 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs @@ -6,16 +6,20 @@ using Common.Configuration.DownloadClient; using Domain.Enums; using Domain.Models.Arr; using Domain.Models.Arr.Queue; +using Infrastructure.Helpers; using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Context; using Infrastructure.Verticals.DownloadClient; +using Infrastructure.Verticals.DownloadRemover.Models; using Infrastructure.Verticals.Jobs; using Infrastructure.Verticals.Notifications; +using MassTransit; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Serilog.Context; +using LogContext = Serilog.Context.LogContext; namespace Infrastructure.Verticals.ContentBlocker; @@ -32,9 +36,9 @@ public sealed class ContentBlocker : GenericHandler IOptions sonarrConfig, IOptions radarrConfig, IOptions lidarrConfig, - SonarrClient sonarrClient, - RadarrClient radarrClient, - LidarrClient lidarrClient, + IMemoryCache cache, + IBus messageBus, + ArrClientFactory arrClientFactory, ArrQueueIterator arrArrQueueIterator, BlocklistProvider blocklistProvider, DownloadServiceFactory downloadServiceFactory, @@ -43,8 +47,7 @@ public sealed class ContentBlocker : GenericHandler ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, - sonarrClient, radarrClient, lidarrClient, - arrArrQueueIterator, downloadServiceFactory, + cache, messageBus, arrClientFactory, arrArrQueueIterator, downloadServiceFactory, notifier ) { @@ -81,15 +84,10 @@ public sealed class ContentBlocker : GenericHandler using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); - HashSet itemsToBeRefreshed = []; - IArrClient arrClient = GetClient(instanceType); + IArrClient arrClient = _arrClientFactory.GetClient(instanceType); BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType); ConcurrentBag patterns = _blocklistProvider.GetPatterns(instanceType); ConcurrentBag regexes = _blocklistProvider.GetRegexes(instanceType); - - // push to context - ContextProvider.Set(nameof(ArrInstance) + nameof(ArrInstance.Url), instance.Url); - ContextProvider.Set(nameof(InstanceType), instanceType); await _arrArrQueueIterator.Iterate(arrClient, instance, async items => { @@ -117,9 +115,14 @@ public sealed class ContentBlocker : GenericHandler _logger.LogInformation("skip | {title} | ignored", record.Title); continue; } + + string downloadRemovalKey = CacheKeys.DownloadMarkedForRemoval(record.DownloadId, instance.Url); - // push record to context - ContextProvider.Set(nameof(QueueRecord), record); + if (_cache.TryGetValue(downloadRemovalKey, out bool _)) + { + _logger.LogDebug("skip | already marked for removal | {title}", record.Title); + continue; + } _logger.LogDebug("searching unwanted files for {title}", record.Title); @@ -133,8 +136,6 @@ public sealed class ContentBlocker : GenericHandler _logger.LogDebug("all files are marked as unwanted | {hash}", record.Title); - itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1)); - bool removeFromClient = true; if (result.IsPrivate && !_config.DeletePrivate) @@ -142,11 +143,16 @@ public sealed class ContentBlocker : GenericHandler removeFromClient = false; } - await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, DeleteReason.AllFilesBlocked); - await _notifier.NotifyQueueItemDeleted(removeFromClient, DeleteReason.AllFilesBlocked); + await PublishQueueItemRemoveRequest( + downloadRemovalKey, + instanceType, + instance, + record, + group.Count() > 1, + removeFromClient, + DeleteReason.AllFilesBlocked + ); } }); - - await arrClient.RefreshItemsAsync(instance, itemsToBeRefreshed); } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs index 1c609cea..c06a5e3e 100644 --- a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs +++ b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs @@ -9,9 +9,11 @@ using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.DownloadClient; using Infrastructure.Verticals.Jobs; using Infrastructure.Verticals.Notifications; +using MassTransit; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Serilog.Context; +using LogContext = Serilog.Context.LogContext; namespace Infrastructure.Verticals.DownloadCleaner; @@ -30,9 +32,9 @@ public sealed class DownloadCleaner : GenericHandler IOptions sonarrConfig, IOptions radarrConfig, IOptions lidarrConfig, - SonarrClient sonarrClient, - RadarrClient radarrClient, - LidarrClient lidarrClient, + IMemoryCache cache, + IBus messageBus, + ArrClientFactory arrClientFactory, ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, INotificationPublisher notifier, @@ -40,8 +42,7 @@ public sealed class DownloadCleaner : GenericHandler ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, - sonarrClient, radarrClient, lidarrClient, - arrArrQueueIterator, downloadServiceFactory, + cache, messageBus, arrClientFactory, arrArrQueueIterator, downloadServiceFactory, notifier ) { @@ -131,7 +132,7 @@ public sealed class DownloadCleaner : GenericHandler { using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); - IArrClient arrClient = GetClient(instanceType); + IArrClient arrClient = _arrClientFactory.GetClient(instanceType); await _arrArrQueueIterator.Iterate(arrClient, instance, async items => { diff --git a/code/Infrastructure/Verticals/DownloadRemover/Consumers/DownloadRemoverConsumer.cs b/code/Infrastructure/Verticals/DownloadRemover/Consumers/DownloadRemoverConsumer.cs new file mode 100644 index 00000000..390538ed --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadRemover/Consumers/DownloadRemoverConsumer.cs @@ -0,0 +1,39 @@ +using Domain.Models.Arr; +using Infrastructure.Verticals.DownloadRemover.Interfaces; +using Infrastructure.Verticals.DownloadRemover.Models; +using MassTransit; +using Microsoft.Extensions.Logging; + +namespace Infrastructure.Verticals.DownloadRemover.Consumers; + +public class DownloadRemoverConsumer : IConsumer> + where T : SearchItem +{ + private readonly ILogger> _logger; + private readonly IQueueItemRemover _queueItemRemover; + + public DownloadRemoverConsumer( + ILogger> logger, + IQueueItemRemover queueItemRemover + ) + { + _logger = logger; + _queueItemRemover = queueItemRemover; + } + + public async Task Consume(ConsumeContext> context) + { + try + { + await _queueItemRemover.RemoveQueueItemAsync(context.Message); + } + catch (Exception exception) + { + _logger.LogError(exception, + "failed to remove queue item| {title} | {url}", + context.Message.Record.Title, + context.Message.Instance.Url + ); + } + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadRemover/Interfaces/IQueueItemRemover.cs b/code/Infrastructure/Verticals/DownloadRemover/Interfaces/IQueueItemRemover.cs new file mode 100644 index 00000000..f7012e9d --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadRemover/Interfaces/IQueueItemRemover.cs @@ -0,0 +1,9 @@ +using Domain.Models.Arr; +using Infrastructure.Verticals.DownloadRemover.Models; + +namespace Infrastructure.Verticals.DownloadRemover.Interfaces; + +public interface IQueueItemRemover +{ + Task RemoveQueueItemAsync(QueueItemRemoveRequest request) where T : SearchItem; +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadRemover/Models/QueueItemRemoveRequest.cs b/code/Infrastructure/Verticals/DownloadRemover/Models/QueueItemRemoveRequest.cs new file mode 100644 index 00000000..b696819b --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadRemover/Models/QueueItemRemoveRequest.cs @@ -0,0 +1,22 @@ +using Common.Configuration.Arr; +using Domain.Enums; +using Domain.Models.Arr; +using Domain.Models.Arr.Queue; + +namespace Infrastructure.Verticals.DownloadRemover.Models; + +public sealed record QueueItemRemoveRequest + where T : SearchItem +{ + public required InstanceType InstanceType { get; init; } + + public required ArrInstance Instance { get; init; } + + public required T SearchItem { get; init; } + + public required QueueRecord Record { get; init; } + + public required bool RemoveFromClient { get; init; } + + public required DeleteReason DeleteReason { get; init; } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadRemover/QueueItemRemover.cs b/code/Infrastructure/Verticals/DownloadRemover/QueueItemRemover.cs new file mode 100644 index 00000000..12376bbc --- /dev/null +++ b/code/Infrastructure/Verticals/DownloadRemover/QueueItemRemover.cs @@ -0,0 +1,66 @@ +using Common.Configuration.Arr; +using Common.Configuration.General; +using Domain.Enums; +using Domain.Models.Arr; +using Domain.Models.Arr.Queue; +using Infrastructure.Helpers; +using Infrastructure.Verticals.Arr; +using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.DownloadRemover.Interfaces; +using Infrastructure.Verticals.DownloadRemover.Models; +using Infrastructure.Verticals.Notifications; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Options; + +namespace Infrastructure.Verticals.DownloadRemover; + +public sealed class QueueItemRemover : IQueueItemRemover +{ + private readonly SearchConfig _searchConfig; + private readonly IMemoryCache _cache; + private readonly ArrClientFactory _arrClientFactory; + private readonly INotificationPublisher _notifier; + + public QueueItemRemover( + IOptions searchConfig, + IMemoryCache cache, + ArrClientFactory arrClientFactory, + INotificationPublisher notifier + ) + { + _searchConfig = searchConfig.Value; + _cache = cache; + _arrClientFactory = arrClientFactory; + _notifier = notifier; + } + + public async Task RemoveQueueItemAsync(QueueItemRemoveRequest request) + where T : SearchItem + { + try + { + var arrClient = _arrClientFactory.GetClient(request.InstanceType); + await arrClient.DeleteQueueItemAsync(request.Instance, request.Record, request.RemoveFromClient, request.DeleteReason); + + // push to context + ContextProvider.Set(nameof(QueueRecord), request.Record); + ContextProvider.Set(nameof(ArrInstance) + nameof(ArrInstance.Url), request.Instance.Url); + ContextProvider.Set(nameof(InstanceType), request.InstanceType); + await _notifier.NotifyQueueItemDeleted(request.RemoveFromClient, request.DeleteReason); + + if (!_searchConfig.SearchEnabled) + { + return; + } + + await arrClient.SearchItemsAsync(request.Instance, [request.SearchItem]); + + // prevent tracker spamming + await Task.Delay(TimeSpan.FromSeconds(_searchConfig.SearchDelay)); + } + finally + { + _cache.Remove(CacheKeys.DownloadMarkedForRemoval(request.Record.DownloadId, request.Instance.Url)); + } + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Jobs/GenericHandler.cs b/code/Infrastructure/Verticals/Jobs/GenericHandler.cs index f561c6b6..7a21f46f 100644 --- a/code/Infrastructure/Verticals/Jobs/GenericHandler.cs +++ b/code/Infrastructure/Verticals/Jobs/GenericHandler.cs @@ -4,9 +4,11 @@ using Domain.Enums; using Domain.Models.Arr; using Domain.Models.Arr.Queue; using Infrastructure.Verticals.Arr; -using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.DownloadClient; +using Infrastructure.Verticals.DownloadRemover.Models; using Infrastructure.Verticals.Notifications; +using MassTransit; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -19,9 +21,9 @@ public abstract class GenericHandler : IHandler, IDisposable protected readonly SonarrConfig _sonarrConfig; protected readonly RadarrConfig _radarrConfig; protected readonly LidarrConfig _lidarrConfig; - protected readonly ISonarrClient _sonarrClient; - protected readonly IRadarrClient _radarrClient; - protected readonly ILidarrClient _lidarrClient; + protected readonly IMemoryCache _cache; + protected readonly IBus _messageBus; + protected readonly ArrClientFactory _arrClientFactory; protected readonly ArrQueueIterator _arrArrQueueIterator; protected readonly IDownloadService _downloadService; protected readonly INotificationPublisher _notifier; @@ -32,9 +34,9 @@ public abstract class GenericHandler : IHandler, IDisposable IOptions sonarrConfig, IOptions radarrConfig, IOptions lidarrConfig, - ISonarrClient sonarrClient, - IRadarrClient radarrClient, - ILidarrClient lidarrClient, + IMemoryCache cache, + IBus messageBus, + ArrClientFactory arrClientFactory, ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, INotificationPublisher notifier @@ -45,9 +47,9 @@ public abstract class GenericHandler : IHandler, IDisposable _sonarrConfig = sonarrConfig.Value; _radarrConfig = radarrConfig.Value; _lidarrConfig = lidarrConfig.Value; - _sonarrClient = sonarrClient; - _radarrClient = radarrClient; - _lidarrClient = lidarrClient; + _cache = cache; + _messageBus = messageBus; + _arrClientFactory = arrClientFactory; _arrArrQueueIterator = arrArrQueueIterator; _downloadService = downloadServiceFactory.CreateDownloadClient(); _notifier = notifier; @@ -93,16 +95,50 @@ public abstract class GenericHandler : IHandler, IDisposable } } } - - protected IArrClient GetClient(InstanceType type) => - type switch - { - InstanceType.Sonarr => _sonarrClient, - InstanceType.Radarr => _radarrClient, - InstanceType.Lidarr => _lidarrClient, - _ => throw new NotImplementedException($"instance type {type} is not yet supported") - }; + protected async Task PublishQueueItemRemoveRequest( + string downloadRemovalKey, + InstanceType instanceType, + ArrInstance instance, + QueueRecord record, + bool isPack, + bool removeFromClient, + DeleteReason deleteReason + ) + { + if (instanceType is InstanceType.Sonarr) + { + QueueItemRemoveRequest removeRequest = new() + { + InstanceType = instanceType, + Instance = instance, + Record = record, + SearchItem = (SonarrSearchItem)GetRecordSearchItem(instanceType, record, isPack), + RemoveFromClient = removeFromClient, + DeleteReason = deleteReason + }; + + await _messageBus.Publish(removeRequest); + } + else + { + QueueItemRemoveRequest removeRequest = new() + { + InstanceType = instanceType, + Instance = instance, + Record = record, + SearchItem = GetRecordSearchItem(instanceType, record, isPack), + RemoveFromClient = removeFromClient, + DeleteReason = deleteReason + }; + + await _messageBus.Publish(removeRequest); + } + + _cache.Set(downloadRemovalKey, true); + _logger.LogInformation("item marked for removal | {title} | {url}", record.Title, instance.Url); + } + protected SearchItem GetRecordSearchItem(InstanceType type, QueueRecord record, bool isPack = false) { return type switch diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index 3cb7ac21..6e98081c 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -4,22 +4,27 @@ using Common.Configuration.QueueCleaner; using Domain.Enums; using Domain.Models.Arr; using Domain.Models.Arr.Queue; +using Infrastructure.Helpers; using Infrastructure.Providers; using Infrastructure.Verticals.Arr; using Infrastructure.Verticals.Arr.Interfaces; using Infrastructure.Verticals.Context; using Infrastructure.Verticals.DownloadClient; +using Infrastructure.Verticals.DownloadRemover.Models; using Infrastructure.Verticals.Jobs; using Infrastructure.Verticals.Notifications; +using MassTransit; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Serilog.Context; +using LogContext = Serilog.Context.LogContext; namespace Infrastructure.Verticals.QueueCleaner; public sealed class QueueCleaner : GenericHandler { private readonly QueueCleanerConfig _config; + private readonly IMemoryCache _cache; private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; public QueueCleaner( @@ -29,9 +34,9 @@ public sealed class QueueCleaner : GenericHandler IOptions sonarrConfig, IOptions radarrConfig, IOptions lidarrConfig, - SonarrClient sonarrClient, - RadarrClient radarrClient, - LidarrClient lidarrClient, + IMemoryCache cache, + IBus messageBus, + ArrClientFactory arrClientFactory, ArrQueueIterator arrArrQueueIterator, DownloadServiceFactory downloadServiceFactory, INotificationPublisher notifier, @@ -39,13 +44,13 @@ public sealed class QueueCleaner : GenericHandler ) : base( logger, downloadClientConfig, sonarrConfig, radarrConfig, lidarrConfig, - sonarrClient, radarrClient, lidarrClient, - arrArrQueueIterator, downloadServiceFactory, + cache, messageBus, arrClientFactory, arrArrQueueIterator, downloadServiceFactory, notifier ) { _config = config.Value; _config.Validate(); + _cache = cache; _ignoredDownloadsProvider = ignoredDownloadsProvider; } @@ -55,8 +60,7 @@ public sealed class QueueCleaner : GenericHandler using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString()); - HashSet itemsToBeRefreshed = []; - IArrClient arrClient = GetClient(instanceType); + IArrClient arrClient = _arrClientFactory.GetClient(instanceType); // push to context ContextProvider.Set(nameof(ArrInstance) + nameof(ArrInstance.Url), instance.Url); @@ -90,6 +94,14 @@ public sealed class QueueCleaner : GenericHandler continue; } + string downloadRemovalKey = CacheKeys.DownloadMarkedForRemoval(record.DownloadId, instance.Url); + + if (_cache.TryGetValue(downloadRemovalKey, out bool _)) + { + _logger.LogDebug("skip | already marked for removal | {title}", record.Title); + continue; + } + // push record to context ContextProvider.Set(nameof(QueueRecord), record); @@ -116,8 +128,6 @@ public sealed class QueueCleaner : GenericHandler _logger.LogInformation("skip | {title}", record.Title); continue; } - - itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1)); bool removeFromClient = true; @@ -140,11 +150,16 @@ public sealed class QueueCleaner : GenericHandler } } - await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, deleteReason); - await _notifier.NotifyQueueItemDeleted(removeFromClient, deleteReason); + await PublishQueueItemRemoveRequest( + downloadRemovalKey, + instanceType, + instance, + record, + group.Count() > 1, + removeFromClient, + deleteReason + ); } }); - - await arrClient.RefreshItemsAsync(instance, itemsToBeRefreshed); } } \ No newline at end of file diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index 3327019b..f592b06c 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -186,6 +186,9 @@ services: - HTTP_MAX_RETRIES=0 - HTTP_TIMEOUT=20 + - SEARCH_ENABLED=true + - SEARCH_DELAY=5 + - TRIGGERS__QUEUECLEANER=0/30 * * * * ? - TRIGGERS__CONTENTBLOCKER=0/30 * * * * ? - TRIGGERS__DOWNLOADCLEANER=0/30 * * * * ? diff --git a/docs/docs/configuration/1_general.mdx b/docs/docs/configuration/1_general.mdx index ea17ec7b..73aba914 100644 --- a/docs/docs/configuration/1_general.mdx +++ b/docs/docs/configuration/1_general.mdx @@ -4,7 +4,7 @@ sidebar_position: 2 import GeneralSettings from '@site/src/components/configuration/GeneralSettings'; -# General Settings +# General settings These are the general configuration settings that apply to the entire application. diff --git a/docs/docs/configuration/2_search.mdx b/docs/docs/configuration/2_search.mdx new file mode 100644 index 00000000..ab8522f3 --- /dev/null +++ b/docs/docs/configuration/2_search.mdx @@ -0,0 +1,11 @@ +--- +sidebar_position: 3 +--- + +import SearchSettings from '@site/src/components/configuration/SearchSettings'; + +# Search settings + +These are the search configuration settings when searching for replacements. + + \ No newline at end of file diff --git a/docs/docs/configuration/examples/1_docker.mdx b/docs/docs/configuration/examples/1_docker.mdx index c7565899..5b014d84 100644 --- a/docs/docs/configuration/examples/1_docker.mdx +++ b/docs/docs/configuration/examples/1_docker.mdx @@ -58,6 +58,13 @@ import styles from './examples.module.css'; LOGGING__ENHANCED {`=true + - `} + SEARCH_ENABLED + {`=true + - `} + SEARCH_DELAY + {`=30 + - `} TRIGGERS__QUEUECLEANER {`=0 0/5 * * * ? diff --git a/docs/docs/configuration/examples/2_config-file.mdx b/docs/docs/configuration/examples/2_config-file.mdx index 380b1cfd..e5b6f3e2 100644 --- a/docs/docs/configuration/examples/2_config-file.mdx +++ b/docs/docs/configuration/examples/2_config-file.mdx @@ -24,6 +24,8 @@ import { Note } from '@site/src/components/Admonition'; "Path": "/var/logs" } }, + "SEARCH_ENABLED": true, + "SEARCH_DELAY": 30, "Triggers": { "QueueCleaner": "0 0/5 * * * ?", "ContentBlocker": "0 0/5 * * * ?", diff --git a/docs/src/components/configuration/SearchSettings.tsx b/docs/src/components/configuration/SearchSettings.tsx new file mode 100644 index 00000000..af92bf9b --- /dev/null +++ b/docs/src/components/configuration/SearchSettings.tsx @@ -0,0 +1,35 @@ +import React from "react"; +import EnvVars, { EnvVarProps } from "./EnvVars"; + +const settings: EnvVarProps[] = [ + { + name: "SEARCH_ENABLED", + description: [ + "Enabled searching for replacements after a download has been removed from an arr." + ], + type: "boolean", + defaultValue: "true", + required: false, + acceptedValues: ["true", "false"], + notes: [ + "If you are using [Huntarr](https://github.com/plexguide/Huntarr.io), this setting should be set to false to let Huntarr do the searching.", + ] + }, + { + name: "SEARCH_DELAY", + description: [ + "If searching for replacements is enabled, this setting will delay the searches by the specified number of seconds.", + "This is useful to avoid overwhelming the indexer with too many requests at once.", + ], + type: "positive integer number", + defaultValue: "30", + required: false, + important: [ + "A lower value or `0` will result in faster searches, but may cause issues such as being rate limited or banned by the indexer.", + ] + }, +]; + +export default function SearchSettings() { + return ; +}