mirror/Cleanuparr
1
0
Fork 0
mirror of https://github.com/Cleanuparr/Cleanuparr.git synced 2026-01-29 08:01:01 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add download cleaner and dry run (#58)

Browse Source
This commit is contained in:
Marius Nechifor
2025-02-16 03:09:07 +02:00
parent 19b3675701
commit 596a5aed8d
87 changed files with 2507 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
code/Infrastructure/Infrastructure.csproj
View File

@@ -12,13 +12,15 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Castle.Core.AsyncInterceptor" Version="2.1.0" />
<PackageReference Include="FLM.QBittorrent" Version="1.0.0" />
<PackageReference Include="FLM.Transmission" Version="1.0.2" />
<PackageReference Include="Mapster" Version="7.4.0" />
<PackageReference Include="MassTransit" Version="8.3.6" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.2" />
<PackageReference Include="Quartz" Version="3.13.1" />
<PackageReference Include="Scrutor" Version="6.0.1" />
</ItemGroup>
</Project>

49
code/Infrastructure/Interceptors/DryRunInterceptor.cs Normal file
View File

@@ -0,0 +1,49 @@
using System.Reflection;
using Castle.DynamicProxy;
using Common.Attributes;
using Common.Configuration.General;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Infrastructure.Interceptors;
public class DryRunAsyncInterceptor : AsyncInterceptorBase
{
private readonly ILogger<DryRunAsyncInterceptor> _logger;
private readonly DryRunConfig _config;
public DryRunAsyncInterceptor(ILogger<DryRunAsyncInterceptor> logger, IOptions<DryRunConfig> config)
{
_logger = logger;
_config = config.Value;
}
protected override async Task InterceptAsync(IInvocation invocation, IInvocationProceedInfo proceedInfo, Func<IInvocation, IInvocationProceedInfo, Task> proceed)
{
MethodInfo? method = invocation.MethodInvocationTarget ?? invocation.Method;
if (IsDryRun(method))
{
_logger.LogInformation("[DRY RUN] skipping method: {name}", method.Name);
return;
}
await proceed(invocation, proceedInfo);
}
protected override async Task<TResult> InterceptAsync<TResult>(IInvocation invocation, IInvocationProceedInfo proceedInfo, Func<IInvocation, IInvocationProceedInfo, Task<TResult>> proceed)
{
MethodInfo? method = invocation.MethodInvocationTarget ?? invocation.Method;
if (IsDryRun(method))
{
_logger.LogInformation("[DRY RUN] skipping method: {name}", method.Name);
return default!;
}
return await proceed(invocation, proceedInfo);
}
private bool IsDryRun(MethodInfo method)
{
return method.GetCustomAttributes(typeof(DryRunSafeguardAttribute), true).Any() && _config.IsDryRun;
}
}

5
code/Infrastructure/Interceptors/IDryRunService.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Interceptors;
public interface IDryRunService : IInterceptedService
{
}

6
code/Infrastructure/Interceptors/IInterceptedService.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace Infrastructure.Interceptors;
public interface IInterceptedService
{
public object Proxy { get; set; }
}

21
code/Infrastructure/Interceptors/InterceptedService.cs Normal file
View File

@@ -0,0 +1,21 @@
namespace Infrastructure.Interceptors;
public class InterceptedService : IInterceptedService
{
private object? _proxy;
public object Proxy
{
get
{
if (_proxy is null)
{
throw new Exception("Proxy is not set");
}
return _proxy;
}
set => _proxy = value;
}
}

37
code/Infrastructure/Verticals/Arr/ArrClient.cs
View File

@@ -1,3 +1,4 @@
using Common.Attributes;
using Common.Configuration.Arr;
using Common.Configuration.Logging;
using Common.Configuration.QueueCleaner;
@@ -5,6 +6,8 @@ using Common.Helpers;
using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -12,24 +15,30 @@ using Newtonsoft.Json;
namespace Infrastructure.Verticals.Arr;
public abstract class ArrClient
public abstract class ArrClient : InterceptedService, IArrClient, IDryRunService
{
protected readonly ILogger<ArrClient> _logger;
protected readonly HttpClient _httpClient;
protected readonly LoggingConfig _loggingConfig;
protected readonly QueueCleanerConfig _queueCleanerConfig;
protected readonly Striker _striker;
protected readonly IStriker _striker;
/// <summary>
/// Constructor to be used by interceptors.
/// </summary>
protected ArrClient()
{
}
protected ArrClient(
ILogger<ArrClient> logger,
IHttpClientFactory httpClientFactory,
IOptions<LoggingConfig> loggingConfig,
IOptions<QueueCleanerConfig> queueCleanerConfig,
Striker striker
IStriker striker
)
{
_logger = logger;
_striker = striker;
_httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
_loggingConfig = loggingConfig.Value;
_queueCleanerConfig = queueCleanerConfig.Value;
@@ -110,16 +119,14 @@ public abstract class ArrClient
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient)
{
Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id, removeFromClient));
using HttpRequestMessage request = new(HttpMethod.Delete, uri);
SetApiKey(request, arrInstance.ApiKey);
using HttpResponseMessage response = await _httpClient.SendAsync(request);
try
{
response.EnsureSuccessStatusCode();
using HttpRequestMessage request = new(HttpMethod.Delete, uri);
SetApiKey(request, arrInstance.ApiKey);
using var _ = await ((ArrClient)Proxy).SendRequestAsync(request);
_logger.LogInformation(
removeFromClient
? "queue item deleted | {url} | {title}"
@@ -157,6 +164,16 @@ public abstract class ArrClient
request.Headers.Add("x-api-key", apiKey);
}
[DryRunSafeguard]
protected virtual async Task<HttpResponseMessage> SendRequestAsync(HttpRequestMessage request)
{
HttpResponseMessage response = await _httpClient.SendAsync(request);
response.EnsureSuccessStatusCode();
return response;
}
private bool HasIgnoredPatterns(QueueRecord record)
{
if (_queueCleanerConfig.ImportFailedIgnorePatterns?.Count is null or 0)

3
code/Infrastructure/Verticals/Arr/ArrQueueIterator.cs
View File

@@ -1,6 +1,7 @@
using Common.Configuration;
using Common.Configuration.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Verticals.Arr.Interfaces;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.Arr;
@@ -14,7 +15,7 @@ public sealed class ArrQueueIterator
_logger = logger;
}
public async Task Iterate(ArrClient arrClient, ArrInstance arrInstance, Func<IReadOnlyList<QueueRecord>, Task> action)
public async Task Iterate(IArrClient arrClient, ArrInstance arrInstance, Func<IReadOnlyList<QueueRecord>, Task> action)
{
const ushort maxPage = 100;
ushort page = 1;

19
code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs Normal file
View File

@@ -0,0 +1,19 @@
using Common.Configuration.Arr;
using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
namespace Infrastructure.Verticals.Arr.Interfaces;
public interface IArrClient
{
Task<QueueListResponse> GetQueueItemsAsync(ArrInstance arrInstance, int page);
Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload);
Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient);
Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items);
bool IsRecordValid(QueueRecord record);
}

5
code/Infrastructure/Verticals/Arr/Interfaces/ILidarrClient.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Arr.Interfaces;
public interface ILidarrClient : IArrClient
{
}

5
code/Infrastructure/Verticals/Arr/Interfaces/IRadarrClient.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Arr.Interfaces;
public interface IRadarrClient : IArrClient
{
}

5
code/Infrastructure/Verticals/Arr/Interfaces/ISonarrClient.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Arr.Interfaces;
public interface ISonarrClient : IArrClient
{
}

15
code/Infrastructure/Verticals/Arr/LidarrClient.cs
View File

@@ -5,6 +5,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Domain.Models.Lidarr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -12,14 +13,19 @@ using Newtonsoft.Json;
namespace Infrastructure.Verticals.Arr;
public sealed class LidarrClient : ArrClient
public class LidarrClient : ArrClient, ILidarrClient
{
/// <inheritdoc/>
public LidarrClient()
{
}
public LidarrClient(
ILogger<LidarrClient> logger,
IHttpClientFactory httpClientFactory,
IOptions<LoggingConfig> loggingConfig,
IOptions<QueueCleanerConfig> queueCleanerConfig,
Striker striker
IStriker striker
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker)
{
}
@@ -54,13 +60,12 @@ public sealed class LidarrClient : ArrClient
);
SetApiKey(request, arrInstance.ApiKey);
using var response = await _httpClient.SendAsync(request);
string? logContext = await ComputeCommandLogContextAsync(arrInstance, command);
try
{
response.EnsureSuccessStatusCode();
using var _ = await ((LidarrClient)Proxy).SendRequestAsync(request);
_logger.LogInformation("{log}", GetSearchLog(arrInstance.Url, command, true, logContext));
}
catch

13
code/Infrastructure/Verticals/Arr/RadarrClient.cs
View File

@@ -5,6 +5,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Domain.Models.Radarr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -12,14 +13,19 @@ using Newtonsoft.Json;
namespace Infrastructure.Verticals.Arr;
public sealed class RadarrClient : ArrClient
public class RadarrClient : ArrClient, IRadarrClient
{
/// <inheritdoc/>
public RadarrClient()
{
}
public RadarrClient(
ILogger<ArrClient> logger,
IHttpClientFactory httpClientFactory,
IOptions<LoggingConfig> loggingConfig,
IOptions<QueueCleanerConfig> queueCleanerConfig,
Striker striker
IStriker striker
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker)
{
}
@@ -62,12 +68,11 @@ public sealed class RadarrClient : ArrClient
);
SetApiKey(request, arrInstance.ApiKey);
using HttpResponseMessage response = await _httpClient.SendAsync(request);
string? logContext = await ComputeCommandLogContextAsync(arrInstance, command);
try
{
response.EnsureSuccessStatusCode();
using var _ = await ((RadarrClient)Proxy).SendRequestAsync(request);
_logger.LogInformation("{log}", GetSearchLog(arrInstance.Url, command, true, logContext));
}

15
code/Infrastructure/Verticals/Arr/SonarrClient.cs
View File

@@ -5,6 +5,7 @@ using Common.Configuration.QueueCleaner;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Domain.Models.Sonarr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -13,14 +14,19 @@ using Series = Domain.Models.Sonarr.Series;
namespace Infrastructure.Verticals.Arr;
public sealed class SonarrClient : ArrClient
public class SonarrClient : ArrClient, ISonarrClient
{
/// <inheritdoc/>
public SonarrClient()
{
}
public SonarrClient(
ILogger<SonarrClient> logger,
IHttpClientFactory httpClientFactory,
IOptions<LoggingConfig> loggingConfig,
IOptions<QueueCleanerConfig> queueCleanerConfig,
Striker striker
IStriker striker
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker)
{
}
@@ -58,13 +64,12 @@ public sealed class SonarrClient : ArrClient
);
SetApiKey(request, arrInstance.ApiKey);
using HttpResponseMessage response = await _httpClient.SendAsync(request);
string? logContext = await ComputeCommandLogContextAsync(arrInstance, command, command.SearchType);
try
{
response.EnsureSuccessStatusCode();
using var _ = await ((SonarrClient)Proxy).SendRequestAsync(request);
_logger.LogInformation("{log}", GetSearchLog(command.SearchType, arrInstance.Url, command, true, logContext));
}
catch

6
code/Infrastructure/Verticals/ContentBlocker/ContentBlocker.cs
View File

@@ -7,6 +7,7 @@ using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Jobs;
@@ -36,7 +37,6 @@ public sealed class ContentBlocker : GenericHandler
BlocklistProvider blocklistProvider,
DownloadServiceFactory downloadServiceFactory,
NotificationPublisher notifier
) : base(
logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig,
@@ -76,7 +76,7 @@ public sealed class ContentBlocker : GenericHandler
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = [];
ArrClient arrClient = GetClient(instanceType);
IArrClient arrClient = GetClient(instanceType);
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
@@ -131,7 +131,7 @@ public sealed class ContentBlocker : GenericHandler
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
await _notifier.NotifyQueueItemDelete(removeFromClient, DeleteReason.AllFilesBlocked);
await _notifier.NotifyQueueItemDeleted(removeFromClient, DeleteReason.AllFilesBlocked);
}
});

6
code/Infrastructure/Verticals/ContentBlocker/FilenameEvaluator.cs
View File

@@ -5,7 +5,7 @@ using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.ContentBlocker;
public sealed class FilenameEvaluator
public class FilenameEvaluator : IFilenameEvaluator
{
private readonly ILogger<FilenameEvaluator> _logger;
@@ -31,7 +31,6 @@ public sealed class FilenameEvaluator
{
BlocklistType.Blacklist => !patterns.Any(pattern => MatchesPattern(filename, pattern)),
BlocklistType.Whitelist => patterns.Any(pattern => MatchesPattern(filename, pattern)),
_ => true
};
}
@@ -46,7 +45,6 @@ public sealed class FilenameEvaluator
{
BlocklistType.Blacklist => !regexes.Any(regex => regex.IsMatch(filename)),
BlocklistType.Whitelist => regexes.Any(regex => regex.IsMatch(filename)),
_ => true
};
}
@@ -76,6 +74,6 @@ public sealed class FilenameEvaluator
);
}
return filename == pattern;
return filename.Equals(pattern, StringComparison.InvariantCultureIgnoreCase);
}
}

10
code/Infrastructure/Verticals/ContentBlocker/IFilenameEvaluator.cs Normal file
View File

@@ -0,0 +1,10 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
namespace Infrastructure.Verticals.ContentBlocker;
public interface IFilenameEvaluator
{
bool IsValid(string filename, BlocklistType type, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes);
}

4
code/Infrastructure/Verticals/Context/ContextProvider.cs
View File

@@ -17,8 +17,8 @@ public static class ContextProvider
return _asyncLocalDict.Value?.TryGetValue(key, out object? value) is true ? value : null;
}
public static T? Get<T>(string key) where T : class
public static T Get<T>(string key) where T : class
{
return Get(key) as T;
return Get(key) as T ?? throw new Exception($"failed to get \"{key}\" from context");
}
}

99
code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs Normal file
View File

@@ -0,0 +1,99 @@
using Common.Configuration.Arr;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Domain.Enums;
using Domain.Models.Arr.Queue;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Jobs;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Serilog.Context;
namespace Infrastructure.Verticals.DownloadCleaner;
public sealed class DownloadCleaner : GenericHandler
{
private readonly DownloadCleanerConfig _config;
private readonly HashSet<string> _excludedHashes = [];
public DownloadCleaner(
ILogger<DownloadCleaner> logger,
IOptions<DownloadCleanerConfig> config,
IOptions<DownloadClientConfig> downloadClientConfig,
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
IOptions<LidarrConfig> lidarrConfig,
SonarrClient sonarrClient,
RadarrClient radarrClient,
LidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory,
NotificationPublisher notifier
) : base(
logger, downloadClientConfig,
sonarrConfig, radarrConfig, lidarrConfig,
sonarrClient, radarrClient, lidarrClient,
arrArrQueueIterator, downloadServiceFactory,
notifier
)
{
_config = config.Value;
_config.Validate();
}
public override async Task ExecuteAsync()
{
if (_config.Categories?.Count is null or 0)
{
_logger.LogWarning("no categories configured");
return;
}
await _downloadService.LoginAsync();
List<object>? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories);
if (downloads?.Count is null or 0)
{
_logger.LogDebug("no downloads found in the download client");
return;
}
// wait for the downloads to appear in the arr queue
await Task.Delay(10 * 1000);
await ProcessArrConfigAsync(_sonarrConfig, InstanceType.Sonarr, true);
await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true);
await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true);
await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes);
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
{
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
IArrClient arrClient = GetClient(instanceType);
await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
{
var groups = items
.Where(x => !string.IsNullOrEmpty(x.DownloadId))
.GroupBy(x => x.DownloadId)
.ToList();
foreach (QueueRecord record in groups.Select(group => group.First()))
{
_excludedHashes.Add(record.DownloadId.ToLowerInvariant());
}
});
}
public override void Dispose()
{
_downloadService.Dispose();
}
}

24
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs
View File

@@ -62,6 +62,26 @@ public sealed class DelugeClient
await ListTorrentsExtended(new Dictionary<string, string> { { "hash", hash } });
return torrents.FirstOrDefault();
}
public async Task<TorrentStatus?> GetTorrentStatus(string hash)
{
return await SendRequest<TorrentStatus?>(
"web.get_torrent_status",
hash,
new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" }
);
}
public async Task<List<TorrentStatus>?> GetStatusForAllTorrents()
{
Dictionary<string, TorrentStatus>? downloads = await SendRequest<Dictionary<string, TorrentStatus>?>(
"core.get_torrents_status",
"",
new[] { "hash", "state", "name", "eta", "private", "total_done", "label", "seeding_time", "ratio" }
);
return downloads?.Values.ToList();
}
public async Task<DelugeContents?> GetTorrentFiles(string hash)
{
@@ -78,9 +98,9 @@ public sealed class DelugeClient
await SendRequest<DelugeResponse<object>>("core.set_torrent_options", hash, filePriorities);
}
public async Task<DelugeResponse<object>> DeleteTorrent(string hash)
public async Task DeleteTorrents(List<string> hashes)
{
return await SendRequest<DelugeResponse<object>>("core.remove_torrents", new List<string> { hash }, true);
await SendRequest<DelugeResponse<object>>("core.remove_torrents", hashes, true);
}
private async Task<String> PostJson(String json)

111
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs
View File

@@ -1,21 +1,31 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Domain.Enums;
using Domain.Models.Deluge.Response;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using MassTransit.Configuration;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Infrastructure.Verticals.DownloadClient.Deluge;
public sealed class DelugeService : DownloadServiceBase
public class DelugeService : DownloadService, IDelugeService
{
private readonly DelugeClient _client;
/// <inheritdoc/>
public DelugeService()
{
}
public DelugeService(
ILogger<DelugeService> logger,
@@ -23,10 +33,12 @@ public sealed class DelugeService : DownloadServiceBase
IHttpClientFactory httpClientFactory,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
IFilenameEvaluator filenameEvaluator,
IStriker striker,
NotificationPublisher notifier
) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier)
{
config.Value.Validate();
_client = new (config, httpClientFactory);
@@ -45,7 +57,7 @@ public sealed class DelugeService : DownloadServiceBase
DelugeContents? contents = null;
StalledResult result = new();
TorrentStatus? status = await GetTorrentStatus(hash);
TorrentStatus? status = await _client.GetTorrentStatus(hash);
if (status?.Hash is null)
{
@@ -98,7 +110,7 @@ public sealed class DelugeService : DownloadServiceBase
{
hash = hash.ToLowerInvariant();
TorrentStatus? status = await GetTorrentStatus(hash);
TorrentStatus? status = await _client.GetTorrentStatus(hash);
BlockFilesResult result = new();
if (status?.Hash is null)
@@ -178,17 +190,89 @@ public sealed class DelugeService : DownloadServiceBase
return result;
}
await _client.ChangeFilesPriority(hash, sortedPriorities);
await ((DelugeService)Proxy).ChangeFilesPriority(hash, sortedPriorities);
return result;
}
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
{
return (await _client.GetStatusForAllTorrents())
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true)
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes)
{
foreach (TorrentStatus download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
Category? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.Private)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SeedingTime);
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, seedingTime, category);
if (!result.ShouldClean)
{
continue;
}
await ((DelugeService)Proxy).DeleteDownload(download.Hash);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _notifier.NotifyDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason);
}
}
/// <inheritdoc/>
public override async Task Delete(string hash)
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
hash = hash.ToLowerInvariant();
await _client.DeleteTorrent(hash);
await _client.DeleteTorrents([hash]);
}
[DryRunSafeguard]
protected virtual async Task ChangeFilesPriority(string hash, List<int> sortedPriorities)
{
await _client.ChangeFilesPriority(hash, sortedPriorities);
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentStatus status)
@@ -219,15 +303,6 @@ public sealed class DelugeService : DownloadServiceBase
return await StrikeAndCheckLimit(status.Hash!, status.Name!);
}
private async Task<TorrentStatus?> GetTorrentStatus(string hash)
{
return await _client.SendRequest<TorrentStatus?>(
"web.get_torrent_status",
hash,
new[] { "hash", "state", "name", "eta", "private", "total_done" }
);
}
private static void ProcessFiles(Dictionary<string, DelugeFileOrDirectory>? contents, Action<string, DelugeFileOrDirectory> processFile)
{

5
code/Infrastructure/Verticals/DownloadClient/Deluge/IDelugeService.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.DownloadClient.Deluge;
public interface IDelugeService : IDownloadService
{
}

183
code/Infrastructure/Verticals/DownloadClient/DownloadService.cs Normal file
View File

@@ -0,0 +1,183 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Domain.Enums;
using Domain.Models.Cache;
using Infrastructure.Helpers;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Infrastructure.Verticals.DownloadClient;
public abstract class DownloadService : InterceptedService, IDownloadService
{
protected readonly ILogger<DownloadService> _logger;
protected readonly QueueCleanerConfig _queueCleanerConfig;
protected readonly ContentBlockerConfig _contentBlockerConfig;
protected readonly DownloadCleanerConfig _downloadCleanerConfig;
protected readonly IMemoryCache _cache;
protected readonly IFilenameEvaluator _filenameEvaluator;
protected readonly IStriker _striker;
protected readonly MemoryCacheEntryOptions _cacheOptions;
protected readonly NotificationPublisher _notifier;
/// <summary>
/// Constructor to be used by interceptors.
/// </summary>
protected DownloadService()
{
}
protected DownloadService(
ILogger<DownloadService> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
IFilenameEvaluator filenameEvaluator,
IStriker striker,
NotificationPublisher notifier)
{
_logger = logger;
_queueCleanerConfig = queueCleanerConfig.Value;
_contentBlockerConfig = contentBlockerConfig.Value;
_downloadCleanerConfig = downloadCleanerConfig.Value;
_cache = cache;
_filenameEvaluator = filenameEvaluator;
_striker = striker;
_notifier = notifier;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
}
public abstract void Dispose();
public abstract Task LoginAsync();
public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
/// <inheritdoc/>
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
);
/// <inheritdoc/>
public abstract Task DeleteDownload(string hash);
/// <inheritdoc/>
public abstract Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
/// <inheritdoc/>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes);
protected void ResetStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
{
return;
}
if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded)
{
// cache item found
_cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
_logger.LogDebug("resetting strikes for {hash} due to progress", hash);
}
_cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions);
}
/// <summary>
/// Strikes an item and checks if the limit has been reached.
/// </summary>
/// <param name="hash">The torrent hash.</param>
/// <param name="itemName">The name or title of the item.</param>
/// <returns>True if the limit has been reached; otherwise, false.</returns>
protected async Task<bool> StrikeAndCheckLimit(string hash, string itemName)
{
return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled);
}
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category)
{
// check ratio
if (DownloadReachedRatio(ratio, seedingTime, category))
{
return new()
{
ShouldClean = true,
Reason = CleanReason.MaxRatioReached
};
}
// check max seed time
if (DownloadReachedMaxSeedTime(seedingTime, category))
{
return new()
{
ShouldClean = true,
Reason = CleanReason.MaxSeedTimeReached
};
}
return new();
}
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category)
{
if (category.MaxRatio < 0)
{
return false;
}
string downloadName = ContextProvider.Get<string>("downloadName");
TimeSpan minSeedingTime = TimeSpan.FromHours(category.MinSeedTime);
if (category.MinSeedTime > 0 && seedingTime < minSeedingTime)
{
_logger.LogDebug("skip | download has not reached MIN_SEED_TIME | {name}", downloadName);
return false;
}
if (ratio < category.MaxRatio)
{
_logger.LogDebug("skip | download has not reached MAX_RATIO | {name}", downloadName);
return false;
}
// max ration is 0 or reached
return true;
}
private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, Category category)
{
if (category.MaxSeedTime < 0)
{
return false;
}
string downloadName = ContextProvider.Get<string>("downloadName");
TimeSpan maxSeedingTime = TimeSpan.FromHours(category.MaxSeedTime);
if (category.MaxSeedTime > 0 && seedingTime < maxSeedingTime)
{
_logger.LogDebug("skip | download has not reached MAX_SEED_TIME | {name}", downloadName);
return false;
}
// max seed time is 0 or reached
return true;
}
}

90
code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs
View File

@@ -1,90 +0,0 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Domain.Enums;
using Domain.Models.Cache;
using Infrastructure.Helpers;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Infrastructure.Verticals.DownloadClient;
public abstract class DownloadServiceBase : IDownloadService
{
protected readonly ILogger<DownloadServiceBase> _logger;
protected readonly QueueCleanerConfig _queueCleanerConfig;
protected readonly ContentBlockerConfig _contentBlockerConfig;
protected readonly IMemoryCache _cache;
protected readonly FilenameEvaluator _filenameEvaluator;
protected readonly Striker _striker;
protected readonly MemoryCacheEntryOptions _cacheOptions;
protected DownloadServiceBase(
ILogger<DownloadServiceBase> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
)
{
_logger = logger;
_queueCleanerConfig = queueCleanerConfig.Value;
_contentBlockerConfig = contentBlockerConfig.Value;
_cache = cache;
_filenameEvaluator = filenameEvaluator;
_striker = striker;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
}
public abstract void Dispose();
public abstract Task LoginAsync();
public abstract Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash);
/// <inheritdoc/>
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(
string hash,
BlocklistType blocklistType,
ConcurrentBag<string> patterns,
ConcurrentBag<Regex> regexes
);
/// <inheritdoc/>
public abstract Task Delete(string hash);
protected void ResetStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
{
return;
}
if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded)
{
// cache item found
_cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
_logger.LogDebug("resetting strikes for {hash} due to progress", hash);
}
_cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions);
}
/// <summary>
/// Strikes an item and checks if the limit has been reached.
/// </summary>
/// <param name="hash">The torrent hash.</param>
/// <param name="itemName">The name or title of the item.</param>
/// <returns>True if the limit has been reached; otherwise, false.</returns>
protected async Task<bool> StrikeAndCheckLimit(string hash, string itemName)
{
return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled);
}
}

18
code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs
View File

@@ -1,18 +1,20 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Infrastructure.Verticals.DownloadClient;
public sealed class DummyDownloadService : DownloadServiceBase
public sealed class DummyDownloadService : DownloadService
{
public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IMemoryCache cache, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
public DummyDownloadService(ILogger<DownloadService> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IOptions<DownloadCleanerConfig> downloadCleanerConfig, IMemoryCache cache, IFilenameEvaluator filenameEvaluator, IStriker striker, NotificationPublisher notifier) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier)
{
}
@@ -35,7 +37,17 @@ public sealed class DummyDownloadService : DownloadServiceBase
throw new NotImplementedException();
}
public override Task Delete(string hash)
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
{
throw new NotImplementedException();
}
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes)
{
throw new NotImplementedException();
}
public override Task DeleteDownload(string hash)
{
throw new NotImplementedException();
}

21
code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs
View File

@@ -1,10 +1,12 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Infrastructure.Interceptors;
namespace Infrastructure.Verticals.DownloadClient;
public interface IDownloadService : IDisposable
public interface IDownloadService : IDisposable, IDryRunService
{
public Task LoginAsync();
@@ -29,8 +31,23 @@ public interface IDownloadService : IDisposable
ConcurrentBag<Regex> regexes
);
/// <summary>
/// Fetches all downloads.
/// </summary>
/// <param name="categories">The categories by which to filter the downloads.</param>
/// <returns>A list of downloads for the provided categories.</returns>
Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
/// <summary>
/// Cleans the downloads.
/// </summary>
/// <param name="downloads"></param>
/// <param name="categoriesToClean">The categories that should be cleaned.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes);
/// <summary>
/// Deletes a download item.
/// </summary>
public Task Delete(string hash);
public Task DeleteDownload(string hash);
}

5
code/Infrastructure/Verticals/DownloadClient/QBittorrent/IQBitService.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
public interface IQBitService : IDownloadService
{
}

105
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs
View File

@@ -1,34 +1,46 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using QBittorrent.Client;
using Category = Common.Configuration.DownloadCleaner.Category;
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
public sealed class QBitService : DownloadServiceBase
public class QBitService : DownloadService, IQBitService
{
private readonly QBitConfig _config;
private readonly QBittorrentClient _client;
/// <inheritdoc/>
public QBitService()
{
}
public QBitService(
ILogger<QBitService> logger,
IHttpClientFactory httpClientFactory,
IOptions<QBitConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
IFilenameEvaluator filenameEvaluator,
IStriker striker,
NotificationPublisher notifier
) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier)
{
_config = config.Value;
_config.Validate();
@@ -188,17 +200,98 @@ public sealed class QBitService : DownloadServiceBase
foreach (int fileIndex in unwantedFiles)
{
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
await ((QBitService)Proxy).SkipFile(hash, fileIndex);
}
return result;
}
/// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) =>
(await _client.GetTorrentListAsync(new()
{
Filter = TorrentListFilter.Seeding
}))
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task Delete(string hash)
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes)
{
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
Category? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate)
{
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash);
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
bool.TryParse(dictValue?.ToString(), out bool boolValue)
&& boolValue;
if (isPrivate)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category);
if (!result.ShouldClean)
{
continue;
}
await ((QBitService)Proxy).DeleteDownload(download.Hash);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _notifier.NotifyDownloadCleaned(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category.Name, result.Reason);
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
await _client.DeleteAsync(hash, deleteDownloadedData: true);
}
[DryRunSafeguard]
protected virtual async Task SkipFile(string hash, int fileIndex)
{
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
}
public override void Dispose()
{

9
code/Infrastructure/Verticals/DownloadClient/SeedingCheckResult.cs Normal file
View File

@@ -0,0 +1,9 @@
using Domain.Enums;
namespace Infrastructure.Verticals.DownloadClient;
public sealed record SeedingCheckResult
{
public bool ShouldClean { get; set; }
public CleanReason Reason { get; set; }
}

5
code/Infrastructure/Verticals/DownloadClient/Transmission/ITransmissionService.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.DownloadClient.Transmission;
public interface ITransmissionService : IDownloadService
{
}

127
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs
View File

@@ -1,12 +1,16 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Attributes;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.Helpers;
using Domain.Enums;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -16,22 +20,29 @@ using Transmission.API.RPC.Entity;
namespace Infrastructure.Verticals.DownloadClient.Transmission;
public sealed class TransmissionService : DownloadServiceBase
public class TransmissionService : DownloadService, ITransmissionService
{
private readonly TransmissionConfig _config;
private readonly Client _client;
private TorrentInfo[]? _torrentsCache;
/// <inheritdoc/>
public TransmissionService()
{
}
public TransmissionService(
IHttpClientFactory httpClientFactory,
ILogger<TransmissionService> logger,
IOptions<TransmissionConfig> config,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
FilenameEvaluator filenameEvaluator,
Striker striker
) : base(logger, queueCleanerConfig, contentBlockerConfig, cache, filenameEvaluator, striker)
IFilenameEvaluator filenameEvaluator,
IStriker striker,
NotificationPublisher notifier
) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier)
{
_config = config.Value;
_config.Validate();
@@ -164,16 +175,96 @@ public sealed class TransmissionService : DownloadServiceBase
_logger.LogDebug("changing priorities | torrent {hash}", hash);
await _client.TorrentSetAsync(new TorrentSettings
{
Ids = [ torrent.Id ],
FilesUnwanted = unwantedFiles.ToArray(),
});
await ((TransmissionService)Proxy).SetUnwantedFiles(torrent.Id, unwantedFiles.ToArray());
return result;
}
public override async Task Delete(string hash)
/// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
{
string[] fields = [
TorrentFields.FILES,
TorrentFields.FILE_STATS,
TorrentFields.HASH_STRING,
TorrentFields.ID,
TorrentFields.ETA,
TorrentFields.NAME,
TorrentFields.STATUS,
TorrentFields.IS_PRIVATE,
TorrentFields.DOWNLOADED_EVER,
TorrentFields.DOWNLOAD_DIR,
TorrentFields.SECONDS_SEEDING,
TorrentFields.UPLOAD_RATIO
];
return (await _client.TorrentGetAsync(fields))
?.Torrents
?.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => x.Status is 5 or 6)
.Where(x => categories
.Any(cat => x.DownloadDir?.EndsWith(cat.Name, StringComparison.InvariantCultureIgnoreCase) is true)
)
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes)
{
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.HashString))
{
continue;
}
Category? category = categoriesToClean
.FirstOrDefault(x => download.DownloadDir?.EndsWith(x.Name, StringComparison.InvariantCultureIgnoreCase) is true);
if (category is null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.HashString);
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SecondsSeeding ?? 0);
SeedingCheckResult result = ShouldCleanDownload(download.uploadRatio ?? 0, seedingTime, category);
if (!result.ShouldClean)
{
continue;
}
await ((TransmissionService)Proxy).RemoveDownloadAsync(download.Id);
_logger.LogInformation(
"download cleaned | {reason} reached | {name}",
result.Reason is CleanReason.MaxRatioReached
? "MAX_RATIO & MIN_SEED_TIME"
: "MAX_SEED_TIME",
download.Name
);
await _notifier.NotifyDownloadCleaned(download.uploadRatio ?? 0, seedingTime, category.Name, result.Reason);
}
}
public override async Task DeleteDownload(string hash)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);
@@ -189,6 +280,22 @@ public sealed class TransmissionService : DownloadServiceBase
{
}
[DryRunSafeguard]
protected virtual async Task RemoveDownloadAsync(long downloadId)
{
await _client.TorrentRemoveAsync([downloadId], true);
}
[DryRunSafeguard]
protected virtual async Task SetUnwantedFiles(long downloadId, long[] unwantedFiles)
{
await _client.TorrentSetAsync(new TorrentSettings
{
Ids = [downloadId],
FilesUnwanted = unwantedFiles,
});
}
private async Task<bool> IsItemStuckAndShouldRemove(TorrentInfo torrent)
{
if (_queueCleanerConfig.StalledMaxStrikes is 0)

8
code/Infrastructure/Verticals/ItemStriker/IStriker.cs Normal file
View File

@@ -0,0 +1,8 @@
using Domain.Enums;
namespace Infrastructure.Verticals.ItemStriker;
public interface IStriker
{
Task<bool> StrikeAndCheckLimit(string hash, string itemName, ushort maxStrikes, StrikeType strikeType);
}

2
code/Infrastructure/Verticals/ItemStriker/Striker.cs
View File

@@ -8,7 +8,7 @@ using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.ItemStriker;
public class Striker
public sealed class Striker : IStriker
{
private readonly ILogger<Striker> _logger;
private readonly IMemoryCache _cache;

24
code/Infrastructure/Verticals/Jobs/GenericHandler.cs
View File

@@ -4,6 +4,7 @@ using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Logging;
@@ -11,16 +12,16 @@ using Microsoft.Extensions.Options;
namespace Infrastructure.Verticals.Jobs;
public abstract class GenericHandler : IDisposable
public abstract class GenericHandler : IHandler, IDisposable
{
protected readonly ILogger<GenericHandler> _logger;
protected readonly DownloadClientConfig _downloadClientConfig;
protected readonly SonarrConfig _sonarrConfig;
protected readonly RadarrConfig _radarrConfig;
protected readonly LidarrConfig _lidarrConfig;
protected readonly SonarrClient _sonarrClient;
protected readonly RadarrClient _radarrClient;
protected readonly LidarrClient _lidarrClient;
protected readonly ISonarrClient _sonarrClient;
protected readonly IRadarrClient _radarrClient;
protected readonly ILidarrClient _lidarrClient;
protected readonly ArrQueueIterator _arrArrQueueIterator;
protected readonly IDownloadService _downloadService;
protected readonly NotificationPublisher _notifier;
@@ -31,9 +32,9 @@ public abstract class GenericHandler : IDisposable
IOptions<SonarrConfig> sonarrConfig,
IOptions<RadarrConfig> radarrConfig,
IOptions<LidarrConfig> lidarrConfig,
SonarrClient sonarrClient,
RadarrClient radarrClient,
LidarrClient lidarrClient,
ISonarrClient sonarrClient,
IRadarrClient radarrClient,
ILidarrClient lidarrClient,
ArrQueueIterator arrArrQueueIterator,
DownloadServiceFactory downloadServiceFactory,
NotificationPublisher notifier
@@ -68,7 +69,7 @@ public abstract class GenericHandler : IDisposable
protected abstract Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType);
private async Task ProcessArrConfigAsync(ArrConfig config, InstanceType instanceType)
protected async Task ProcessArrConfigAsync(ArrConfig config, InstanceType instanceType, bool throwOnFailure = false)
{
if (!config.Enabled)
{
@@ -84,11 +85,16 @@ public abstract class GenericHandler : IDisposable
catch (Exception exception)
{
_logger.LogError(exception, "failed to clean {type} instance | {url}", instanceType, arrInstance.Url);
if (throwOnFailure)
{
throw;
}
}
}
}
protected ArrClient GetClient(InstanceType type) =>
protected IArrClient GetClient(InstanceType type) =>
type switch
{
InstanceType.Sonarr => _sonarrClient,

6
code/Infrastructure/Verticals/Jobs/IHandler.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace Infrastructure.Verticals.Jobs;
public interface IHandler
{
Task ExecuteAsync();
}

5
code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs
View File

@@ -27,9 +27,12 @@ public sealed class NotificationConsumer<T> : IConsumer<T> where T : Notificatio
case StalledStrikeNotification stalledMessage:
await _notificationService.Notify(stalledMessage);
break;
case QueueItemDeleteNotification queueItemDeleteMessage:
case QueueItemDeletedNotification queueItemDeleteMessage:
await _notificationService.Notify(queueItemDeleteMessage);
break;
case DownloadCleanedNotification downloadCleanedNotification:
await _notificationService.Notify(downloadCleanedNotification);
break;
default:
throw new NotImplementedException();
}

4
code/Infrastructure/Verticals/Notifications/INotificationFactory.cs
View File

@@ -6,5 +6,7 @@ public interface INotificationFactory
List<INotificationProvider> OnStalledStrikeEnabled();
List<INotificationProvider> OnQueueItemDeleteEnabled();
List<INotificationProvider> OnQueueItemDeletedEnabled();
List<INotificationProvider> OnDownloadCleanedEnabled();
}

4
code/Infrastructure/Verticals/Notifications/INotificationProvider.cs
View File

@@ -13,5 +13,7 @@ public interface INotificationProvider
Task OnStalledStrike(StalledStrikeNotification notification);
Task OnQueueItemDelete(QueueItemDeleteNotification notification);
Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
Task OnDownloadCleaned(DownloadCleanedNotification notification);
}

14
code/Infrastructure/Verticals/Notifications/Models/ArrNotification.cs Normal file
View File

@@ -0,0 +1,14 @@
using Domain.Enums;
namespace Infrastructure.Verticals.Notifications.Models;
public record ArrNotification : Notification
{
public required InstanceType InstanceType { get; init; }
public required Uri InstanceUrl { get; init; }
public required string Hash { get; init; }
public Uri? Image { get; init; }
}

5
code/Infrastructure/Verticals/Notifications/Models/DownloadCleanedNotification.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record DownloadCleanedNotification : Notification
{
}

2
code/Infrastructure/Verticals/Notifications/Models/FailedImportStrikeNotification.cs
View File

@@ -1,5 +1,5 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record FailedImportStrikeNotification : Notification
public sealed record FailedImportStrikeNotification : ArrNotification
{
}

16
code/Infrastructure/Verticals/Notifications/Models/Notification.cs
View File

@@ -1,20 +1,12 @@
using Domain.Enums;
namespace Infrastructure.Verticals.Notifications.Models;
namespace Infrastructure.Verticals.Notifications.Models;
public record Notification
public abstract record Notification
{
public required InstanceType InstanceType { get; init; }
public required Uri InstanceUrl { get; init; }
public required string Hash { get; init; }
public required string Title { get; init; }
public required string Description { get; init; }
public Uri? Image { get; init; }
public List<NotificationField>? Fields { get; init; }
public NotificationLevel Level { get; init; }
}

9
code/Infrastructure/Verticals/Notifications/Models/NotificationLevel.cs Normal file
View File

@@ -0,0 +1,9 @@
namespace Infrastructure.Verticals.Notifications.Models;
public enum NotificationLevel
{
Test,
Information,
Warning,
Important
}

5
code/Infrastructure/Verticals/Notifications/Models/QueueItemDeleteNotification.cs
View File

@@ -1,5 +0,0 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record QueueItemDeleteNotification : Notification
{
}

5
code/Infrastructure/Verticals/Notifications/Models/QueueItemDeletedNotification.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record QueueItemDeletedNotification : ArrNotification
{
}

2
code/Infrastructure/Verticals/Notifications/Models/StalledStrikeNotification.cs
View File

@@ -1,5 +1,5 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record StalledStrikeNotification : Notification
public sealed record StalledStrikeNotification : ArrNotification
{
}

43
code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs
View File

@@ -1,4 +1,3 @@
using Domain.Enums;
using Infrastructure.Verticals.Notifications.Models;
using Mapster;
using Microsoft.Extensions.Options;
@@ -12,6 +11,7 @@ public class NotifiarrProvider : NotificationProvider
private const string WarningColor = "f0ad4e";
private const string ImportantColor = "bb2124";
private const string Logo = "https://github.com/flmorg/cleanuperr/blob/main/Logo/48.png?raw=true";
public NotifiarrProvider(IOptions<NotifiarrConfig> config, INotifiarrProxy proxy)
: base(config)
@@ -32,12 +32,17 @@ public class NotifiarrProvider : NotificationProvider
await _proxy.SendNotification(BuildPayload(notification, WarningColor), _config);
}
public override async Task OnQueueItemDelete(QueueItemDeleteNotification notification)
public override async Task OnQueueItemDeleted(QueueItemDeletedNotification notification)
{
await _proxy.SendNotification(BuildPayload(notification, ImportantColor), _config);
}
private NotifiarrPayload BuildPayload(Notification notification, string color)
public override async Task OnDownloadCleaned(DownloadCleanedNotification notification)
{
await _proxy.SendNotification(BuildPayload(notification), _config);
}
private NotifiarrPayload BuildPayload(ArrNotification notification, string color)
{
NotifiarrPayload payload = new()
{
@@ -47,7 +52,7 @@ public class NotifiarrProvider : NotificationProvider
Text = new()
{
Title = notification.Title,
Icon = "https://github.com/flmorg/cleanuperr/blob/main/Logo/48.png?raw=true",
Icon = Logo,
Description = notification.Description,
Fields = new()
{
@@ -62,7 +67,7 @@ public class NotifiarrProvider : NotificationProvider
},
Images = new()
{
Thumbnail = new Uri("https://github.com/flmorg/cleanuperr/blob/main/Logo/48.png?raw=true"),
Thumbnail = new Uri(Logo),
Image = notification.Image
}
}
@@ -72,4 +77,32 @@ public class NotifiarrProvider : NotificationProvider
return payload;
}
private NotifiarrPayload BuildPayload(DownloadCleanedNotification notification)
{
NotifiarrPayload payload = new()
{
Discord = new()
{
Color = ImportantColor,
Text = new()
{
Title = notification.Title,
Icon = Logo,
Description = notification.Description,
Fields = notification.Fields?.Adapt<List<Field>>() ?? []
},
Ids = new Ids
{
Channel = _config.ChannelId
},
Images = new()
{
Thumbnail = new Uri(Logo)
}
}
};
return payload;
}
}

9
code/Infrastructure/Verticals/Notifications/NotificationFactory.cs
View File

@@ -25,8 +25,13 @@ public class NotificationFactory : INotificationFactory
.Where(n => n.Config.OnStalledStrike)
.ToList();
public List<INotificationProvider> OnQueueItemDeleteEnabled() =>
public List<INotificationProvider> OnQueueItemDeletedEnabled() =>
ActiveProviders()
.Where(n => n.Config.OnQueueItemDelete)
.Where(n => n.Config.OnQueueItemDeleted)
.ToList();
public List<INotificationProvider> OnDownloadCleanedEnabled() =>
ActiveProviders()
.Where(n => n.Config.OnDownloadCleaned)
.ToList();
}

4
code/Infrastructure/Verticals/Notifications/NotificationProvider.cs
View File

@@ -19,5 +19,7 @@ public abstract class NotificationProvider : INotificationProvider
public abstract Task OnStalledStrike(StalledStrikeNotification notification);
public abstract Task OnQueueItemDelete(QueueItemDeleteNotification notification);
public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
public abstract Task OnDownloadCleaned(DownloadCleanedNotification notification);
}

73
code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs
View File

@@ -1,6 +1,9 @@
using Common.Configuration.Arr;
using System.Globalization;
using Common.Attributes;
using Common.Configuration.Arr;
using Domain.Enums;
using Domain.Models.Arr.Queue;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Notifications.Models;
using Mapster;
@@ -9,27 +12,35 @@ using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.Notifications;
public sealed class NotificationPublisher
public class NotificationPublisher : InterceptedService, IDryRunService
{
private readonly ILogger<NotificationPublisher> _logger;
private readonly IBus _messageBus;
/// <summary>
/// Constructor to be used by interceptors.
/// </summary>
public NotificationPublisher()
{
}
public NotificationPublisher(ILogger<NotificationPublisher> logger, IBus messageBus)
{
_logger = logger;
_messageBus = messageBus;
}
public async Task NotifyStrike(StrikeType strikeType, int strikeCount)
[DryRunSafeguard]
public virtual async Task NotifyStrike(StrikeType strikeType, int strikeCount)
{
try
{
QueueRecord record = GetRecordFromContext();
InstanceType instanceType = GetInstanceTypeFromContext();
Uri instanceUrl = GetInstanceUrlFromContext();
Uri? imageUrl = GetImageFromContext(record, instanceType);
QueueRecord record = ContextProvider.Get<QueueRecord>(nameof(QueueRecord));
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
Uri instanceUrl = ContextProvider.Get<Uri>(nameof(ArrInstance) + nameof(ArrInstance.Url));
Uri imageUrl = GetImageFromContext(record, instanceType);
Notification notification = new()
ArrNotification notification = new()
{
InstanceType = instanceType,
InstanceUrl = instanceUrl,
@@ -56,14 +67,15 @@ public sealed class NotificationPublisher
}
}
public async Task NotifyQueueItemDelete(bool removeFromClient, DeleteReason reason)
[DryRunSafeguard]
public virtual async Task NotifyQueueItemDeleted(bool removeFromClient, DeleteReason reason)
{
QueueRecord record = GetRecordFromContext();
InstanceType instanceType = GetInstanceTypeFromContext();
Uri instanceUrl = GetInstanceUrlFromContext();
Uri? imageUrl = GetImageFromContext(record, instanceType);
QueueRecord record = ContextProvider.Get<QueueRecord>(nameof(QueueRecord));
InstanceType instanceType = (InstanceType)ContextProvider.Get<object>(nameof(InstanceType));
Uri instanceUrl = ContextProvider.Get<Uri>(nameof(ArrInstance) + nameof(ArrInstance.Url));
Uri imageUrl = GetImageFromContext(record, instanceType);
Notification notification = new()
QueueItemDeletedNotification notification = new()
{
InstanceType = instanceType,
InstanceUrl = instanceUrl,
@@ -74,20 +86,29 @@ public sealed class NotificationPublisher
Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }]
};
await _messageBus.Publish(notification.Adapt<QueueItemDeleteNotification>());
await _messageBus.Publish(notification);
}
private static QueueRecord GetRecordFromContext() =>
ContextProvider.Get<QueueRecord>(nameof(QueueRecord)) ?? throw new Exception("failed to get record from context");
[DryRunSafeguard]
public virtual async Task NotifyDownloadCleaned(double ratio, TimeSpan seedingTime, string categoryName, CleanReason reason)
{
DownloadCleanedNotification notification = new()
{
Title = $"Cleaned item from download client with reason: {reason}",
Description = ContextProvider.Get<string>("downloadName"),
Fields =
[
new() { Title = "Hash", Text = ContextProvider.Get<string>("hash").ToLowerInvariant() },
new() { Title = "Category", Text = categoryName.ToLowerInvariant() },
new() { Title = "Ratio", Text = $"{ratio.ToString(CultureInfo.InvariantCulture)}%" },
new() { Title = "Seeding hours", Text = $"{Math.Round(seedingTime.TotalHours, 0).ToString(CultureInfo.InvariantCulture)}h" }
],
Level = NotificationLevel.Important
};
await _messageBus.Publish(notification);
}
private static InstanceType GetInstanceTypeFromContext() =>
(InstanceType)(ContextProvider.Get<object>(nameof(InstanceType)) ??
throw new Exception("failed to get instance type from context"));
private static Uri GetInstanceUrlFromContext() =>
ContextProvider.Get<Uri>(nameof(ArrInstance) + nameof(ArrInstance.Url)) ??
throw new Exception("failed to get instance url from context");
private static Uri GetImageFromContext(QueueRecord record, InstanceType instanceType) =>
instanceType switch
{

21
code/Infrastructure/Verticals/Notifications/NotificationService.cs
View File

@@ -44,13 +44,28 @@ public class NotificationService
}
}
public async Task Notify(QueueItemDeleteNotification notification)
public async Task Notify(QueueItemDeletedNotification notification)
{
foreach (INotificationProvider provider in _notificationFactory.OnQueueItemDeleteEnabled())
foreach (INotificationProvider provider in _notificationFactory.OnQueueItemDeletedEnabled())
{
try
{
await provider.OnQueueItemDelete(notification);
await provider.OnQueueItemDeleted(notification);
}
catch (Exception exception)
{
_logger.LogWarning(exception, "failed to send notification | provider {provider}", provider.Name);
}
}
}
public async Task Notify(DownloadCleanedNotification notification)
{
foreach (INotificationProvider provider in _notificationFactory.OnDownloadCleanedEnabled())
{
try
{
await provider.OnDownloadCleaned(notification);
}
catch (Exception exception)
{

5
code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs
View File

@@ -5,6 +5,7 @@ using Domain.Enums;
using Domain.Models.Arr;
using Domain.Models.Arr.Queue;
using Infrastructure.Verticals.Arr;
using Infrastructure.Verticals.Arr.Interfaces;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.Jobs;
@@ -48,7 +49,7 @@ public sealed class QueueCleaner : GenericHandler
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
HashSet<SearchItem> itemsToBeRefreshed = [];
ArrClient arrClient = GetClient(instanceType);
IArrClient arrClient = GetClient(instanceType);
// push to context
ContextProvider.Set(nameof(ArrInstance) + nameof(ArrInstance.Url), instance.Url);
@@ -113,7 +114,7 @@ public sealed class QueueCleaner : GenericHandler
}
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient);
await _notifier.NotifyQueueItemDelete(removeFromClient, deleteReason);
await _notifier.NotifyQueueItemDeleted(removeFromClient, deleteReason);
}
});