mirror/Cleanuparr
1
0
Fork 0
mirror of https://github.com/Cleanuparr/Cleanuparr.git synced 2026-04-16 03:58:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add category change for downloads with no additional hardlinks (#65)

Browse Source
This commit is contained in:
Flaminel
2025-05-04 17:26:51 +03:00
committed by GitHub
parent 8cfc73213a
commit 693f80fe6a
49 changed files with 1255 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
code/Infrastructure/Infrastructure.csproj
View File

@@ -18,6 +18,7 @@
<PackageReference Include="MassTransit" Version="8.3.6" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.2" />
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
<PackageReference Include="Quartz" Version="3.13.1" />
<PackageReference Include="Scrutor" Version="6.0.1" />
</ItemGroup>

2
code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs
View File

@@ -43,7 +43,7 @@ public sealed class BlocklistProvider
{
if (_initialized)
{
_logger.LogDebug("blocklists already loaded");
_logger.LogTrace("blocklists already loaded");
return;
}

32
code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs
View File

@@ -21,6 +21,8 @@ public sealed class DownloadCleaner : GenericHandler
private readonly IgnoredDownloadsProvider<DownloadCleanerConfig> _ignoredDownloadsProvider;
private readonly HashSet<string> _excludedHashes = [];
private static bool _hardLinkCategoryCreated;
public DownloadCleaner(
ILogger<DownloadCleaner> logger,
IOptions<DownloadCleanerConfig> config,
@@ -65,13 +67,20 @@ public sealed class DownloadCleaner : GenericHandler
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
await _downloadService.LoginAsync();
List<object>? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories);
if (downloads?.Count is null or 0)
List<object>? downloads = await _downloadService.GetSeedingDownloads();
List<object>? downloadsToChangeCategory = null;
if (!string.IsNullOrEmpty(_config.UnlinkedTargetCategory) && _config.UnlinkedCategories?.Count > 0)
{
_logger.LogDebug("no downloads found in the download client");
return;
if (!_hardLinkCategoryCreated)
{
_logger.LogDebug("creating category {cat}", _config.UnlinkedTargetCategory);
await _downloadService.CreateCategoryAsync(_config.UnlinkedTargetCategory);
_hardLinkCategoryCreated = true;
}
downloadsToChangeCategory = _downloadService.FilterDownloadsToChangeCategoryAsync(downloads, _config.UnlinkedCategories);
}
// wait for the downloads to appear in the arr queue
@@ -81,7 +90,16 @@ public sealed class DownloadCleaner : GenericHandler
await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true);
await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true);
await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes, ignoredDownloads);
_logger.LogTrace("looking for downloads to change category");
await _downloadService.ChangeCategoryForNoHardLinksAsync(downloadsToChangeCategory, _excludedHashes, ignoredDownloads);
List<object>? downloadsToClean = _downloadService.FilterDownloadsToBeCleanedAsync(downloads, _config.Categories);
// release unused objects
downloads = null;
_logger.LogTrace("looking for downloads to clean");
await _downloadService.CleanDownloadsAsync(downloadsToClean, _config.Categories, _excludedHashes, ignoredDownloads);
}
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)

52
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs
View File

@@ -2,6 +2,7 @@
using System.Text.Json.Serialization;
using Common.Configuration;
using Common.Configuration.DownloadClient;
using Common.Exceptions;
using Domain.Models.Deluge.Exceptions;
using Domain.Models.Deluge.Request;
using Domain.Models.Deluge.Response;
@@ -29,7 +30,8 @@ public sealed class DelugeClient
"ratio",
"trackers",
"download_payload_rate",
"total_size"
"total_size",
"download_location"
];
public DelugeClient(IOptions<DelugeConfig> config, IHttpClientFactory httpClientFactory)
@@ -44,11 +46,42 @@ public sealed class DelugeClient
return await SendRequest<bool>("auth.login", _config.Password);
}
public async Task<bool> IsConnected()
{
return await SendRequest<bool>("web.connected");
}
public async Task<bool> Connect()
{
string? firstHost = await GetHost();
if (string.IsNullOrEmpty(firstHost))
{
return false;
}
var result = await SendRequest<List<string>?>("web.connect", firstHost);
return result?.Count > 0;
}
public async Task<bool> Logout()
{
return await SendRequest<bool>("auth.delete_session");
}
public async Task<string?> GetHost()
{
var hosts = await SendRequest<List<List<string>?>?>("web.get_hosts");
if (hosts?.Count > 1)
{
throw new FatalException("multiple Deluge hosts found - please connect to only one host");
}
return hosts?.FirstOrDefault()?.FirstOrDefault();
}
public async Task<List<DelugeTorrent>> ListTorrents(Dictionary<string, string>? filters = null)
{
filters ??= new Dictionary<string, string>();
@@ -149,7 +182,7 @@ public sealed class DelugeClient
return responseJson;
}
private DelugeRequest CreateRequest(string method, params object[] parameters)
private static DelugeRequest CreateRequest(string method, params object[] parameters)
{
if (String.IsNullOrWhiteSpace(method))
{
@@ -195,4 +228,19 @@ public sealed class DelugeClient
return webResponse.Result;
}
public async Task<IReadOnlyList<string>> GetLabels()
{
return await SendRequest<IReadOnlyList<string>>("label.get_labels");
}
public async Task CreateLabel(string label)
{
await SendRequest<DelugeResponse<object>>("label.add", label);
}
public async Task SetTorrentLabel(string hash, string newLabel)
{
await SendRequest<DelugeResponse<object>>("label.set_torrent", hash, newLabel);
}
}

165
code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs
View File

@@ -7,12 +7,14 @@ using Common.Configuration.DownloadCleaner;
using Common.Configuration.DownloadClient;
using Common.Configuration.QueueCleaner;
using Common.CustomDataTypes;
using Common.Exceptions;
using Domain.Enums;
using Domain.Models.Deluge.Response;
using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -36,10 +38,11 @@ public class DelugeService : DownloadService, IDelugeService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
config.Value.Validate();
@@ -49,6 +52,11 @@ public class DelugeService : DownloadService, IDelugeService
public override async Task LoginAsync()
{
await _client.LoginAsync();
if (!await _client.IsConnected() && !await _client.Connect())
{
throw new FatalException("Deluge WebUI is not connected to the daemon");
}
}
/// <inheritdoc/>
@@ -208,26 +216,51 @@ public class DelugeService : DownloadService, IDelugeService
return result;
}
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
public override async Task<List<object>?> GetSeedingDownloads()
{
return (await _client.GetStatusForAllTorrents())
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true)
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
}
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<DownloadStatus>()
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<DownloadStatus>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (DownloadStatus download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
@@ -235,19 +268,13 @@ public class DelugeService : DownloadService, IDelugeService
continue;
}
Category? category = categoriesToClean
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.Private)
{
@@ -279,7 +306,107 @@ public class DelugeService : DownloadService, IDelugeService
await _notifier.NotifyDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason);
}
}
public override async Task CreateCategoryAsync(string name)
{
IReadOnlyList<string> existingLabels = await _client.GetLabels();
if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateLabel, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (DownloadStatus download in downloads.Cast<DownloadStatus>())
{
if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
DelugeContents? contents = null;
try
{
contents = await _client.GetTorrentFiles(download.Hash);
}
catch (Exception exception)
{
_logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name);
continue;
}
bool hasHardlinks = false;
ProcessFiles(contents?.Contents, (_, file) =>
{
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/']));
if (file.Priority <= 0)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
return;
}
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
return;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
}
});
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
_logger.LogInformation("category changed for {name}", download.Name);
await _notifier.NotifyCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory);
download.Label = _downloadCleanerConfig.UnlinkedTargetCategory;
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
@@ -288,6 +415,12 @@ public class DelugeService : DownloadService, IDelugeService
await _client.DeleteTorrents([hash]);
}
[DryRunSafeguard]
protected async Task CreateLabel(string name)
{
await _client.CreateLabel(name);
}
[DryRunSafeguard]
protected virtual async Task ChangeFilesPriority(string hash, List<int> sortedPriorities)
@@ -295,6 +428,12 @@ public class DelugeService : DownloadService, IDelugeService
await _client.ChangeFilesPriority(hash, sortedPriorities);
}
[DryRunSafeguard]
protected virtual async Task ChangeLabel(string hash, string newLabel)
{
await _client.SetTorrentLabel(hash, newLabel);
}
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(DownloadStatus status)
{
(bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(status);

51
code/Infrastructure/Verticals/DownloadClient/DownloadService.cs
View File

@@ -1,4 +1,4 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
@@ -11,6 +11,7 @@ using Infrastructure.Helpers;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -31,6 +32,7 @@ public abstract class DownloadService : IDownloadService
protected readonly MemoryCacheEntryOptions _cacheOptions;
protected readonly INotificationPublisher _notifier;
protected readonly IDryRunInterceptor _dryRunInterceptor;
protected readonly IHardLinkFileService _hardLinkFileService;
protected DownloadService(
ILogger<DownloadService> logger,
@@ -41,7 +43,8 @@ public abstract class DownloadService : IDownloadService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
)
{
_logger = logger;
@@ -53,6 +56,7 @@ public abstract class DownloadService : IDownloadService
_striker = striker;
_notifier = notifier;
_dryRunInterceptor = dryRunInterceptor;
_hardLinkFileService = hardLinkFileService;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
}
@@ -73,12 +77,23 @@ public abstract class DownloadService : IDownloadService
public abstract Task DeleteDownload(string hash);
/// <inheritdoc/>
public abstract Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
public abstract Task<List<object>?> GetSeedingDownloads();
/// <inheritdoc/>
public abstract List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories);
/// <inheritdoc/>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads);
public abstract List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories);
/// <inheritdoc/>
public abstract Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <inheritdoc/>
public abstract Task CreateCategoryAsync(string name);
protected void ResetStalledStrikesOnProgress(string hash, long downloaded)
{
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
@@ -179,7 +194,7 @@ public abstract class DownloadService : IDownloadService
return (false, DeleteReason.None);
}
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category)
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category)
{
// check ratio
if (DownloadReachedRatio(ratio, seedingTime, category))
@@ -203,8 +218,28 @@ public abstract class DownloadService : IDownloadService
return new();
}
protected string? GetRootWithFirstDirectory(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category)
string? root = Path.GetPathRoot(path);
if (root is null)
{
return null;
}
string relativePath = path[root.Length..].TrimStart(Path.DirectorySeparatorChar);
string[] parts = relativePath.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries);
return parts.Length > 0 ? Path.Combine(root, parts[0]) : root;
}
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, CleanCategory category)
{
if (category.MaxRatio < 0)
{
@@ -230,7 +265,7 @@ public abstract class DownloadService : IDownloadService
return true;
}
private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, Category category)
private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, CleanCategory category)
{
if (category.MaxSeedTime < 0)
{

44
code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs
View File

@@ -1,10 +1,11 @@
using System.Collections.Concurrent;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -15,7 +16,21 @@ namespace Infrastructure.Verticals.DownloadClient;
public class DummyDownloadService : DownloadService
{
public DummyDownloadService(ILogger<DownloadService> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, IOptions<DownloadCleanerConfig> downloadCleanerConfig, IMemoryCache cache, IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, IDryRunInterceptor dryRunInterceptor) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier, dryRunInterceptor)
public DummyDownloadService(
ILogger<DownloadService> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig,
cache, filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
}
@@ -39,13 +54,32 @@ public class DummyDownloadService : DownloadService
throw new NotImplementedException();
}
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
public override Task<List<object>?> GetSeedingDownloads()
{
throw new NotImplementedException();
}
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
{
throw new NotImplementedException();
}
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
{
throw new NotImplementedException();
}
public override Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
throw new NotImplementedException();
}
public override Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
throw new NotImplementedException();
}
public override Task CreateCategoryAsync(string name)
{
throw new NotImplementedException();
}

40
code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs
View File

@@ -34,24 +34,52 @@ public interface IDownloadService : IDisposable
);
/// <summary>
/// Fetches all downloads.
/// Fetches all seeding downloads.
/// </summary>
/// <returns>A list of downloads that are seeding.</returns>
Task<List<object>?> GetSeedingDownloads();
/// <summary>
/// Filters downloads that should be cleaned.
/// </summary>
/// <param name="downloads">The downloads to filter.</param>
/// <param name="categories">The categories by which to filter the downloads.</param>
/// <returns>A list of downloads for the provided categories.</returns>
Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories);
List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories);
/// <summary>
/// Filters downloads that should have their category changed.
/// </summary>
/// <param name="downloads">The downloads to filter.</param>
/// <param name="categories">The categories by which to filter the downloads.</param>
/// <returns>A list of downloads for the provided categories.</returns>
List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories);
/// <summary>
/// Cleans the downloads.
/// </summary>
/// <param name="downloads"></param>
/// <param name="downloads">The downloads to clean.</param>
/// <param name="categoriesToClean">The categories that should be cleaned.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
public abstract Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads);
/// <param name="ignoredDownloads">The downloads to ignore from processing.</param>
Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Changes the category for downloads that have no hardlinks.
/// </summary>
/// <param name="downloads">The downloads to change.</param>
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
/// <param name="ignoredDownloads">The downloads to ignore from processing.</param>
Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
/// <summary>
/// Deletes a download item.
/// </summary>
public Task DeleteDownload(string hash);
/// <summary>
/// Creates a category.
/// </summary>
/// <param name="name">The category name.</param>
public Task CreateCategoryAsync(string name);
}

172
code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs
View File

@@ -12,13 +12,13 @@ using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using QBittorrent.Client;
using Category = Common.Configuration.DownloadCleaner.Category;
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
@@ -38,10 +38,11 @@ public class QBitService : DownloadService, IQBitService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
_config = config.Value;
@@ -226,20 +227,42 @@ public class QBitService : DownloadService, IQBitService
}
/// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) =>
public override async Task<List<object>?> GetSeedingDownloads() =>
(await _client.GetTorrentListAsync(new()
{
Filter = TorrentListFilter.Seeding
}))
?.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.Hash))
.Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
@@ -247,16 +270,22 @@ public class QBitService : DownloadService, IQBitService
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
Category? category = categoriesToClean
CleanCategory? category = categoriesToClean
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
if (category is null)
@@ -264,12 +293,6 @@ public class QBitService : DownloadService, IQBitService
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate)
{
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash);
@@ -315,12 +338,125 @@ public class QBitService : DownloadService, IQBitService
}
}
public override async Task CreateCategoryAsync(string name)
{
IReadOnlyDictionary<string, Category>? existingCategories = await _client.GetCategoriesAsync();
if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
await _dryRunInterceptor.InterceptAsync(CreateCategory, name);
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.Hash))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
if (ignoredDownloads.Count > 0 &&
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
{
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
continue;
}
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(download.Hash);
if (files is null)
{
_logger.LogDebug("failed to find files for {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.Hash);
bool hasHardlinks = false;
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
{
_logger.LogDebug("skip | file index is null for {name}", download.Name);
hasHardlinks = true;
break;
}
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/']));
if (file.Priority is TorrentContentPriority.Skip)
{
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
continue;
}
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
break;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
break;
}
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
_logger.LogInformation("category changed for {name}", download.Name);
await _notifier.NotifyCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory);
download.Category = _downloadCleanerConfig.UnlinkedTargetCategory;
}
}
/// <inheritdoc/>
[DryRunSafeguard]
public override async Task DeleteDownload(string hash)
{
await _client.DeleteAsync(hash, deleteDownloadedData: true);
}
[DryRunSafeguard]
protected async Task CreateCategory(string name)
{
await _client.AddCategoryAsync(name);
}
[DryRunSafeguard]
protected virtual async Task SkipFile(string hash, int fileIndex)
@@ -328,6 +464,12 @@ public class QBitService : DownloadService, IQBitService
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
}
[DryRunSafeguard]
protected virtual async Task ChangeCategory(string hash, string newCategory)
{
await _client.SetTorrentCategoryAsync([hash], newCategory);
}
public override void Dispose()
{
_client.Dispose();

171
code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs
View File

@@ -12,6 +12,7 @@ using Infrastructure.Extensions;
using Infrastructure.Interceptors;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.Files;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
@@ -44,7 +45,7 @@ public class TransmissionService : DownloadService, ITransmissionService
TorrentFields.UPLOAD_RATIO,
TorrentFields.TRACKERS,
TorrentFields.RATE_DOWNLOAD,
TorrentFields.TOTAL_SIZE
TorrentFields.TOTAL_SIZE,
];
public TransmissionService(
@@ -58,10 +59,11 @@ public class TransmissionService : DownloadService, ITransmissionService
IFilenameEvaluator filenameEvaluator,
IStriker striker,
INotificationPublisher notifier,
IDryRunInterceptor dryRunInterceptor
IDryRunInterceptor dryRunInterceptor,
IHardLinkFileService hardLinkFileService
) : base(
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
filenameEvaluator, striker, notifier, dryRunInterceptor
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
)
{
_config = config.Value;
@@ -211,40 +213,59 @@ public class TransmissionService : DownloadService, ITransmissionService
return result;
}
public override async Task<List<object>?> GetSeedingDownloads() =>
(await _client.TorrentGetAsync(Fields))
?.Torrents
?.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => x.Status is 5 or 6)
.Cast<object>()
.ToList();
/// <inheritdoc/>
public override async Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories)
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
{
return (await _client.TorrentGetAsync(Fields))
?.Torrents
?.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => x.Status is 5 or 6)
return downloads
?
.Cast<TorrentInfo>()
.Where(x => categories
.Any(cat =>
{
if (x.DownloadDir is null)
{
return false;
}
return Path.GetFileName(Path.TrimEndingDirectorySeparator(x.DownloadDir))
.Equals(cat.Name, StringComparison.InvariantCultureIgnoreCase);
})
.Any(cat => cat.Name.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase))
)
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override async Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes,
IReadOnlyList<string> ignoredDownloads)
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
{
return downloads
?.Cast<TorrentInfo>()
.Where(x => !string.IsNullOrEmpty(x.HashString))
.Where(x => categories.Any(cat => cat.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase)))
.Cast<object>()
.ToList();
}
/// <inheritdoc/>
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
foreach (TorrentInfo download in downloads)
{
if (string.IsNullOrEmpty(download.HashString))
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
@@ -252,7 +273,7 @@ public class TransmissionService : DownloadService, ITransmissionService
continue;
}
Category? category = categoriesToClean
CleanCategory? category = categoriesToClean
.FirstOrDefault(x =>
{
if (download.DownloadDir is null)
@@ -269,12 +290,6 @@ public class TransmissionService : DownloadService, ITransmissionService
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true)
{
_logger.LogDebug("skip | download is private | {name}", download.Name);
@@ -306,6 +321,106 @@ public class TransmissionService : DownloadService, ITransmissionService
}
}
public override async Task CreateCategoryAsync(string name)
{
await Task.CompletedTask;
}
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
{
if (downloads?.Count is null or 0)
{
return;
}
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
{
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
}
foreach (TorrentInfo download in downloads.Cast<TorrentInfo>())
{
if (string.IsNullOrEmpty(download.HashString) || string.IsNullOrEmpty(download.Name) || download.DownloadDir == null)
{
continue;
}
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
{
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
continue;
}
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
{
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
continue;
}
ContextProvider.Set("downloadName", download.Name);
ContextProvider.Set("hash", download.HashString);
bool hasHardlinks = false;
if (download.Files is null || download.FileStats is null)
{
_logger.LogDebug("skip | download has no files | {name}", download.Name);
continue;
}
for (int i = 0; i < download.Files.Length; i++)
{
TransmissionTorrentFiles file = download.Files[i];
TransmissionTorrentFileStats stats = download.FileStats[i];
if (stats.Wanted is null or false || string.IsNullOrEmpty(file.Name))
{
continue;
}
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, file.Name).Split(['\\', '/']));
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
if (hardlinkCount < 0)
{
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
hasHardlinks = true;
break;
}
if (hardlinkCount > 0)
{
hasHardlinks = true;
break;
}
}
if (hasHardlinks)
{
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
continue;
}
string currentCategory = download.GetCategory();
string newLocation = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, _downloadCleanerConfig.UnlinkedTargetCategory).Split(['\\', '/']));
await _dryRunInterceptor.InterceptAsync(ChangeDownloadLocation, download.Id, newLocation);
_logger.LogInformation("category changed for {name}", download.Name);
await _notifier.NotifyCategoryChanged(currentCategory, _downloadCleanerConfig.UnlinkedTargetCategory);
download.DownloadDir = newLocation;
}
}
[DryRunSafeguard]
protected virtual async Task ChangeDownloadLocation(long downloadId, string newLocation)
{
await _client.TorrentSetLocationAsync([downloadId], newLocation, true);
}
public override async Task DeleteDownload(string hash)
{
TorrentInfo? torrent = await GetTorrentAsync(hash);

51
code/Infrastructure/Verticals/Files/HardLinkFileService.cs Normal file
View File

@@ -0,0 +1,51 @@
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
namespace Infrastructure.Verticals.Files;
public class HardLinkFileService : IHardLinkFileService
{
private readonly ILogger<HardLinkFileService> _logger;
private readonly UnixHardLinkFileService _unixHardLinkFileService;
private readonly WindowsHardLinkFileService _windowsHardLinkFileService;
public HardLinkFileService(
ILogger<HardLinkFileService> logger,
UnixHardLinkFileService unixHardLinkFileService,
WindowsHardLinkFileService windowsHardLinkFileService
)
{
_logger = logger;
_unixHardLinkFileService = unixHardLinkFileService;
_windowsHardLinkFileService = windowsHardLinkFileService;
}
public void PopulateFileCounts(string directoryPath)
{
_logger.LogTrace("populating file counts from {dir}", directoryPath);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_windowsHardLinkFileService.PopulateFileCounts(directoryPath);
return;
}
_unixHardLinkFileService.PopulateFileCounts(directoryPath);
}
public long GetHardLinkCount(string filePath, bool ignoreRootDir)
{
if (!File.Exists(filePath))
{
_logger.LogDebug("file {file} does not exist", filePath);
return -1;
}
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return _windowsHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir);
}
return _unixHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir);
}
}

19
code/Infrastructure/Verticals/Files/IHardLinkFileService.cs Normal file
View File

@@ -0,0 +1,19 @@
namespace Infrastructure.Verticals.Files;
public interface IHardLinkFileService
{
/// <summary>
/// Populates the inode counts for Unix and the file index counts for Windows.
/// Needs to be called before <see cref="GetHardLinkCount"/> to populate the inode counts.
/// </summary>
/// <param name="directoryPath">The root directory where to search for hardlinks.</param>
void PopulateFileCounts(string directoryPath);
/// <summary>
/// Get the hardlink count of a file.
/// </summary>
/// <param name="filePath">File path.</param>
/// <param name="ignoreRootDir">Whether to ignore hardlinks found in the same root dir.</param>
/// <returns>-1 on error, 0 if there are no hardlinks and 1 otherwise.</returns>
long GetHardLinkCount(string filePath, bool ignoreRootDir);
}

87
code/Infrastructure/Verticals/Files/UnixHardLinkFileService.cs Normal file
View File

@@ -0,0 +1,87 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Mono.Unix.Native;
namespace Infrastructure.Verticals.Files;
public class UnixHardLinkFileService : IHardLinkFileService, IDisposable
{
private readonly ILogger<UnixHardLinkFileService> _logger;
private readonly ConcurrentDictionary<ulong, int> _inodeCounts = new();
public UnixHardLinkFileService(ILogger<UnixHardLinkFileService> logger)
{
_logger = logger;
}
/// <inheritdoc/>
public long GetHardLinkCount(string filePath, bool ignoreRootDir)
{
try
{
if (Syscall.stat(filePath, out Stat stat) != 0)
{
_logger.LogDebug("failed to stat file {file}", filePath);
return -1;
}
if (!ignoreRootDir)
{
_logger.LogDebug("stat file | hardlinks: {nlink} | {file}", stat.st_nlink, filePath);
return (long)stat.st_nlink == 1 ? 0 : 1;
}
// get the number of hardlinks in the same root directory
int linksInIgnoredDir = _inodeCounts.TryGetValue(stat.st_ino, out int count)
? count
: 1; // default to 1 if not found
_logger.LogDebug("stat file | hardlinks: {nlink} | ignored: {ignored} | {file}", stat.st_nlink, linksInIgnoredDir, filePath);
return (long)stat.st_nlink - linksInIgnoredDir;
}
catch (Exception exception)
{
_logger.LogError(exception, "failed to stat file {file}", filePath);
return -1;
}
}
/// <inheritdoc/>
public void PopulateFileCounts(string directoryPath)
{
try
{
// traverse all files in the ignored path and subdirectories
foreach (string file in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories))
{
AddInodeToCount(file);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "failed to populate inode counts from {dir}", directoryPath);
throw;
}
}
private void AddInodeToCount(string path)
{
try
{
if (Syscall.stat(path, out Stat stat) == 0)
{
_inodeCounts.AddOrUpdate(stat.st_ino, 1, (_, count) => count + 1);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "could not stat {path} during inode counting", path);
throw;
}
}
public void Dispose()
{
_inodeCounts.Clear();
}
}

113
code/Infrastructure/Verticals/Files/WindowsHardLinkFileService.cs Normal file
View File

@@ -0,0 +1,113 @@
using System.Collections.Concurrent;
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
using Microsoft.Win32.SafeHandles;
namespace Infrastructure.Verticals.Files;
public class WindowsHardLinkFileService : IHardLinkFileService, IDisposable
{
private readonly ILogger<WindowsHardLinkFileService> _logger;
private readonly ConcurrentDictionary<ulong, int> _fileIndexCounts = new();
public WindowsHardLinkFileService(ILogger<WindowsHardLinkFileService> logger)
{
_logger = logger;
}
/// <inheritdoc/>
public long GetHardLinkCount(string filePath, bool ignoreRootDir)
{
try
{
using SafeFileHandle fileStream = File.OpenHandle(filePath);
if (!GetFileInformationByHandle(fileStream, out var file))
{
_logger.LogDebug("failed to get file handle {file}", filePath);
return -1;
}
if (!ignoreRootDir)
{
_logger.LogDebug("stat file | hardlinks: {nlink} | {file}", file.NumberOfLinks, filePath);
return file.NumberOfLinks == 1 ? 0 : 1;
}
// Get unique file ID (combination of high and low indices)
ulong fileIndex = ((ulong)file.FileIndexHigh << 32) | file.FileIndexLow;
// get the number of hardlinks in the same root directory
int linksInIgnoredDir = _fileIndexCounts.TryGetValue(fileIndex, out int count)
? count
: 1; // default to 1 if not found
_logger.LogDebug("stat file | hardlinks: {links} | ignored: {ignored} | {file}", file.NumberOfLinks, linksInIgnoredDir, filePath);
return file.NumberOfLinks - linksInIgnoredDir;
}
catch (Exception exception)
{
_logger.LogError(exception, "failed to stat file {file}", filePath);
return -1;
}
}
/// <inheritdoc/>
public void PopulateFileCounts(string directoryPath)
{
try
{
// traverse all files in the ignored path and subdirectories
foreach (string file in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories))
{
AddFileIndexToCount(file);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to populate file index counts from {dir}", directoryPath);
}
}
private void AddFileIndexToCount(string path)
{
try
{
using SafeFileHandle fileStream = File.OpenHandle(path);
if (GetFileInformationByHandle(fileStream, out var file))
{
ulong fileIndex = ((ulong)file.FileIndexHigh << 32) | file.FileIndexLow;
_fileIndexCounts.AddOrUpdate(fileIndex, 1, (_, count) => count + 1);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Couldn't stat {path} during file index counting", path);
}
}
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool GetFileInformationByHandle(
SafeFileHandle hFile,
out BY_HANDLE_FILE_INFORMATION lpFileInformation
);
private struct BY_HANDLE_FILE_INFORMATION
{
public uint FileAttributes;
public System.Runtime.InteropServices.ComTypes.FILETIME CreationTime;
public System.Runtime.InteropServices.ComTypes.FILETIME LastAccessTime;
public System.Runtime.InteropServices.ComTypes.FILETIME LastWriteTime;
public uint VolumeSerialNumber;
public uint FileSizeHigh;
public uint FileSizeLow;
public uint NumberOfLinks;
public uint FileIndexHigh;
public uint FileIndexLow;
}
public void Dispose()
{
_fileIndexCounts.Clear();
}
}

4
code/Infrastructure/Verticals/ItemStriker/Striker.cs
View File

@@ -15,14 +15,12 @@ public sealed class Striker : IStriker
private readonly IMemoryCache _cache;
private readonly MemoryCacheEntryOptions _cacheOptions;
private readonly INotificationPublisher _notifier;
private readonly IDryRunInterceptor _dryRunInterceptor;
public Striker(ILogger<Striker> logger, IMemoryCache cache, INotificationPublisher notifier, IDryRunInterceptor dryRunInterceptor)
public Striker(ILogger<Striker> logger, IMemoryCache cache, INotificationPublisher notifier)
{
_logger = logger;
_cache = cache;
_notifier = notifier;
_dryRunInterceptor = dryRunInterceptor;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
}

5
code/Infrastructure/Verticals/Notifications/Apprise/AppriseProvider.cs
View File

@@ -43,6 +43,11 @@ public sealed class AppriseProvider : NotificationProvider
await _proxy.SendNotification(BuildPayload(notification, NotificationType.Warning), _config);
}
public override async Task OnCategoryChanged(CategoryChangedNotification notification)
{
await _proxy.SendNotification(BuildPayload(notification, NotificationType.Warning), _config);
}
private static ApprisePayload BuildPayload(ArrNotification notification, NotificationType notificationType)
{
StringBuilder body = new();

3
code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs
View File

@@ -36,6 +36,9 @@ public sealed class NotificationConsumer<T> : IConsumer<T> where T : Notificatio
case DownloadCleanedNotification downloadCleanedNotification:
await _notificationService.Notify(downloadCleanedNotification);
break;
case CategoryChangedNotification categoryChangedNotification:
await _notificationService.Notify(categoryChangedNotification);
break;
default:
throw new NotImplementedException();
}

2
code/Infrastructure/Verticals/Notifications/INotificationFactory.cs
View File

@@ -11,4 +11,6 @@ public interface INotificationFactory
List<INotificationProvider> OnQueueItemDeletedEnabled();
List<INotificationProvider> OnDownloadCleanedEnabled();
List<INotificationProvider> OnCategoryChangedEnabled();
}

2
code/Infrastructure/Verticals/Notifications/INotificationProvider.cs
View File

@@ -18,4 +18,6 @@ public interface INotificationProvider
Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
Task OnDownloadCleaned(DownloadCleanedNotification notification);
Task OnCategoryChanged(CategoryChangedNotification notification);
}

2
code/Infrastructure/Verticals/Notifications/INotificationPublisher.cs
View File

@@ -9,4 +9,6 @@ public interface INotificationPublisher
Task NotifyQueueItemDeleted(bool removeFromClient, DeleteReason reason);
Task NotifyDownloadCleaned(double ratio, TimeSpan seedingTime, string categoryName, CleanReason reason);
Task NotifyCategoryChanged(string oldCategory, string newCategory);
}

5
code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs Normal file
View File

@@ -0,0 +1,5 @@
namespace Infrastructure.Verticals.Notifications.Models;
public sealed record CategoryChangedNotification : Notification
{
}

33
code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs
View File

@@ -46,6 +46,11 @@ public class NotifiarrProvider : NotificationProvider
{
await _proxy.SendNotification(BuildPayload(notification), _config);
}
public override async Task OnCategoryChanged(CategoryChangedNotification notification)
{
await _proxy.SendNotification(BuildPayload(notification), _config);
}
private NotifiarrPayload BuildPayload(ArrNotification notification, string color)
{
@@ -110,4 +115,32 @@ public class NotifiarrProvider : NotificationProvider
return payload;
}
private NotifiarrPayload BuildPayload(CategoryChangedNotification notification)
{
NotifiarrPayload payload = new()
{
Discord = new()
{
Color = WarningColor,
Text = new()
{
Title = notification.Title,
Icon = Logo,
Description = notification.Description,
Fields = notification.Fields?.Adapt<List<Field>>() ?? []
},
Ids = new Ids
{
Channel = _config.ChannelId
},
Images = new()
{
Thumbnail = new Uri(Logo)
}
}
};
return payload;
}
}

7
code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProxy.cs
View File

@@ -1,5 +1,6 @@
using System.Text;
using Common.Helpers;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
@@ -7,12 +8,14 @@ namespace Infrastructure.Verticals.Notifications.Notifiarr;
public sealed class NotifiarrProxy : INotifiarrProxy
{
private readonly ILogger<NotifiarrProxy> _logger;
private readonly HttpClient _httpClient;
private const string Url = "https://notifiarr.com/api/v1/notification/passthrough/";
public NotifiarrProxy(IHttpClientFactory httpClientFactory)
public NotifiarrProxy(ILogger<NotifiarrProxy> logger, IHttpClientFactory httpClientFactory)
{
_logger = logger;
_httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
}
@@ -25,6 +28,8 @@ public sealed class NotifiarrProxy : INotifiarrProxy
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
_logger.LogTrace("sending notification to Notifiarr: {content}", content);
using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, $"{Url}{config.ApiKey}");
request.Method = HttpMethod.Post;
request.Content = new StringContent(content, Encoding.UTF8, "application/json");

5
code/Infrastructure/Verticals/Notifications/NotificationFactory.cs
View File

@@ -39,4 +39,9 @@ public class NotificationFactory : INotificationFactory
ActiveProviders()
.Where(n => n.Config.OnDownloadCleaned)
.ToList();
public List<INotificationProvider> OnCategoryChangedEnabled() =>
ActiveProviders()
.Where(n => n.Config.OnCategoryChanged)
.ToList();
}

2
code/Infrastructure/Verticals/Notifications/NotificationProvider.cs
View File

@@ -24,4 +24,6 @@ public abstract class NotificationProvider : INotificationProvider
public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification);
public abstract Task OnDownloadCleaned(DownloadCleanedNotification notification);
public abstract Task OnCategoryChanged(CategoryChangedNotification notification);
}

33
code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs
View File

@@ -49,14 +49,14 @@ public class NotificationPublisher : INotificationPublisher
{
case StrikeType.Stalled:
case StrikeType.DownloadingMetadata:
await _dryRunInterceptor.InterceptAsync(Notify<StalledStrikeNotification>, notification.Adapt<StalledStrikeNotification>());
await NotifyInternal(notification.Adapt<StalledStrikeNotification>());
break;
case StrikeType.ImportFailed:
await _dryRunInterceptor.InterceptAsync(Notify<FailedImportStrikeNotification>, notification.Adapt<FailedImportStrikeNotification>());
await NotifyInternal(notification.Adapt<FailedImportStrikeNotification>());
break;
case StrikeType.SlowSpeed:
case StrikeType.SlowTime:
await _dryRunInterceptor.InterceptAsync(Notify<SlowStrikeNotification>, notification.Adapt<SlowStrikeNotification>());
await NotifyInternal(notification.Adapt<SlowStrikeNotification>());
break;
}
}
@@ -86,7 +86,7 @@ public class NotificationPublisher : INotificationPublisher
Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }]
};
await _dryRunInterceptor.InterceptAsync(Notify<QueueItemDeletedNotification>, notification);
await NotifyInternal(notification);
}
catch (Exception ex)
{
@@ -115,13 +115,36 @@ public class NotificationPublisher : INotificationPublisher
Level = NotificationLevel.Important
};
await _dryRunInterceptor.InterceptAsync(Notify<DownloadCleanedNotification>, notification);
await NotifyInternal(notification);
}
catch (Exception ex)
{
_logger.LogError(ex, "failed to notify download cleaned");
}
}
public virtual async Task NotifyCategoryChanged(string oldCategory, string newCategory)
{
CategoryChangedNotification notification = new()
{
Title = "Category changed",
Description = ContextProvider.Get<string>("downloadName"),
Fields =
[
new() { Title = "Hash", Text = ContextProvider.Get<string>("hash").ToLowerInvariant() },
new() { Title = "Old category", Text = oldCategory },
new() { Title = "New category", Text = newCategory }
],
Level = NotificationLevel.Important
};
await NotifyInternal(notification);
}
private Task NotifyInternal<T>(T message) where T: notnull
{
return _dryRunInterceptor.InterceptAsync(Notify<T>, message);
}
[DryRunSafeguard]
private Task Notify<T>(T message) where T: notnull

15
code/Infrastructure/Verticals/Notifications/NotificationService.cs
View File

@@ -88,4 +88,19 @@ public class NotificationService
}
}
}
public async Task Notify(CategoryChangedNotification notification)
{
foreach (INotificationProvider provider in _notificationFactory.OnCategoryChangedEnabled())
{
try
{
await provider.OnCategoryChanged(notification);
}
catch (Exception exception)
{
_logger.LogWarning(exception, "failed to send notification | provider {provider}", provider.Name);
}
}
}
}