diff --git a/README.md b/README.md index 56735861..813728a5 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ Cleanuperr was created primarily to address malicious files, such as `*.lnk` or > - Remove and block downloads that have a low download speed or high estimated completion time. > - Remove downloads blocked by qBittorrent or by Cleanuperr's **content blocker**. > - Trigger a search for downloads removed from the *arrs. -> - Clean up downloads that have been seeding for a certain amount of time. +> - Remove downloads that have been seeding for a certain amount of time. +> - Remove downloads that have no hardlinks (have been upgraded by the *arrs). > - Notify on strike or download removal. > - Ignore certain torrent hashes, categories, tags or trackers from being processed by Cleanuperr. diff --git a/code/Common/Configuration/DownloadCleaner/Category.cs b/code/Common/Configuration/DownloadCleaner/CleanCategory.cs similarity index 96% rename from code/Common/Configuration/DownloadCleaner/Category.cs rename to code/Common/Configuration/DownloadCleaner/CleanCategory.cs index 67d12c56..48574cfe 100644 --- a/code/Common/Configuration/DownloadCleaner/Category.cs +++ b/code/Common/Configuration/DownloadCleaner/CleanCategory.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Configuration; namespace Common.Configuration.DownloadCleaner; -public sealed record Category : IConfig +public sealed record CleanCategory : IConfig { public required string Name { get; init; } diff --git a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs index b5658fad..cfede35c 100644 --- a/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs +++ b/code/Common/Configuration/DownloadCleaner/DownloadCleanerConfig.cs @@ -8,8 +8,8 @@ public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig public const string SectionName = "DownloadCleaner"; public bool Enabled { get; init; } - - public List? Categories { get; init; } + + public List? Categories { get; init; } [ConfigurationKeyName("DELETE_PRIVATE")] public bool DeletePrivate { get; init; } @@ -17,6 +17,15 @@ public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig [ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")] public string? IgnoredDownloadsPath { get; init; } + [ConfigurationKeyName("UNLINKED_TARGET_CATEGORY")] + public string UnlinkedTargetCategory { get; init; } = "cleanuperr-unlinked"; + + [ConfigurationKeyName("UNLINKED_IGNORED_ROOT_DIR")] + public string UnlinkedIgnoredRootDir { get; init; } = string.Empty; + + [ConfigurationKeyName("UNLINKED_CATEGORIES")] + public List? UnlinkedCategories { get; init; } + public void Validate() { if (!Enabled) @@ -31,9 +40,34 @@ public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig if (Categories?.GroupBy(x => x.Name).Any(x => x.Count() > 1) is true) { - throw new ValidationException("duplicated categories found"); + throw new ValidationException("duplicated clean categories found"); } Categories?.ForEach(x => x.Validate()); + + if (string.IsNullOrEmpty(UnlinkedTargetCategory)) + { + return; + } + + if (UnlinkedCategories?.Count is null or 0) + { + throw new ValidationException("no unlinked categories configured"); + } + + if (UnlinkedCategories.Contains(UnlinkedTargetCategory)) + { + throw new ValidationException($"{SectionName.ToUpperInvariant()}__UNLINKED_TARGET_CATEGORY should not be present in {SectionName.ToUpperInvariant()}__UNLINKED_CATEGORIES"); + } + + if (UnlinkedCategories.Any(string.IsNullOrEmpty)) + { + throw new ValidationException("empty unlinked category filter found"); + } + + if (!string.IsNullOrEmpty(UnlinkedIgnoredRootDir) && !Directory.Exists(UnlinkedIgnoredRootDir)) + { + throw new ValidationException($"{UnlinkedIgnoredRootDir} root directory does not exist"); + } } } \ No newline at end of file diff --git a/code/Common/Configuration/Notification/NotificationConfig.cs b/code/Common/Configuration/Notification/NotificationConfig.cs index 369b084b..7b51f59f 100644 --- a/code/Common/Configuration/Notification/NotificationConfig.cs +++ b/code/Common/Configuration/Notification/NotificationConfig.cs @@ -18,8 +18,17 @@ public abstract record NotificationConfig [ConfigurationKeyName("ON_DOWNLOAD_CLEANED")] public bool OnDownloadCleaned { get; init; } + + [ConfigurationKeyName("ON_CATEGORY_CHANGED")] + public bool OnCategoryChanged { get; init; } - public bool IsEnabled => OnImportFailedStrike || OnStalledStrike || OnSlowStrike || OnQueueItemDeleted || OnDownloadCleaned; + public bool IsEnabled => + OnImportFailedStrike || + OnStalledStrike || + OnSlowStrike || + OnQueueItemDeleted || + OnDownloadCleaned || + OnCategoryChanged; public abstract bool IsValid(); } \ No newline at end of file diff --git a/code/Common/Exceptions/FatalException.cs b/code/Common/Exceptions/FatalException.cs new file mode 100644 index 00000000..1966150c --- /dev/null +++ b/code/Common/Exceptions/FatalException.cs @@ -0,0 +1,12 @@ +namespace Common.Exceptions; + +public class FatalException : Exception +{ + public FatalException() + { + } + + public FatalException(string message) : base(message) + { + } +} \ No newline at end of file diff --git a/code/Domain/Models/Deluge/Response/DownloadStatus.cs b/code/Domain/Models/Deluge/Response/DownloadStatus.cs index 99428530..b86d5330 100644 --- a/code/Domain/Models/Deluge/Response/DownloadStatus.cs +++ b/code/Domain/Models/Deluge/Response/DownloadStatus.cs @@ -23,7 +23,7 @@ public sealed record DownloadStatus [JsonProperty("total_done")] public long TotalDone { get; init; } - public string? Label { get; init; } + public string? Label { get; set; } [JsonProperty("seeding_time")] public long SeedingTime { get; init; } @@ -31,6 +31,9 @@ public sealed record DownloadStatus public float Ratio { get; init; } public required IReadOnlyList Trackers { get; init; } + + [JsonProperty("download_location")] + public required string DownloadLocation { get; init; } } public sealed record Tracker diff --git a/code/Executable/DependencyInjection/MainDI.cs b/code/Executable/DependencyInjection/MainDI.cs index 4cbf7354..d9f53db5 100644 --- a/code/Executable/DependencyInjection/MainDI.cs +++ b/code/Executable/DependencyInjection/MainDI.cs @@ -17,7 +17,9 @@ public static class MainDI .AddLogging(builder => builder.ClearProviders().AddConsole()) .AddHttpClients(configuration) .AddConfiguration(configuration) - .AddMemoryCache() + .AddMemoryCache(options => { + options.ExpirationScanFrequency = TimeSpan.FromMinutes(1); + }) .AddServices() .AddQuartzServices(configuration) .AddNotifications(configuration) @@ -28,6 +30,7 @@ public static class MainDI config.AddConsumer>(); config.AddConsumer>(); config.AddConsumer>(); + config.AddConsumer>(); config.UsingInMemory((context, cfg) => { @@ -38,6 +41,7 @@ public static class MainDI e.ConfigureConsumer>(context); e.ConfigureConsumer>(context); e.ConfigureConsumer>(context); + e.ConfigureConsumer>(context); e.ConcurrentMessageLimit = 1; e.PrefetchCount = 1; }); diff --git a/code/Executable/DependencyInjection/ServicesDI.cs b/code/Executable/DependencyInjection/ServicesDI.cs index 395d164c..fcc56b39 100644 --- a/code/Executable/DependencyInjection/ServicesDI.cs +++ b/code/Executable/DependencyInjection/ServicesDI.cs @@ -10,6 +10,7 @@ using Infrastructure.Verticals.DownloadClient; using Infrastructure.Verticals.DownloadClient.Deluge; using Infrastructure.Verticals.DownloadClient.QBittorrent; using Infrastructure.Verticals.DownloadClient.Transmission; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.QueueCleaner; @@ -27,14 +28,17 @@ public static class ServicesDI .AddTransient() .AddTransient() .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() .AddTransient() .AddTransient() .AddTransient() .AddTransient() .AddTransient() .AddTransient() + .AddTransient() .AddSingleton() - .AddSingleton() .AddSingleton>() .AddSingleton>() .AddSingleton>(); diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index dbbf50eb..ec956d22 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -52,9 +52,15 @@ "Name": "tv-sonarr", "MAX_RATIO": -1, "MIN_SEED_TIME": 0, - "MAX_SEED_TIME": -1 + "MAX_SEED_TIME": 240 } ], + "UNLINKED_TARGET_CATEGORY": "cleanuperr-unlinked", + "UNLINKED_IGNORED_ROOT_DIR": "", + "UNLINKED_CATEGORIES": [ + "tv-sonarr", + "radarr" + ], "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads" }, "DOWNLOAD_CLIENT": "qbittorrent", @@ -121,6 +127,7 @@ "ON_SLOW_STRIKE": true, "ON_QUEUE_ITEM_DELETED": true, "ON_DOWNLOAD_CLEANED": true, + "ON_CATEGORY_CHANGED": true, "API_KEY": "", "CHANNEL_ID": "" }, @@ -130,6 +137,7 @@ "ON_SLOW_STRIKE": true, "ON_QUEUE_ITEM_DELETED": true, "ON_DOWNLOAD_CLEANED": true, + "ON_CATEGORY_CHANGED": true, "URL": "http://localhost:8000", "KEY": "" } diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index 00e872a5..1b1ac2ff 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -38,6 +38,9 @@ "Enabled": false, "DELETE_PRIVATE": false, "CATEGORIES": [], + "UNLINKED_TARGET_CATEGORY": "cleanuperr-unlinked", + "UNLINKED_IGNORED_ROOT_DIR": "", + "UNLINKED_CATEGORIES": [], "IGNORED_DOWNLOADS_PATH": "" }, "DOWNLOAD_CLIENT": "none", @@ -104,6 +107,7 @@ "ON_SLOW_STRIKE": false, "ON_QUEUE_ITEM_DELETED": false, "ON_DOWNLOAD_CLEANED": false, + "ON_CATEGORY_CHANGED": false, "API_KEY": "", "CHANNEL_ID": "" }, @@ -113,6 +117,7 @@ "ON_SLOW_STRIKE": false, "ON_QUEUE_ITEM_DELETED": false, "ON_DOWNLOAD_CLEANED": false, + "ON_CATEGORY_CHANGED": false, "URL": "", "KEY": "" } diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs index e1d35596..5b89934a 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceFixture.cs @@ -1,9 +1,10 @@ -using Common.Configuration.ContentBlocker; +using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.QueueCleaner; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadClient; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -56,6 +57,7 @@ public class DownloadServiceFixture : IDisposable var filenameEvaluator = Substitute.For(); var notifier = Substitute.For(); var dryRunInterceptor = Substitute.For(); + var hardlinkFileService = Substitute.For(); return new TestDownloadService( Logger, @@ -66,7 +68,8 @@ public class DownloadServiceFixture : IDisposable filenameEvaluator, Striker, notifier, - dryRunInterceptor + dryRunInterceptor, + hardlinkFileService ); } diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs index 19c564ff..e69c21c2 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs @@ -111,7 +111,7 @@ public class DownloadServiceTests : IClassFixture public void WhenRatioAndMinSeedTimeReached_ShouldReturnTrue() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = 1.0, @@ -137,7 +137,7 @@ public class DownloadServiceTests : IClassFixture public void WhenRatioReachedAndMinSeedTimeNotReached_ShouldReturnFalse() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = 1.0, @@ -163,7 +163,7 @@ public class DownloadServiceTests : IClassFixture public void WhenMaxSeedTimeReached_ShouldReturnTrue() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = -1, @@ -189,7 +189,7 @@ public class DownloadServiceTests : IClassFixture public void WhenNeitherConditionMet_ShouldReturnFalse() { // Arrange - Category category = new() + CleanCategory category = new() { Name = "test", MaxRatio = 2.0, diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs index 4defffab..5869ba14 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; @@ -7,6 +7,7 @@ using Domain.Enums; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.DownloadClient; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -26,10 +27,11 @@ public class TestDownloadService : DownloadService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardLinkFileService hardLinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService ) { } @@ -40,11 +42,13 @@ public class TestDownloadService : DownloadService public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, ConcurrentBag regexes, IReadOnlyList ignoredDownloads) => Task.FromResult(new BlockFilesResult()); public override Task DeleteDownload(string hash) => Task.CompletedTask; - public override Task?> GetAllDownloadsToBeCleaned(List categories) => Task.FromResult?>(null); - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, - IReadOnlyList ignoredDownloads) => Task.CompletedTask; - + public override Task CreateCategoryAsync(string name) => Task.CompletedTask; + public override Task?> GetSeedingDownloads() => Task.FromResult?>(null); + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) => null; + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) => null; + public override Task CleanDownloadsAsync(List? downloads, List categoriesToClean, HashSet excludedHashes, IReadOnlyList ignoredDownloads) => Task.CompletedTask; + public override Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) => Task.CompletedTask; // Expose protected methods for testing public new void ResetStalledStrikesOnProgress(string hash, long downloaded) => base.ResetStalledStrikesOnProgress(hash, downloaded); - public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) => base.ShouldCleanDownload(ratio, seedingTime, category); + public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category) => base.ShouldCleanDownload(ratio, seedingTime, category); } \ No newline at end of file diff --git a/code/Infrastructure/Infrastructure.csproj b/code/Infrastructure/Infrastructure.csproj index 3f44a525..fa2e2e62 100644 --- a/code/Infrastructure/Infrastructure.csproj +++ b/code/Infrastructure/Infrastructure.csproj @@ -18,6 +18,7 @@ + diff --git a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs index f0f5dbb0..31e88ff4 100644 --- a/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs +++ b/code/Infrastructure/Verticals/ContentBlocker/BlocklistProvider.cs @@ -43,7 +43,7 @@ public sealed class BlocklistProvider { if (_initialized) { - _logger.LogDebug("blocklists already loaded"); + _logger.LogTrace("blocklists already loaded"); return; } diff --git a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs index 25c79eac..cb486744 100644 --- a/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs +++ b/code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs @@ -21,6 +21,8 @@ public sealed class DownloadCleaner : GenericHandler private readonly IgnoredDownloadsProvider _ignoredDownloadsProvider; private readonly HashSet _excludedHashes = []; + private static bool _hardLinkCategoryCreated; + public DownloadCleaner( ILogger logger, IOptions config, @@ -65,13 +67,20 @@ public sealed class DownloadCleaner : GenericHandler IReadOnlyList ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads(); await _downloadService.LoginAsync(); - - List? downloads = await _downloadService.GetAllDownloadsToBeCleaned(_config.Categories); - - if (downloads?.Count is null or 0) + List? downloads = await _downloadService.GetSeedingDownloads(); + List? downloadsToChangeCategory = null; + + if (!string.IsNullOrEmpty(_config.UnlinkedTargetCategory) && _config.UnlinkedCategories?.Count > 0) { - _logger.LogDebug("no downloads found in the download client"); - return; + if (!_hardLinkCategoryCreated) + { + _logger.LogDebug("creating category {cat}", _config.UnlinkedTargetCategory); + + await _downloadService.CreateCategoryAsync(_config.UnlinkedTargetCategory); + _hardLinkCategoryCreated = true; + } + + downloadsToChangeCategory = _downloadService.FilterDownloadsToChangeCategoryAsync(downloads, _config.UnlinkedCategories); } // wait for the downloads to appear in the arr queue @@ -81,7 +90,16 @@ public sealed class DownloadCleaner : GenericHandler await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true); await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true); - await _downloadService.CleanDownloads(downloads, _config.Categories, _excludedHashes, ignoredDownloads); + _logger.LogTrace("looking for downloads to change category"); + await _downloadService.ChangeCategoryForNoHardLinksAsync(downloadsToChangeCategory, _excludedHashes, ignoredDownloads); + + List? downloadsToClean = _downloadService.FilterDownloadsToBeCleanedAsync(downloads, _config.Categories); + + // release unused objects + downloads = null; + + _logger.LogTrace("looking for downloads to clean"); + await _downloadService.CleanDownloadsAsync(downloadsToClean, _config.Categories, _excludedHashes, ignoredDownloads); } protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType) diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs index 6c5a9a0d..b075a67c 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs @@ -2,6 +2,7 @@ using System.Text.Json.Serialization; using Common.Configuration; using Common.Configuration.DownloadClient; +using Common.Exceptions; using Domain.Models.Deluge.Exceptions; using Domain.Models.Deluge.Request; using Domain.Models.Deluge.Response; @@ -29,7 +30,8 @@ public sealed class DelugeClient "ratio", "trackers", "download_payload_rate", - "total_size" + "total_size", + "download_location" ]; public DelugeClient(IOptions config, IHttpClientFactory httpClientFactory) @@ -44,11 +46,42 @@ public sealed class DelugeClient return await SendRequest("auth.login", _config.Password); } + public async Task IsConnected() + { + return await SendRequest("web.connected"); + } + + public async Task Connect() + { + string? firstHost = await GetHost(); + + if (string.IsNullOrEmpty(firstHost)) + { + return false; + } + + var result = await SendRequest?>("web.connect", firstHost); + + return result?.Count > 0; + } + public async Task Logout() { return await SendRequest("auth.delete_session"); } + public async Task GetHost() + { + var hosts = await SendRequest?>?>("web.get_hosts"); + + if (hosts?.Count > 1) + { + throw new FatalException("multiple Deluge hosts found - please connect to only one host"); + } + + return hosts?.FirstOrDefault()?.FirstOrDefault(); + } + public async Task> ListTorrents(Dictionary? filters = null) { filters ??= new Dictionary(); @@ -149,7 +182,7 @@ public sealed class DelugeClient return responseJson; } - private DelugeRequest CreateRequest(string method, params object[] parameters) + private static DelugeRequest CreateRequest(string method, params object[] parameters) { if (String.IsNullOrWhiteSpace(method)) { @@ -195,4 +228,19 @@ public sealed class DelugeClient return webResponse.Result; } + + public async Task> GetLabels() + { + return await SendRequest>("label.get_labels"); + } + + public async Task CreateLabel(string label) + { + await SendRequest>("label.add", label); + } + + public async Task SetTorrentLabel(string hash, string newLabel) + { + await SendRequest>("label.set_torrent", hash, newLabel); + } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index 09158607..7bf30636 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -7,12 +7,14 @@ using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; using Common.CustomDataTypes; +using Common.Exceptions; using Domain.Enums; using Domain.Models.Deluge.Response; using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -36,10 +38,11 @@ public class DelugeService : DownloadService, IDelugeService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardLinkFileService hardLinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService ) { config.Value.Validate(); @@ -49,6 +52,11 @@ public class DelugeService : DownloadService, IDelugeService public override async Task LoginAsync() { await _client.LoginAsync(); + + if (!await _client.IsConnected() && !await _client.Connect()) + { + throw new FatalException("Deluge WebUI is not connected to the daemon"); + } } /// @@ -208,26 +216,51 @@ public class DelugeService : DownloadService, IDelugeService return result; } - public override async Task?> GetAllDownloadsToBeCleaned(List categories) + public override async Task?> GetSeedingDownloads() { return (await _client.GetStatusForAllTorrents()) ?.Where(x => !string.IsNullOrEmpty(x.Hash)) .Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true) - .Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase))) .Cast() .ToList(); } + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.Hash)) + .Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, + public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, HashSet excludedHashes, IReadOnlyList ignoredDownloads) { + if (downloads?.Count is null or 0) + { + return; + } + foreach (DownloadStatus download in downloads) { if (string.IsNullOrEmpty(download.Hash)) { continue; } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) { @@ -235,19 +268,13 @@ public class DelugeService : DownloadService, IDelugeService continue; } - Category? category = categoriesToClean + CleanCategory? category = categoriesToClean .FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase)); if (category is null) { continue; } - - if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } if (!_downloadCleanerConfig.DeletePrivate && download.Private) { @@ -279,7 +306,107 @@ public class DelugeService : DownloadService, IDelugeService await _notifier.NotifyDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason); } } - + + public override async Task CreateCategoryAsync(string name) + { + IReadOnlyList existingLabels = await _client.GetLabels(); + + if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase)) + { + return; + } + + await _dryRunInterceptor.InterceptAsync(CreateLabel, name); + } + + public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + + if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) + { + _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); + } + + foreach (DownloadStatus download in downloads.Cast()) + { + if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label)) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.Hash); + + DelugeContents? contents = null; + try + { + contents = await _client.GetTorrentFiles(download.Hash); + } + catch (Exception exception) + { + _logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name); + continue; + } + + bool hasHardlinks = false; + + ProcessFiles(contents?.Contents, (_, file) => + { + string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/'])); + + if (file.Priority <= 0) + { + _logger.LogDebug("skip | file is not downloaded | {file}", filePath); + return; + } + + long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); + + if (hardlinkCount < 0) + { + _logger.LogDebug("skip | could not get file properties | {file}", filePath); + hasHardlinks = true; + return; + } + + if (hardlinkCount > 0) + { + hasHardlinks = true; + } + }); + + if (hasHardlinks) + { + _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); + continue; + } + + await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory); + + _logger.LogInformation("category changed for {name}", download.Name); + + await _notifier.NotifyCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory); + + download.Label = _downloadCleanerConfig.UnlinkedTargetCategory; + } + } + /// [DryRunSafeguard] public override async Task DeleteDownload(string hash) @@ -288,6 +415,12 @@ public class DelugeService : DownloadService, IDelugeService await _client.DeleteTorrents([hash]); } + + [DryRunSafeguard] + protected async Task CreateLabel(string name) + { + await _client.CreateLabel(name); + } [DryRunSafeguard] protected virtual async Task ChangeFilesPriority(string hash, List sortedPriorities) @@ -295,6 +428,12 @@ public class DelugeService : DownloadService, IDelugeService await _client.ChangeFilesPriority(hash, sortedPriorities); } + [DryRunSafeguard] + protected virtual async Task ChangeLabel(string hash, string newLabel) + { + await _client.SetTorrentLabel(hash, newLabel); + } + private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(DownloadStatus status) { (bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(status); diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs index bf217dbf..a498c812 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; @@ -11,6 +11,7 @@ using Infrastructure.Helpers; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -31,6 +32,7 @@ public abstract class DownloadService : IDownloadService protected readonly MemoryCacheEntryOptions _cacheOptions; protected readonly INotificationPublisher _notifier; protected readonly IDryRunInterceptor _dryRunInterceptor; + protected readonly IHardLinkFileService _hardLinkFileService; protected DownloadService( ILogger logger, @@ -41,7 +43,8 @@ public abstract class DownloadService : IDownloadService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardLinkFileService hardLinkFileService ) { _logger = logger; @@ -53,6 +56,7 @@ public abstract class DownloadService : IDownloadService _striker = striker; _notifier = notifier; _dryRunInterceptor = dryRunInterceptor; + _hardLinkFileService = hardLinkFileService; _cacheOptions = new MemoryCacheEntryOptions() .SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer); } @@ -73,12 +77,23 @@ public abstract class DownloadService : IDownloadService public abstract Task DeleteDownload(string hash); /// - public abstract Task?> GetAllDownloadsToBeCleaned(List categories); + public abstract Task?> GetSeedingDownloads(); + + /// + public abstract List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories); /// - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, - IReadOnlyList ignoredDownloads); + public abstract List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories); + /// + public abstract Task CleanDownloadsAsync(List? downloads, List categoriesToClean, HashSet excludedHashes, IReadOnlyList ignoredDownloads); + + /// + public abstract Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads); + + /// + public abstract Task CreateCategoryAsync(string name); + protected void ResetStalledStrikesOnProgress(string hash, long downloaded) { if (!_queueCleanerConfig.StalledResetStrikesOnProgress) @@ -179,7 +194,7 @@ public abstract class DownloadService : IDownloadService return (false, DeleteReason.None); } - protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) + protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category) { // check ratio if (DownloadReachedRatio(ratio, seedingTime, category)) @@ -203,8 +218,28 @@ public abstract class DownloadService : IDownloadService return new(); } + + protected string? GetRootWithFirstDirectory(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return null; + } - private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, Category category) + string? root = Path.GetPathRoot(path); + + if (root is null) + { + return null; + } + + string relativePath = path[root.Length..].TrimStart(Path.DirectorySeparatorChar); + string[] parts = relativePath.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries); + + return parts.Length > 0 ? Path.Combine(root, parts[0]) : root; + } + + private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, CleanCategory category) { if (category.MaxRatio < 0) { @@ -230,7 +265,7 @@ public abstract class DownloadService : IDownloadService return true; } - private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, Category category) + private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, CleanCategory category) { if (category.MaxSeedTime < 0) { diff --git a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs index 5832a378..86ada43e 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs @@ -1,10 +1,11 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.QueueCleaner; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -15,7 +16,21 @@ namespace Infrastructure.Verticals.DownloadClient; public class DummyDownloadService : DownloadService { - public DummyDownloadService(ILogger logger, IOptions queueCleanerConfig, IOptions contentBlockerConfig, IOptions downloadCleanerConfig, IMemoryCache cache, IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, IDryRunInterceptor dryRunInterceptor) : base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, filenameEvaluator, striker, notifier, dryRunInterceptor) + public DummyDownloadService( + ILogger logger, + IOptions queueCleanerConfig, + IOptions contentBlockerConfig, + IOptions downloadCleanerConfig, + IMemoryCache cache, + IFilenameEvaluator filenameEvaluator, + IStriker striker, + INotificationPublisher notifier, + IDryRunInterceptor dryRunInterceptor, + IHardLinkFileService hardLinkFileService + ) : base( + logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, + cache, filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService + ) { } @@ -39,13 +54,32 @@ public class DummyDownloadService : DownloadService throw new NotImplementedException(); } - public override Task?> GetAllDownloadsToBeCleaned(List categories) + public override Task?> GetSeedingDownloads() { throw new NotImplementedException(); } - public override Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, - IReadOnlyList ignoredDownloads) + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) + { + throw new NotImplementedException(); + } + + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) + { + throw new NotImplementedException(); + } + + public override Task CleanDownloadsAsync(List? downloads, List categoriesToClean, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + throw new NotImplementedException(); + } + + public override Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + throw new NotImplementedException(); + } + + public override Task CreateCategoryAsync(string name) { throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs index 4f592ddc..ebdb6a45 100644 --- a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs @@ -34,24 +34,52 @@ public interface IDownloadService : IDisposable ); /// - /// Fetches all downloads. + /// Fetches all seeding downloads. /// + /// A list of downloads that are seeding. + Task?> GetSeedingDownloads(); + + /// + /// Filters downloads that should be cleaned. + /// + /// The downloads to filter. /// The categories by which to filter the downloads. /// A list of downloads for the provided categories. - Task?> GetAllDownloadsToBeCleaned(List categories); + List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories); + + /// + /// Filters downloads that should have their category changed. + /// + /// The downloads to filter. + /// The categories by which to filter the downloads. + /// A list of downloads for the provided categories. + List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories); /// /// Cleans the downloads. /// - /// + /// The downloads to clean. /// The categories that should be cleaned. /// The hashes that should not be cleaned. - /// Downloads to ignore from processing. - public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, - IReadOnlyList ignoredDownloads); + /// The downloads to ignore from processing. + Task CleanDownloadsAsync(List? downloads, List categoriesToClean, HashSet excludedHashes, IReadOnlyList ignoredDownloads); + /// + /// Changes the category for downloads that have no hardlinks. + /// + /// The downloads to change. + /// The hashes that should not be cleaned. + /// The downloads to ignore from processing. + Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads); + /// /// Deletes a download item. /// public Task DeleteDownload(string hash); + + /// + /// Creates a category. + /// + /// The category name. + public Task CreateCategoryAsync(string name); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 141d9f78..2a94099e 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -12,13 +12,13 @@ using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using QBittorrent.Client; -using Category = Common.Configuration.DownloadCleaner.Category; namespace Infrastructure.Verticals.DownloadClient.QBittorrent; @@ -38,10 +38,11 @@ public class QBitService : DownloadService, IQBitService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardLinkFileService hardLinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService ) { _config = config.Value; @@ -226,20 +227,42 @@ public class QBitService : DownloadService, IQBitService } /// - public override async Task?> GetAllDownloadsToBeCleaned(List categories) => + public override async Task?> GetSeedingDownloads() => (await _client.GetTorrentListAsync(new() { Filter = TorrentListFilter.Seeding })) ?.Where(x => !string.IsNullOrEmpty(x.Hash)) - .Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase))) .Cast() .ToList(); /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, - IReadOnlyList ignoredDownloads) + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.Hash)) + .Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + + /// + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) => + downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.Hash)) + .Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + + /// + public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, + HashSet excludedHashes, IReadOnlyList ignoredDownloads) { + if (downloads?.Count is null or 0) + { + return; + } + foreach (TorrentInfo download in downloads) { if (string.IsNullOrEmpty(download.Hash)) @@ -247,16 +270,22 @@ public class QBitService : DownloadService, IQBitService continue; } + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + IReadOnlyList trackers = await GetTrackersAsync(download.Hash); if (ignoredDownloads.Count > 0 && - (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true)) + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)))) { _logger.LogInformation("skip | download is ignored | {name}", download.Name); continue; } - Category? category = categoriesToClean + CleanCategory? category = categoriesToClean .FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase)); if (category is null) @@ -264,12 +293,6 @@ public class QBitService : DownloadService, IQBitService continue; } - if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - if (!_downloadCleanerConfig.DeletePrivate) { TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash); @@ -315,12 +338,125 @@ public class QBitService : DownloadService, IQBitService } } + public override async Task CreateCategoryAsync(string name) + { + IReadOnlyDictionary? existingCategories = await _client.GetCategoriesAsync(); + + if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase))) + { + return; + } + + await _dryRunInterceptor.InterceptAsync(CreateCategory, name); + } + + public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + + if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) + { + _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); + } + + foreach (TorrentInfo download in downloads) + { + if (string.IsNullOrEmpty(download.Hash)) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + IReadOnlyList trackers = await GetTrackersAsync(download.Hash); + + if (ignoredDownloads.Count > 0 && + (download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)))) + { + _logger.LogInformation("skip | download is ignored | {name}", download.Name); + continue; + } + + IReadOnlyList? files = await _client.GetTorrentContentsAsync(download.Hash); + + if (files is null) + { + _logger.LogDebug("failed to find files for {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.Hash); + bool hasHardlinks = false; + + foreach (TorrentContent file in files) + { + if (!file.Index.HasValue) + { + _logger.LogDebug("skip | file index is null for {name}", download.Name); + hasHardlinks = true; + break; + } + + string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/'])); + + if (file.Priority is TorrentContentPriority.Skip) + { + _logger.LogDebug("skip | file is not downloaded | {file}", filePath); + continue; + } + + long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); + + if (hardlinkCount < 0) + { + _logger.LogDebug("skip | could not get file properties | {file}", filePath); + hasHardlinks = true; + break; + } + + if (hardlinkCount > 0) + { + hasHardlinks = true; + break; + } + } + + if (hasHardlinks) + { + _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); + continue; + } + + await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory); + + _logger.LogInformation("category changed for {name}", download.Name); + + await _notifier.NotifyCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory); + + download.Category = _downloadCleanerConfig.UnlinkedTargetCategory; + } + } + /// [DryRunSafeguard] public override async Task DeleteDownload(string hash) { await _client.DeleteAsync(hash, deleteDownloadedData: true); } + + [DryRunSafeguard] + protected async Task CreateCategory(string name) + { + await _client.AddCategoryAsync(name); + } [DryRunSafeguard] protected virtual async Task SkipFile(string hash, int fileIndex) @@ -328,6 +464,12 @@ public class QBitService : DownloadService, IQBitService await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip); } + [DryRunSafeguard] + protected virtual async Task ChangeCategory(string hash, string newCategory) + { + await _client.SetTorrentCategoryAsync([hash], newCategory); + } + public override void Dispose() { _client.Dispose(); diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 16985bd7..80ae704d 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -12,6 +12,7 @@ using Infrastructure.Extensions; using Infrastructure.Interceptors; using Infrastructure.Verticals.ContentBlocker; using Infrastructure.Verticals.Context; +using Infrastructure.Verticals.Files; using Infrastructure.Verticals.ItemStriker; using Infrastructure.Verticals.Notifications; using Microsoft.Extensions.Caching.Memory; @@ -44,7 +45,7 @@ public class TransmissionService : DownloadService, ITransmissionService TorrentFields.UPLOAD_RATIO, TorrentFields.TRACKERS, TorrentFields.RATE_DOWNLOAD, - TorrentFields.TOTAL_SIZE + TorrentFields.TOTAL_SIZE, ]; public TransmissionService( @@ -58,10 +59,11 @@ public class TransmissionService : DownloadService, ITransmissionService IFilenameEvaluator filenameEvaluator, IStriker striker, INotificationPublisher notifier, - IDryRunInterceptor dryRunInterceptor + IDryRunInterceptor dryRunInterceptor, + IHardLinkFileService hardLinkFileService ) : base( logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache, - filenameEvaluator, striker, notifier, dryRunInterceptor + filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService ) { _config = config.Value; @@ -211,40 +213,59 @@ public class TransmissionService : DownloadService, ITransmissionService return result; } + + public override async Task?> GetSeedingDownloads() => + (await _client.TorrentGetAsync(Fields)) + ?.Torrents + ?.Where(x => !string.IsNullOrEmpty(x.HashString)) + .Where(x => x.Status is 5 or 6) + .Cast() + .ToList(); /// - public override async Task?> GetAllDownloadsToBeCleaned(List categories) + public override List? FilterDownloadsToBeCleanedAsync(List? downloads, List categories) { - return (await _client.TorrentGetAsync(Fields)) - ?.Torrents - ?.Where(x => !string.IsNullOrEmpty(x.HashString)) - .Where(x => x.Status is 5 or 6) + return downloads + ? + .Cast() .Where(x => categories - .Any(cat => - { - if (x.DownloadDir is null) - { - return false; - } - - return Path.GetFileName(Path.TrimEndingDirectorySeparator(x.DownloadDir)) - .Equals(cat.Name, StringComparison.InvariantCultureIgnoreCase); - }) + .Any(cat => cat.Name.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase)) ) .Cast() .ToList(); } - /// - public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, - IReadOnlyList ignoredDownloads) + public override List? FilterDownloadsToChangeCategoryAsync(List? downloads, List categories) { + return downloads + ?.Cast() + .Where(x => !string.IsNullOrEmpty(x.HashString)) + .Where(x => categories.Any(cat => cat.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase))) + .Cast() + .ToList(); + } + + /// + public override async Task CleanDownloadsAsync(List? downloads, List categoriesToClean, + HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + foreach (TorrentInfo download in downloads) { if (string.IsNullOrEmpty(download.HashString)) { continue; } + + if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) { @@ -252,7 +273,7 @@ public class TransmissionService : DownloadService, ITransmissionService continue; } - Category? category = categoriesToClean + CleanCategory? category = categoriesToClean .FirstOrDefault(x => { if (download.DownloadDir is null) @@ -269,12 +290,6 @@ public class TransmissionService : DownloadService, ITransmissionService continue; } - if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase))) - { - _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); - continue; - } - if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true) { _logger.LogDebug("skip | download is private | {name}", download.Name); @@ -306,6 +321,106 @@ public class TransmissionService : DownloadService, ITransmissionService } } + public override async Task CreateCategoryAsync(string name) + { + await Task.CompletedTask; + } + + public override async Task ChangeCategoryForNoHardLinksAsync(List? downloads, HashSet excludedHashes, IReadOnlyList ignoredDownloads) + { + if (downloads?.Count is null or 0) + { + return; + } + + if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)) + { + _hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir); + } + + foreach (TorrentInfo download in downloads.Cast()) + { + if (string.IsNullOrEmpty(download.HashString) || string.IsNullOrEmpty(download.Name) || download.DownloadDir == null) + { + continue; + } + + if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase))) + { + _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); + continue; + } + + if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) + { + _logger.LogDebug("skip | download is ignored | {name}", download.Name); + continue; + } + + ContextProvider.Set("downloadName", download.Name); + ContextProvider.Set("hash", download.HashString); + + bool hasHardlinks = false; + + if (download.Files is null || download.FileStats is null) + { + _logger.LogDebug("skip | download has no files | {name}", download.Name); + continue; + } + + for (int i = 0; i < download.Files.Length; i++) + { + TransmissionTorrentFiles file = download.Files[i]; + TransmissionTorrentFileStats stats = download.FileStats[i]; + + if (stats.Wanted is null or false || string.IsNullOrEmpty(file.Name)) + { + continue; + } + + string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, file.Name).Split(['\\', '/'])); + + long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir)); + + if (hardlinkCount < 0) + { + _logger.LogDebug("skip | could not get file properties | {file}", filePath); + hasHardlinks = true; + break; + } + + if (hardlinkCount > 0) + { + hasHardlinks = true; + break; + } + } + + if (hasHardlinks) + { + _logger.LogDebug("skip | download has hardlinks | {name}", download.Name); + continue; + } + + string currentCategory = download.GetCategory(); + string newLocation = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, _downloadCleanerConfig.UnlinkedTargetCategory).Split(['\\', '/'])); + + await _dryRunInterceptor.InterceptAsync(ChangeDownloadLocation, download.Id, newLocation); + + _logger.LogInformation("category changed for {name}", download.Name); + + await _notifier.NotifyCategoryChanged(currentCategory, _downloadCleanerConfig.UnlinkedTargetCategory); + + download.DownloadDir = newLocation; + } + } + + [DryRunSafeguard] + protected virtual async Task ChangeDownloadLocation(long downloadId, string newLocation) + { + await _client.TorrentSetLocationAsync([downloadId], newLocation, true); + } + public override async Task DeleteDownload(string hash) { TorrentInfo? torrent = await GetTorrentAsync(hash); diff --git a/code/Infrastructure/Verticals/Files/HardLinkFileService.cs b/code/Infrastructure/Verticals/Files/HardLinkFileService.cs new file mode 100644 index 00000000..0f4c3e14 --- /dev/null +++ b/code/Infrastructure/Verticals/Files/HardLinkFileService.cs @@ -0,0 +1,51 @@ +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; + +namespace Infrastructure.Verticals.Files; + +public class HardLinkFileService : IHardLinkFileService +{ + private readonly ILogger _logger; + private readonly UnixHardLinkFileService _unixHardLinkFileService; + private readonly WindowsHardLinkFileService _windowsHardLinkFileService; + + public HardLinkFileService( + ILogger logger, + UnixHardLinkFileService unixHardLinkFileService, + WindowsHardLinkFileService windowsHardLinkFileService + ) + { + _logger = logger; + _unixHardLinkFileService = unixHardLinkFileService; + _windowsHardLinkFileService = windowsHardLinkFileService; + } + + public void PopulateFileCounts(string directoryPath) + { + _logger.LogTrace("populating file counts from {dir}", directoryPath); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _windowsHardLinkFileService.PopulateFileCounts(directoryPath); + return; + } + + _unixHardLinkFileService.PopulateFileCounts(directoryPath); + } + + public long GetHardLinkCount(string filePath, bool ignoreRootDir) + { + if (!File.Exists(filePath)) + { + _logger.LogDebug("file {file} does not exist", filePath); + return -1; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return _windowsHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir); + } + + return _unixHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Files/IHardLinkFileService.cs b/code/Infrastructure/Verticals/Files/IHardLinkFileService.cs new file mode 100644 index 00000000..1639c57c --- /dev/null +++ b/code/Infrastructure/Verticals/Files/IHardLinkFileService.cs @@ -0,0 +1,19 @@ +namespace Infrastructure.Verticals.Files; + +public interface IHardLinkFileService +{ + /// + /// Populates the inode counts for Unix and the file index counts for Windows. + /// Needs to be called before to populate the inode counts. + /// + /// The root directory where to search for hardlinks. + void PopulateFileCounts(string directoryPath); + + /// + /// Get the hardlink count of a file. + /// + /// File path. + /// Whether to ignore hardlinks found in the same root dir. + /// -1 on error, 0 if there are no hardlinks and 1 otherwise. + long GetHardLinkCount(string filePath, bool ignoreRootDir); +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Files/UnixHardLinkFileService.cs b/code/Infrastructure/Verticals/Files/UnixHardLinkFileService.cs new file mode 100644 index 00000000..7720b640 --- /dev/null +++ b/code/Infrastructure/Verticals/Files/UnixHardLinkFileService.cs @@ -0,0 +1,87 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Mono.Unix.Native; + +namespace Infrastructure.Verticals.Files; + +public class UnixHardLinkFileService : IHardLinkFileService, IDisposable +{ + private readonly ILogger _logger; + private readonly ConcurrentDictionary _inodeCounts = new(); + + public UnixHardLinkFileService(ILogger logger) + { + _logger = logger; + } + + /// + public long GetHardLinkCount(string filePath, bool ignoreRootDir) + { + try + { + if (Syscall.stat(filePath, out Stat stat) != 0) + { + _logger.LogDebug("failed to stat file {file}", filePath); + return -1; + } + + if (!ignoreRootDir) + { + _logger.LogDebug("stat file | hardlinks: {nlink} | {file}", stat.st_nlink, filePath); + return (long)stat.st_nlink == 1 ? 0 : 1; + } + + // get the number of hardlinks in the same root directory + int linksInIgnoredDir = _inodeCounts.TryGetValue(stat.st_ino, out int count) + ? count + : 1; // default to 1 if not found + + _logger.LogDebug("stat file | hardlinks: {nlink} | ignored: {ignored} | {file}", stat.st_nlink, linksInIgnoredDir, filePath); + return (long)stat.st_nlink - linksInIgnoredDir; + } + catch (Exception exception) + { + _logger.LogError(exception, "failed to stat file {file}", filePath); + return -1; + } + } + + /// + public void PopulateFileCounts(string directoryPath) + { + try + { + // traverse all files in the ignored path and subdirectories + foreach (string file in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories)) + { + AddInodeToCount(file); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "failed to populate inode counts from {dir}", directoryPath); + throw; + } + } + + private void AddInodeToCount(string path) + { + try + { + if (Syscall.stat(path, out Stat stat) == 0) + { + _inodeCounts.AddOrUpdate(stat.st_ino, 1, (_, count) => count + 1); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "could not stat {path} during inode counting", path); + throw; + } + } + + public void Dispose() + { + _inodeCounts.Clear(); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Files/WindowsHardLinkFileService.cs b/code/Infrastructure/Verticals/Files/WindowsHardLinkFileService.cs new file mode 100644 index 00000000..fde441db --- /dev/null +++ b/code/Infrastructure/Verticals/Files/WindowsHardLinkFileService.cs @@ -0,0 +1,113 @@ +using System.Collections.Concurrent; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Microsoft.Win32.SafeHandles; + +namespace Infrastructure.Verticals.Files; + +public class WindowsHardLinkFileService : IHardLinkFileService, IDisposable +{ + private readonly ILogger _logger; + private readonly ConcurrentDictionary _fileIndexCounts = new(); + + public WindowsHardLinkFileService(ILogger logger) + { + _logger = logger; + } + + /// + public long GetHardLinkCount(string filePath, bool ignoreRootDir) + { + try + { + using SafeFileHandle fileStream = File.OpenHandle(filePath); + + if (!GetFileInformationByHandle(fileStream, out var file)) + { + _logger.LogDebug("failed to get file handle {file}", filePath); + return -1; + } + + if (!ignoreRootDir) + { + _logger.LogDebug("stat file | hardlinks: {nlink} | {file}", file.NumberOfLinks, filePath); + return file.NumberOfLinks == 1 ? 0 : 1; + } + + // Get unique file ID (combination of high and low indices) + ulong fileIndex = ((ulong)file.FileIndexHigh << 32) | file.FileIndexLow; + + // get the number of hardlinks in the same root directory + int linksInIgnoredDir = _fileIndexCounts.TryGetValue(fileIndex, out int count) + ? count + : 1; // default to 1 if not found + + _logger.LogDebug("stat file | hardlinks: {links} | ignored: {ignored} | {file}", file.NumberOfLinks, linksInIgnoredDir, filePath); + return file.NumberOfLinks - linksInIgnoredDir; + } + catch (Exception exception) + { + _logger.LogError(exception, "failed to stat file {file}", filePath); + return -1; + } + } + + /// + public void PopulateFileCounts(string directoryPath) + { + try + { + // traverse all files in the ignored path and subdirectories + foreach (string file in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories)) + { + AddFileIndexToCount(file); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to populate file index counts from {dir}", directoryPath); + } + } + + private void AddFileIndexToCount(string path) + { + try + { + using SafeFileHandle fileStream = File.OpenHandle(path); + if (GetFileInformationByHandle(fileStream, out var file)) + { + ulong fileIndex = ((ulong)file.FileIndexHigh << 32) | file.FileIndexLow; + _fileIndexCounts.AddOrUpdate(fileIndex, 1, (_, count) => count + 1); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Couldn't stat {path} during file index counting", path); + } + } + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool GetFileInformationByHandle( + SafeFileHandle hFile, + out BY_HANDLE_FILE_INFORMATION lpFileInformation + ); + + private struct BY_HANDLE_FILE_INFORMATION + { + public uint FileAttributes; + public System.Runtime.InteropServices.ComTypes.FILETIME CreationTime; + public System.Runtime.InteropServices.ComTypes.FILETIME LastAccessTime; + public System.Runtime.InteropServices.ComTypes.FILETIME LastWriteTime; + public uint VolumeSerialNumber; + public uint FileSizeHigh; + public uint FileSizeLow; + public uint NumberOfLinks; + public uint FileIndexHigh; + public uint FileIndexLow; + } + + public void Dispose() + { + _fileIndexCounts.Clear(); + } +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/ItemStriker/Striker.cs b/code/Infrastructure/Verticals/ItemStriker/Striker.cs index 04e5bdcf..a5e2c96c 100644 --- a/code/Infrastructure/Verticals/ItemStriker/Striker.cs +++ b/code/Infrastructure/Verticals/ItemStriker/Striker.cs @@ -15,14 +15,12 @@ public sealed class Striker : IStriker private readonly IMemoryCache _cache; private readonly MemoryCacheEntryOptions _cacheOptions; private readonly INotificationPublisher _notifier; - private readonly IDryRunInterceptor _dryRunInterceptor; - public Striker(ILogger logger, IMemoryCache cache, INotificationPublisher notifier, IDryRunInterceptor dryRunInterceptor) + public Striker(ILogger logger, IMemoryCache cache, INotificationPublisher notifier) { _logger = logger; _cache = cache; _notifier = notifier; - _dryRunInterceptor = dryRunInterceptor; _cacheOptions = new MemoryCacheEntryOptions() .SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer); } diff --git a/code/Infrastructure/Verticals/Notifications/Apprise/AppriseProvider.cs b/code/Infrastructure/Verticals/Notifications/Apprise/AppriseProvider.cs index 10384db3..43807ccd 100644 --- a/code/Infrastructure/Verticals/Notifications/Apprise/AppriseProvider.cs +++ b/code/Infrastructure/Verticals/Notifications/Apprise/AppriseProvider.cs @@ -43,6 +43,11 @@ public sealed class AppriseProvider : NotificationProvider await _proxy.SendNotification(BuildPayload(notification, NotificationType.Warning), _config); } + public override async Task OnCategoryChanged(CategoryChangedNotification notification) + { + await _proxy.SendNotification(BuildPayload(notification, NotificationType.Warning), _config); + } + private static ApprisePayload BuildPayload(ArrNotification notification, NotificationType notificationType) { StringBuilder body = new(); diff --git a/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs b/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs index f3c6f148..3d2344a0 100644 --- a/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs +++ b/code/Infrastructure/Verticals/Notifications/Consumers/NotificationConsumer.cs @@ -36,6 +36,9 @@ public sealed class NotificationConsumer : IConsumer where T : Notificatio case DownloadCleanedNotification downloadCleanedNotification: await _notificationService.Notify(downloadCleanedNotification); break; + case CategoryChangedNotification categoryChangedNotification: + await _notificationService.Notify(categoryChangedNotification); + break; default: throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs b/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs index e68a9617..460354f7 100644 --- a/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs +++ b/code/Infrastructure/Verticals/Notifications/INotificationFactory.cs @@ -11,4 +11,6 @@ public interface INotificationFactory List OnQueueItemDeletedEnabled(); List OnDownloadCleanedEnabled(); + + List OnCategoryChangedEnabled(); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs b/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs index c2cf2d8e..02b94709 100644 --- a/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs +++ b/code/Infrastructure/Verticals/Notifications/INotificationProvider.cs @@ -18,4 +18,6 @@ public interface INotificationProvider Task OnQueueItemDeleted(QueueItemDeletedNotification notification); Task OnDownloadCleaned(DownloadCleanedNotification notification); + + Task OnCategoryChanged(CategoryChangedNotification notification); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/INotificationPublisher.cs b/code/Infrastructure/Verticals/Notifications/INotificationPublisher.cs index 5720a339..aadadca1 100644 --- a/code/Infrastructure/Verticals/Notifications/INotificationPublisher.cs +++ b/code/Infrastructure/Verticals/Notifications/INotificationPublisher.cs @@ -9,4 +9,6 @@ public interface INotificationPublisher Task NotifyQueueItemDeleted(bool removeFromClient, DeleteReason reason); Task NotifyDownloadCleaned(double ratio, TimeSpan seedingTime, string categoryName, CleanReason reason); + + Task NotifyCategoryChanged(string oldCategory, string newCategory); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs b/code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs new file mode 100644 index 00000000..4da1dc8d --- /dev/null +++ b/code/Infrastructure/Verticals/Notifications/Models/CategoryChangedNotification.cs @@ -0,0 +1,5 @@ +namespace Infrastructure.Verticals.Notifications.Models; + +public sealed record CategoryChangedNotification : Notification +{ +} \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs b/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs index 9a278774..12dd9b41 100644 --- a/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs +++ b/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProvider.cs @@ -46,6 +46,11 @@ public class NotifiarrProvider : NotificationProvider { await _proxy.SendNotification(BuildPayload(notification), _config); } + + public override async Task OnCategoryChanged(CategoryChangedNotification notification) + { + await _proxy.SendNotification(BuildPayload(notification), _config); + } private NotifiarrPayload BuildPayload(ArrNotification notification, string color) { @@ -110,4 +115,32 @@ public class NotifiarrProvider : NotificationProvider return payload; } + + private NotifiarrPayload BuildPayload(CategoryChangedNotification notification) + { + NotifiarrPayload payload = new() + { + Discord = new() + { + Color = WarningColor, + Text = new() + { + Title = notification.Title, + Icon = Logo, + Description = notification.Description, + Fields = notification.Fields?.Adapt>() ?? [] + }, + Ids = new Ids + { + Channel = _config.ChannelId + }, + Images = new() + { + Thumbnail = new Uri(Logo) + } + } + }; + + return payload; + } } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProxy.cs b/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProxy.cs index f41c4ed6..4c48c42f 100644 --- a/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProxy.cs +++ b/code/Infrastructure/Verticals/Notifications/Notifiarr/NotifiarrProxy.cs @@ -1,5 +1,6 @@ using System.Text; using Common.Helpers; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; @@ -7,12 +8,14 @@ namespace Infrastructure.Verticals.Notifications.Notifiarr; public sealed class NotifiarrProxy : INotifiarrProxy { + private readonly ILogger _logger; private readonly HttpClient _httpClient; private const string Url = "https://notifiarr.com/api/v1/notification/passthrough/"; - public NotifiarrProxy(IHttpClientFactory httpClientFactory) + public NotifiarrProxy(ILogger logger, IHttpClientFactory httpClientFactory) { + _logger = logger; _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName); } @@ -25,6 +28,8 @@ public sealed class NotifiarrProxy : INotifiarrProxy ContractResolver = new CamelCasePropertyNamesContractResolver() }); + _logger.LogTrace("sending notification to Notifiarr: {content}", content); + using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, $"{Url}{config.ApiKey}"); request.Method = HttpMethod.Post; request.Content = new StringContent(content, Encoding.UTF8, "application/json"); diff --git a/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs b/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs index 691d8079..34b7170d 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationFactory.cs @@ -39,4 +39,9 @@ public class NotificationFactory : INotificationFactory ActiveProviders() .Where(n => n.Config.OnDownloadCleaned) .ToList(); + + public List OnCategoryChangedEnabled() => + ActiveProviders() + .Where(n => n.Config.OnCategoryChanged) + .ToList(); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs b/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs index c71bb3fe..169c5eeb 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationProvider.cs @@ -24,4 +24,6 @@ public abstract class NotificationProvider : INotificationProvider public abstract Task OnQueueItemDeleted(QueueItemDeletedNotification notification); public abstract Task OnDownloadCleaned(DownloadCleanedNotification notification); + + public abstract Task OnCategoryChanged(CategoryChangedNotification notification); } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs index 68f9dea2..a9ea4852 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationPublisher.cs @@ -49,14 +49,14 @@ public class NotificationPublisher : INotificationPublisher { case StrikeType.Stalled: case StrikeType.DownloadingMetadata: - await _dryRunInterceptor.InterceptAsync(Notify, notification.Adapt()); + await NotifyInternal(notification.Adapt()); break; case StrikeType.ImportFailed: - await _dryRunInterceptor.InterceptAsync(Notify, notification.Adapt()); + await NotifyInternal(notification.Adapt()); break; case StrikeType.SlowSpeed: case StrikeType.SlowTime: - await _dryRunInterceptor.InterceptAsync(Notify, notification.Adapt()); + await NotifyInternal(notification.Adapt()); break; } } @@ -86,7 +86,7 @@ public class NotificationPublisher : INotificationPublisher Fields = [new() { Title = "Removed from download client?", Text = removeFromClient ? "Yes" : "No" }] }; - await _dryRunInterceptor.InterceptAsync(Notify, notification); + await NotifyInternal(notification); } catch (Exception ex) { @@ -115,13 +115,36 @@ public class NotificationPublisher : INotificationPublisher Level = NotificationLevel.Important }; - await _dryRunInterceptor.InterceptAsync(Notify, notification); + await NotifyInternal(notification); } catch (Exception ex) { _logger.LogError(ex, "failed to notify download cleaned"); } } + + public virtual async Task NotifyCategoryChanged(string oldCategory, string newCategory) + { + CategoryChangedNotification notification = new() + { + Title = "Category changed", + Description = ContextProvider.Get("downloadName"), + Fields = + [ + new() { Title = "Hash", Text = ContextProvider.Get("hash").ToLowerInvariant() }, + new() { Title = "Old category", Text = oldCategory }, + new() { Title = "New category", Text = newCategory } + ], + Level = NotificationLevel.Important + }; + + await NotifyInternal(notification); + } + + private Task NotifyInternal(T message) where T: notnull + { + return _dryRunInterceptor.InterceptAsync(Notify, message); + } [DryRunSafeguard] private Task Notify(T message) where T: notnull diff --git a/code/Infrastructure/Verticals/Notifications/NotificationService.cs b/code/Infrastructure/Verticals/Notifications/NotificationService.cs index 4efadaac..d26b8f81 100644 --- a/code/Infrastructure/Verticals/Notifications/NotificationService.cs +++ b/code/Infrastructure/Verticals/Notifications/NotificationService.cs @@ -88,4 +88,19 @@ public class NotificationService } } } + + public async Task Notify(CategoryChangedNotification notification) + { + foreach (INotificationProvider provider in _notificationFactory.OnCategoryChangedEnabled()) + { + try + { + await provider.OnCategoryChanged(notification); + } + catch (Exception exception) + { + _logger.LogWarning(exception, "failed to send notification | provider {provider}", provider.Name); + } + } + } } \ No newline at end of file diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index 86059c97..6423a3a2 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -224,11 +224,15 @@ services: - DOWNLOADCLEANER__CATEGORIES__0__NAME=tv-sonarr - DOWNLOADCLEANER__CATEGORIES__0__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__0__MIN_SEED_TIME=0 - - DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=0.01 - - DOWNLOADCLEANER__CATEGORIES__1__NAME=radarr + - DOWNLOADCLEANER__CATEGORIES__0__MAX_SEED_TIME=99999 + - DOWNLOADCLEANER__CATEGORIES__1__NAME=nohardlink - DOWNLOADCLEANER__CATEGORIES__1__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__1__MIN_SEED_TIME=0 - - DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=0.01 + - DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=99999 + - DOWNLOADCLEANER__UNLINKED_TARGET_CATEGORY=cleanuperr-unlinked + - DOWNLOADCLEANER__UNLINKED_IGNORED_ROOT_DIR=/downloads + - DOWNLOADCLEANER__UNLINKED_CATEGORIES__0=tv-sonarr + - DOWNLOADCLEANER__UNLINKED_CATEGORIES__1=radarr - DOWNLOAD_CLIENT=qbittorrent - QBITTORRENT__URL=http://qbittorrent:8080 @@ -268,6 +272,7 @@ services: # - NOTIFIARR__ON_SLOW_STRIKE=true # - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true # - NOTIFIARR__ON_DOWNLOAD_CLEANED=true + # - NOTIFIARR__ON_CATEGORY_CHANGED=true # - NOTIFIARR__API_KEY=notifiarr_secret # - NOTIFIARR__CHANNEL_ID=discord_channel_id @@ -281,6 +286,7 @@ services: volumes: - ./data/cleanuperr/logs:/var/logs - ./data/cleanuperr/ignored_downloads:/ignored + - ./data/qbittorrent/downloads:/downloads restart: unless-stopped depends_on: - qbittorrent diff --git a/docs/docs/4_how_it_works.mdx b/docs/docs/4_how_it_works.mdx index 10747d42..6f30cff1 100644 --- a/docs/docs/4_how_it_works.mdx +++ b/docs/docs/4_how_it_works.mdx @@ -32,4 +32,5 @@ This is a detailed explanation of how the recurring cleanup jobs work. - A new search will be triggered for the *arr item. #### 3. **Download cleaner** will: - Run every hour (or configured cron). - - Automatically clean up downloads that have been seeding for a certain amount of time. \ No newline at end of file + - Automatically clean up downloads that have been seeding for a certain amount of time. + - Automatically changes the category of downloads that have no hardlinks. \ No newline at end of file diff --git a/docs/docs/configuration/download-cleaner/3_hardlinks.mdx b/docs/docs/configuration/download-cleaner/3_hardlinks.mdx new file mode 100644 index 00000000..b0864503 --- /dev/null +++ b/docs/docs/configuration/download-cleaner/3_hardlinks.mdx @@ -0,0 +1,19 @@ +--- +sidebar_position: 3 +--- + +import DownloadCleanerHardlinksSettings from '@site/src/components/configuration/download-cleaner/DownloadCleanerHardlinksSettings'; +import { Important } from '@site/src/components/Admonition'; + +# Hardlinks Settings + +These settings control how the Download Cleaner handles downloads with no hardlinks remaining (they are not available in the arrs anymore). + +The Download Cleaner will change the category of a download that has no hardlinks and the new category can be cleaned based on the rules configured [here](/docs/configuration/download-cleaner/categories). + + + If you are using Docker, make sure to mount the downloads directory the same way it is mounted for the download client. + If your download client's download directory is `/downloads`, it should be the same for Cleanuperr. + + + \ No newline at end of file diff --git a/docs/docs/configuration/examples/1_docker.mdx b/docs/docs/configuration/examples/1_docker.mdx index 5b772ecb..763a93c7 100644 --- a/docs/docs/configuration/examples/1_docker.mdx +++ b/docs/docs/configuration/examples/1_docker.mdx @@ -79,6 +79,17 @@ services: - DOWNLOADCLEANER__CATEGORIES__1__MAX_RATIO=-1 - DOWNLOADCLEANER__CATEGORIES__1__MIN_SEED_TIME=0 - DOWNLOADCLEANER__CATEGORIES__1__MAX_SEED_TIME=240 + # remove downloads with no hardlinks + - DOWNLOADCLEANER__CATEGORIES__2__NAME=cleanuperr-unlinked + - DOWNLOADCLEANER__CATEGORIES__2__MAX_RATIO=-1 + - DOWNLOADCLEANER__CATEGORIES__2__MIN_SEED_TIME=0 + - DOWNLOADCLEANER__CATEGORIES__2__MAX_SEED_TIME=0 + + # change category for downloads with no hardlinks + - DOWNLOADCLEANER__UNLINKED_TARGET_CATEGORY=cleanuperr-unlinked + - DOWNLOADCLEANER__UNLINKED_IGNORED_ROOT_DIR=/downloads + - DOWNLOADCLEANER__UNLINKED_CATEGORIES__0=tv-sonarr + - DOWNLOADCLEANER__UNLINKED_CATEGORIES__1=radarr - DOWNLOAD_CLIENT=none # OR @@ -131,6 +142,7 @@ services: - NOTIFIARR__ON_SLOW_STRIKE=true - NOTIFIARR__ON_QUEUE_ITEM_DELETED=true - NOTIFIARR__ON_DOWNLOAD_CLEANED=true + - NOTIFIARR__ON_CATEGORY_CHANGED=true - NOTIFIARR__API_KEY=notifiarr_secret - NOTIFIARR__CHANNEL_ID=discord_channel_id @@ -139,6 +151,7 @@ services: - APPRISE__ON_SLOW_STRIKE=true - APPRISE__ON_QUEUE_ITEM_DELETED=true - APPRISE__ON_DOWNLOAD_CLEANED=true + - NOTIFIARR__ON_CATEGORY_CHANGED=true - APPRISE__URL=http://apprise:8000 - APPRISE__KEY=myConfigKey ``` \ No newline at end of file diff --git a/docs/docs/configuration/examples/2_config-file.mdx b/docs/docs/configuration/examples/2_config-file.mdx index 4e898358..b0f5fde0 100644 --- a/docs/docs/configuration/examples/2_config-file.mdx +++ b/docs/docs/configuration/examples/2_config-file.mdx @@ -69,8 +69,20 @@ import { Note } from '@site/src/components/Admonition'; "MAX_RATIO": 1, "MIN_SEED_TIME": 0, "MAX_SEED_TIME": 240 + }, + { + "Name": "cleanuperr-unlinked", + "MAX_RATIO": 1, + "MIN_SEED_TIME": 0, + "MAX_SEED_TIME": 240 } ], + "UNLINKED_TARGET_CATEGORY": "cleanuperr-unlinked", + "UNLINKED_IGNORED_ROOT_DIR": "/downloads", + "UNLINKED_CATEGORIES": [ + "tv-sonarr", + "radarr" + ], "IGNORED_DOWNLOADS_PATH": "/ignored.txt" }, "DOWNLOAD_CLIENT": "none", @@ -149,6 +161,7 @@ import { Note } from '@site/src/components/Admonition'; "ON_SLOW_STRIKE": true, "ON_QUEUE_ITEM_DELETED": true, "ON_DOWNLOAD_CLEANED": true, + "ON_CATEGORY_CHANGED": true, "API_KEY": "notifiarr_secret", "CHANNEL_ID": "discord_channel_id" }, @@ -158,6 +171,7 @@ import { Note } from '@site/src/components/Admonition'; "ON_SLOW_STRIKE": true, "ON_QUEUE_ITEM_DELETED": true, "ON_DOWNLOAD_CLEANED": true, + "ON_CATEGORY_CHANGED": true, "URL": "http://localhost:8000", "KEY": "myConfigKey" } diff --git a/docs/src/components/configuration/download-cleaner/DownloadCleanerHardlinksSettings.tsx b/docs/src/components/configuration/download-cleaner/DownloadCleanerHardlinksSettings.tsx new file mode 100644 index 00000000..cca58352 --- /dev/null +++ b/docs/src/components/configuration/download-cleaner/DownloadCleanerHardlinksSettings.tsx @@ -0,0 +1,43 @@ +import React from "react"; +import EnvVars, { EnvVarProps } from "../EnvVars"; + +const settings: EnvVarProps[] = [ + { + name: "DOWNLOADCLEANER__UNLINKED_TARGET_CATEGORY", + description: [ + "The category to set on downloads that do not have hardlinks." + ], + type: "text", + defaultValue: "cleanuperr-unlinked", + required: false, + }, + { + name: "DOWNLOADCLEANER__UNLINKED_IGNORED_ROOT_DIR", + description: [ + "This is useful if you are using [cross-seed](https://www.cross-seed.org/).", + "The downloads root directory where the original and cross-seed hardlinks reside. All other hardlinks from this directory will be treated as if they do not exist (e.g. if you have a download with the original file and a cross-seed hardlink, it will be deleted).", + ], + type: "text", + defaultValue: "Empty", + required: false, + }, + { + name: "DOWNLOADCLEANER__UNLINKED_CATEGORIES__0", + description: [ + "The categories of downloads to check for available hardlinks.", + { + type: "code", + title: "Multiple patterns can be specified using incrementing numbers starting from 0.", + content: `DOWNLOADCLEANER__UNLINKED_CATEGORIES__0=tv-sonarr +DOWNLOADCLEANER__UNLINKED_CATEGORIES__1=radarr` + } + ], + type: "text", + defaultValue: "Empty", + required: false, + } +]; + +export default function DownloadCleanerHardlinksSettings() { + return ; +} diff --git a/docs/src/components/configuration/notifications/AppriseSettings.tsx b/docs/src/components/configuration/notifications/AppriseSettings.tsx index 758e27cc..f7b59565 100644 --- a/docs/src/components/configuration/notifications/AppriseSettings.tsx +++ b/docs/src/components/configuration/notifications/AppriseSettings.tsx @@ -68,6 +68,16 @@ const settings: EnvVarProps[] = [ required: false, acceptedValues: ["true", "false"], }, + { + name: "APPRISE__ON_CATEGORY_CHANGED", + description: [ + "Controls whether to notify when a download's category is changed." + ], + type: "boolean", + defaultValue: "false", + required: false, + acceptedValues: ["true", "false"], + } ]; export default function AppriseSettings() { diff --git a/docs/src/components/configuration/notifications/NotifiarrSettings.tsx b/docs/src/components/configuration/notifications/NotifiarrSettings.tsx index 931abbc0..4cdfb1c1 100644 --- a/docs/src/components/configuration/notifications/NotifiarrSettings.tsx +++ b/docs/src/components/configuration/notifications/NotifiarrSettings.tsx @@ -70,6 +70,16 @@ const settings: EnvVarProps[] = [ defaultValue: "false", required: false, acceptedValues: ["true", "false"], + }, + { + name: "NOTIFIARR__ON_CATEGORY_CHANGED", + description: [ + "Controls whether to notify when a download's category is changed." + ], + type: "boolean", + defaultValue: "false", + required: false, + acceptedValues: ["true", "false"], } ]; diff --git a/docs/src/pages/index.tsx b/docs/src/pages/index.tsx index 61d2edbc..697e9cb0 100644 --- a/docs/src/pages/index.tsx +++ b/docs/src/pages/index.tsx @@ -3,7 +3,6 @@ import clsx from "clsx"; import Link from "@docusaurus/Link"; import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; import Layout from "@theme/Layout"; -import HomepageFeatures from "@site/src/components/HomepageFeatures"; import Heading from "@theme/Heading"; import styles from "./index.module.css";