diff --git a/.github/ISSUE_TEMPLATE/1-bug.yml b/.github/ISSUE_TEMPLATE/1-bug.yml index 63c5f7c4..ee50073b 100644 --- a/.github/ISSUE_TEMPLATE/1-bug.yml +++ b/.github/ISSUE_TEMPLATE/1-bug.yml @@ -18,7 +18,7 @@ body: required: true - label: Ensured I am using the latest version. required: true - - label: Enabled debug logging. + - label: Enabled verbose logging. required: true - type: textarea id: what-happened diff --git a/.github/ISSUE_TEMPLATE/3-help.yml b/.github/ISSUE_TEMPLATE/3-help.yml index cece2d9c..2e7828c4 100644 --- a/.github/ISSUE_TEMPLATE/3-help.yml +++ b/.github/ISSUE_TEMPLATE/3-help.yml @@ -18,7 +18,7 @@ body: required: true - label: Ensured I am using the latest version. required: true - - label: Enabled debug logging. + - label: Enabled verbose logging. required: true - type: textarea id: description diff --git a/chart/values.yaml b/chart/values.yaml index 5262cd70..73787c41 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -14,7 +14,7 @@ deployment: value: "false" - name: LOGGING__LOGLEVEL - value: Debug + value: Verbose - name: LOGGING__FILE__ENABLED value: "true" - name: LOGGING__FILE__PATH diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index 90062537..9d4f6c9e 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -1,4 +1,5 @@ -using Common.Exceptions; +using Common.CustomDataTypes; +using Common.Exceptions; using Microsoft.Extensions.Configuration; namespace Common.Configuration.QueueCleaner; @@ -37,17 +38,74 @@ public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig [ConfigurationKeyName("STALLED_DELETE_PRIVATE")] public bool StalledDeletePrivate { get; init; } + + [ConfigurationKeyName("SLOW_MAX_STRIKES")] + public ushort SlowMaxStrikes { get; init; } + + [ConfigurationKeyName("SLOW_RESET_STRIKES_ON_PROGRESS")] + public bool SlowResetStrikesOnProgress { get; init; } + + [ConfigurationKeyName("SLOW_IGNORE_PRIVATE")] + public bool SlowIgnorePrivate { get; init; } + + [ConfigurationKeyName("SLOW_DELETE_PRIVATE")] + public bool SlowDeletePrivate { get; init; } + + [ConfigurationKeyName("SLOW_MIN_SPEED")] + public string SlowMinSpeed { get; init; } = string.Empty; + + public ByteSize SlowMinSpeedByteSize => string.IsNullOrEmpty(SlowMinSpeed) ? new ByteSize(0) : ByteSize.Parse(SlowMinSpeed); + + [ConfigurationKeyName("SLOW_MAX_TIME")] + public double SlowMaxTime { get; init; } + + [ConfigurationKeyName("SLOW_IGNORE_ABOVE_SIZE")] + public string SlowIgnoreAboveSize { get; init; } = string.Empty; + + public ByteSize? SlowIgnoreAboveSizeByteSize => string.IsNullOrEmpty(SlowIgnoreAboveSize) ? null : ByteSize.Parse(SlowIgnoreAboveSize); public void Validate() { if (ImportFailedMaxStrikes is > 0 and < 3) { - throw new ValidationException("the minimum value for IMPORT_FAILED_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {SectionName}__IMPORT_FAILED_MAX_STRIKES must be 3"); } if (StalledMaxStrikes is > 0 and < 3) { - throw new ValidationException("the minimum value for STALLED_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {SectionName}__STALLED_MAX_STRIKES must be 3"); + } + + if (SlowMaxStrikes is > 0 and < 3) + { + throw new ValidationException($"the minimum value for {SectionName}__SLOW_MAX_STRIKES must be 3"); + } + + if (SlowMaxStrikes > 0) + { + bool isSlowSpeedSet = !string.IsNullOrEmpty(SlowMinSpeed); + + if (isSlowSpeedSet && ByteSize.TryParse(SlowMinSpeed, out _) is false) + { + throw new ValidationException($"invalid value for {SectionName}__SLOW_MIN_SPEED"); + } + + if (SlowMaxTime < 0) + { + throw new ValidationException($"invalid value for {SectionName}__SLOW_MAX_TIME"); + } + + if (!isSlowSpeedSet && SlowMaxTime is 0) + { + throw new ValidationException($"either {SectionName}__SLOW_MIN_SPEED or {SectionName}__SLOW_MAX_STRIKES must be set"); + } + + bool isSlowIgnoreAboveSizeSet = !string.IsNullOrEmpty(SlowIgnoreAboveSize); + + if (isSlowIgnoreAboveSizeSet && ByteSize.TryParse(SlowIgnoreAboveSize, out _) is false) + { + throw new ValidationException($"invalid value for {SectionName}__SLOW_IGNORE_ABOVE_SIZE"); + } } } } \ No newline at end of file diff --git a/code/Common/CustomDataTypes/ByteSize.cs b/code/Common/CustomDataTypes/ByteSize.cs new file mode 100644 index 00000000..72511242 --- /dev/null +++ b/code/Common/CustomDataTypes/ByteSize.cs @@ -0,0 +1,115 @@ +using System.Globalization; + +namespace Common.CustomDataTypes; + +public readonly struct ByteSize : IComparable, IEquatable +{ + public long Bytes { get; } + + private const long BytesPerKB = 1024; + private const long BytesPerMB = 1024 * 1024; + private const long BytesPerGB = 1024 * 1024 * 1024; + + public ByteSize(long bytes) + { + if (bytes < 0) + { + throw new ArgumentOutOfRangeException(nameof(bytes), "bytes can not be negative"); + } + + Bytes = bytes; + } + + public static ByteSize FromKilobytes(double kilobytes) => new((long)(kilobytes * BytesPerKB)); + public static ByteSize FromMegabytes(double megabytes) => new((long)(megabytes * BytesPerMB)); + public static ByteSize FromGigabytes(double gigabytes) => new((long)(gigabytes * BytesPerGB)); + + public static ByteSize Parse(string input) + { + if (string.IsNullOrWhiteSpace(input)) + { + throw new ArgumentNullException(nameof(input)); + } + + input = input.Trim().ToUpperInvariant(); + double value; + if (input.EndsWith("KB", StringComparison.InvariantCultureIgnoreCase)) + { + value = double.Parse(input[..^2], CultureInfo.InvariantCulture); + return FromKilobytes(value); + } + + if (input.EndsWith("MB", StringComparison.InvariantCultureIgnoreCase)) + { + value = double.Parse(input[..^2], CultureInfo.InvariantCulture); + return FromMegabytes(value); + } + + if (input.EndsWith("GB", StringComparison.InvariantCultureIgnoreCase)) + { + value = double.Parse(input[..^2], CultureInfo.InvariantCulture); + return FromGigabytes(value); + } + + throw new FormatException("invalid size format | only KB, MB and GB are supported"); + } + + public static bool TryParse(string? input, out ByteSize? result) + { + result = default; + + if (string.IsNullOrWhiteSpace(input)) + { + return false; + } + + input = input.Trim().ToUpperInvariant(); + + if (input.EndsWith("KB", StringComparison.InvariantCultureIgnoreCase) && + double.TryParse(input[..^2], NumberStyles.Float, CultureInfo.InvariantCulture, out double kb)) + { + result = FromKilobytes(kb); + return true; + } + + if (input.EndsWith("MB", StringComparison.InvariantCultureIgnoreCase) && + double.TryParse(input[..^2], NumberStyles.Float, CultureInfo.InvariantCulture, out double mb)) + { + result = FromMegabytes(mb); + return true; + } + + if (input.EndsWith("GB", StringComparison.InvariantCultureIgnoreCase) && + double.TryParse(input[..^2], NumberStyles.Float, CultureInfo.InvariantCulture, out double gb)) + { + result = FromGigabytes(gb); + return true; + } + + return false; + } + + public override string ToString() => + Bytes switch + { + >= BytesPerGB => $"{Bytes / (double)BytesPerGB:0.##} GB", + >= BytesPerMB => $"{Bytes / (double)BytesPerMB:0.##} MB", + _ => $"{Bytes / (double)BytesPerKB:0.##} KB" + }; + + public int CompareTo(ByteSize other) => Bytes.CompareTo(other.Bytes); + public bool Equals(ByteSize other) => Bytes == other.Bytes; + + public override bool Equals(object? obj) => obj is ByteSize other && Equals(other); + public override int GetHashCode() => Bytes.GetHashCode(); + + public static bool operator ==(ByteSize left, ByteSize right) => left.Equals(right); + public static bool operator !=(ByteSize left, ByteSize right) => !(left == right); + public static bool operator <(ByteSize left, ByteSize right) => left.Bytes < right.Bytes; + public static bool operator >(ByteSize left, ByteSize right) => left.Bytes > right.Bytes; + public static bool operator <=(ByteSize left, ByteSize right) => left.Bytes <= right.Bytes; + public static bool operator >=(ByteSize left, ByteSize right) => left.Bytes >= right.Bytes; + + public static ByteSize operator +(ByteSize left, ByteSize right) => new(left.Bytes + right.Bytes); + public static ByteSize operator -(ByteSize left, ByteSize right) => new(Math.Max(left.Bytes - right.Bytes, 0)); +} \ No newline at end of file diff --git a/code/Common/CustomDataTypes/SmartTimeSpan.cs b/code/Common/CustomDataTypes/SmartTimeSpan.cs new file mode 100644 index 00000000..40778e5b --- /dev/null +++ b/code/Common/CustomDataTypes/SmartTimeSpan.cs @@ -0,0 +1,66 @@ +using System.Text; + +namespace Common.CustomDataTypes; + +public readonly struct SmartTimeSpan : IComparable, IEquatable +{ + public TimeSpan Time { get; } + + public SmartTimeSpan(TimeSpan time) + { + Time = time; + } + + public override string ToString() + { + if (Time == TimeSpan.Zero) + { + return "0 seconds"; + } + + StringBuilder sb = new(); + + if (Time.Days > 0) + { + sb.Append($"{Time.Days} day{(Time.Days > 1 ? "s" : "")} "); + } + + if (Time.Hours > 0) + { + sb.Append($"{Time.Hours} hour{(Time.Hours > 1 ? "s" : "")} "); + } + + if (Time.Minutes > 0) + { + sb.Append($"{Time.Minutes} minute{(Time.Minutes > 1 ? "s" : "")} "); + } + + if (Time.Seconds > 0) + { + sb.Append($"{Time.Seconds} second{(Time.Seconds > 1 ? "s" : "")}"); + } + + return sb.ToString().TrimEnd(); + } + + public static SmartTimeSpan FromMinutes(double minutes) => new(TimeSpan.FromMinutes(minutes)); + public static SmartTimeSpan FromSeconds(double seconds) => new(TimeSpan.FromSeconds(seconds)); + public static SmartTimeSpan FromHours(double hours) => new(TimeSpan.FromHours(hours)); + public static SmartTimeSpan FromDays(double days) => new(TimeSpan.FromDays(days)); + + public int CompareTo(SmartTimeSpan other) => Time.CompareTo(other.Time); + public bool Equals(SmartTimeSpan other) => Time.Equals(other.Time); + + public override bool Equals(object? obj) => obj is SmartTimeSpan other && Equals(other); + public override int GetHashCode() => Time.GetHashCode(); + + public static bool operator ==(SmartTimeSpan left, SmartTimeSpan right) => left.Equals(right); + public static bool operator !=(SmartTimeSpan left, SmartTimeSpan right) => !left.Equals(right); + public static bool operator <(SmartTimeSpan left, SmartTimeSpan right) => left.Time < right.Time; + public static bool operator >(SmartTimeSpan left, SmartTimeSpan right) => left.Time > right.Time; + public static bool operator <=(SmartTimeSpan left, SmartTimeSpan right) => left.Time <= right.Time; + public static bool operator >=(SmartTimeSpan left, SmartTimeSpan right) => left.Time >= right.Time; + + public static SmartTimeSpan operator +(SmartTimeSpan left, SmartTimeSpan right) => new(left.Time + right.Time); + public static SmartTimeSpan operator -(SmartTimeSpan left, SmartTimeSpan right) => new(left.Time - right.Time); +} \ No newline at end of file diff --git a/code/Domain/Enums/DeleteReason.cs b/code/Domain/Enums/DeleteReason.cs index baeddd31..c80585eb 100644 --- a/code/Domain/Enums/DeleteReason.cs +++ b/code/Domain/Enums/DeleteReason.cs @@ -6,6 +6,8 @@ public enum DeleteReason Stalled, ImportFailed, DownloadingMetadata, + SlowSpeed, + SlowTime, AllFilesSkipped, AllFilesSkippedByQBit, AllFilesBlocked, diff --git a/code/Domain/Enums/StrikeType.cs b/code/Domain/Enums/StrikeType.cs index 234401d8..71cc8c2f 100644 --- a/code/Domain/Enums/StrikeType.cs +++ b/code/Domain/Enums/StrikeType.cs @@ -4,5 +4,7 @@ public enum StrikeType { Stalled, DownloadingMetadata, - ImportFailed + ImportFailed, + SlowSpeed, + SlowTime, } \ No newline at end of file diff --git a/code/Domain/Models/Cache/CacheItem.cs b/code/Domain/Models/Cache/StalledCacheItem.cs similarity index 81% rename from code/Domain/Models/Cache/CacheItem.cs rename to code/Domain/Models/Cache/StalledCacheItem.cs index 8d5b2aef..7fa99526 100644 --- a/code/Domain/Models/Cache/CacheItem.cs +++ b/code/Domain/Models/Cache/StalledCacheItem.cs @@ -1,6 +1,6 @@ namespace Domain.Models.Cache; -public sealed record CacheItem +public sealed record StalledCacheItem { /// /// The amount of bytes that have been downloaded. diff --git a/code/Domain/Models/Deluge/Response/TorrentStatus.cs b/code/Domain/Models/Deluge/Response/DownloadStatus.cs similarity index 76% rename from code/Domain/Models/Deluge/Response/TorrentStatus.cs rename to code/Domain/Models/Deluge/Response/DownloadStatus.cs index b33dd8f5..f9fa0841 100644 --- a/code/Domain/Models/Deluge/Response/TorrentStatus.cs +++ b/code/Domain/Models/Deluge/Response/DownloadStatus.cs @@ -2,7 +2,7 @@ namespace Domain.Models.Deluge.Response; -public sealed record TorrentStatus +public sealed record DownloadStatus { public string? Hash { get; init; } @@ -12,8 +12,14 @@ public sealed record TorrentStatus public ulong Eta { get; init; } + [JsonProperty("download_payload_rate")] + public long DownloadSpeed { get; init; } + public bool Private { get; init; } + [JsonProperty("total_size")] + public long Size { get; init; } + [JsonProperty("total_done")] public long TotalDone { get; init; } diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index f5d87202..81fc79ef 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -3,7 +3,7 @@ "HTTP_MAX_RETRIES": 0, "HTTP_TIMEOUT": 10, "Logging": { - "LogLevel": "Debug", + "LogLevel": "Verbose", "Enhanced": true, "File": { "Enabled": false, @@ -34,7 +34,14 @@ "STALLED_MAX_STRIKES": 5, "STALLED_RESET_STRIKES_ON_PROGRESS": true, "STALLED_IGNORE_PRIVATE": true, - "STALLED_DELETE_PRIVATE": false + "STALLED_DELETE_PRIVATE": false, + "SLOW_MAX_STRIKES": 5, + "SLOW_RESET_STRIKES_ON_PROGRESS": true, + "SLOW_IGNORE_PRIVATE": false, + "SLOW_DELETE_PRIVATE": false, + "SLOW_MIN_SPEED": "1MB", + "SLOW_MAX_TIME": 20, + "SLOW_IGNORE_ABOVE_SIZE": "4GB" }, "DownloadCleaner": { "Enabled": false, diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs index d63fd3d8..19c564ff 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/DownloadServiceTests.cs @@ -39,7 +39,7 @@ public class DownloadServiceTests : IClassFixture }); // Act - sut.ResetStrikesOnProgress("test-hash", 100); + sut.ResetStalledStrikesOnProgress("test-hash", 100); // Assert _fixture.Cache.ReceivedCalls().ShouldBeEmpty(); @@ -50,19 +50,19 @@ public class DownloadServiceTests : IClassFixture { // Arrange const string hash = "test-hash"; - CacheItem cacheItem = new CacheItem { Downloaded = 100 }; + StalledCacheItem stalledCacheItem = new StalledCacheItem { Downloaded = 100 }; _fixture.Cache.TryGetValue(Arg.Any(), out Arg.Any()) .Returns(x => { - x[1] = cacheItem; + x[1] = stalledCacheItem; return true; }); TestDownloadService sut = _fixture.CreateSut(); // Act - sut.ResetStrikesOnProgress(hash, 200); + sut.ResetStalledStrikesOnProgress(hash, 200); // Assert _fixture.Cache.Received(1).Remove(CacheKeys.Strike(StrikeType.Stalled, hash)); @@ -73,20 +73,20 @@ public class DownloadServiceTests : IClassFixture { // Arrange const string hash = "test-hash"; - CacheItem cacheItem = new CacheItem { Downloaded = 200 }; + StalledCacheItem stalledCacheItem = new StalledCacheItem { Downloaded = 200 }; _fixture.Cache .TryGetValue(Arg.Any(), out Arg.Any()) .Returns(x => { - x[1] = cacheItem; + x[1] = stalledCacheItem; return true; }); TestDownloadService sut = _fixture.CreateSut(); // Act - sut.ResetStrikesOnProgress(hash, 100); + sut.ResetStalledStrikesOnProgress(hash, 100); // Assert _fixture.Cache.DidNotReceive().Remove(Arg.Any()); @@ -98,28 +98,6 @@ public class DownloadServiceTests : IClassFixture public StrikeAndCheckLimitTests(DownloadServiceFixture fixture) : base(fixture) { } - - [Fact] - public async Task ShouldDelegateCallToStriker() - { - // Arrange - const string hash = "test-hash"; - const string itemName = "test-item"; - StrikeType strikeType = StrikeType.Stalled; - _fixture.Striker.StrikeAndCheckLimit(hash, itemName, 3, strikeType) - .Returns(true); - - TestDownloadService sut = _fixture.CreateSut(); - - // Act - bool result = await sut.StrikeAndCheckLimit(hash, itemName, strikeType); - - // Assert - result.ShouldBeTrue(); - await _fixture.Striker - .Received(1) - .StrikeAndCheckLimit(hash, itemName, 3, StrikeType.Stalled); - } } public class ShouldCleanDownloadTests : DownloadServiceTests diff --git a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs index fc0a125f..4defffab 100644 --- a/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs +++ b/code/Infrastructure.Tests/Verticals/DownloadClient/TestDownloadService.cs @@ -36,7 +36,7 @@ public class TestDownloadService : DownloadService public override void Dispose() { } public override Task LoginAsync() => Task.CompletedTask; - public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) => Task.FromResult(new StalledResult()); + public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) => Task.FromResult(new DownloadCheckResult()); public override Task BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag patterns, ConcurrentBag regexes, IReadOnlyList ignoredDownloads) => Task.FromResult(new BlockFilesResult()); public override Task DeleteDownload(string hash) => Task.CompletedTask; @@ -45,7 +45,6 @@ public class TestDownloadService : DownloadService IReadOnlyList ignoredDownloads) => Task.CompletedTask; // Expose protected methods for testing - public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded); - public new Task StrikeAndCheckLimit(string hash, string itemName, StrikeType strikeType) => base.StrikeAndCheckLimit(hash, itemName, strikeType); + public new void ResetStalledStrikesOnProgress(string hash, long downloaded) => base.ResetStalledStrikesOnProgress(hash, downloaded); public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) => base.ShouldCleanDownload(ratio, seedingTime, category); } \ No newline at end of file diff --git a/code/Infrastructure/Extensions/DelugeExtensions.cs b/code/Infrastructure/Extensions/DelugeExtensions.cs index 0c20a458..b012598a 100644 --- a/code/Infrastructure/Extensions/DelugeExtensions.cs +++ b/code/Infrastructure/Extensions/DelugeExtensions.cs @@ -4,7 +4,7 @@ namespace Infrastructure.Extensions; public static class DelugeExtensions { - public static bool ShouldIgnore(this TorrentStatus download, IReadOnlyList ignoredDownloads) + public static bool ShouldIgnore(this DownloadStatus download, IReadOnlyList ignoredDownloads) { foreach (string value in ignoredDownloads) { diff --git a/code/Infrastructure/Helpers/CacheKeys.cs b/code/Infrastructure/Helpers/CacheKeys.cs index 2f0d726a..61314fca 100644 --- a/code/Infrastructure/Helpers/CacheKeys.cs +++ b/code/Infrastructure/Helpers/CacheKeys.cs @@ -10,7 +10,7 @@ public static class CacheKeys public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns"; public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes"; - public static string Item(string hash) => $"item_{hash}"; + public static string StrikeItem(string hash, StrikeType strikeType) => $"item_{hash}_{strikeType.ToString()}"; public static string IgnoredDownloads(string name) => $"{name}_ignored"; } \ No newline at end of file diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs index 195c08b0..6c5a9a0d 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeClient.cs @@ -27,7 +27,9 @@ public sealed class DelugeClient "label", "seeding_time", "ratio", - "trackers" + "trackers", + "download_payload_rate", + "total_size" ]; public DelugeClient(IOptions config, IHttpClientFactory httpClientFactory) @@ -78,11 +80,11 @@ public sealed class DelugeClient return torrents.FirstOrDefault(); } - public async Task GetTorrentStatus(string hash) + public async Task GetTorrentStatus(string hash) { try { - return await SendRequest( + return await SendRequest( "web.get_torrent_status", hash, Fields @@ -100,9 +102,9 @@ public sealed class DelugeClient } } - public async Task?> GetStatusForAllTorrents() + public async Task?> GetStatusForAllTorrents() { - Dictionary? downloads = await SendRequest?>( + Dictionary? downloads = await SendRequest?>( "core.get_torrents_status", "", Fields diff --git a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs index 689518eb..09158607 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs @@ -1,10 +1,12 @@ using System.Collections.Concurrent; +using System.Globalization; using System.Text.RegularExpressions; using Common.Attributes; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; +using Common.CustomDataTypes; using Domain.Enums; using Domain.Models.Deluge.Response; using Infrastructure.Extensions; @@ -50,14 +52,14 @@ public class DelugeService : DownloadService, IDelugeService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { hash = hash.ToLowerInvariant(); DelugeContents? contents = null; - StalledResult result = new(); + DownloadCheckResult result = new(); - TorrentStatus? download = await _client.GetTorrentStatus(hash); + DownloadStatus? download = await _client.GetTorrentStatus(hash); if (download?.Hash is null) { @@ -102,7 +104,7 @@ public class DelugeService : DownloadService, IDelugeService } // remove if download is stuck - (result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download); + (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download); return result; } @@ -115,7 +117,7 @@ public class DelugeService : DownloadService, IDelugeService { hash = hash.ToLowerInvariant(); - TorrentStatus? download = await _client.GetTorrentStatus(hash); + DownloadStatus? download = await _client.GetTorrentStatus(hash); BlockFilesResult result = new(); if (download?.Hash is null) @@ -124,9 +126,6 @@ public class DelugeService : DownloadService, IDelugeService return result; } - var ceva = await _client.GetTorrentExtended(hash); - - if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads)) { _logger.LogInformation("skip | download is ignored | {name}", download.Name); @@ -223,7 +222,7 @@ public class DelugeService : DownloadService, IDelugeService public override async Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, IReadOnlyList ignoredDownloads) { - foreach (TorrentStatus download in downloads) + foreach (DownloadStatus download in downloads) { if (string.IsNullOrEmpty(download.Hash)) { @@ -296,33 +295,90 @@ public class DelugeService : DownloadService, IDelugeService await _client.ChangeFilesPriority(hash, sortedPriorities); } - private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentStatus status) + private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(DownloadStatus status) + { + (bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(status); + + if (result.ShouldRemove) + { + return result; + } + + return await CheckIfStuck(status); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(DownloadStatus download) + { + if (_queueCleanerConfig.SlowMaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (download.State is null || !download.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) + { + return (false, DeleteReason.None); + } + + if (download.DownloadSpeed <= 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.SlowIgnorePrivate && download.Private) + { + // ignore private trackers + _logger.LogDebug("skip slow check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) + { + _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); + return (false, DeleteReason.None); + } + + ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize; + ByteSize currentSpeed = new ByteSize(download.DownloadSpeed); + SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime); + SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta); + + return await CheckIfSlow( + download.Hash!, + download.Name!, + minSpeed, + currentSpeed, + maxTime, + currentTime + ); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(DownloadStatus status) { if (_queueCleanerConfig.StalledMaxStrikes is 0) { - return (false, default); + return (false, DeleteReason.None); } if (_queueCleanerConfig.StalledIgnorePrivate && status.Private) { // ignore private trackers _logger.LogDebug("skip stalled check | download is private | {name}", status.Name); - return (false, default); + return (false, DeleteReason.None); } if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase)) { - return (false, default); + return (false, DeleteReason.None); } if (status.Eta > 0) { - return (false, default); + return (false, DeleteReason.None); } - ResetStrikesOnProgress(status.Hash!, status.TotalDone); - - return (await StrikeAndCheckLimit(status.Hash!, status.Name!, StrikeType.Stalled), DeleteReason.Stalled); + ResetStalledStrikesOnProgress(status.Hash!, status.TotalDone); + + return (await _striker.StrikeAndCheckLimit(status.Hash!, status.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); } private static void ProcessFiles(Dictionary? contents, Action processFile) diff --git a/code/Infrastructure/Verticals/DownloadClient/StalledResult.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadCheckResult.cs similarity index 90% rename from code/Infrastructure/Verticals/DownloadClient/StalledResult.cs rename to code/Infrastructure/Verticals/DownloadClient/DownloadCheckResult.cs index a1f35f81..a9d78662 100644 --- a/code/Infrastructure/Verticals/DownloadClient/StalledResult.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadCheckResult.cs @@ -2,7 +2,7 @@ namespace Infrastructure.Verticals.DownloadClient; -public sealed record StalledResult +public sealed record DownloadCheckResult { /// /// True if the download should be removed; otherwise false. diff --git a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs index 7b703bcd..bf217dbf 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DownloadService.cs @@ -3,6 +3,7 @@ using System.Text.RegularExpressions; using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.QueueCleaner; +using Common.CustomDataTypes; using Common.Helpers; using Domain.Enums; using Domain.Models.Cache; @@ -60,7 +61,7 @@ public abstract class DownloadService : IDownloadService public abstract Task LoginAsync(); - public abstract Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); + public abstract Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); /// public abstract Task BlockUnwantedFilesAsync(string hash, @@ -78,33 +79,104 @@ public abstract class DownloadService : IDownloadService public abstract Task CleanDownloads(List downloads, List categoriesToClean, HashSet excludedHashes, IReadOnlyList ignoredDownloads); - protected void ResetStrikesOnProgress(string hash, long downloaded) + protected void ResetStalledStrikesOnProgress(string hash, long downloaded) { if (!_queueCleanerConfig.StalledResetStrikesOnProgress) { return; } - - if (_cache.TryGetValue(CacheKeys.Item(hash), out CacheItem? cachedItem) && cachedItem is not null && downloaded > cachedItem.Downloaded) + + if (_cache.TryGetValue(CacheKeys.StrikeItem(hash, StrikeType.Stalled), out StalledCacheItem? cachedItem) && + cachedItem is not null && downloaded > cachedItem.Downloaded) { // cache item found _cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash)); - _logger.LogDebug("resetting strikes for {hash} due to progress", hash); + _logger.LogDebug("resetting stalled strikes for {hash} due to progress", hash); } - _cache.Set(CacheKeys.Item(hash), new CacheItem { Downloaded = downloaded }, _cacheOptions); + _cache.Set(CacheKeys.StrikeItem(hash, StrikeType.Stalled), new StalledCacheItem { Downloaded = downloaded }, _cacheOptions); + } + + protected void ResetSlowSpeedStrikesOnProgress(string downloadName, string hash) + { + if (!_queueCleanerConfig.SlowResetStrikesOnProgress) + { + return; + } + + string key = CacheKeys.Strike(StrikeType.SlowSpeed, hash); + + if (!_cache.TryGetValue(key, out object? value) || value is null) + { + return; + } + + _cache.Remove(key); + _logger.LogDebug("resetting slow speed strikes due to progress | {name}", downloadName); + } + + protected void ResetSlowTimeStrikesOnProgress(string downloadName, string hash) + { + if (!_queueCleanerConfig.SlowResetStrikesOnProgress) + { + return; + } + + string key = CacheKeys.Strike(StrikeType.SlowTime, hash); + + if (!_cache.TryGetValue(key, out object? value) || value is null) + { + return; + } + + _cache.Remove(key); + _logger.LogDebug("resetting slow time strikes due to progress | {name}", downloadName); } - /// - /// Strikes an item and checks if the limit has been reached. - /// - /// The torrent hash. - /// The name or title of the item. - /// - /// True if the limit has been reached; otherwise, false. - protected async Task StrikeAndCheckLimit(string hash, string itemName, StrikeType strikeType) + protected async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow( + string downloadHash, + string downloadName, + ByteSize minSpeed, + ByteSize currentSpeed, + SmartTimeSpan maxTime, + SmartTimeSpan currentTime + ) { - return await _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, strikeType); + if (minSpeed.Bytes > 0 && currentSpeed < minSpeed) + { + _logger.LogTrace("slow speed | {speed}/s | {name}", currentSpeed.ToString(), downloadName); + + bool shouldRemove = await _striker + .StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowSpeed); + + if (shouldRemove) + { + return (true, DeleteReason.SlowSpeed); + } + } + else + { + ResetSlowSpeedStrikesOnProgress(downloadName, downloadHash); + } + + if (maxTime.Time > TimeSpan.Zero && currentTime > maxTime) + { + _logger.LogTrace("slow estimated time | {time} | {name}", currentTime.ToString(), downloadName); + + bool shouldRemove = await _striker + .StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowTime); + + if (shouldRemove) + { + return (true, DeleteReason.SlowTime); + } + } + else + { + ResetSlowTimeStrikesOnProgress(downloadName, downloadHash); + } + + return (false, DeleteReason.None); } protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) diff --git a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs index 1d0e87e0..5832a378 100644 --- a/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs @@ -28,7 +28,7 @@ public class DummyDownloadService : DownloadService return Task.CompletedTask; } - public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) + public override Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { throw new NotImplementedException(); } diff --git a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs index 5ac9db7e..4f592ddc 100644 --- a/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs @@ -15,7 +15,7 @@ public interface IDownloadService : IDisposable /// /// The download hash. /// Downloads to ignore from processing. - public Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); + public Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads); /// /// Blocks unwanted files from being fully downloaded. diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index 34c27d4b..a459030c 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -5,6 +5,7 @@ using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; +using Common.CustomDataTypes; using Common.Helpers; using Domain.Enums; using Infrastructure.Extensions; @@ -63,9 +64,9 @@ public class QBitService : DownloadService, IQBitService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { - StalledResult result = new(); + DownloadCheckResult result = new(); TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] })) .FirstOrDefault(); @@ -114,8 +115,7 @@ public class QBitService : DownloadService, IQBitService return result; } - // remove if download is stuck - (result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download, result.IsPrivate); + (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, result.IsPrivate); return result; } @@ -333,35 +333,96 @@ public class QBitService : DownloadService, IQBitService _client.Dispose(); } - private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate) + private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent, bool isPrivate) + { + (bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent, isPrivate); + + if (result.ShouldRemove) + { + return result; + } + + return await CheckIfStuck(torrent, isPrivate); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download, bool isPrivate) + { + if (_queueCleanerConfig.SlowMaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (download.State is not (TorrentState.Downloading or TorrentState.ForcedDownload)) + { + return (false, DeleteReason.None); + } + + if (download.DownloadSpeed <= 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.SlowIgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip slow check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) + { + _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); + return (false, DeleteReason.None); + } + + ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize; + ByteSize currentSpeed = new ByteSize(download.DownloadSpeed); + SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime); + SmartTimeSpan currentTime = new SmartTimeSpan(download.EstimatedTime ?? TimeSpan.Zero); + + return await CheckIfSlow( + download.Hash, + download.Name, + minSpeed, + currentSpeed, + maxTime, + currentTime + ); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo torrent, bool isPrivate) { if (_queueCleanerConfig.StalledMaxStrikes is 0) { - return (false, default); - } - - if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) - { - // ignore private trackers - _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); - return (false, default); + return (false, DeleteReason.None); } if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata and not TorrentState.ForcedFetchingMetadata) { // ignore other states - return (false, default); + return (false, DeleteReason.None); } - - ResetStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); - + + if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); + return (false, DeleteReason.None); + } + if (torrent.State is TorrentState.StalledDownload) { - return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.Stalled), DeleteReason.Stalled); + _logger.LogTrace("stalled download | {name}", torrent.Name); + + ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); + + return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); } - return (await StrikeAndCheckLimit(torrent.Hash, torrent.Name, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata); + _logger.LogTrace("downloading metadata | {name}", torrent.Name); + + return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata); } private async Task> GetTrackersAsync(string hash) diff --git a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs index 05d9114b..16985bd7 100644 --- a/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/Transmission/TransmissionService.cs @@ -5,6 +5,7 @@ using Common.Configuration.ContentBlocker; using Common.Configuration.DownloadCleaner; using Common.Configuration.DownloadClient; using Common.Configuration.QueueCleaner; +using Common.CustomDataTypes; using Common.Helpers; using Domain.Enums; using Infrastructure.Extensions; @@ -41,7 +42,9 @@ public class TransmissionService : DownloadService, ITransmissionService TorrentFields.DOWNLOAD_DIR, TorrentFields.SECONDS_SEEDING, TorrentFields.UPLOAD_RATIO, - TorrentFields.TRACKERS + TorrentFields.TRACKERS, + TorrentFields.RATE_DOWNLOAD, + TorrentFields.TOTAL_SIZE ]; public TransmissionService( @@ -81,9 +84,9 @@ public class TransmissionService : DownloadService, ITransmissionService } /// - public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) + public override async Task ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList ignoredDownloads) { - StalledResult result = new(); + DownloadCheckResult result = new(); TorrentInfo? download = await GetTorrentAsync(hash); if (download is null) @@ -125,7 +128,7 @@ public class TransmissionService : DownloadService, ITransmissionService } // remove if download is stuck - (result.ShouldRemove, result.DeleteReason) = await IsItemStuckAndShouldRemove(download); + (result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download); return result; } @@ -335,34 +338,92 @@ public class TransmissionService : DownloadService, ITransmissionService }); } - private async Task<(bool, DeleteReason)> IsItemStuckAndShouldRemove(TorrentInfo torrent) + private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent) + { + (bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent); + + if (result.ShouldRemove) + { + return result; + } + + return await CheckIfStuck(torrent); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download) + { + if (_queueCleanerConfig.SlowMaxStrikes is 0) + { + return (false, DeleteReason.None); + } + + if (download.Status is not 4) + { + // not in downloading state + return (false, DeleteReason.None); + } + + if (download.RateDownload <= 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.SlowIgnorePrivate && download.IsPrivate is true) + { + // ignore private trackers + _logger.LogDebug("skip slow check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + if (download.TotalSize > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue)) + { + _logger.LogDebug("skip slow check | download is too large | {name}", download.Name); + return (false, DeleteReason.None); + } + + ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize; + ByteSize currentSpeed = new ByteSize(download.RateDownload ?? long.MaxValue); + SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime); + SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta ?? 0); + + return await CheckIfSlow( + download.HashString!, + download.Name!, + minSpeed, + currentSpeed, + maxTime, + currentTime + ); + } + + private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo download) { if (_queueCleanerConfig.StalledMaxStrikes is 0) { - return (false, default); + return (false, DeleteReason.None); } - if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false)) - { - // ignore private trackers - _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); - return (false, default); - } - - if (torrent.Status is not 4) + if (download.Status is not 4) { // not in downloading state - return (false, default); - } - - if (torrent.Eta > 0) - { - return (false, default); + return (false, DeleteReason.None); } - ResetStrikesOnProgress(torrent.HashString!, torrent.DownloadedEver ?? 0); - - return (await StrikeAndCheckLimit(torrent.HashString!, torrent.Name!, StrikeType.Stalled), DeleteReason.Stalled); + if (download.RateDownload > 0 || download.Eta > 0) + { + return (false, DeleteReason.None); + } + + if (_queueCleanerConfig.StalledIgnorePrivate && (download.IsPrivate ?? false)) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", download.Name); + return (false, DeleteReason.None); + } + + ResetStalledStrikesOnProgress(download.HashString!, download.DownloadedEver ?? 0); + + return (await _striker.StrikeAndCheckLimit(download.HashString!, download.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); } private async Task GetTorrentAsync(string hash) => diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index 32aadcad..c4ab67c5 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -77,6 +77,8 @@ public sealed class QueueCleaner : GenericHandler QueueRecord record = group.First(); + _logger.LogTrace("processing | {title} | {id}", record.Title, record.DownloadId); + if (!arrClient.IsRecordValid(record)) { continue; @@ -91,7 +93,7 @@ public sealed class QueueCleaner : GenericHandler // push record to context ContextProvider.Set(nameof(QueueRecord), record); - StalledResult stalledCheckResult = new(); + DownloadCheckResult downloadCheckResult = new(); if (record.Protocol is "torrent" && _downloadClientConfig.DownloadClient is not Common.Enums.DownloadClient.Disabled) { @@ -102,14 +104,14 @@ public sealed class QueueCleaner : GenericHandler } // stalled download check - stalledCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads); + downloadCheckResult = await _downloadService.ShouldRemoveFromArrQueueAsync(record.DownloadId, ignoredDownloads); } // failed import check - bool shouldRemoveFromArr = await arrClient.ShouldRemoveFromQueue(instanceType, record, stalledCheckResult.IsPrivate); - DeleteReason deleteReason = stalledCheckResult.ShouldRemove ? stalledCheckResult.DeleteReason : DeleteReason.ImportFailed; + bool shouldRemoveFromArr = await arrClient.ShouldRemoveFromQueue(instanceType, record, downloadCheckResult.IsPrivate); + DeleteReason deleteReason = downloadCheckResult.ShouldRemove ? downloadCheckResult.DeleteReason : DeleteReason.ImportFailed; - if (!shouldRemoveFromArr && !stalledCheckResult.ShouldRemove) + if (!shouldRemoveFromArr && !downloadCheckResult.ShouldRemove) { _logger.LogInformation("skip | {title}", record.Title); continue; @@ -119,14 +121,20 @@ public sealed class QueueCleaner : GenericHandler bool removeFromClient = true; - if (stalledCheckResult.IsPrivate) + if (downloadCheckResult.IsPrivate) { - if (stalledCheckResult.ShouldRemove && !_config.StalledDeletePrivate) - { - removeFromClient = false; - } + bool isStalledWithoutPruneFlag = + downloadCheckResult.DeleteReason is DeleteReason.Stalled && + !_config.StalledDeletePrivate; + + bool isSlowWithoutPruneFlag = + downloadCheckResult.DeleteReason is DeleteReason.SlowSpeed or DeleteReason.SlowTime && + !_config.SlowDeletePrivate; + + bool shouldKeepDueToDeleteRules = downloadCheckResult.ShouldRemove && (isStalledWithoutPruneFlag || isSlowWithoutPruneFlag); + bool shouldKeepDueToImportRules = shouldRemoveFromArr && !_config.ImportFailedDeletePrivate; - if (shouldRemoveFromArr && !_config.ImportFailedDeletePrivate) + if (shouldKeepDueToDeleteRules || shouldKeepDueToImportRules) { removeFromClient = false; } diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index fb8c186b..d8a2be73 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -178,7 +178,7 @@ services: - TZ=Europe/Bucharest - DRY_RUN=false - - LOGGING__LOGLEVEL=Debug + - LOGGING__LOGLEVEL=Verbose - LOGGING__FILE__ENABLED=true - LOGGING__FILE__PATH=/var/logs - LOGGING__ENHANCED=true @@ -193,14 +193,25 @@ services: - QUEUECLEANER__ENABLED=true - QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored - QUEUECLEANER__RUNSEQUENTIALLY=true + - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true - QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=false - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=file is a sample + - QUEUECLEANER__STALLED_MAX_STRIKES=5 + - QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS=true - QUEUECLEANER__STALLED_IGNORE_PRIVATE=true - QUEUECLEANER__STALLED_DELETE_PRIVATE=false + - QUEUECLEANER__SLOW_MAX_STRIKES=5 + - QUEUECLEANER__SLOW_RESET_STRIKES_ON_PROGRESS=true + - QUEUECLEANER__SLOW_IGNORE_PRIVATE=false + - QUEUECLEANER__SLOW_DELETE_PRIVATE=false + - QUEUECLEANER__SLOW_MIN_SPEED=1MB + - QUEUECLEANER__SLOW_MAX_TIME=20 + - QUEUECLEANER__SLOW_IGNORE_ABOVE_SIZE=1KB + - CONTENTBLOCKER__ENABLED=true - CONTENTBLOCKER__IGNORED_DOWNLOADS_PATH=/ignored - CONTENTBLOCKER__IGNORE_PRIVATE=true diff --git a/variables.md b/variables.md index e5567056..7b1750c3 100644 --- a/variables.md +++ b/variables.md @@ -160,7 +160,7 @@ > If not set to `0`, the minimum value is `3`. #### **`QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS`** -- Controls whether to remove strikes if any download progress was made since last checked. +- Controls whether to remove the given strikes if any download progress was made since last checked. - Type: Boolean - Possible values: `true`, `false` - Default: `false` @@ -174,7 +174,7 @@ - Required: No. #### **`QUEUECLEANER__STALLED_DELETE_PRIVATE`** -- Controls whether to delete stalled private downloads from the download client. +- Controls whether stalled downloads from private trackers should be removed from the download client. - Has no effect if `QUEUECLEANER__STALLED_IGNORE_PRIVATE` is `true`. - Type: Boolean - Possible values: `true`, `false` @@ -184,6 +184,65 @@ > [!WARNING] > Setting `QUEUECLEANER__STALLED_DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account. +#### **`QUEUECLEANER__SLOW_MAX_STRIKES`** +- Number of strikes before removing a slow download. +- Set to `0` to never remove slow downloads. +- A strike is given when an item is slow. +- Type: Integer +- Default: `0` +- Required: No. +> [!NOTE] +> If not set to `0`, the minimum value is `3`. + +#### **`QUEUECLEANER__SLOW_RESET_STRIKES_ON_PROGRESS`** +- Controls whether to remove the given strikes if the download speed or estimated time are not slow anymore. +- Type: Boolean +- Possible values: `true`, `false` +- Default: `false` +- Required: No. + +#### **`QUEUECLEANER__SLOW_IGNORE_PRIVATE`** +- Controls whether to ignore slow downloads from private trackers. +- Type: Boolean +- Possible values: `true`, `false` +- Default: `false` +- Required: No. + +#### **`QUEUECLEANER__SLOW_DELETE_PRIVATE`** +- Controls whether slow downloads from private trackers should be removed from the download client. +- Has no effect if `QUEUECLEANER__SLOW_IGNORE_PRIVATE` is `true`. +- Type: Boolean +- Possible values: `true`, `false` +- Default: `false` +- Required: No. + +> [!WARNING] +> Setting `QUEUECLEANER__SLOW_DELETE_PRIVATE=true` means you don't care about seeding, ratio, H&R and potentially losing your private tracker account. + +#### **`QUEUECLEANER__SLOW_MIN_SPEED`** +- The minimum speed a download should have. +- Downloads receive strikes if their speed falls bellow this value. +- If not specified, downloads will not receive strikes for slow download speed. +- Type: String. +- Default: Empty. +- Required: No. +- Value examples: `1.5KB`, `400KB`, `2MB` + +#### **`QUEUECLEANER__SLOW_MAX_TIME`** +- The maximum estimated hours a download should take to finish. +- Downloads receive strikes if their estimated finish time is above this value. +- If not specified (or `0`), downloads will not receive strikes for slow estimated finish time. +- Type: Integer. +- Default: `0`. +- Required: No. + +#### **`QUEUECLEANER__SLOW_IGNORE_ABOVE_SIZE`** +- Downloads above this size will not be removed for being slow. +- Type: String. +- Default: Empty. +- Required: No. +- Value examples: `10KB`, `200MB`, `3GB`. + # ### Content Blocker settings