From 8cfc73213a749dc1c310d5ac28b7b83dec841749 Mon Sep 17 00:00:00 2001 From: Flaminel Date: Sun, 4 May 2025 17:11:38 +0300 Subject: [PATCH] Add separate strikes for downloading metadata (#104) --- .../QueueCleaner/QueueCleanerConfig.cs | 26 +++++++++----- code/Domain/Enums/DeleteReason.cs | 2 +- code/Executable/appsettings.Development.json | 5 +-- code/Executable/appsettings.json | 3 +- .../DownloadClient/QBittorrent/QBitService.cs | 36 ++++++++++--------- .../Verticals/QueueCleaner/QueueCleaner.cs | 4 +-- code/test/docker-compose.yml | 5 +-- .../QueueCleanerSlowSettings.tsx | 2 +- .../QueueCleanerStalledSettings.tsx | 14 +++++++- 9 files changed, 61 insertions(+), 36 deletions(-) diff --git a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs index 9d4f6c9e..b8f746b8 100644 --- a/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs +++ b/code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs @@ -25,7 +25,7 @@ public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig public bool ImportFailedDeletePrivate { get; init; } [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")] - public List? ImportFailedIgnorePatterns { get; init; } + public IReadOnlyList? ImportFailedIgnorePatterns { get; init; } [ConfigurationKeyName("STALLED_MAX_STRIKES")] public ushort StalledMaxStrikes { get; init; } @@ -39,6 +39,9 @@ public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig [ConfigurationKeyName("STALLED_DELETE_PRIVATE")] public bool StalledDeletePrivate { get; init; } + [ConfigurationKeyName("DOWNLOADING_METADATA_MAX_STRIKES")] + public ushort DownloadingMetadataMaxStrikes { get; init; } + [ConfigurationKeyName("SLOW_MAX_STRIKES")] public ushort SlowMaxStrikes { get; init; } @@ -63,22 +66,27 @@ public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig public string SlowIgnoreAboveSize { get; init; } = string.Empty; public ByteSize? SlowIgnoreAboveSizeByteSize => string.IsNullOrEmpty(SlowIgnoreAboveSize) ? null : ByteSize.Parse(SlowIgnoreAboveSize); - + public void Validate() { if (ImportFailedMaxStrikes is > 0 and < 3) { - throw new ValidationException($"the minimum value for {SectionName}__IMPORT_FAILED_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__IMPORT_FAILED_MAX_STRIKES must be 3"); } if (StalledMaxStrikes is > 0 and < 3) { - throw new ValidationException($"the minimum value for {SectionName}__STALLED_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__STALLED_MAX_STRIKES must be 3"); + } + + if (DownloadingMetadataMaxStrikes is > 0 and < 3) + { + throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__DOWNLOADING_METADATA_MAX_STRIKES must be 3"); } if (SlowMaxStrikes is > 0 and < 3) { - throw new ValidationException($"the minimum value for {SectionName}__SLOW_MAX_STRIKES must be 3"); + throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__SLOW_MAX_STRIKES must be 3"); } if (SlowMaxStrikes > 0) @@ -87,24 +95,24 @@ public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig if (isSlowSpeedSet && ByteSize.TryParse(SlowMinSpeed, out _) is false) { - throw new ValidationException($"invalid value for {SectionName}__SLOW_MIN_SPEED"); + throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_MIN_SPEED"); } if (SlowMaxTime < 0) { - throw new ValidationException($"invalid value for {SectionName}__SLOW_MAX_TIME"); + throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_MAX_TIME"); } if (!isSlowSpeedSet && SlowMaxTime is 0) { - throw new ValidationException($"either {SectionName}__SLOW_MIN_SPEED or {SectionName}__SLOW_MAX_STRIKES must be set"); + throw new ValidationException($"either {SectionName.ToUpperInvariant()}__SLOW_MIN_SPEED or {SectionName.ToUpperInvariant()}__SLOW_MAX_STRIKES must be set"); } bool isSlowIgnoreAboveSizeSet = !string.IsNullOrEmpty(SlowIgnoreAboveSize); if (isSlowIgnoreAboveSizeSet && ByteSize.TryParse(SlowIgnoreAboveSize, out _) is false) { - throw new ValidationException($"invalid value for {SectionName}__SLOW_IGNORE_ABOVE_SIZE"); + throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_IGNORE_ABOVE_SIZE"); } } } diff --git a/code/Domain/Enums/DeleteReason.cs b/code/Domain/Enums/DeleteReason.cs index c80585eb..ba6ea660 100644 --- a/code/Domain/Enums/DeleteReason.cs +++ b/code/Domain/Enums/DeleteReason.cs @@ -1,4 +1,4 @@ -namespace Domain.Enums; +namespace Domain.Enums; public enum DeleteReason { diff --git a/code/Executable/appsettings.Development.json b/code/Executable/appsettings.Development.json index 872be2c9..dbbf50eb 100644 --- a/code/Executable/appsettings.Development.json +++ b/code/Executable/appsettings.Development.json @@ -25,16 +25,17 @@ "Enabled": true, "RunSequentially": true, "IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads", - "IMPORT_FAILED_MAX_STRIKES": 5, + "IMPORT_FAILED_MAX_STRIKES": 3, "IMPORT_FAILED_IGNORE_PRIVATE": true, "IMPORT_FAILED_DELETE_PRIVATE": false, "IMPORT_FAILED_IGNORE_PATTERNS": [ "file is a sample" ], - "STALLED_MAX_STRIKES": 5, + "STALLED_MAX_STRIKES": 3, "STALLED_RESET_STRIKES_ON_PROGRESS": true, "STALLED_IGNORE_PRIVATE": true, "STALLED_DELETE_PRIVATE": false, + "DOWNLOADING_METADATA_MAX_STRIKES": 3, "SLOW_MAX_STRIKES": 5, "SLOW_RESET_STRIKES_ON_PROGRESS": true, "SLOW_IGNORE_PRIVATE": false, diff --git a/code/Executable/appsettings.json b/code/Executable/appsettings.json index 3158a02c..00e872a5 100644 --- a/code/Executable/appsettings.json +++ b/code/Executable/appsettings.json @@ -31,7 +31,8 @@ "STALLED_MAX_STRIKES": 0, "STALLED_RESET_STRIKES_ON_PROGRESS": false, "STALLED_IGNORE_PRIVATE": false, - "STALLED_DELETE_PRIVATE": false + "STALLED_DELETE_PRIVATE": false, + "DOWNLOADING_METADATA_MAX_STRIKES": 0 }, "DownloadCleaner": { "Enabled": false, diff --git a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs index cfe87096..141d9f78 100644 --- a/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs +++ b/code/Infrastructure/Verticals/DownloadClient/QBittorrent/QBitService.cs @@ -392,7 +392,7 @@ public class QBitService : DownloadService, IQBitService private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo torrent, bool isPrivate) { - if (_queueCleanerConfig.StalledMaxStrikes is 0) + if (_queueCleanerConfig.StalledMaxStrikes is 0 && _queueCleanerConfig.DownloadingMetadataMaxStrikes is 0) { return (false, DeleteReason.None); } @@ -403,26 +403,28 @@ public class QBitService : DownloadService, IQBitService // ignore other states return (false, DeleteReason.None); } - - if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) + + if (_queueCleanerConfig.StalledMaxStrikes > 0 && torrent.State is TorrentState.StalledDownload) { - // ignore private trackers - _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); - return (false, DeleteReason.None); - } - - if (torrent.State is TorrentState.StalledDownload) - { - _logger.LogTrace("stalled download | {name}", torrent.Name); + if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate) + { + // ignore private trackers + _logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name); + } + else + { + ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); - ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0); - - return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); + return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled); + } } - _logger.LogTrace("downloading metadata | {name}", torrent.Name); - - return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata); + if (_queueCleanerConfig.DownloadingMetadataMaxStrikes > 0 && torrent.State is not TorrentState.StalledDownload) + { + return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.DownloadingMetadataMaxStrikes, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata); + } + + return (false, DeleteReason.None); } private async Task> GetTrackersAsync(string hash) diff --git a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs index c4ab67c5..d7752831 100644 --- a/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs +++ b/code/Infrastructure/Verticals/QueueCleaner/QueueCleaner.cs @@ -124,11 +124,11 @@ public sealed class QueueCleaner : GenericHandler if (downloadCheckResult.IsPrivate) { bool isStalledWithoutPruneFlag = - downloadCheckResult.DeleteReason is DeleteReason.Stalled && + downloadCheckResult.DeleteReason is DeleteReason.Stalled && !_config.StalledDeletePrivate; bool isSlowWithoutPruneFlag = - downloadCheckResult.DeleteReason is DeleteReason.SlowSpeed or DeleteReason.SlowTime && + downloadCheckResult.DeleteReason is DeleteReason.SlowSpeed or DeleteReason.SlowTime && !_config.SlowDeletePrivate; bool shouldKeepDueToDeleteRules = downloadCheckResult.ShouldRemove && (isStalledWithoutPruneFlag || isSlowWithoutPruneFlag); diff --git a/code/test/docker-compose.yml b/code/test/docker-compose.yml index 0f031a9f..86059c97 100644 --- a/code/test/docker-compose.yml +++ b/code/test/docker-compose.yml @@ -194,15 +194,16 @@ services: - QUEUECLEANER__IGNORED_DOWNLOADS_PATH=/ignored - QUEUECLEANER__RUNSEQUENTIALLY=true - - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5 + - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=3 - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=true - QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE=false - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=file is a sample - - QUEUECLEANER__STALLED_MAX_STRIKES=5 + - QUEUECLEANER__STALLED_MAX_STRIKES=3 - QUEUECLEANER__STALLED_RESET_STRIKES_ON_PROGRESS=true - QUEUECLEANER__STALLED_IGNORE_PRIVATE=true - QUEUECLEANER__STALLED_DELETE_PRIVATE=false + - QUEUECLEANER__DOWNLOADING_METADATA_MAX_STRIKES=3 - QUEUECLEANER__SLOW_MAX_STRIKES=5 - QUEUECLEANER__SLOW_RESET_STRIKES_ON_PROGRESS=true diff --git a/docs/src/components/configuration/queue-cleaner/QueueCleanerSlowSettings.tsx b/docs/src/components/configuration/queue-cleaner/QueueCleanerSlowSettings.tsx index 2ed760fd..8a1d7a79 100644 --- a/docs/src/components/configuration/queue-cleaner/QueueCleanerSlowSettings.tsx +++ b/docs/src/components/configuration/queue-cleaner/QueueCleanerSlowSettings.tsx @@ -13,7 +13,7 @@ const settings: EnvVarProps[] = [ required: false, examples: ["0", "3", "10"], notes: [ - "If not set to 0, the minimum value is 3." + "If not set to 0, the minimum value is `3`." ] }, { diff --git a/docs/src/components/configuration/queue-cleaner/QueueCleanerStalledSettings.tsx b/docs/src/components/configuration/queue-cleaner/QueueCleanerStalledSettings.tsx index b68fab1c..cbf84eab 100644 --- a/docs/src/components/configuration/queue-cleaner/QueueCleanerStalledSettings.tsx +++ b/docs/src/components/configuration/queue-cleaner/QueueCleanerStalledSettings.tsx @@ -13,7 +13,7 @@ const settings: EnvVarProps[] = [ required: false, examples: ["0", "3", "10"], notes: [ - "If not set to 0, the minimum value is 3." + "If not set to 0, the minimum value is `3`." ] }, { @@ -49,6 +49,18 @@ const settings: EnvVarProps[] = [ important: [ "Setting this to true means you don't care about seeding, ratio, H&R and potentially losing your private tracker account." ] + }, + { + name: "QUEUECLEANER__DOWNLOADING_METADATA_MAX_STRIKES", + description: [ + "Number of strikes before removing a download stuck while downloading metadata.", + ], + type: "positive integer number", + defaultValue: "0", + required: false, + notes: [ + "If not set to `0`, the minimum value is `3`." + ] } ];