Compare commits

...

3 Commits

Author SHA1 Message Date
Flaminel
2971445090 Add handling type of malware when containing thepirateheaven.org file (#232) 2025-07-07 14:29:39 +03:00
Flaminel
55c23419cd Improve download removal to be separate from download search (#233) 2025-07-07 14:28:34 +03:00
Flaminel
c4b9d9503a Add more logs for debug (#201) 2025-07-07 14:28:15 +03:00
22 changed files with 257 additions and 32 deletions

View File

@@ -1,5 +1,6 @@
using System.Text.Json.Serialization;
using Cleanuparr.Domain.Entities.Arr;
using Cleanuparr.Infrastructure.Features.DownloadHunter.Consumers;
using Cleanuparr.Infrastructure.Features.DownloadRemover.Consumers;
using Cleanuparr.Infrastructure.Features.Notifications.Consumers;
using Cleanuparr.Infrastructure.Features.Notifications.Models;
@@ -28,6 +29,8 @@ public static class MainDI
{
config.AddConsumer<DownloadRemoverConsumer<SearchItem>>();
config.AddConsumer<DownloadRemoverConsumer<SeriesSearchItem>>();
config.AddConsumer<DownloadHunterConsumer<SearchItem>>();
config.AddConsumer<DownloadHunterConsumer<SeriesSearchItem>>();
config.AddConsumer<NotificationConsumer<FailedImportStrikeNotification>>();
config.AddConsumer<NotificationConsumer<StalledStrikeNotification>>();
@@ -50,6 +53,14 @@ public static class MainDI
{
e.ConfigureConsumer<DownloadRemoverConsumer<SearchItem>>(context);
e.ConfigureConsumer<DownloadRemoverConsumer<SeriesSearchItem>>(context);
e.ConcurrentMessageLimit = 2;
e.PrefetchCount = 2;
});
cfg.ReceiveEndpoint("download-hunter-queue", e =>
{
e.ConfigureConsumer<DownloadHunterConsumer<SearchItem>>(context);
e.ConfigureConsumer<DownloadHunterConsumer<SeriesSearchItem>>(context);
e.ConcurrentMessageLimit = 1;
e.PrefetchCount = 1;
});

View File

@@ -5,6 +5,8 @@ using Cleanuparr.Infrastructure.Events;
using Cleanuparr.Infrastructure.Features.Arr;
using Cleanuparr.Infrastructure.Features.ContentBlocker;
using Cleanuparr.Infrastructure.Features.DownloadClient;
using Cleanuparr.Infrastructure.Features.DownloadHunter;
using Cleanuparr.Infrastructure.Features.DownloadHunter.Interfaces;
using Cleanuparr.Infrastructure.Features.DownloadRemover;
using Cleanuparr.Infrastructure.Features.DownloadRemover.Interfaces;
using Cleanuparr.Infrastructure.Features.Files;
@@ -44,6 +46,7 @@ public static class ServicesDI
.AddTransient<ContentBlocker>()
.AddTransient<DownloadCleaner>()
.AddTransient<IQueueItemRemover, QueueItemRemover>()
.AddTransient<IDownloadHunter, DownloadHunter>()
.AddTransient<IFilenameEvaluator, FilenameEvaluator>()
.AddTransient<IHardLinkFileService, HardLinkFileService>()
.AddTransient<UnixHardLinkFileService>()

View File

@@ -52,7 +52,7 @@ public sealed class ContentBlocker : GenericHandler
var config = ContextProvider.Get<ContentBlockerConfig>();
if (!config.Sonarr.Enabled && !config.Radarr.Enabled && !config.Lidarr.Enabled)
if (!config.Sonarr.Enabled && !config.Radarr.Enabled && !config.Lidarr.Enabled && !config.Readarr.Enabled && !config.Whisparr.Enabled)
{
_logger.LogWarning("No blocklists are enabled");
return;
@@ -183,6 +183,10 @@ public sealed class ContentBlocker : GenericHandler
_logger.LogWarning("Download not found in any torrent client | {title}", record.Title);
}
}
else
{
_logger.LogDebug("No torrent clients enabled");
}
}
if (!result.ShouldRemove)
@@ -206,7 +210,7 @@ public sealed class ContentBlocker : GenericHandler
record,
group.Count() > 1,
removeFromClient,
DeleteReason.AllFilesBlocked
result.DeleteReason
);
}
});

View File

@@ -141,6 +141,10 @@ public sealed class QueueCleaner : GenericHandler
_logger.LogWarning("Download not found in any torrent client | {title}", record.Title);
}
}
else
{
_logger.LogDebug("No torrent clients enabled");
}
}
var config = ContextProvider.Get<QueueCleanerConfig>();

View File

@@ -11,4 +11,5 @@ public enum DeleteReason
AllFilesSkipped,
AllFilesSkippedByQBit,
AllFilesBlocked,
MalwareFileFound,
}

View File

@@ -6,10 +6,6 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="Features\" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="FLM.QBittorrent" Version="1.0.1" />
<PackageReference Include="FLM.Transmission" Version="1.0.3" />

View File

@@ -106,6 +106,17 @@ public sealed class BlocklistProvider
changedCount++;
}
// Check and update Whisparr blocklist if needed
string whisparrHash = GenerateSettingsHash(contentBlockerConfig.Whisparr);
if (shouldReload || !_configHashes.TryGetValue(InstanceType.Whisparr, out string? oldWhisparrHash) || whisparrHash != oldWhisparrHash)
{
_logger.LogDebug("Loading Whisparr blocklist");
await LoadPatternsAndRegexesAsync(contentBlockerConfig.Whisparr, InstanceType.Whisparr);
_configHashes[InstanceType.Whisparr] = whisparrHash;
changedCount++;
}
if (changedCount > 0)
{
_logger.LogInformation("Successfully loaded {count} blocklists", changedCount);

View File

@@ -1,4 +1,6 @@
namespace Cleanuparr.Infrastructure.Features.DownloadClient;
using Cleanuparr.Domain.Enums;
namespace Cleanuparr.Infrastructure.Features.DownloadClient;
public sealed record BlockFilesResult
{
@@ -13,4 +15,6 @@ public sealed record BlockFilesResult
public bool IsPrivate { get; set; }
public bool Found { get; set; }
public DeleteReason DeleteReason { get; set; } = DeleteReason.None;
}

View File

@@ -75,8 +75,21 @@ public partial class DelugeService
totalFiles++;
int priority = file.Priority;
if (result.ShouldRemove)
{
return;
}
if (IsDefinitelyMalware(name))
{
_logger.LogInformation("malware file found | {file} | {title}", file.Path, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
}
if (file.Priority is 0)
{
_logger.LogTrace("File is already skipped | {file}", file.Path);
totalUnwantedFiles++;
}
@@ -88,9 +101,15 @@ public partial class DelugeService
_logger.LogInformation("unwanted file found | {file}", file.Path);
}
_logger.LogTrace("File is valid | {file}", file.Path);
priorities.Add(file.Index, priority);
});
if (result.ShouldRemove)
{
return result;
}
if (!hasPriorityUpdates)
{
return result;
@@ -105,8 +124,12 @@ public partial class DelugeService
if (totalUnwantedFiles == totalFiles)
{
_logger.LogDebug("All files are blocked for {name}", download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
}
_logger.LogDebug("Marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name);
await _dryRunInterceptor.InterceptAsync(ChangeFilesPriority, hash, sortedPriorities);

View File

@@ -1,6 +1,4 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Cleanuparr.Domain.Entities;
using Cleanuparr.Domain.Entities;
using Cleanuparr.Domain.Entities.Deluge.Response;
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Extensions;
@@ -61,6 +59,7 @@ public partial class DelugeService
if (shouldRemove)
{
// remove if all files are unwanted
_logger.LogTrace("all files are unwanted | removing download | {name}", download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
@@ -95,11 +94,13 @@ public partial class DelugeService
if (download.State is null || !download.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
_logger.LogTrace("skip slow check | item is in {state} state | {name}", download.State, download.Name);
return (false, DeleteReason.None);
}
if (download.DownloadSpeed <= 0)
{
_logger.LogTrace("skip slow check | download speed is 0 | {name}", download.Name);
return (false, DeleteReason.None);
}
@@ -137,6 +138,7 @@ public partial class DelugeService
if (queueCleanerConfig.Stalled.MaxStrikes is 0)
{
_logger.LogTrace("skip stalled check | max strikes is 0 | {name}", status.Name);
return (false, DeleteReason.None);
}
@@ -149,11 +151,13 @@ public partial class DelugeService
if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
{
_logger.LogTrace("skip stalled check | download is in {state} state | {name}", status.State, status.Name);
return (false, DeleteReason.None);
}
if (status.Eta > 0)
{
_logger.LogTrace("skip stalled check | download is not stalled | {name}", status.Name);
return (false, DeleteReason.None);
}

View File

@@ -100,6 +100,16 @@ public abstract class DownloadService : IDownloadService
/// <inheritdoc/>
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, IReadOnlyList<string> ignoredDownloads);
protected bool IsDefinitelyMalware(string filename)
{
if (filename.Contains("thepirateheaven.org", StringComparison.InvariantCultureIgnoreCase))
{
return true;
}
return false;
}
protected void ResetStalledStrikesOnProgress(string hash, long downloaded)
{

View File

@@ -62,6 +62,7 @@ public partial class QBitService
if (files is null)
{
_logger.LogDebug("torrent {hash} has no files", hash);
return result;
}
@@ -78,19 +79,30 @@ public partial class QBitService
{
if (!file.Index.HasValue)
{
_logger.LogTrace("Skipping file with no index | {file}", file.Name);
continue;
}
totalFiles++;
if (IsDefinitelyMalware(file.Name))
{
_logger.LogInformation("malware file found | {file} | {title}", file.Name, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
return result;
}
if (file.Priority is TorrentContentPriority.Skip)
{
_logger.LogTrace("File is already skipped | {file}", file.Name);
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(file.Name, blocklistType, patterns, regexes))
{
_logger.LogTrace("File is valid | {file}", file.Name);
continue;
}
@@ -101,13 +113,18 @@ public partial class QBitService
if (unwantedFiles.Count is 0)
{
_logger.LogDebug("No unwanted files found for {name}", download.Name);
return result;
}
if (totalUnwantedFiles == totalFiles)
{
_logger.LogDebug("All files are blocked for {name}", download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
}
_logger.LogDebug("Marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name);
foreach (int fileIndex in unwantedFiles)
{

View File

@@ -55,11 +55,13 @@ public partial class QBitService
// if all files were blocked by qBittorrent
if (download is { CompletionOn: not null, Downloaded: null or 0 })
{
_logger.LogDebug("all files are unwanted by qBit | removing download | {name}", download.Name);
result.DeleteReason = DeleteReason.AllFilesSkippedByQBit;
return result;
}
// remove if all files are unwanted
_logger.LogDebug("all files are unwanted | removing download | {name}", download.Name);
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
}
@@ -87,16 +89,19 @@ public partial class QBitService
if (queueCleanerConfig.Slow.MaxStrikes is 0)
{
_logger.LogDebug("skip slow check | max strikes is 0 | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.State is not (TorrentState.Downloading or TorrentState.ForcedDownload))
{
_logger.LogDebug("skip slow check | download is in {state} state | {name}", download.State, download.Name);
return (false, DeleteReason.None);
}
if (download.DownloadSpeed <= 0)
{
_logger.LogDebug("skip slow check | download speed is 0 | {name}", download.Name);
return (false, DeleteReason.None);
}
@@ -134,6 +139,7 @@ public partial class QBitService
if (queueCleanerConfig.Stalled.MaxStrikes is 0 && queueCleanerConfig.Stalled.DownloadingMetadataMaxStrikes is 0)
{
_logger.LogDebug("skip stalled check | max strikes is 0 | {name}", torrent.Name);
return (false, DeleteReason.None);
}
@@ -141,6 +147,7 @@ public partial class QBitService
and not TorrentState.ForcedFetchingMetadata)
{
// ignore other states
_logger.LogDebug("skip stalled check | download is in {state} state | {name}", torrent.State, torrent.Name);
return (false, DeleteReason.None);
}
@@ -168,6 +175,7 @@ public partial class QBitService
StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata);
}
_logger.LogDebug("skip stalled check | download is not stalled | {name}", torrent.Name);
return (false, DeleteReason.None);
}
}

View File

@@ -62,19 +62,30 @@ public partial class TransmissionService
{
if (download.FileStats?[i].Wanted == null)
{
_logger.LogTrace("Skipping file with no stats | {file}", download.Files[i].Name);
continue;
}
totalFiles++;
if (IsDefinitelyMalware(download.Files[i].Name))
{
_logger.LogInformation("malware file found | {file} | {title}", download.Files[i].Name, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
return result;
}
if (!download.FileStats[i].Wanted.Value)
{
_logger.LogTrace("File is already skipped | {file}", download.Files[i].Name);
totalUnwantedFiles++;
continue;
}
if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes))
{
_logger.LogTrace("File is valid | {file}", download.Files[i].Name);
continue;
}
@@ -85,15 +96,18 @@ public partial class TransmissionService
if (unwantedFiles.Count is 0)
{
_logger.LogDebug("No unwanted files found for {name}", download.Name);
return result;
}
if (totalUnwantedFiles == totalFiles)
{
_logger.LogDebug("All files are blocked for {name}", download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesBlocked;
}
_logger.LogDebug("marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name);
_logger.LogDebug("Marking {count} unwanted files as skipped for {name}", totalUnwantedFiles, download.Name);
await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray());

View File

@@ -1,6 +1,4 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Cleanuparr.Domain.Entities;
using Cleanuparr.Domain.Entities;
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Extensions;
using Cleanuparr.Infrastructure.Features.Context;
@@ -56,6 +54,7 @@ public partial class TransmissionService
if (shouldRemove)
{
// remove if all files are unwanted
_logger.LogDebug("all files are unwanted | removing download | {name}", download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.AllFilesSkipped;
return result;
@@ -100,11 +99,13 @@ public partial class TransmissionService
if (download.Status is not 4)
{
// not in downloading state
_logger.LogTrace("skip slow check | download is in {state} state | {name}", download.Status, download.Name);
return (false, DeleteReason.None);
}
if (download.RateDownload <= 0)
{
_logger.LogTrace("skip slow check | download speed is 0 | {name}", download.Name);
return (false, DeleteReason.None);
}
@@ -142,17 +143,20 @@ public partial class TransmissionService
if (queueCleanerConfig.Stalled.MaxStrikes is 0)
{
_logger.LogTrace("skip stalled check | max strikes is 0 | {name}", download.Name);
return (false, DeleteReason.None);
}
if (download.Status is not 4)
{
// not in downloading state
_logger.LogTrace("skip stalled check | download is in {state} state | {name}", download.Status, download.Name);
return (false, DeleteReason.None);
}
if (download.RateDownload > 0 || download.Eta > 0)
{
_logger.LogTrace("skip stalled check | download is not stalled | {name}", download.Name);
return (false, DeleteReason.None);
}

View File

@@ -0,0 +1,36 @@
using Cleanuparr.Infrastructure.Features.DownloadHunter.Interfaces;
using Cleanuparr.Infrastructure.Features.DownloadHunter.Models;
using Data.Models.Arr;
using MassTransit;
using Microsoft.Extensions.Logging;
namespace Cleanuparr.Infrastructure.Features.DownloadHunter.Consumers;
public class DownloadHunterConsumer<T> : IConsumer<DownloadHuntRequest<T>>
where T : SearchItem
{
private readonly ILogger<DownloadHunterConsumer<T>> _logger;
private readonly IDownloadHunter _downloadHunter;
public DownloadHunterConsumer(ILogger<DownloadHunterConsumer<T>> logger, IDownloadHunter downloadHunter)
{
_logger = logger;
_downloadHunter = downloadHunter;
}
public async Task Consume(ConsumeContext<DownloadHuntRequest<T>> context)
{
try
{
await _downloadHunter.HuntDownloadsAsync(context.Message);
}
catch (Exception exception)
{
_logger.LogError(exception,
"failed to search for replacement | {title} | {url}",
context.Message.Record.Title,
context.Message.Instance.Url
);
}
}
}

View File

@@ -0,0 +1,42 @@
using Cleanuparr.Infrastructure.Features.Arr;
using Cleanuparr.Infrastructure.Features.DownloadHunter.Interfaces;
using Cleanuparr.Infrastructure.Features.DownloadHunter.Models;
using Cleanuparr.Persistence;
using Data.Models.Arr;
using Microsoft.EntityFrameworkCore;
namespace Cleanuparr.Infrastructure.Features.DownloadHunter;
public sealed class DownloadHunter : IDownloadHunter
{
private readonly DataContext _dataContext;
private readonly ArrClientFactory _arrClientFactory;
public DownloadHunter(
DataContext dataContext,
ArrClientFactory arrClientFactory
)
{
_dataContext = dataContext;
_arrClientFactory = arrClientFactory;
}
public async Task HuntDownloadsAsync<T>(DownloadHuntRequest<T> request)
where T : SearchItem
{
var generalConfig = await _dataContext.GeneralConfigs
.AsNoTracking()
.FirstAsync();
if (!generalConfig.SearchEnabled)
{
return;
}
var arrClient = _arrClientFactory.GetClient(request.InstanceType);
await arrClient.SearchItemsAsync(request.Instance, [request.SearchItem]);
// prevent tracker spamming
await Task.Delay(TimeSpan.FromSeconds(generalConfig.SearchDelay));
}
}

View File

@@ -0,0 +1,9 @@
using Cleanuparr.Infrastructure.Features.DownloadHunter.Models;
using Data.Models.Arr;
namespace Cleanuparr.Infrastructure.Features.DownloadHunter.Interfaces;
public interface IDownloadHunter
{
Task HuntDownloadsAsync<T>(DownloadHuntRequest<T> request) where T : SearchItem;
}

View File

@@ -0,0 +1,18 @@
using Cleanuparr.Domain.Entities.Arr.Queue;
using Cleanuparr.Domain.Enums;
using Cleanuparr.Persistence.Models.Configuration.Arr;
using Data.Models.Arr;
namespace Cleanuparr.Infrastructure.Features.DownloadHunter.Models;
public sealed record DownloadHuntRequest<T>
where T : SearchItem
{
public required InstanceType InstanceType { get; init; }
public required ArrInstance Instance { get; init; }
public required T SearchItem { get; init; }
public required QueueRecord Record { get; init; }
}

View File

@@ -30,7 +30,7 @@ public class DownloadRemoverConsumer<T> : IConsumer<QueueItemRemoveRequest<T>>
catch (Exception exception)
{
_logger.LogError(exception,
"failed to remove queue item| {title} | {url}",
"failed to remove queue item | {title} | {url}",
context.Message.Record.Title,
context.Message.Instance.Url
);

View File

@@ -1,34 +1,35 @@
using Cleanuparr.Domain.Entities.Arr.Queue;
using System.Net;
using Cleanuparr.Domain.Entities.Arr.Queue;
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Events;
using Cleanuparr.Infrastructure.Features.Arr;
using Cleanuparr.Infrastructure.Features.Context;
using Cleanuparr.Infrastructure.Features.DownloadHunter.Models;
using Cleanuparr.Infrastructure.Features.DownloadRemover.Interfaces;
using Cleanuparr.Infrastructure.Features.DownloadRemover.Models;
using Cleanuparr.Infrastructure.Helpers;
using Cleanuparr.Persistence;
using Cleanuparr.Persistence.Models.Configuration.Arr;
using Data.Models.Arr;
using Microsoft.EntityFrameworkCore;
using MassTransit;
using Microsoft.Extensions.Caching.Memory;
namespace Cleanuparr.Infrastructure.Features.DownloadRemover;
public sealed class QueueItemRemover : IQueueItemRemover
{
private readonly DataContext _dataContext;
private readonly IBus _messageBus;
private readonly IMemoryCache _cache;
private readonly ArrClientFactory _arrClientFactory;
private readonly EventPublisher _eventPublisher;
public QueueItemRemover(
DataContext dataContext,
IBus messageBus,
IMemoryCache cache,
ArrClientFactory arrClientFactory,
EventPublisher eventPublisher
)
{
_dataContext = dataContext;
_messageBus = messageBus;
_cache = cache;
_arrClientFactory = arrClientFactory;
_eventPublisher = eventPublisher;
@@ -39,31 +40,35 @@ public sealed class QueueItemRemover : IQueueItemRemover
{
try
{
var generalConfig = await _dataContext.GeneralConfigs
.AsNoTracking()
.FirstAsync();
var arrClient = _arrClientFactory.GetClient(request.InstanceType);
await arrClient.DeleteQueueItemAsync(request.Instance, request.Record, request.RemoveFromClient, request.DeleteReason);
// Set context for EventPublisher
ContextProvider.Set("downloadName", request.Record.Title);
ContextProvider.Set("hash", request.Record.DownloadId);
ContextProvider.Set(nameof(QueueRecord), request.Record);
ContextProvider.Set(nameof(ArrInstance) + nameof(ArrInstance.Url), request.Instance.Url);
ContextProvider.Set(nameof(InstanceType), request.InstanceType);
// Use the new centralized EventPublisher method
await _eventPublisher.PublishQueueItemDeleted(request.RemoveFromClient, request.DeleteReason);
if (!generalConfig.SearchEnabled)
await _messageBus.Publish(new DownloadHuntRequest<T>
{
return;
InstanceType = request.InstanceType,
Instance = request.Instance,
SearchItem = request.SearchItem,
Record = request.Record
});
}
catch (HttpRequestException exception)
{
if (exception.StatusCode is not HttpStatusCode.NotFound)
{
throw;
}
await arrClient.SearchItemsAsync(request.Instance, [request.SearchItem]);
// prevent tracker spamming
await Task.Delay(TimeSpan.FromSeconds(generalConfig.SearchDelay));
throw new Exception($"Item might have already been deleted by your {request.InstanceType} instance", exception);
}
finally
{

View File

@@ -27,6 +27,7 @@ public sealed class Striker : IStriker
{
if (maxStrikes is 0)
{
_logger.LogTrace("skip striking for {reason} | max strikes is 0 | {name}", strikeType, itemName);
return false;
}