Remove the known malware feature (#489)

This commit is contained in:
Flaminel
2026-03-06 17:53:04 +02:00
committed by GitHub
parent 33d1756fdd
commit f51973bb7b
29 changed files with 1385 additions and 212 deletions

View File

@@ -1,2 +0,0 @@
thepirateheaven.org
RARBG.work

View File

@@ -16,8 +16,6 @@ public sealed record UpdateMalwareBlockerConfigRequest
public bool DeletePrivate { get; init; }
public bool DeleteKnownMalware { get; init; }
public BlocklistSettings Sonarr { get; init; } = new();
public BlocklistSettings Radarr { get; init; } = new();
@@ -37,7 +35,6 @@ public sealed record UpdateMalwareBlockerConfigRequest
config.UseAdvancedScheduling = UseAdvancedScheduling;
config.IgnorePrivate = IgnorePrivate;
config.DeletePrivate = DeletePrivate;
config.DeleteKnownMalware = DeleteKnownMalware;
config.Sonarr = Sonarr;
config.Radarr = Radarr;
config.Lidarr = Lidarr;

View File

@@ -11,5 +11,4 @@ public enum DeleteReason
AllFilesSkipped,
AllFilesSkippedByQBit,
AllFilesBlocked,
MalwareFileFound,
}

View File

@@ -377,7 +377,6 @@ public class QueueItemRemoverTests : IDisposable
[InlineData(DeleteReason.SlowSpeed)]
[InlineData(DeleteReason.SlowTime)]
[InlineData(DeleteReason.DownloadingMetadata)]
[InlineData(DeleteReason.MalwareFileFound)]
public async Task RemoveQueueItemAsync_PassesCorrectDeleteReason(DeleteReason deleteReason)
{
// Arrange

View File

@@ -159,24 +159,21 @@ public class MalwareBlockerTests : IDisposable
_fixture.ArrClientFactory.Verify(x => x.GetClient(InstanceType.Sonarr, It.IsAny<float>()), Times.Once);
}
[Fact]
public async Task ExecuteInternalAsync_WhenDeleteKnownMalwareEnabled_ProcessesAllArrs()
[Theory]
[InlineData(InstanceType.Radarr)]
[InlineData(InstanceType.Lidarr)]
[InlineData(InstanceType.Readarr)]
[InlineData(InstanceType.Whisparr)]
public async Task ExecuteInternalAsync_WhenArrTypeEnabled_ProcessesCorrectInstances(InstanceType instanceType)
{
// Arrange
TestDataContextFactory.AddDownloadClient(_fixture.DataContext);
var contentBlockerConfig = _fixture.DataContext.ContentBlockerConfigs.First();
contentBlockerConfig.DeleteKnownMalware = true;
// Need at least one blocklist enabled for processing to occur
contentBlockerConfig.Sonarr = new BlocklistSettings { Enabled = true };
_fixture.DataContext.SaveChanges();
TestDataContextFactory.AddSonarrInstance(_fixture.DataContext);
TestDataContextFactory.AddRadarrInstance(_fixture.DataContext);
EnableBlocklist(instanceType);
AddArrInstance(instanceType);
var mockArrClient = new Mock<IArrClient>();
_fixture.ArrClientFactory
.Setup(x => x.GetClient(It.IsAny<InstanceType>(), It.IsAny<float>()))
.Setup(x => x.GetClient(instanceType, It.IsAny<float>()))
.Returns(mockArrClient.Object);
_fixture.ArrQueueIterator
@@ -192,9 +189,8 @@ public class MalwareBlockerTests : IDisposable
// Act
await sut.ExecuteAsync();
// Assert - Sonarr and Radarr processed because DeleteKnownMalware is true
_fixture.ArrClientFactory.Verify(x => x.GetClient(InstanceType.Sonarr, It.IsAny<float>()), Times.Once);
_fixture.ArrClientFactory.Verify(x => x.GetClient(InstanceType.Radarr, It.IsAny<float>()), Times.Once);
// Assert
_fixture.ArrClientFactory.Verify(x => x.GetClient(instanceType, It.IsAny<float>()), Times.Once);
}
#endregion
@@ -605,5 +601,32 @@ public class MalwareBlockerTests : IDisposable
_fixture.DataContext.SaveChanges();
}
private void EnableBlocklist(InstanceType instanceType)
{
var config = _fixture.DataContext.ContentBlockerConfigs.First();
var settings = new BlocklistSettings { Enabled = true };
switch (instanceType)
{
case InstanceType.Radarr: config.Radarr = settings; break;
case InstanceType.Lidarr: config.Lidarr = settings; break;
case InstanceType.Readarr: config.Readarr = settings; break;
case InstanceType.Whisparr: config.Whisparr = settings; break;
default: throw new ArgumentOutOfRangeException(nameof(instanceType));
}
_fixture.DataContext.SaveChanges();
}
private void AddArrInstance(InstanceType instanceType)
{
switch (instanceType)
{
case InstanceType.Radarr: TestDataContextFactory.AddRadarrInstance(_fixture.DataContext); break;
case InstanceType.Lidarr: TestDataContextFactory.AddLidarrInstance(_fixture.DataContext); break;
case InstanceType.Readarr: TestDataContextFactory.AddReadarrInstance(_fixture.DataContext); break;
case InstanceType.Whisparr: TestDataContextFactory.AddWhisparrInstance(_fixture.DataContext); break;
default: throw new ArgumentOutOfRangeException(nameof(instanceType));
}
}
#endregion
}

View File

@@ -75,7 +75,6 @@ public static class TestDataContextFactory
{
Id = Guid.NewGuid(),
IgnoredDownloads = [],
DeleteKnownMalware = false,
DeletePrivate = false,
Sonarr = new BlocklistSettings { Enabled = false },
Radarr = new BlocklistSettings { Enabled = false },

View File

@@ -118,34 +118,6 @@ public class BlocklistProviderTests : IDisposable
result.Count.ShouldBe(2);
}
[Fact]
public void GetMalwarePatterns_NotInCache_ReturnsEmptyBag()
{
// Act
var result = _provider.GetMalwarePatterns();
// Assert
result.ShouldNotBeNull();
result.ShouldBeEmpty();
}
[Fact]
public void GetMalwarePatterns_InCache_ReturnsCachedPatterns()
{
// Arrange
var patterns = new ConcurrentBag<string> { "known_malware.exe", "trojan*", "virus.dll" };
_cache.Set(CacheKeys.KnownMalwarePatterns(), patterns);
// Act
var result = _provider.GetMalwarePatterns();
// Assert
result.Count.ShouldBe(3);
result.ShouldContain("known_malware.exe");
result.ShouldContain("trojan*");
result.ShouldContain("virus.dll");
}
[Theory]
[InlineData(InstanceType.Sonarr)]
[InlineData(InstanceType.Radarr)]

View File

@@ -271,12 +271,12 @@ public class NotificationPublisherTests
.Returns(providerMock.Object);
// Act
await _publisher.NotifyQueueItemDeleted(false, DeleteReason.MalwareFileFound);
await _publisher.NotifyQueueItemDeleted(false, DeleteReason.AllFilesBlocked);
// Assert
providerMock.Verify(p => p.SendNotificationAsync(It.Is<NotificationContext>(
c => c.Data["Removed from client?"] == "False" &&
c.Data["Reason"] == "MalwareFileFound")), Times.Once);
c.Data["Reason"] == "AllFilesBlocked")), Times.Once);
}
#endregion

View File

@@ -68,7 +68,6 @@ public partial class DelugeService
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
ConcurrentBag<string> malwarePatterns = _blocklistProvider.GetMalwarePatterns();
ProcessFiles(contents.Contents, (name, file) =>
{
@@ -79,13 +78,6 @@ public partial class DelugeService
{
return;
}
if (malwareBlockerConfig.DeleteKnownMalware && _filenameEvaluator.IsKnownMalware(name, malwarePatterns))
{
_logger.LogInformation("malware file found | {file} | {title}", file.Path, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
}
if (file.Priority is 0)
{

View File

@@ -73,8 +73,7 @@ public partial class QBitService
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
ConcurrentBag<string> malwarePatterns = _blocklistProvider.GetMalwarePatterns();
foreach (TorrentContent file in files)
{
if (!file.Index.HasValue)
@@ -84,14 +83,6 @@ public partial class QBitService
}
totalFiles++;
if (malwareBlockerConfig.DeleteKnownMalware && _filenameEvaluator.IsKnownMalware(file.Name, malwarePatterns))
{
_logger.LogInformation("malware file found | {file} | {title}", file.Name, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
return result;
}
if (file.Priority is TorrentContentPriority.Skip)
{

View File

@@ -71,7 +71,6 @@ public partial class RTorrentService
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
ConcurrentBag<string> malwarePatterns = _blocklistProvider.GetMalwarePatterns();
List<(int Index, int Priority)> priorityUpdates = [];
@@ -85,13 +84,6 @@ public partial class RTorrentService
continue;
}
if (malwareBlockerConfig.DeleteKnownMalware && _filenameEvaluator.IsKnownMalware(fileName, malwarePatterns))
{
_logger.LogInformation("malware file found | {file} | {title}", file.Path, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
}
if (file.Priority == 0)
{
_logger.LogTrace("File is already skipped | {file}", file.Path);

View File

@@ -56,8 +56,7 @@ public partial class TransmissionService
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
ConcurrentBag<string> malwarePatterns = _blocklistProvider.GetMalwarePatterns();
for (int i = 0; i < download.Files.Length; i++)
{
if (download.FileStats?[i].Wanted == null)
@@ -67,15 +66,7 @@ public partial class TransmissionService
}
totalFiles++;
if (malwareBlockerConfig.DeleteKnownMalware && _filenameEvaluator.IsKnownMalware(download.Files[i].Name, malwarePatterns))
{
_logger.LogInformation("malware file found | {file} | {title}", download.Files[i].Name, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
return result;
}
if (!download.FileStats[i].Wanted.Value)
{
_logger.LogTrace("File is already skipped | {file}", download.Files[i].Name);

View File

@@ -61,18 +61,9 @@ public partial class UTorrentService
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
ConcurrentBag<string> malwarePatterns = _blocklistProvider.GetMalwarePatterns();
for (int i = 0; i < files.Count; i++)
{
if (malwareBlockerConfig.DeleteKnownMalware && _filenameEvaluator.IsKnownMalware(files[i].Name, malwarePatterns))
{
_logger.LogInformation("malware file found | {file} | {title}", files[i].Name, download.Name);
result.ShouldRemove = true;
result.DeleteReason = DeleteReason.MalwareFileFound;
return result;
}
var file = files[i];
if (file.Priority == 0) // Already skipped

View File

@@ -64,27 +64,27 @@ public sealed class MalwareBlocker : GenericHandler
var readarrConfig = ContextProvider.Get<ArrConfig>(nameof(InstanceType.Readarr));
var whisparrConfig = ContextProvider.Get<ArrConfig>(nameof(InstanceType.Whisparr));
if (config.Sonarr.Enabled || config.DeleteKnownMalware)
if (config.Sonarr.Enabled)
{
await ProcessArrConfigAsync(sonarrConfig);
}
if (config.Radarr.Enabled || config.DeleteKnownMalware)
if (config.Radarr.Enabled)
{
await ProcessArrConfigAsync(radarrConfig);
}
if (config.Lidarr.Enabled || config.DeleteKnownMalware)
if (config.Lidarr.Enabled)
{
await ProcessArrConfigAsync(lidarrConfig);
}
if (config.Readarr.Enabled || config.DeleteKnownMalware)
if (config.Readarr.Enabled)
{
await ProcessArrConfigAsync(readarrConfig);
}
if (config.Whisparr.Enabled || config.DeleteKnownMalware)
if (config.Whisparr.Enabled)
{
await ProcessArrConfigAsync(whisparrConfig);
}

View File

@@ -23,8 +23,6 @@ public sealed class BlocklistProvider : IBlocklistProvider
private readonly Dictionary<string, DateTime> _lastLoadTimes = new();
private const int DefaultLoadIntervalHours = 4;
private const int FastLoadIntervalMinutes = 5;
private const string MalwareListUrl = "https://cleanuparr.pages.dev/static/known_malware_file_name_patterns";
private const string MalwareListKey = "MALWARE_PATTERNS";
public BlocklistProvider(
ILogger<BlocklistProvider> logger,
@@ -72,10 +70,7 @@ public sealed class BlocklistProvider : IBlocklistProvider
changedCount++;
}
}
// Always check and update malware patterns
await LoadMalwarePatternsAsync(fileReader);
if (changedCount > 0)
{
_logger.LogInformation("Successfully loaded {count} blocklists", changedCount);
@@ -109,17 +104,10 @@ public sealed class BlocklistProvider : IBlocklistProvider
public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
{
_cache.TryGetValue(CacheKeys.BlocklistRegexes(instanceType), out ConcurrentBag<Regex>? regexes);
return regexes ?? [];
}
public ConcurrentBag<string> GetMalwarePatterns()
{
_cache.TryGetValue(CacheKeys.KnownMalwarePatterns(), out ConcurrentBag<string>? patterns);
return patterns ?? [];
}
private async Task<bool> EnsureInstanceLoadedAsync(BlocklistSettings settings, InstanceType instanceType, FileReader fileReader)
{
if (!settings.Enabled || string.IsNullOrEmpty(settings.BlocklistPath))
@@ -165,47 +153,9 @@ public sealed class BlocklistProvider : IBlocklistProvider
{
return true;
}
return DateTime.UtcNow - lastLoad >= interval;
}
private async Task LoadMalwarePatternsAsync(FileReader fileReader)
{
var malwareInterval = TimeSpan.FromMinutes(FastLoadIntervalMinutes);
if (!ShouldReloadBlocklist(MalwareListKey, malwareInterval))
{
return;
}
try
{
_logger.LogDebug("Loading malware patterns");
string[] filePatterns = await fileReader.ReadContentAsync(MalwareListUrl);
long startTime = Stopwatch.GetTimestamp();
ParallelOptions options = new() { MaxDegreeOfParallelism = 5 };
ConcurrentBag<string> patterns = [];
Parallel.ForEach(filePatterns, options, pattern =>
{
patterns.Add(pattern);
});
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
_cache.Set(CacheKeys.KnownMalwarePatterns(), patterns);
_lastLoadTimes[MalwareListKey] = DateTime.UtcNow;
_logger.LogDebug("loaded {count} known malware patterns", patterns.Count);
_logger.LogDebug("malware patterns loaded in {elapsed} ms", elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load malware patterns from {url}", MalwareListUrl);
}
}
private async Task LoadPatternsAndRegexesAsync(BlocklistSettings blocklistSettings, InstanceType instanceType, FileReader fileReader)
{

View File

@@ -20,16 +20,6 @@ public class FilenameEvaluator : IFilenameEvaluator
return IsValidAgainstPatterns(filename, type, patterns) && IsValidAgainstRegexes(filename, type, regexes);
}
public bool IsKnownMalware(string filename, ConcurrentBag<string> malwarePatterns)
{
if (malwarePatterns.Count is 0)
{
return false;
}
return malwarePatterns.Any(pattern => filename.Contains(pattern, StringComparison.InvariantCultureIgnoreCase));
}
private static bool IsValidAgainstPatterns(string filename, BlocklistType type, ConcurrentBag<string> patterns)
{
if (patterns.Count is 0)

View File

@@ -13,6 +13,4 @@ public interface IBlocklistProvider
ConcurrentBag<string> GetPatterns(InstanceType instanceType);
ConcurrentBag<Regex> GetRegexes(InstanceType instanceType);
ConcurrentBag<string> GetMalwarePatterns();
}

View File

@@ -7,6 +7,4 @@ namespace Cleanuparr.Infrastructure.Features.MalwareBlocker;
public interface IFilenameEvaluator
{
bool IsValid(string filename, BlocklistType type, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes);
bool IsKnownMalware(string filename, ConcurrentBag<string> malwarePatterns);
}

View File

@@ -8,8 +8,6 @@ public static class CacheKeys
public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns";
public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes";
public static string KnownMalwarePatterns() => "KNOWN_MALWARE_PATTERNS";
public static string IgnoredDownloads(string name) => $"{name}_ignored";
public static string DownloadMarkedForRemoval(string hash, Uri url) => $"remove_{hash.ToLowerInvariant()}_{url}";

View File

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Cleanuparr.Persistence.Migrations.Data
{
/// <inheritdoc />
public partial class RemoveKnownMalwareOption : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "delete_known_malware",
table: "content_blocker_configs");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "delete_known_malware",
table: "content_blocker_configs",
type: "INTEGER",
nullable: false,
defaultValue: false);
}
}
}

View File

@@ -389,10 +389,6 @@ namespace Cleanuparr.Persistence.Migrations.Data
.HasColumnType("TEXT")
.HasColumnName("cron_expression");
b.Property<bool>("DeleteKnownMalware")
.HasColumnType("INTEGER")
.HasColumnName("delete_known_malware");
b.Property<bool>("DeletePrivate")
.HasColumnType("INTEGER")
.HasColumnName("delete_private");

View File

@@ -19,8 +19,6 @@ public sealed record ContentBlockerConfig : IJobConfig
public bool IgnorePrivate { get; set; }
public bool DeletePrivate { get; set; }
public bool DeleteKnownMalware { get; set; }
public BlocklistSettings Sonarr { get; set; } = new();

View File

@@ -87,7 +87,6 @@ export class DocumentationService {
'cronExpression': 'cron-expression',
'ignorePrivate': 'ignore-private',
'deletePrivate': 'delete-private',
'deleteKnownMalware': 'delete-known-malware',
'sonarr.enabled': 'enable-blocklist',
'sonarr.blocklistPath': 'blocklist-path',
'sonarr.blocklistType': 'blocklist-type',

View File

@@ -23,9 +23,6 @@
hint="When enabled, the Malware blocker will run according to the schedule"
helpKey="malware-blocker:enabled" />
@if (enabled()) {
<app-toggle label="Delete Known Malware" [(checked)]="deleteKnownMalware"
hint="When enabled, downloads matching known malware patterns will be deleted"
helpKey="malware-blocker:deleteKnownMalware" />
<app-toggle label="Ignore Private Torrents" [(checked)]="ignorePrivate"
hint="When enabled, private torrents will not be processed"
helpKey="malware-blocker:ignorePrivate" />

View File

@@ -62,7 +62,6 @@ export class MalwareBlockerComponent implements OnInit, HasPendingChanges {
readonly ignoredDownloads = signal<string[]>([]);
readonly ignorePrivate = signal(false);
readonly deletePrivate = signal(false);
readonly deleteKnownMalware = signal(false);
readonly arrExpanded = signal(false);
readonly scheduleIntervalOptions = computed(() => {
@@ -164,7 +163,6 @@ export class MalwareBlockerComponent implements OnInit, HasPendingChanges {
this.ignoredDownloads.set(config.ignoredDownloads ?? []);
this.ignorePrivate.set(config.ignorePrivate);
this.deletePrivate.set(config.deletePrivate);
this.deleteKnownMalware.set(config.deleteKnownMalware);
const blocklists: Record<string, any> = {};
for (const name of ARR_NAMES) {
@@ -220,7 +218,6 @@ export class MalwareBlockerComponent implements OnInit, HasPendingChanges {
ignoredDownloads: this.ignoredDownloads(),
ignorePrivate: this.ignorePrivate(),
deletePrivate: this.deletePrivate(),
deleteKnownMalware: this.deleteKnownMalware(),
sonarr: { enabled: blocklists['sonarr'].enabled, blocklistPath: blocklists['sonarr'].blocklistPath, blocklistType: blocklists['sonarr'].blocklistType as BlocklistType },
radarr: { enabled: blocklists['radarr'].enabled, blocklistPath: blocklists['radarr'].blocklistPath, blocklistType: blocklists['radarr'].blocklistType as BlocklistType },
lidarr: { enabled: blocklists['lidarr'].enabled, blocklistPath: blocklists['lidarr'].blocklistPath, blocklistType: blocklists['lidarr'].blocklistType as BlocklistType },
@@ -256,7 +253,6 @@ export class MalwareBlockerComponent implements OnInit, HasPendingChanges {
ignoredDownloads: this.ignoredDownloads(),
ignorePrivate: this.ignorePrivate(),
deletePrivate: this.deletePrivate(),
deleteKnownMalware: this.deleteKnownMalware(),
arrBlocklists: this.arrBlocklists(),
});
}

View File

@@ -21,7 +21,6 @@ export interface MalwareBlockerConfig {
ignoredDownloads: string[];
ignorePrivate: boolean;
deletePrivate: boolean;
deleteKnownMalware: boolean;
sonarr: BlocklistSettings;
radarr: BlocklistSettings;
lidarr: BlocklistSettings;

View File

@@ -69,7 +69,6 @@ Advanced download management and automation features for your *arr applications
>
- Remove and block downloads blocked by qBittorrent or by Cleanuparr's **Malware Blocker**.
- Remove and block known malware based on patterns found by the community.
</ConfigSection>

View File

@@ -105,24 +105,6 @@ Setting this to true means private torrents will be permanently deleted, potenti
</ConfigSection>
<ConfigSection
title="Delete Known Malware"
icon="🦠"
>
When enabled, downloads that match known malware patterns will be automatically deleted from the download client.
**Malware Detection Source:**
- List is automatically fetched from: `https://cleanuparr.pages.dev/static/known_malware_file_name_patterns`
- Updates automatically every **5 minutes**
- Contains filename patterns known to be associated with malware
<Warning>
This feature permanently deletes downloads that match malware patterns. While the patterns are carefully curated, false positives are possible. Monitor logs carefully when first enabling this feature.
</Warning>
</ConfigSection>
</div>
<div className={styles.section}>