Add download cleaner and dry run (#58)

This commit is contained in:
Marius Nechifor
2025-02-16 03:09:07 +02:00
parent 19b3675701
commit 596a5aed8d
87 changed files with 2507 additions and 413 deletions

View File

@@ -0,0 +1,33 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
<PackageReference Include="NSubstitute" Version="5.3.0" />
<PackageReference Include="Shouldly" Version="4.3.0" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<Using Include="Xunit"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Infrastructure\Infrastructure.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,20 @@
using Infrastructure.Verticals.ContentBlocker;
using Microsoft.Extensions.Logging;
using NSubstitute;
namespace Infrastructure.Tests.Verticals.ContentBlocker;
public class FilenameEvaluatorFixture
{
public ILogger<FilenameEvaluator> Logger { get; }
public FilenameEvaluatorFixture()
{
Logger = Substitute.For<ILogger<FilenameEvaluator>>();
}
public FilenameEvaluator CreateSut()
{
return new FilenameEvaluator(Logger);
}
}

View File

@@ -0,0 +1,219 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Shouldly;
namespace Infrastructure.Tests.Verticals.ContentBlocker;
public class FilenameEvaluatorTests : IClassFixture<FilenameEvaluatorFixture>
{
private readonly FilenameEvaluatorFixture _fixture;
public FilenameEvaluatorTests(FilenameEvaluatorFixture fixture)
{
_fixture = fixture;
}
public class PatternTests : FilenameEvaluatorTests
{
public PatternTests(FilenameEvaluatorFixture fixture) : base(fixture) { }
[Fact]
public void WhenNoPatterns_ShouldReturnTrue()
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string>();
var regexes = new ConcurrentBag<Regex>();
// Act
var result = sut.IsValid("test.txt", BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBeTrue();
}
[Theory]
[InlineData("test.txt", "test.txt", true)] // Exact match
[InlineData("test.txt", "*.txt", true)] // End wildcard
[InlineData("test.txt", "test.*", true)] // Start wildcard
[InlineData("test.txt", "*test*", true)] // Both wildcards
[InlineData("test.txt", "other.txt", false)] // No match
public void Blacklist_ShouldMatchPatterns(string filename, string pattern, bool shouldBeBlocked)
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string> { pattern };
var regexes = new ConcurrentBag<Regex>();
// Act
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBe(!shouldBeBlocked);
}
[Theory]
[InlineData("test.txt", "test.txt", true)] // Exact match
[InlineData("test.txt", "*.txt", true)] // End wildcard
[InlineData("test.txt", "test.*", true)] // Start wildcard
[InlineData("test.txt", "*test*", true)] // Both wildcards
[InlineData("test.txt", "other.txt", false)] // No match
public void Whitelist_ShouldMatchPatterns(string filename, string pattern, bool shouldBeAllowed)
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string> { pattern };
var regexes = new ConcurrentBag<Regex>();
// Act
var result = sut.IsValid(filename, BlocklistType.Whitelist, patterns, regexes);
// Assert
result.ShouldBe(shouldBeAllowed);
}
[Theory]
[InlineData("TEST.TXT", "test.txt")]
[InlineData("test.txt", "TEST.TXT")]
public void ShouldBeCaseInsensitive(string filename, string pattern)
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string> { pattern };
var regexes = new ConcurrentBag<Regex>();
// Act
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBeFalse();
}
[Fact]
public void MultiplePatterns_ShouldMatchAny()
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string>
{
"other.txt",
"*.pdf",
"test.*"
};
var regexes = new ConcurrentBag<Regex>();
// Act
var result = sut.IsValid("test.txt", BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBeFalse();
}
}
public class RegexTests : FilenameEvaluatorTests
{
public RegexTests(FilenameEvaluatorFixture fixture) : base(fixture) { }
[Fact]
public void WhenNoRegexes_ShouldReturnTrue()
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string>();
var regexes = new ConcurrentBag<Regex>();
// Act
var result = sut.IsValid("test.txt", BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBeTrue();
}
[Theory]
[InlineData(@"test\d+\.txt", "test123.txt", true)]
[InlineData(@"test\d+\.txt", "test.txt", false)]
public void Blacklist_ShouldMatchRegexes(string pattern, string filename, bool shouldBeBlocked)
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string>();
var regexes = new ConcurrentBag<Regex> { new Regex(pattern, RegexOptions.IgnoreCase) };
// Act
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBe(!shouldBeBlocked);
}
[Theory]
[InlineData(@"test\d+\.txt", "test123.txt", true)]
[InlineData(@"test\d+\.txt", "test.txt", false)]
public void Whitelist_ShouldMatchRegexes(string pattern, string filename, bool shouldBeAllowed)
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string>();
var regexes = new ConcurrentBag<Regex> { new Regex(pattern, RegexOptions.IgnoreCase) };
// Act
var result = sut.IsValid(filename, BlocklistType.Whitelist, patterns, regexes);
// Assert
result.ShouldBe(shouldBeAllowed);
}
[Theory]
[InlineData(@"TEST\d+\.TXT", "test123.txt")]
[InlineData(@"test\d+\.txt", "TEST123.TXT")]
public void ShouldBeCaseInsensitive(string pattern, string filename)
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string>();
var regexes = new ConcurrentBag<Regex> { new Regex(pattern, RegexOptions.IgnoreCase) };
// Act
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBeFalse();
}
}
public class CombinedTests : FilenameEvaluatorTests
{
public CombinedTests(FilenameEvaluatorFixture fixture) : base(fixture) { }
[Fact]
public void WhenBothPatternsAndRegexes_ShouldMatchBoth()
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string> { "*.txt" };
var regexes = new ConcurrentBag<Regex> { new Regex(@"test\d+", RegexOptions.IgnoreCase) };
// Act
var result = sut.IsValid("test123.txt", BlocklistType.Blacklist, patterns, regexes);
// Assert
result.ShouldBeFalse();
}
[Fact]
public void WhenPatternMatchesButRegexDoesNot_ShouldReturnFalse()
{
// Arrange
var sut = _fixture.CreateSut();
var patterns = new ConcurrentBag<string> { "*.txt" };
var regexes = new ConcurrentBag<Regex> { new Regex(@"test\d+", RegexOptions.IgnoreCase) };
// Act
var result = sut.IsValid("other.txt", BlocklistType.Whitelist, patterns, regexes);
// Assert
result.ShouldBeFalse();
}
}
}

View File

@@ -0,0 +1,74 @@
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using NSubstitute;
namespace Infrastructure.Tests.Verticals.DownloadClient;
public class DownloadServiceFixture : IDisposable
{
public ILogger<DownloadService> Logger { get; set; }
public IMemoryCache Cache { get; set; }
public IStriker Striker { get; set; }
public DownloadServiceFixture()
{
Logger = Substitute.For<ILogger<DownloadService>>();
Cache = Substitute.For<IMemoryCache>();
Striker = Substitute.For<IStriker>();
}
public TestDownloadService CreateSut(
QueueCleanerConfig? queueCleanerConfig = null,
ContentBlockerConfig? contentBlockerConfig = null
)
{
queueCleanerConfig ??= new QueueCleanerConfig
{
Enabled = true,
RunSequentially = true,
StalledResetStrikesOnProgress = true,
StalledMaxStrikes = 3
};
var queueCleanerOptions = Substitute.For<IOptions<QueueCleanerConfig>>();
queueCleanerOptions.Value.Returns(queueCleanerConfig);
contentBlockerConfig ??= new ContentBlockerConfig
{
Enabled = true
};
var contentBlockerOptions = Substitute.For<IOptions<ContentBlockerConfig>>();
contentBlockerOptions.Value.Returns(contentBlockerConfig);
var downloadCleanerOptions = Substitute.For<IOptions<DownloadCleanerConfig>>();
downloadCleanerOptions.Value.Returns(new DownloadCleanerConfig());
var filenameEvaluator = Substitute.For<IFilenameEvaluator>();
var notifier = Substitute.For<NotificationPublisher>();
return new TestDownloadService(
Logger,
queueCleanerOptions,
contentBlockerOptions,
downloadCleanerOptions,
Cache,
filenameEvaluator,
Striker,
notifier
);
}
public void Dispose()
{
// Cleanup if needed
}
}

View File

@@ -0,0 +1,235 @@
using Common.Configuration.DownloadCleaner;
using Domain.Enums;
using Domain.Models.Cache;
using Infrastructure.Helpers;
using Infrastructure.Verticals.Context;
using Infrastructure.Verticals.DownloadClient;
using NSubstitute;
using NSubstitute.ClearExtensions;
using Shouldly;
namespace Infrastructure.Tests.Verticals.DownloadClient;
public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
{
private readonly DownloadServiceFixture _fixture;
public DownloadServiceTests(DownloadServiceFixture fixture)
{
_fixture = fixture;
_fixture.Cache.ClearSubstitute();
_fixture.Striker.ClearSubstitute();
}
public class ResetStrikesOnProgressTests : DownloadServiceTests
{
public ResetStrikesOnProgressTests(DownloadServiceFixture fixture) : base(fixture)
{
}
[Fact]
public void WhenStalledStrikeDisabled_ShouldNotResetStrikes()
{
// Arrange
TestDownloadService sut = _fixture.CreateSut(queueCleanerConfig: new()
{
Enabled = true,
RunSequentially = true,
StalledResetStrikesOnProgress = false,
});
// Act
sut.ResetStrikesOnProgress("test-hash", 100);
// Assert
_fixture.Cache.ReceivedCalls().ShouldBeEmpty();
}
[Fact]
public void WhenProgressMade_ShouldResetStrikes()
{
// Arrange
const string hash = "test-hash";
CacheItem cacheItem = new CacheItem { Downloaded = 100 };
_fixture.Cache.TryGetValue(Arg.Any<object>(), out Arg.Any<object?>())
.Returns(x =>
{
x[1] = cacheItem;
return true;
});
TestDownloadService sut = _fixture.CreateSut();
// Act
sut.ResetStrikesOnProgress(hash, 200);
// Assert
_fixture.Cache.Received(1).Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
}
[Fact]
public void WhenNoProgress_ShouldNotResetStrikes()
{
// Arrange
const string hash = "test-hash";
CacheItem cacheItem = new CacheItem { Downloaded = 200 };
_fixture.Cache
.TryGetValue(Arg.Any<object>(), out Arg.Any<object?>())
.Returns(x =>
{
x[1] = cacheItem;
return true;
});
TestDownloadService sut = _fixture.CreateSut();
// Act
sut.ResetStrikesOnProgress(hash, 100);
// Assert
_fixture.Cache.DidNotReceive().Remove(Arg.Any<object>());
}
}
public class StrikeAndCheckLimitTests : DownloadServiceTests
{
public StrikeAndCheckLimitTests(DownloadServiceFixture fixture) : base(fixture)
{
}
[Fact]
public async Task ShouldDelegateCallToStriker()
{
// Arrange
const string hash = "test-hash";
const string itemName = "test-item";
_fixture.Striker.StrikeAndCheckLimit(hash, itemName, 3, StrikeType.Stalled)
.Returns(true);
TestDownloadService sut = _fixture.CreateSut();
// Act
bool result = await sut.StrikeAndCheckLimit(hash, itemName);
// Assert
result.ShouldBeTrue();
await _fixture.Striker
.Received(1)
.StrikeAndCheckLimit(hash, itemName, 3, StrikeType.Stalled);
}
}
public class ShouldCleanDownloadTests : DownloadServiceTests
{
public ShouldCleanDownloadTests(DownloadServiceFixture fixture) : base(fixture)
{
ContextProvider.Set("downloadName", "test-download");
}
[Fact]
public void WhenRatioAndMinSeedTimeReached_ShouldReturnTrue()
{
// Arrange
Category category = new()
{
Name = "test",
MaxRatio = 1.0,
MinSeedTime = 1,
MaxSeedTime = -1
};
const double ratio = 1.5;
TimeSpan seedingTime = TimeSpan.FromHours(2);
TestDownloadService sut = _fixture.CreateSut();
// Act
var result = sut.ShouldCleanDownload(ratio, seedingTime, category);
// Assert
result.ShouldSatisfyAllConditions(
() => result.ShouldClean.ShouldBeTrue(),
() => result.Reason.ShouldBe(CleanReason.MaxRatioReached)
);
}
[Fact]
public void WhenRatioReachedAndMinSeedTimeNotReached_ShouldReturnFalse()
{
// Arrange
Category category = new()
{
Name = "test",
MaxRatio = 1.0,
MinSeedTime = 3,
MaxSeedTime = -1
};
const double ratio = 1.5;
TimeSpan seedingTime = TimeSpan.FromHours(2);
TestDownloadService sut = _fixture.CreateSut();
// Act
var result = sut.ShouldCleanDownload(ratio, seedingTime, category);
// Assert
result.ShouldSatisfyAllConditions(
() => result.ShouldClean.ShouldBeFalse(),
() => result.Reason.ShouldBe(CleanReason.None)
);
}
[Fact]
public void WhenMaxSeedTimeReached_ShouldReturnTrue()
{
// Arrange
Category category = new()
{
Name = "test",
MaxRatio = -1,
MinSeedTime = 0,
MaxSeedTime = 1
};
const double ratio = 0.5;
TimeSpan seedingTime = TimeSpan.FromHours(2);
TestDownloadService sut = _fixture.CreateSut();
// Act
SeedingCheckResult result = sut.ShouldCleanDownload(ratio, seedingTime, category);
// Assert
result.ShouldSatisfyAllConditions(
() => result.ShouldClean.ShouldBeTrue(),
() => result.Reason.ShouldBe(CleanReason.MaxSeedTimeReached)
);
}
[Fact]
public void WhenNeitherConditionMet_ShouldReturnFalse()
{
// Arrange
Category category = new()
{
Name = "test",
MaxRatio = 2.0,
MinSeedTime = 0,
MaxSeedTime = 3
};
const double ratio = 1.0;
TimeSpan seedingTime = TimeSpan.FromHours(1);
TestDownloadService sut = _fixture.CreateSut();
// Act
var result = sut.ShouldCleanDownload(ratio, seedingTime, category);
// Assert
result.ShouldSatisfyAllConditions(
() => result.ShouldClean.ShouldBeFalse(),
() => result.Reason.ShouldBe(CleanReason.None)
);
}
}
}

View File

@@ -0,0 +1,45 @@
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Common.Configuration.ContentBlocker;
using Common.Configuration.DownloadCleaner;
using Common.Configuration.QueueCleaner;
using Infrastructure.Verticals.ContentBlocker;
using Infrastructure.Verticals.DownloadClient;
using Infrastructure.Verticals.ItemStriker;
using Infrastructure.Verticals.Notifications;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace Infrastructure.Tests.Verticals.DownloadClient;
public class TestDownloadService : DownloadService
{
public TestDownloadService(
ILogger<DownloadService> logger,
IOptions<QueueCleanerConfig> queueCleanerConfig,
IOptions<ContentBlockerConfig> contentBlockerConfig,
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
IMemoryCache cache,
IFilenameEvaluator filenameEvaluator,
IStriker striker,
NotificationPublisher notifier)
: base(logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig,
cache, filenameEvaluator, striker, notifier)
{
}
public override void Dispose() { }
public override Task LoginAsync() => Task.CompletedTask;
public override Task<StalledResult> ShouldRemoveFromArrQueueAsync(string hash) => Task.FromResult(new StalledResult());
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType,
ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes) => Task.FromResult(new BlockFilesResult());
public override Task DeleteDownload(string hash) => Task.CompletedTask;
public override Task<List<object>?> GetAllDownloadsToBeCleaned(List<Category> categories) => Task.FromResult<List<object>?>(null);
public override Task CleanDownloads(List<object> downloads, List<Category> categoriesToClean, HashSet<string> excludedHashes) => Task.CompletedTask;
// Expose protected methods for testing
public new void ResetStrikesOnProgress(string hash, long downloaded) => base.ResetStrikesOnProgress(hash, downloaded);
public new Task<bool> StrikeAndCheckLimit(string hash, string itemName) => base.StrikeAndCheckLimit(hash, itemName);
public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, Category category) => base.ShouldCleanDownload(ratio, seedingTime, category);
}