fixed failed retries restarting the rest

This commit is contained in:
Flaminel committed 2026-10-23 11:19:04 +03:00
1 parent 6d8d4e0070
commit ae8b5d6dbe
12 files changed
+196 -9

No files matched your search

@@ -1183,6 +1183,95 @@ public class SeekerCommandMonitorTests : IAsyncDisposable
unit.ConsecutiveMisses.ShouldBe(0);
}
[Fact]
public async Task NothingGrabbed_outcome_sets_the_last_miss_to_the_search_time()
{
// Arrange
ArrInstance radarrInstance = TestDataContextFactory.AddRadarrInstance(_dataContext);
DateTimeOffset searchedAt = _timeProvider.GetUtcNow().AddHours(-1);
Guid eventId = SetUpUnitOutcomeScenario(radarrInstance.Id, Guid.NewGuid(), externalItemId: 516, consecutiveMisses: 2, lastSearchedAt: searchedAt);
AddTracker(radarrInstance.Id, eventId, externalItemId: 516);
await _dataContext.SaveChangesAsync();
await _eventsContext.SaveChangesAsync();
StubCommandState(ArrCommandState.Completed);
_arrClient.GetQueueItemsAsync(Arg.Any<ArrInstance>(), Arg.Any<int>())
.Returns(new QueueListResponse());
Task<SearchCommandStatus> publishTask = CaptureNextPublishedStatus();
// Act
await _sut.StartAsync(_cts.Token);
_timeProvider.Advance(TimeSpan.FromSeconds(11));
await publishTask.WaitAsync(TimeSpan.FromSeconds(5));
await WaitForTrackerCountAsync(0);
// Assert
SeekerHistory unit = await _eventsContext.SeekerHistory.AsNoTracking().SingleAsync(h => h.ArrInstanceId == radarrInstance.Id);
unit.LastMissAt.ShouldBe(searchedAt);
}
[Fact]
public async Task Failed_command_leaves_the_last_miss_unchanged()
{
// Arrange
ArrInstance radarrInstance = TestDataContextFactory.AddRadarrInstance(_dataContext);
DateTimeOffset searchedAt = _timeProvider.GetUtcNow().AddHours(-1);
Guid eventId = SetUpUnitOutcomeScenario(radarrInstance.Id, Guid.NewGuid(), externalItemId: 517, consecutiveMisses: 3, lastSearchedAt: searchedAt, lastMissAt: searchedAt.AddDays(-9));
AddTracker(radarrInstance.Id, eventId, externalItemId: 517);
await _dataContext.SaveChangesAsync();
await _eventsContext.SaveChangesAsync();
StubCommandState(ArrCommandState.Failed);
Task<SearchCommandStatus> publishTask = CaptureNextPublishedStatus();
// Act
await _sut.StartAsync(_cts.Token);
_timeProvider.Advance(TimeSpan.FromSeconds(11));
await publishTask.WaitAsync(TimeSpan.FromSeconds(5));
await WaitForTrackerCountAsync(0);
// Assert
SeekerHistory unit = await _eventsContext.SeekerHistory.AsNoTracking().SingleAsync(h => h.ArrInstanceId == radarrInstance.Id);
unit.LastMissAt.ShouldBe(searchedAt.AddDays(-9));
}
[Fact]
public async Task Grabbed_outcome_clears_the_last_miss()
{
// Arrange
ArrInstance radarrInstance = TestDataContextFactory.AddRadarrInstance(_dataContext);
DateTimeOffset searchedAt = _timeProvider.GetUtcNow().AddHours(-1);
Guid eventId = SetUpUnitOutcomeScenario(radarrInstance.Id, Guid.NewGuid(), externalItemId: 515, consecutiveMisses: 4, lastSearchedAt: searchedAt, lastMissAt: searchedAt.AddDays(-9));
AddTracker(radarrInstance.Id, eventId, externalItemId: 515);
await _dataContext.SaveChangesAsync();
await _eventsContext.SaveChangesAsync();
StubCommandState(ArrCommandState.Completed);
_arrClient.GetQueueItemsAsync(Arg.Any<ArrInstance>(), Arg.Any<int>())
.Returns(new QueueListResponse
{
TotalRecords = 1,
Records = [new QueueRecord { Id = 1, MovieId = 515, Title = "Grabbed.Movie", DownloadId = "HASH515", Protocol = "torrent", Status = "downloading" }]
});
Task<SearchCommandStatus> publishTask = CaptureNextPublishedStatus();
// Act
await _sut.StartAsync(_cts.Token);
_timeProvider.Advance(TimeSpan.FromSeconds(11));
await publishTask.WaitAsync(TimeSpan.FromSeconds(5));
await WaitForTrackerCountAsync(0);
// Assert
SeekerHistory unit = await _eventsContext.SeekerHistory.AsNoTracking().SingleAsync(h => h.ArrInstanceId == radarrInstance.Id);
unit.LastMissAt.ShouldBeNull();
}
[Fact]
public async Task Failed_command_leaves_the_consecutive_misses_unchanged()
{
@@ -1521,7 +1610,7 @@ public class SeekerCommandMonitorTests : IAsyncDisposable
/// <summary>
/// Adds the search-triggered event and matching unit history row RecordUnitOutcomeAsync reads to attribute an outcome.
/// </summary>
private Guid SetUpUnitOutcomeScenario(Guid arrInstanceId, Guid? cycleId, long externalItemId, int consecutiveMisses = 0)
private Guid SetUpUnitOutcomeScenario(Guid arrInstanceId, Guid? cycleId, long externalItemId, int consecutiveMisses = 0, DateTimeOffset lastSearchedAt = default, DateTimeOffset? lastMissAt = null)
{
Guid eventId = Guid.NewGuid();
@@ -1547,6 +1636,8 @@ public class SeekerCommandMonitorTests : IAsyncDisposable
CycleId = cycleId ?? Guid.NewGuid(),
ItemTitle = "Test Item",
ConsecutiveMisses = consecutiveMisses,
LastSearchedAt = lastSearchedAt,
LastMissAt = lastMissAt,
});
return eventId;
@@ -3108,7 +3108,7 @@ public class SeekerTests : IDisposable
newCycle.FunnelCountedAt.ShouldBe(_fixture.TimeProvider.GetUtcNow());
}
private void AddRestingMovieHistory(ArrInstance instance, long movieId, Guid cycleId)
private void AddRestingMovieHistory(ArrInstance instance, long movieId, Guid cycleId, DateTimeOffset? lastMissAt = null)
{
_fixture.EventsContext.SeekerHistory.Add(new SeekerHistory
{
@@ -3119,6 +3119,7 @@ public class SeekerTests : IDisposable
LastSearchedAt = _fixture.TimeProvider.GetUtcNow().AddDays(-2),
ItemTitle = $"Movie {movieId}",
ConsecutiveMisses = 3,
LastMissAt = lastMissAt,
});
}
@@ -3140,6 +3141,24 @@ public class SeekerTests : IDisposable
cycle.QueuedUnits.ShouldBe(0);
}
[Fact]
public async Task ExecuteAsync_Radarr_RetryFailedAfterTheRestEnded_SearchesTheMovieAgain()
{
// Arrange: failed retry two days ago, last miss past RestMaxDays
(ArrInstance radarrInstance, IArrClient arrClient) = await SetupFunnelInstanceAsync(InstanceType.Radarr, c => c.RestUnitsEnabled = true);
AddRestingMovieHistory(radarrInstance, 1, Guid.NewGuid(), _fixture.TimeProvider.GetUtcNow().AddDays(-40));
await _fixture.EventsContext.SaveChangesAsync();
StubFunnelMovies(radarrInstance, [ReleasedMovie(1)]);
// Act
await CreateSut().ExecuteAsync();
// Assert
await arrClient.Received(1).SearchItemAsync(radarrInstance, Arg.Any<SearchItem>());
SeekerCycle cycle = await GetCycleAsync(radarrInstance.Id);
cycle.RestingUnits.ShouldBe(0);
}
[Fact]
public async Task ExecuteAsync_Radarr_EveryUnsearchedUnitRests_TriggersNoSearch()
{
@@ -1585,6 +1585,36 @@ public class SeekerUnitEvaluatorTests : IDisposable
rest.Until.ShouldBe(unit.LastSearchedAt.AddDays(30));
}
[Fact]
public void GetRest_LastMissAtSet_AnchorsTheRestOnTheMiss()
{
// Arrange
SeekerHistory unit = Missed(3, RestNow.AddDays(-1));
unit.LastMissAt = RestNow.AddDays(-5);
// Act
SeekerUnitRest? rest = SeekerUnitEvaluator.GetRest(unit, [RestNow.AddDays(-3)], 30, RestNow);
// Assert
rest.ShouldNotBeNull();
rest.CyclesLeft.ShouldBe(1);
rest.Until.ShouldBe(unit.LastMissAt.Value.AddDays(30));
}
[Fact]
public void GetRest_LastMissAtPastRestMaxDays_ReturnsNull()
{
// Arrange
SeekerHistory unit = Missed(3, RestNow.AddDays(-1));
unit.LastMissAt = RestNow.AddDays(-31);
// Act
SeekerUnitRest? rest = SeekerUnitEvaluator.GetRest(unit, [], 30, RestNow);
// Assert
rest.ShouldBeNull();
}
[Fact]
public void GetRest_ThreeMissesAndOneCycleSince_RestsForOneMoreCycleEnd()
{
@@ -404,6 +404,7 @@ public class SeekerCommandMonitor : BackgroundService
if (outcome is SeekerSearchOutcome.NothingGrabbed)
{
unit.ConsecutiveMisses++;
unit.LastMissAt = unit.LastSearchedAt;
}
}
@@ -416,6 +417,7 @@ public class SeekerCommandMonitor : BackgroundService
{
unit.Grabs++;
unit.ConsecutiveMisses = 0;
unit.LastMissAt = null;
}
SeekerCycle? cycle = await eventsContext.SeekerCycles.FindAsync(cycleId.Value);
@@ -128,14 +128,15 @@ public sealed class SeekerUnitEvaluator
return null;
}
DateTimeOffset until = unit.LastSearchedAt.AddDays(maxRestDays);
DateTimeOffset anchor = unit.LastMissAt ?? unit.LastSearchedAt;
DateTimeOffset until = anchor.AddDays(maxRestDays);
if (now >= until)
{
return null;
}
int restCycles = (int)Math.Min(Math.Pow(2, unit.ConsecutiveMisses - MissesBeforeRest), MaxRestCycles);
int cyclesSince = cycleStarts.Count(s => s > unit.LastSearchedAt);
int cyclesSince = cycleStarts.Count(s => s > anchor);
if (cyclesSince > restCycles)
{
return null;
@@ -165,7 +166,7 @@ public sealed class SeekerUnitEvaluator
.Where(h => h.ArrInstanceId == arrInstance.Id
&& !h.IsDryRun
&& h.ConsecutiveMisses >= MissesBeforeRest
&& h.LastSearchedAt > restCap)
&& (h.LastMissAt ?? h.LastSearchedAt) > restCap)
.ToListAsync(cancellationToken);
if (missed.Count == 0)
@@ -173,7 +174,7 @@ public sealed class SeekerUnitEvaluator
return [];
}
DateTimeOffset earliestSearch = missed.Min(h => h.LastSearchedAt);
DateTimeOffset earliestSearch = missed.Min(h => h.LastMissAt ?? h.LastSearchedAt);
List<DateTimeOffset> cycleStarts = await _eventsContext.SeekerCycles
.AsNoTracking()
.Where(c => c.ArrInstanceId == arrInstance.Id && !c.IsDryRun && c.StartedAt > earliestSearch)
@@ -13,7 +13,7 @@ using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
namespace Cleanuparr.Persistence.Postgres.Migrations.Events
{
[DbContext(typeof(EventsContext))]
[Migration("20261022235247_AddSeekerUnitRest")]
[Migration("20261023081626_AddSeekerUnitRest")]
partial class AddSeekerUnitRest
{
/// <inheritdoc />
@@ -752,6 +752,10 @@ namespace Cleanuparr.Persistence.Postgres.Migrations.Events
.HasColumnType("text")
.HasColumnName("item_type");
b.Property<DateTimeOffset?>("LastMissAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_miss_at");
b.Property<string>("LastOutcome")
.HasColumnType("text")
.HasColumnName("last_outcome");
@@ -1,4 +1,5 @@
using Microsoft.EntityFrameworkCore.Migrations;
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
@@ -18,6 +19,13 @@ namespace Cleanuparr.Persistence.Postgres.Migrations.Events
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<DateTimeOffset>(
name: "last_miss_at",
schema: "events",
table: "seeker_history",
type: "timestamp with time zone",
nullable: true);
migrationBuilder.AddColumn<int>(
name: "resting_units",
schema: "events",
@@ -35,6 +43,11 @@ namespace Cleanuparr.Persistence.Postgres.Migrations.Events
schema: "events",
table: "seeker_history");
migrationBuilder.DropColumn(
name: "last_miss_at",
schema: "events",
table: "seeker_history");
migrationBuilder.DropColumn(
name: "resting_units",
schema: "events",
@@ -749,6 +749,10 @@ namespace Cleanuparr.Persistence.Postgres.Migrations.Events
.HasColumnType("text")
.HasColumnName("item_type");
b.Property<DateTimeOffset?>("LastMissAt")
.HasColumnType("timestamp with time zone")
.HasColumnName("last_miss_at");
b.Property<string>("LastOutcome")
.HasColumnType("text")
.HasColumnName("last_outcome");
@@ -11,7 +11,7 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace Cleanuparr.Persistence.Migrations.Events
{
[DbContext(typeof(EventsContext))]
[Migration("20261022235242_AddSeekerUnitRest")]
[Migration("20261023081621_AddSeekerUnitRest")]
partial class AddSeekerUnitRest
{
/// <inheritdoc />
@@ -753,6 +753,10 @@ namespace Cleanuparr.Persistence.Migrations.Events
.HasColumnType("TEXT")
.HasColumnName("item_type");
b.Property<string>("LastMissAt")
.HasColumnType("TEXT")
.HasColumnName("last_miss_at");
b.Property<string>("LastOutcome")
.HasColumnType("TEXT")
.HasColumnName("last_outcome");
@@ -17,6 +17,12 @@ namespace Cleanuparr.Persistence.Migrations.Events
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<string>(
name: "last_miss_at",
table: "seeker_history",
type: "TEXT",
nullable: true);
migrationBuilder.AddColumn<int>(
name: "resting_units",
table: "seeker_cycles",
@@ -32,6 +38,10 @@ namespace Cleanuparr.Persistence.Migrations.Events
name: "consecutive_misses",
table: "seeker_history");
migrationBuilder.DropColumn(
name: "last_miss_at",
table: "seeker_history");
migrationBuilder.DropColumn(
name: "resting_units",
table: "seeker_cycles");
@@ -750,6 +750,10 @@ namespace Cleanuparr.Persistence.Migrations.Events
.HasColumnType("TEXT")
.HasColumnName("item_type");
b.Property<string>("LastMissAt")
.HasColumnType("TEXT")
.HasColumnName("last_miss_at");
b.Property<string>("LastOutcome")
.HasColumnType("TEXT")
.HasColumnName("last_outcome");
@@ -75,6 +75,11 @@ public sealed record SeekerHistory
/// </summary>
public int ConsecutiveMisses { get; set; }
/// <summary>
/// When the last search that grabbed nothing ran. A grab clears it.
/// </summary>
public DateTimeOffset? LastMissAt { get; set; }
/// <summary>
/// Result of the last search. Null until that search finishes.
/// </summary>