using Cleanuparr.Domain.Entities; using Cleanuparr.Domain.Entities.Arr.Queue; using Cleanuparr.Domain.Enums; using Cleanuparr.Infrastructure.Events.Interfaces; using Cleanuparr.Infrastructure.Features.Arr.Interfaces; using Cleanuparr.Infrastructure.Features.Context; using Cleanuparr.Infrastructure.Features.DownloadClient; using Cleanuparr.Infrastructure.Features.Files; using Cleanuparr.Infrastructure.Helpers; using Cleanuparr.Persistence; using Cleanuparr.Persistence.Models.Configuration.Arr; using Cleanuparr.Persistence.Models.Configuration.DownloadCleaner; using Cleanuparr.Persistence.Models.Configuration.General; using MassTransit; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using LogContext = Serilog.Context.LogContext; namespace Cleanuparr.Infrastructure.Features.Jobs; public sealed class DownloadCleaner : GenericHandler { private readonly HashSet _downloadsProcessedByArrs = []; private readonly TimeProvider _timeProvider; private readonly IHardLinkFileService _hardLinkFileService; public DownloadCleaner( ILogger logger, DataContext dataContext, IMemoryCache cache, IBus messageBus, IArrClientFactory arrClientFactory, IArrQueueIterator arrArrQueueIterator, IDownloadServiceFactory downloadServiceFactory, IEventPublisher eventPublisher, TimeProvider timeProvider, IHardLinkFileService hardLinkFileService ) : base( logger, dataContext, cache, messageBus, arrClientFactory, arrArrQueueIterator, downloadServiceFactory, eventPublisher ) { _timeProvider = timeProvider; _hardLinkFileService = hardLinkFileService; } protected override async Task ExecuteInternalAsync(CancellationToken cancellationToken = default) { var downloadServices = await GetInitializedDownloadServicesAsync(); if (downloadServices.Count is 0) { _logger.LogWarning("Processing skipped because no download clients are configured"); return; } var config = ContextProvider.Get(); bool isUnlinkedEnabled = config.UnlinkedEnabled && !string.IsNullOrEmpty(config.UnlinkedTargetCategory) && config.UnlinkedCategories.Count > 0; bool isCleaningEnabled = config.Categories.Count > 0; if (!isUnlinkedEnabled && !isCleaningEnabled) { _logger.LogWarning("No features are enabled for {name}", nameof(DownloadCleaner)); return; } List ignoredDownloads = ContextProvider.Get(nameof(GeneralConfig)).IgnoredDownloads; ignoredDownloads.AddRange(ContextProvider.Get().IgnoredDownloads); var downloadServiceToDownloadsMap = new Dictionary>(); foreach (var downloadService in downloadServices) { using var dcType = LogContext.PushProperty(LogProperties.DownloadClientType, downloadService.ClientConfig.Type.ToString()); using var dcName = LogContext.PushProperty(LogProperties.DownloadClientName, downloadService.ClientConfig.Name); try { await downloadService.LoginAsync(); List clientDownloads = await downloadService.GetSeedingDownloads(); if (clientDownloads.Count > 0) { downloadServiceToDownloadsMap[downloadService] = clientDownloads; } } catch (Exception ex) { _logger.LogError(ex, "Failed to get seeding downloads from download client {clientName}", downloadService.ClientConfig.Name); } } if (downloadServiceToDownloadsMap.Count is 0) { _logger.LogInformation("No seeding downloads found"); return; } int totalDownloads = downloadServiceToDownloadsMap.Values.Sum(x => x.Count); _logger.LogTrace("Found {count} seeding downloads across {clientCount} clients", totalDownloads, downloadServiceToDownloadsMap.Count); // wait for the downloads to appear in the arr queue await Task.Delay(TimeSpan.FromSeconds(10), _timeProvider, cancellationToken); await ProcessArrConfigAsync(ContextProvider.Get(nameof(InstanceType.Sonarr)), true); await ProcessArrConfigAsync(ContextProvider.Get(nameof(InstanceType.Radarr)), true); await ProcessArrConfigAsync(ContextProvider.Get(nameof(InstanceType.Lidarr)), true); await ProcessArrConfigAsync(ContextProvider.Get(nameof(InstanceType.Readarr)), true); await ProcessArrConfigAsync(ContextProvider.Get(nameof(InstanceType.Whisparr)), true); foreach (var pair in downloadServiceToDownloadsMap) { List filteredDownloads = []; foreach (ITorrentItemWrapper download in pair.Value) { if (download.IsIgnored(ignoredDownloads)) { _logger.LogDebug("skip | download is ignored | {name}", download.Name); continue; } if (_downloadsProcessedByArrs.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase))) { _logger.LogDebug("skip | download is used by an arr | {name}", download.Name); continue; } filteredDownloads.Add(download); } downloadServiceToDownloadsMap[pair.Key] = filteredDownloads; } await ChangeUnlinkedCategoriesAsync(isUnlinkedEnabled, downloadServiceToDownloadsMap, config); await CleanDownloadsAsync(downloadServiceToDownloadsMap, config); foreach (var downloadService in downloadServices) { downloadService.Dispose(); } } protected override async Task ProcessInstanceAsync(ArrInstance instance) { using var _ = LogContext.PushProperty(LogProperties.Category, instance.ArrConfig.Type.ToString()); using var _2 = LogContext.PushProperty(LogProperties.InstanceName, instance.Name); IArrClient arrClient = _arrClientFactory.GetClient(instance.ArrConfig.Type, instance.Version); await _arrArrQueueIterator.Iterate(arrClient, instance, async items => { var groups = items .Where(x => !string.IsNullOrEmpty(x.DownloadId)) .GroupBy(x => x.DownloadId) .ToList(); foreach (QueueRecord record in groups.Select(group => group.First())) { _downloadsProcessedByArrs.Add(record.DownloadId.ToLowerInvariant()); } }); } private async Task ChangeUnlinkedCategoriesAsync(bool isUnlinkedEnabled, Dictionary> downloadServiceToDownloadsMap, DownloadCleanerConfig config) { if (!isUnlinkedEnabled) { return; } if (config.UnlinkedIgnoredRootDirs.Count > 0) { _hardLinkFileService.PopulateFileCounts(config.UnlinkedIgnoredRootDirs); } Dictionary> downloadServiceWithDownloads = []; foreach (var (downloadService, clientDownloads) in downloadServiceToDownloadsMap) { try { var downloadsToChangeCategory = downloadService .FilterDownloadsToChangeCategoryAsync(clientDownloads, config.UnlinkedCategories); if (downloadsToChangeCategory?.Count > 0) { downloadServiceWithDownloads.Add(downloadService, downloadsToChangeCategory); } } catch (Exception ex) { _logger.LogError( ex, "Failed to filter downloads for hardlinks evaluation for download client {clientName}", downloadService.ClientConfig.Name ); } } if (downloadServiceWithDownloads.Count is 0) { _logger.LogInformation("No downloads found to evaluate for hardlinks"); return; } _logger.LogInformation( "Evaluating {count} downloads for hardlinks", downloadServiceWithDownloads.Sum(x => x.Value.Count) ); // Process each client with its own filtered downloads foreach (var (downloadService, downloadsToChangeCategory) in downloadServiceWithDownloads) { using var dcType = LogContext.PushProperty(LogProperties.DownloadClientType, downloadService.ClientConfig.Type.ToString()); using var dcName = LogContext.PushProperty(LogProperties.DownloadClientName, downloadService.ClientConfig.Name); try { await downloadService.CreateCategoryAsync(config.UnlinkedTargetCategory); } catch (Exception ex) { _logger.LogError( ex, "Failed to create category {category} for download client {clientName}", config.UnlinkedTargetCategory, downloadService.ClientConfig.Name ); } try { await downloadService.ChangeCategoryForNoHardLinksAsync(downloadsToChangeCategory); } catch (Exception ex) { _logger.LogError(ex, "Failed to change category for download client {clientName}", downloadService.ClientConfig.Name); } } _logger.LogInformation("Finished hardlinks evaluation"); } private async Task CleanDownloadsAsync(Dictionary> downloadServiceToDownloadsMap, DownloadCleanerConfig config) { if (config.Categories.Count is 0) { return; } Dictionary> downloadServiceWithDownloads = []; foreach (var (downloadService, clientDownloads) in downloadServiceToDownloadsMap) { try { var downloadsToClean = downloadService .FilterDownloadsToBeCleanedAsync(clientDownloads, config.Categories); if (downloadsToClean?.Count > 0) { downloadServiceWithDownloads.Add(downloadService, downloadsToClean); } } catch (Exception ex) { _logger.LogError( ex, "Failed to filter downloads for cleaning for download client {clientName}", downloadService.ClientConfig.Name ); } } _logger.LogInformation( "Evaluating {count} downloads for cleanup", downloadServiceWithDownloads.Sum(x => x.Value.Count) ); // Process cleaning for each client foreach (var (downloadService, downloadsToClean) in downloadServiceWithDownloads) { using var dcType = LogContext.PushProperty(LogProperties.DownloadClientType, downloadService.ClientConfig.Type.ToString()); using var dcName = LogContext.PushProperty(LogProperties.DownloadClientName, downloadService.ClientConfig.Name); try { await downloadService.CleanDownloadsAsync(downloadsToClean, config.Categories); } catch (Exception ex) { _logger.LogError(ex, "Failed to clean downloads for download client {clientName}", downloadService.ClientConfig.Name); } } _logger.LogInformation("Finished cleanup evaluation"); } }