Cleanuparr/code/backend/Cleanuparr.Infrastructure/Features/Arr/ArrClient.cs

using System.Net;
using System.Text;
using Cleanuparr.Domain.Entities.Arr;
using Cleanuparr.Domain.Entities.Arr.History;
using Cleanuparr.Domain.Entities.Arr.ManualImport;
using Cleanuparr.Domain.Entities.Arr.Queue;
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Features.Arr.Interfaces;
using Cleanuparr.Infrastructure.Features.Context;
using Cleanuparr.Infrastructure.Features.ItemStriker;
using Cleanuparr.Infrastructure.Interceptors;
using Cleanuparr.Persistence.Models.Configuration.Arr;
using Cleanuparr.Persistence.Models.Configuration.QueueCleaner;
using Cleanuparr.Shared.Helpers;
using Microsoft.Extensions.Logging;
using System.Text.Json;
using System.Text.Json.Serialization;
using Cleanuparr.Infrastructure.Json;

namespace Cleanuparr.Infrastructure.Features.Arr;

public abstract class ArrClient : IArrClient
{
    /// <summary>
    /// What every arr calls importing a file it downloaded, as its history event id.
    /// </summary>
    private const int DownloadFolderImportedEvent = 3;

    /// <summary>
    /// What every arr calls grabbing a release, as its history event id.
    /// </summary>
    private const int GrabbedEvent = 1;

    /// <summary>
    /// Page size used when paging through an *arr instance's queue.
    /// </summary>
    protected const int QueuePageSize = 200;

    /// <summary>
    /// Command API path for this app's API version.
    /// </summary>
    protected string CommandUrlPath => $"/api/{ApiVersion}/command";

    /// <summary>
    /// REST API version segment for this app, e.g. "v3".
    /// </summary>
    protected abstract string ApiVersion { get; }

    protected readonly ILogger<ArrClient> _logger;
    protected readonly HttpClient _httpClient;
    protected readonly IStriker _striker;
    protected readonly IDryRunInterceptor _dryRunInterceptor;
    
    protected ArrClient(
        ILogger<ArrClient> logger,
        IHttpClientFactory httpClientFactory,
        IStriker striker,
        IDryRunInterceptor dryRunInterceptor
    )
    {
        _logger = logger;
        _httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
        _striker = striker;
        _dryRunInterceptor = dryRunInterceptor;
    }

    public virtual async Task<QueueListResponse> GetQueueItemsAsync(ArrInstance arrInstance, int page)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueUrlPath().TrimStart('/')}";
        uriBuilder.Query = GetQueueUrlQuery(page);

        using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
        SetApiKey(request, arrInstance.ApiKey);

        using HttpResponseMessage response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);

        try
        {
            response.EnsureSuccessStatusCode();
        }
        catch
        {
            _logger.LogError("queue list failed | {uri}", uriBuilder.Uri);
            throw;
        }

        QueueListResponse? queueResponse = await DeserializeStreamAsync<QueueListResponse>(response);

        if (queueResponse is null)
        {
            throw new Exception($"unrecognized queue list response | {uriBuilder.Uri}");
        }

        return queueResponse;
    }

    public async Task<int> GetActiveDownloadCountAsync(ArrInstance arrInstance)
    {
        int count = 0;
        int page = 1;
        int processed = 0;

        while (true)
        {
            QueueListResponse response = await GetQueueItemsAsync(arrInstance, page);

            if (response.Records.Count == 0)
            {
                break;
            }

            count += response.Records.Count(r => r.SizeLeft > 0);
            processed += response.Records.Count;

            if (processed >= response.TotalRecords)
            {
                break;
            }

            page++;
        }

        return count;
    }

    public virtual async Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload, short arrMaxStrikes)
    {
        var queueCleanerConfig = ContextProvider.Get<QueueCleanerConfig>();
        
        if (queueCleanerConfig.FailedImport.IgnorePrivate && isPrivateDownload)
        {
            // ignore private trackers
            _logger.LogDebug("skip failed import check | download is private | {name}", record.Title);
            return false;
        }
        
        bool HasWarn() => record.TrackedDownloadStatus
            .Equals("warning", StringComparison.InvariantCultureIgnoreCase);
        bool IsImportBlocked() => record.TrackedDownloadState
            .Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase);
        bool IsImportPending() => record.TrackedDownloadState
            .Equals("importPending", StringComparison.InvariantCultureIgnoreCase);
        bool IsImportFailed() => record.TrackedDownloadState
            .Equals("importFailed", StringComparison.InvariantCultureIgnoreCase);
        bool IsFailedLidarr() => instanceType is InstanceType.Lidarr &&
                                 (record.Status.Equals("failed", StringComparison.InvariantCultureIgnoreCase) ||
                                  record.Status.Equals("completed", StringComparison.InvariantCultureIgnoreCase)) &&
                                 HasWarn();
        bool IsDownloading() => record.TrackedDownloadState
            .Equals("downloading", StringComparison.InvariantCultureIgnoreCase);
        bool HasFailedImportMessage() => record.StatusMessages
            ?.Any(status => status.Messages
                ?.Any(message => message.StartsWith("Unable to import automatically", StringComparison.InvariantCultureIgnoreCase)) is true
            ) is true;
        bool IsEdgeCase() => IsDownloading() && HasFailedImportMessage();
            
        
        if (HasWarn() && (IsImportBlocked() || IsImportPending() || IsImportFailed()) || IsFailedLidarr() || IsEdgeCase())
        {
            if (!ShouldStrikeFailedImport(queueCleanerConfig, record))
            {
                return false;
            }

            if (arrMaxStrikes is 0)
            {
                _logger.LogDebug("skip failed import check | arr max strikes is 0 | {name}", record.Title);
                return false;
            }
            
            ushort maxStrikes = arrMaxStrikes > 0 ? (ushort)arrMaxStrikes : queueCleanerConfig.FailedImport.MaxStrikes;
            
            _logger.LogInformation(
                "Item {title} has failed import status with the following reason(s):\n{messages}",
                record.Title,
                string.Join("\n",  record.StatusMessages?.Select(m => JsonSerializer.Serialize(m, CleanuparrJsonOptions.Outbound)) ?? [])
            );
            
            return await _striker.StrikeAndCheckLimit(
                record.DownloadId,
                record.Title,
                maxStrikes,
                StrikeType.FailedImport
            );
        }
        
        _logger.LogDebug("skip | not a failed import | {name}", record.Title);

        return false;
    }
    
    /// <inheritdoc/>
    public virtual bool SupportsForceImport => false;

    /// <inheritdoc/>
    public async Task<List<ManualImportCandidate>> GetManualImportCandidatesAsync(ArrInstance arrInstance, string downloadId)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/manualimport";
        // The arr matches the download id exactly, so the record's own casing goes back unchanged.
        uriBuilder.Query = $"downloadId={Uri.EscapeDataString(downloadId)}&filterExistingFiles=true";

        using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
        SetApiKey(request, arrInstance.ApiKey);

        using HttpResponseMessage response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);

        try
        {
            response.EnsureSuccessStatusCode();
        }
        catch
        {
            _logger.LogError("manual import candidates failed | {Uri}", uriBuilder.Uri);
            throw;
        }

        return await DeserializeStreamAsync<List<ManualImportCandidate>>(response) ?? [];
    }

    /// <inheritdoc/>
    public async Task<bool> ForceImportAsync(ArrInstance arrInstance, List<ManualImportFile> files)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/command";

        ManualImportCommand command = new() { Files = files };

        using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri);
        request.Content = new StringContent(
            JsonSerializer.Serialize(command, CleanuparrJsonOptions.Outbound),
            Encoding.UTF8,
            "application/json"
        );
        SetApiKey(request, arrInstance.ApiKey);

        try
        {
            HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync(() => SendRequestAsync(request));
            response?.Dispose();
            return response is not null;
        }
        catch
        {
            _logger.LogError("force import failed | {Uri}", uriBuilder.Uri);
            throw;
        }
    }

    /// <inheritdoc/>
    public async Task<int> GetImportedCountAsync(ArrInstance arrInstance, string downloadId)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/history";
        // The arr counts the rows it matched, so one download's imports come back as a number.
        uriBuilder.Query = $"downloadId={Uri.EscapeDataString(downloadId)}&eventType={DownloadFolderImportedEvent}&page=1&pageSize=1";

        using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
        SetApiKey(request, arrInstance.ApiKey);

        using HttpResponseMessage response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);

        try
        {
            response.EnsureSuccessStatusCode();
        }
        catch
        {
            _logger.LogError("history failed | {Uri}", uriBuilder.Uri);
            throw;
        }

        ArrHistoryResponse? history = await DeserializeStreamAsync<ArrHistoryResponse>(response);

        return history?.TotalRecords ?? 0;
    }

    /// <summary>
    /// Reads the grabbed releases an arr recorded in its history since a given time.
    /// </summary>
    /// <param name="arrInstance">The arr instance to query.</param>
    /// <param name="since">Only rows at or after this time are returned.</param>
    /// <param name="extraQuery">Additional query parameters, appended as-is (e.g. "&amp;includeEpisode=true").</param>
    protected async Task<List<ArrHistoryRecord>> GetGrabHistorySinceAsync(ArrInstance arrInstance, DateTimeOffset since, string extraQuery)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/{ApiVersion}/history/since";
        string date = Uri.EscapeDataString(since.UtcDateTime.ToString("o"));
        uriBuilder.Query = $"date={date}&eventType={GrabbedEvent}{extraQuery}";

        using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
        SetApiKey(request, arrInstance.ApiKey);

        using HttpResponseMessage response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);

        try
        {
            response.EnsureSuccessStatusCode();
        }
        catch
        {
            _logger.LogError("grab history failed | {Uri}", uriBuilder.Uri);
            throw;
        }

        return await DeserializeStreamAsync<List<ArrHistoryRecord>>(response) ?? [];
    }

    /// <inheritdoc/>
    public virtual ManualImportFile? MapCandidate(QueueRecord record, ManualImportCandidate candidate) => null;

    public virtual async Task<bool> DeleteQueueItemAsync(
        ArrInstance arrInstance,
        QueueRecord record,
        bool removeFromClient,
        bool changeCategory,
        DeleteReason deleteReason
    )
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueDeleteUrlPath(record.Id).TrimStart('/')}";
        uriBuilder.Query = GetQueueDeleteUrlQuery(removeFromClient, changeCategory);

        try
        {
            using HttpRequestMessage request = new(HttpMethod.Delete, uriBuilder.Uri);
            SetApiKey(request, arrInstance.ApiKey);

            HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync(() => SendRequestAsync(request));
            response?.Dispose();

            // A null response means the interceptor skipped the request.
            string prefix = response is null ? "[DRY RUN] " : string.Empty;

            string logMessage;
            if (changeCategory)
            {
                logMessage = prefix + "queue item category changed in arr with reason {reason} | {url} | {title}";
            }
            else if (removeFromClient)
            {
                logMessage = prefix + "queue item deleted with reason {reason} | {url} | {title}";
            }
            else
            {
                logMessage = prefix + "queue item removed from arr with reason {reason} | {url} | {title}";
            }

            _logger.LogInformation(
                logMessage,
                deleteReason.ToString(),
                arrInstance.Url,
                record.Title
            );

            return response is not null;
        }
        catch
        {
            _logger.LogError("queue delete failed | {uri} | {title}", uriBuilder.Uri, record.Title);
            throw;
        }
    }

    public abstract Task<List<long>> SearchItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items);

    public virtual async Task<long> SearchItemAsync(ArrInstance arrInstance, SearchItem item)
    {
        List<long> ids = await SearchItemsAsync(arrInstance, [item]);

        if (await _dryRunInterceptor.IsDryRunEnabled())
        {
            return ids.FirstOrDefault();
        }

        return ids.First();
    }

    public bool IsRecordValid(QueueRecord record)
    {
        if (string.IsNullOrEmpty(record.DownloadId))
        {
            _logger.LogDebug("skip | download id is null for {title}", record.Title);
            return false;
        }

        return true;
    }

    public abstract bool HasContentId(QueueRecord record);

    /// <inheritdoc/>
    public virtual async Task HealthCheckAsync(ArrInstance arrInstance)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}{GetSystemStatusUrlPath()}";

        using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
        SetApiKey(request, arrInstance.ApiKey);
        
        using HttpResponseMessage response = await _httpClient.SendAsync(request);
        
        response.EnsureSuccessStatusCode();
        
        _logger.LogDebug("Connection test successful for {url}", arrInstance.Url);
    }

    /// <inheritdoc/>
    public virtual async Task<ArrCommandStatus> GetCommandStatusAsync(ArrInstance arrInstance, long commandId)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}{CommandUrlPath}/{commandId}";

        using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
        SetApiKey(request, arrInstance.ApiKey);

        using HttpResponseMessage response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
        response.EnsureSuccessStatusCode();

        var result = await DeserializeStreamAsync<ArrCommandStatus>(response);

        return result ?? new ArrCommandStatus(commandId, ArrCommandState.Unknown, null);
    }

    /// <inheritdoc/>
    public async Task<List<ArrCommandStatus>> GetCommandsAsync(ArrInstance arrInstance)
    {
        UriBuilder uriBuilder = new(arrInstance.Url);
        uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}{CommandUrlPath}";

        using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
        SetApiKey(request, arrInstance.ApiKey);

        using HttpResponseMessage response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
        response.EnsureSuccessStatusCode();

        List<ArrCommandStatus>? result = await DeserializeStreamAsync<List<ArrCommandStatus>>(response);

        return result ?? [];
    }

    protected abstract string GetSystemStatusUrlPath();
    
    protected abstract string GetQueueUrlPath();

    protected abstract string GetQueueUrlQuery(int page);

    protected abstract string GetQueueDeleteUrlPath(long recordId);
    
    protected virtual string GetQueueDeleteUrlQuery(bool removeFromClient, bool changeCategory)
    {
        string query = "blocklist=true&skipRedownload=true&";

        if (changeCategory)
        {
            query += "changeCategory=true&removeFromClient=false";
            return query;
        }

        query += "changeCategory=false";
        query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";

        return query;
    }
    
    protected virtual void SetApiKey(HttpRequestMessage request, string apiKey)
    {
        request.Headers.Add("x-api-key", apiKey);
    }

    protected virtual async Task<HttpResponseMessage> SendRequestAsync(HttpRequestMessage request)
    {
        HttpResponseMessage response = await _httpClient.SendAsync(request);
        
        response.EnsureSuccessStatusCode();
        
        return response;
    }
    
    /// <summary>
    /// Reads the body of a response from an arr.
    /// </summary>
    /// <remarks>
    /// An arr answers some requests with an empty body. An empty body gives null,
    /// because each caller has a value for a null result.
    /// </remarks>
    protected static async Task<T?> DeserializeStreamAsync<T>(HttpResponseMessage response, CancellationToken cancellationToken = default)
    {
        if (response.StatusCode is HttpStatusCode.NoContent || response.Content.Headers.ContentLength is 0)
        {
            return default;
        }

        await using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken);
        return await JsonSerializer.DeserializeAsync<T>(stream, CleanuparrJsonOptions.ExternalApiRead, cancellationToken);
    }

    protected static async Task<long?> ReadCommandIdAsync(HttpResponseMessage response)
    {
        CommandIdResponse? result = await DeserializeStreamAsync<CommandIdResponse>(response);
        return result?.Id;
    }

    private sealed class CommandIdResponse
    {
        [JsonPropertyName("id")]
        public long? Id { get; init; }
    }

    /// <summary>
    /// Determines whether the failed import record should be skipped
    /// </summary>
    private bool ShouldStrikeFailedImport(QueueCleanerConfig queueCleanerConfig, QueueRecord record)
    {
        if (record.StatusMessages?.Count is null or 0)
        {
            _logger.LogWarning("skip failed import check | no status message found | {name}", record.Title);
            return false;
        }
        
        HashSet<string> messages = record.StatusMessages
            .SelectMany(x => x.Messages ?? Enumerable.Empty<string>())
            .ToHashSet();
        record.StatusMessages.Select(x => x.Title)
            .ToList()
            .ForEach(x => messages.Add(x));
        
        var patterns = queueCleanerConfig.FailedImport.Patterns;
        var patternMode = queueCleanerConfig.FailedImport.PatternMode;
        
        var matched = messages.Any(
            m => patterns.Any(
                p => !string.IsNullOrWhiteSpace(p?.Trim()) && m.Contains(p, StringComparison.InvariantCultureIgnoreCase)
            )
        );

        if (patternMode is PatternMode.Exclude && matched)
        {
            // contains an excluded/ignored pattern -> skip
            _logger.LogTrace("skip failed import check | excluded pattern matched | {name}", record.Title);
            return false;
        }

        if (patternMode is PatternMode.Include && (!matched || patterns.Count is 0))
        {
            // does not match any included patterns -> skip
            _logger.LogTrace("skip failed import check | no included pattern matched | {name}", record.Title);
            return false;
        }
        
        return true;
    }

    public abstract Task<List<Tag>> GetAllTagsAsync(ArrInstance arrInstance);
}