Add option to send the User-Agent header on all HTTP requests (#750)

This commit is contained in:
Flaminel authored and GitHub committed 2026-08-22 22:18:49 +03:00
1 parent 8f8f30c37a
commit 41dbbf155b
38 files changed
+4878 -67

No files matched your search

@@ -69,6 +69,7 @@ public class GeneralConfigControllerTests : IDisposable
DryRun = false,
HttpMaxRetries = 5,
HttpTimeout = 60,
HttpSendUserAgent = true,
StatusCheckEnabled = false,
EncryptionKey = existing.EncryptionKey,
IgnoredDownloads = new List<string> { "ignored-item" },
@@ -88,6 +89,7 @@ public class GeneralConfigControllerTests : IDisposable
saved.DisplaySupportBanner.ShouldBeFalse();
saved.HttpMaxRetries.ShouldBe((ushort)5);
saved.HttpTimeout.ShouldBe((ushort)60);
saved.HttpSendUserAgent.ShouldBeTrue();
saved.StrikeInactivityWindowHours.ShouldBe((ushort)48);
saved.IgnoredDownloads.ShouldContain("ignored-item");
}
@@ -75,12 +75,6 @@ public static class MainDI
// Add the dynamic HTTP client provider that uses the new system
services.AddSingleton<IDynamicHttpClientProvider, DynamicHttpClientProvider>();
// Add HTTP client for Plex authentication
services.AddHttpClient(Constants.HttpClientPlexAuthName);
// Add HTTP client for OIDC authentication
services.AddHttpClient(Constants.HttpClientOidcAuthName);
return services;
}
@@ -17,6 +17,8 @@ public sealed record UpdateGeneralConfigRequest
public CertificateValidationType HttpCertificateValidation { get; init; } = CertificateValidationType.Enabled;
public bool HttpSendUserAgent { get; init; }
public bool StatusCheckEnabled { get; init; } = true;
public string EncryptionKey { get; init; } = Guid.NewGuid().ToString();
@@ -42,6 +44,7 @@ public sealed record UpdateGeneralConfigRequest
existingConfig.HttpMaxRetries = HttpMaxRetries;
existingConfig.HttpTimeout = HttpTimeout;
existingConfig.HttpCertificateValidation = HttpCertificateValidation;
existingConfig.HttpSendUserAgent = HttpSendUserAgent;
existingConfig.StatusCheckEnabled = StatusCheckEnabled;
existingConfig.EncryptionKey = EncryptionKey;
existingConfig.IgnoredDownloads = IgnoredDownloads;
@@ -1,5 +1,6 @@
using System.Net;
using Cleanuparr.Infrastructure.Http.DynamicHttpClientSystem;
using Cleanuparr.Shared.Helpers;
using Shouldly;
using Xunit;
@@ -40,4 +41,14 @@ public sealed class DynamicHttpClientConfigurationTests
DynamicHttpClientConfiguration.IsRetryable(response, excludeUnauthorized: true).ShouldBeTrue();
}
[Fact]
public void ApplyUserAgent_ShouldSetTheProductToken()
{
using HttpClient client = new();
DynamicHttpClientConfiguration.ApplyUserAgent(client);
client.DefaultRequestHeaders.UserAgent.ToString().ShouldBe(AppUserAgent.Value);
}
}
@@ -0,0 +1,127 @@
using System.Net;
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Http.DynamicHttpClientSystem;
using Cleanuparr.Persistence.Models.Configuration.General;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using Shouldly;
using Xunit;
namespace Cleanuparr.Infrastructure.Tests.Http;
/// <summary>
/// Covers the configurations the factory stores, without going through the HTTP pipeline.
/// </summary>
public sealed class DynamicHttpClientFactoryTests
{
private readonly IHttpClientConfigStore _store = new HttpClientConfigStore();
private readonly DynamicHttpClientFactory _factory;
public DynamicHttpClientFactoryTests()
{
IHttpClientFactory httpClientFactory = Substitute.For<IHttpClientFactory>();
httpClientFactory.CreateClient(Arg.Any<string>()).Returns(_ => new HttpClient());
_factory = new DynamicHttpClientFactory(
httpClientFactory,
_store,
new OptionsCache<HttpClientFactoryOptions>(),
NullLogger<DynamicHttpClientFactory>.Instance);
}
[Fact]
public void CreateClient_ShouldThrow_WhenTheNameWasNeverRegistered()
{
Should.Throw<InvalidOperationException>(() => _factory.CreateClient("unknown"));
}
[Fact]
public void RegisterDelugeClient_ShouldAllowRedirectsAndDecompression()
{
_factory.RegisterDelugeClient("deluge", timeout: 30, new RetryConfig { MaxRetries = 2 },
CertificateValidationType.Enabled, sendUserAgent: true);
HttpClientConfig config = Stored("deluge");
config.Type.ShouldBe(HttpClientType.Deluge);
config.AllowAutoRedirect.ShouldBeTrue();
config.AutomaticDecompression.ShouldBe(DecompressionMethods.GZip | DecompressionMethods.Deflate);
config.SendUserAgent.ShouldBeTrue();
}
[Theory]
[InlineData(HttpClientType.Deluge)]
[InlineData(HttpClientType.UTorrent)]
[InlineData(HttpClientType.WithRetry)]
public void RegisterDownloadClient_ShouldStoreTheRequestedType(HttpClientType clientType)
{
_factory.RegisterDownloadClient("client", timeout: 30, clientType, new RetryConfig { MaxRetries = 1 },
CertificateValidationType.Enabled, sendUserAgent: true);
HttpClientConfig config = Stored("client");
config.Type.ShouldBe(clientType);
config.Timeout.ShouldBe(30);
config.RetryConfig!.MaxRetries.ShouldBe(1);
config.SendUserAgent.ShouldBeTrue();
}
[Fact]
public void RegisterPlainClient_ShouldStoreNoRetryPolicy()
{
_factory.RegisterPlainClient("plain", sendUserAgent: true);
HttpClientConfig config = Stored("plain");
config.Type.ShouldBe(HttpClientType.Plain);
config.RetryConfig.ShouldBeNull();
config.SendUserAgent.ShouldBeTrue();
}
[Fact]
public void UnregisterConfiguration_ShouldDropTheStoredConfiguration()
{
_factory.RegisterPlainClient("plain", sendUserAgent: true);
_factory.UnregisterConfiguration("plain");
_store.TryGetConfiguration("plain", out _).ShouldBeFalse();
}
[Fact]
public void UpdateAllClientsFromGeneralConfig_ShouldRefreshEveryStoredClient()
{
_factory.RegisterRetryClient("retry", timeout: 1, new RetryConfig { MaxRetries = 0 },
CertificateValidationType.Enabled, sendUserAgent: false);
_factory.RegisterPlainClient("plain", sendUserAgent: false);
_factory.UpdateAllClientsFromGeneralConfig(new GeneralConfig
{
HttpTimeout = 55,
HttpMaxRetries = 9,
HttpCertificateValidation = CertificateValidationType.Disabled,
HttpSendUserAgent = true,
});
HttpClientConfig retry = Stored("retry");
retry.Timeout.ShouldBe(55);
retry.CertificateValidationType.ShouldBe(CertificateValidationType.Disabled);
retry.RetryConfig!.MaxRetries.ShouldBe(9);
retry.SendUserAgent.ShouldBeTrue();
Stored("plain").SendUserAgent.ShouldBeTrue();
}
[Fact]
public void UpdateAllClientsFromGeneralConfig_ShouldDoNothing_WhenNothingIsRegistered()
{
Should.NotThrow(() => _factory.UpdateAllClientsFromGeneralConfig(new GeneralConfig()));
_store.GetAllConfigurations().ShouldBeEmpty();
}
private HttpClientConfig Stored(string clientName)
{
_store.TryGetConfiguration(clientName, out HttpClientConfig? config).ShouldBeTrue();
return config!;
}
}
@@ -0,0 +1,155 @@
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Http;
using Cleanuparr.Infrastructure.Http.DynamicHttpClientSystem;
using Cleanuparr.Persistence;
using Cleanuparr.Persistence.Models.Configuration;
using Cleanuparr.Persistence.Models.Configuration.General;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using Shouldly;
using Xunit;
namespace Cleanuparr.Infrastructure.Tests.Http;
/// <summary>
/// Covers the second registration site for the general HTTP settings.
/// </summary>
public sealed class DynamicHttpClientProviderTests : IDisposable
{
private readonly SqliteConnection _connection;
private readonly DbContextOptions<DataContext> _options;
private readonly IDynamicHttpClientFactory _clientFactory = Substitute.For<IDynamicHttpClientFactory>();
public DynamicHttpClientProviderTests()
{
_connection = new SqliteConnection("DataSource=:memory:");
_connection.Open();
_options = new DbContextOptionsBuilder<DataContext>()
.UseSqlite(_connection)
.Options;
using DataContext context = new(_options);
context.Database.EnsureCreated();
_clientFactory.CreateClient(Arg.Any<string>()).Returns(_ => new HttpClient());
}
[Theory]
[InlineData(DownloadClientTypeName.Deluge, HttpClientType.Deluge)]
[InlineData(DownloadClientTypeName.uTorrent, HttpClientType.UTorrent)]
[InlineData(DownloadClientTypeName.qBittorrent, HttpClientType.WithRetry)]
[InlineData(DownloadClientTypeName.Transmission, HttpClientType.WithRetry)]
[InlineData(DownloadClientTypeName.rTorrent, HttpClientType.WithRetry)]
public void CreateClient_ShouldMapTheDownloadClientType(DownloadClientTypeName typeName, HttpClientType expected)
{
Seed(sendUserAgent: true);
DownloadClientConfig config = BuildConfig(typeName);
using HttpClient client = BuildProvider().CreateClient(config);
_clientFactory.Received(1).RegisterDownloadClient(
$"DownloadClient_{config.Id}",
Arg.Any<int>(),
expected,
Arg.Any<RetryConfig>(),
Arg.Any<CertificateValidationType>(),
Arg.Any<bool>());
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void CreateClient_ShouldPassTheUserAgentSetting(bool sendUserAgent)
{
Seed(sendUserAgent: sendUserAgent);
using HttpClient client = BuildProvider().CreateClient(BuildConfig(DownloadClientTypeName.qBittorrent));
_clientFactory.Received(1).RegisterDownloadClient(
Arg.Any<string>(),
Arg.Any<int>(),
Arg.Any<HttpClientType>(),
Arg.Any<RetryConfig>(),
Arg.Any<CertificateValidationType>(),
sendUserAgent);
}
[Fact]
public void CreateClient_ShouldCarryTheStoredHttpSettings()
{
Seed(sendUserAgent: false, timeout: 33, maxRetries: 4, certificate: CertificateValidationType.DisabledForLocalAddresses);
using HttpClient client = BuildProvider().CreateClient(BuildConfig(DownloadClientTypeName.qBittorrent));
_clientFactory.Received(1).RegisterDownloadClient(
Arg.Any<string>(),
33,
Arg.Any<HttpClientType>(),
Arg.Is<RetryConfig>(retry => retry.MaxRetries == 4 && retry.ExcludeUnauthorized),
CertificateValidationType.DisabledForLocalAddresses,
false);
}
[Fact]
public void CreateClient_ShouldSetTheBaseAddress()
{
Seed(sendUserAgent: false);
using HttpClient client = BuildProvider().CreateClient(BuildConfig(DownloadClientTypeName.qBittorrent));
client.BaseAddress.ShouldNotBeNull();
client.BaseAddress!.ToString().ShouldStartWith("http://localhost:8080/");
}
private DynamicHttpClientProvider BuildProvider()
{
ServiceCollection services = new();
services.AddScoped(_ => new DataContext(_options));
return new DynamicHttpClientProvider(
NullLogger<DynamicHttpClientProvider>.Instance,
services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>(),
_clientFactory);
}
private static DownloadClientConfig BuildConfig(DownloadClientTypeName typeName) => new()
{
Id = Guid.NewGuid(),
Name = $"test-{typeName}",
TypeName = typeName,
Type = DownloadClientType.Torrent,
Enabled = true,
Host = new Uri("http://localhost:8080"),
};
private void Seed(
bool sendUserAgent,
ushort timeout = 100,
ushort maxRetries = 0,
CertificateValidationType certificate = CertificateValidationType.Enabled)
{
using DataContext context = new(_options);
context.GeneralConfigs.RemoveRange(context.GeneralConfigs);
context.GeneralConfigs.Add(new GeneralConfig
{
Id = Guid.NewGuid(),
HttpSendUserAgent = sendUserAgent,
HttpTimeout = timeout,
HttpMaxRetries = maxRetries,
HttpCertificateValidation = certificate,
IgnoredDownloads = [],
Log = new LoggingConfig()
});
context.SaveChanges();
}
public void Dispose()
{
_connection.Dispose();
}
}
@@ -0,0 +1,125 @@
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Features.DownloadClient.Deluge;
using Cleanuparr.Infrastructure.Http.DynamicHttpClientSystem;
using Cleanuparr.Persistence;
using Cleanuparr.Persistence.Models.Configuration.General;
using Cleanuparr.Shared.Helpers;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using Shouldly;
using Xunit;
namespace Cleanuparr.Infrastructure.Tests.Http;
/// <summary>
/// Covers the startup registrations that put every client in reach of the general HTTP settings.
/// </summary>
public sealed class HttpClientConfigurationServiceTests : IDisposable
{
private readonly SqliteConnection _connection;
private readonly DbContextOptions<DataContext> _options;
private readonly IDynamicHttpClientFactory _clientFactory = Substitute.For<IDynamicHttpClientFactory>();
public HttpClientConfigurationServiceTests()
{
_connection = new SqliteConnection("DataSource=:memory:");
_connection.Open();
_options = new DbContextOptionsBuilder<DataContext>()
.UseSqlite(_connection)
.Options;
using DataContext context = new(_options);
context.Database.EnsureCreated();
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task StartAsync_ShouldPassTheUserAgentSetting_ToEveryRegistration(bool sendUserAgent)
{
Seed(sendUserAgent: sendUserAgent);
await BuildService().StartAsync(CancellationToken.None);
_clientFactory.Received(1).RegisterRetryClient(
Constants.HttpClientWithRetryName,
Arg.Any<int>(),
Arg.Any<RetryConfig>(),
Arg.Any<CertificateValidationType>(),
sendUserAgent);
_clientFactory.Received(1).RegisterDelugeClient(
nameof(DelugeService),
Arg.Any<int>(),
Arg.Any<RetryConfig>(),
Arg.Any<CertificateValidationType>(),
sendUserAgent);
_clientFactory.Received(1).RegisterPlainClient(Constants.HttpClientPlexAuthName, sendUserAgent);
_clientFactory.Received(1).RegisterPlainClient(Constants.HttpClientOidcAuthName, sendUserAgent);
_clientFactory.Received(1).RegisterPlainClient(Constants.HttpClientConnectivityName, sendUserAgent);
}
[Fact]
public async Task StartAsync_ShouldCarryTheStoredHttpSettings_IntoTheRetryClient()
{
Seed(sendUserAgent: true, timeout: 42, maxRetries: 7, certificate: CertificateValidationType.Disabled);
await BuildService().StartAsync(CancellationToken.None);
_clientFactory.Received(1).RegisterRetryClient(
Constants.HttpClientWithRetryName,
42,
Arg.Is<RetryConfig>(retry => retry.MaxRetries == 7 && retry.ExcludeUnauthorized),
CertificateValidationType.Disabled,
true);
}
[Fact]
public async Task StartAsync_ShouldThrow_WhenNoGeneralConfigExists()
{
await Should.ThrowAsync<InvalidOperationException>(
() => BuildService().StartAsync(CancellationToken.None));
}
private HttpClientConfigurationService BuildService()
{
ServiceCollection services = new();
services.AddScoped(_ => new DataContext(_options));
return new HttpClientConfigurationService(
_clientFactory,
NullLogger<HttpClientConfigurationService>.Instance,
services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>());
}
private void Seed(
bool sendUserAgent,
ushort timeout = 100,
ushort maxRetries = 0,
CertificateValidationType certificate = CertificateValidationType.Enabled)
{
using DataContext context = new(_options);
context.GeneralConfigs.Add(new GeneralConfig
{
Id = Guid.NewGuid(),
HttpSendUserAgent = sendUserAgent,
HttpTimeout = timeout,
HttpMaxRetries = maxRetries,
HttpCertificateValidation = certificate,
IgnoredDownloads = [],
Log = new LoggingConfig()
});
context.SaveChanges();
}
public void Dispose()
{
_connection.Dispose();
}
}
@@ -0,0 +1,108 @@
using Cleanuparr.Domain.Enums;
using Cleanuparr.Infrastructure.Http.DynamicHttpClientSystem;
using Cleanuparr.Infrastructure.Services;
using Cleanuparr.Persistence.Models.Configuration.General;
using Cleanuparr.Shared.Helpers;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Shouldly;
using Xunit;
namespace Cleanuparr.Infrastructure.Tests.Http;
/// <summary>
/// Drives the real IHttpClientFactory pipeline.
/// Covers both the handler-configured clients and the plain ones.
/// </summary>
public sealed class UserAgentHeaderPipelineTests
{
[Fact]
public void RetryClient_ShouldCarryTheUserAgent_WhenTheSettingIsOn()
{
using ServiceProvider provider = BuildProvider();
IDynamicHttpClientFactory dynamicFactory = provider.GetRequiredService<IDynamicHttpClientFactory>();
RegisterRetryClient(dynamicFactory, sendUserAgent: true);
using HttpClient client = dynamicFactory.CreateClient(Constants.HttpClientWithRetryName);
client.DefaultRequestHeaders.UserAgent.ToString().ShouldBe(AppUserAgent.Value);
}
[Fact]
public void PlainClient_ShouldCarryTheUserAgent_WhenTheSettingIsOn()
{
using ServiceProvider provider = BuildProvider();
IDynamicHttpClientFactory dynamicFactory = provider.GetRequiredService<IDynamicHttpClientFactory>();
dynamicFactory.RegisterPlainClient(Constants.HttpClientOidcAuthName, sendUserAgent: true);
using HttpClient client = dynamicFactory.CreateClient(Constants.HttpClientOidcAuthName);
client.DefaultRequestHeaders.UserAgent.ToString().ShouldBe(AppUserAgent.Value);
}
[Fact]
public void PlainClient_ShouldKeepTheDefaultTimeout()
{
using ServiceProvider provider = BuildProvider();
IDynamicHttpClientFactory dynamicFactory = provider.GetRequiredService<IDynamicHttpClientFactory>();
dynamicFactory.RegisterPlainClient(Constants.HttpClientOidcAuthName, sendUserAgent: true);
using HttpClient client = dynamicFactory.CreateClient(Constants.HttpClientOidcAuthName);
client.Timeout.ShouldBe(TimeSpan.FromSeconds(100));
}
[Fact]
public void NoClient_ShouldCarryAUserAgent_WhenTheSettingIsOff()
{
using ServiceProvider provider = BuildProvider();
IDynamicHttpClientFactory dynamicFactory = provider.GetRequiredService<IDynamicHttpClientFactory>();
RegisterRetryClient(dynamicFactory, sendUserAgent: false);
dynamicFactory.RegisterPlainClient(Constants.HttpClientOidcAuthName, sendUserAgent: false);
using HttpClient retry = dynamicFactory.CreateClient(Constants.HttpClientWithRetryName);
using HttpClient plain = dynamicFactory.CreateClient(Constants.HttpClientOidcAuthName);
retry.DefaultRequestHeaders.UserAgent.ShouldBeEmpty();
plain.DefaultRequestHeaders.UserAgent.ShouldBeEmpty();
}
[Fact]
public void PlainClient_ShouldPickUpTheSetting_WithoutARestart()
{
using ServiceProvider provider = BuildProvider();
IDynamicHttpClientFactory dynamicFactory = provider.GetRequiredService<IDynamicHttpClientFactory>();
dynamicFactory.RegisterPlainClient(Constants.HttpClientOidcAuthName, sendUserAgent: false);
using HttpClient before = dynamicFactory.CreateClient(Constants.HttpClientOidcAuthName);
before.DefaultRequestHeaders.UserAgent.ShouldBeEmpty();
dynamicFactory.UpdateAllClientsFromGeneralConfig(new GeneralConfig { HttpSendUserAgent = true });
using HttpClient after = dynamicFactory.CreateClient(Constants.HttpClientOidcAuthName);
after.DefaultRequestHeaders.UserAgent.ToString().ShouldBe(AppUserAgent.Value);
// The setting reaches the next client, never one already handed out.
// Every consumer is scoped or disposes per call, so none outlives a change.
before.DefaultRequestHeaders.UserAgent.ShouldBeEmpty();
}
private static void RegisterRetryClient(IDynamicHttpClientFactory dynamicFactory, bool sendUserAgent) =>
dynamicFactory.RegisterRetryClient(
Constants.HttpClientWithRetryName,
timeout: 100,
new RetryConfig { MaxRetries = 0 },
CertificateValidationType.Enabled,
sendUserAgent);
private static ServiceProvider BuildProvider()
{
ServiceCollection services = new();
services.AddLogging();
services.AddSingleton<CertificateValidationService>();
services.AddDynamicHttpClients();
return services.BuildServiceProvider();
}
}
@@ -0,0 +1,22 @@
using Cleanuparr.Shared.Helpers;
using Shouldly;
using Xunit;
namespace Cleanuparr.Infrastructure.Tests.Shared;
public sealed class AppUserAgentTests
{
[Theory]
[InlineData(1, 2, 3, 4)]
[InlineData(1, 2, 3, 0)]
public void Format_ShouldDropTheRevision(int major, int minor, int build, int revision)
{
AppUserAgent.Format(new Version(major, minor, build, revision)).ShouldBe("Cleanuparr/1.2.3");
}
[Fact]
public void Format_ShouldEmitTheBareProductToken_WhenTheVersionIsUnknown()
{
AppUserAgent.Format(null).ShouldBe("Cleanuparr");
}
}
@@ -75,7 +75,8 @@ public class DynamicHttpClientProvider : IDynamicHttpClientProvider
httpConfig.HttpTimeout,
clientType,
retryConfig,
httpConfig.HttpCertificateValidation
httpConfig.HttpCertificateValidation,
httpConfig.HttpSendUserAgent
);
// Create and configure the client
@@ -1,5 +1,6 @@
using System.Net;
using Cleanuparr.Infrastructure.Services;
using Cleanuparr.Shared.Helpers;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Http;
using Microsoft.Extensions.Options;
@@ -24,10 +25,20 @@ public class DynamicHttpClientConfiguration : IConfigureNamedOptions<HttpClientF
{
using var scope = _scopeFactory.CreateScope();
var configStore = scope.ServiceProvider.GetRequiredService<IHttpClientConfigStore>();
if (!configStore.TryGetConfiguration(name, out HttpClientConfig? config))
return;
if (config.SendUserAgent)
{
options.HttpClientActions.Add(ApplyUserAgent);
}
if (config.Type is HttpClientType.Plain)
{
return;
}
// Configure the HttpClient
options.HttpClientActions.Add(httpClient =>
{
@@ -47,6 +58,11 @@ public class DynamicHttpClientConfiguration : IConfigureNamedOptions<HttpClientF
});
}
internal static void ApplyUserAgent(HttpClient client)
{
client.DefaultRequestHeaders.UserAgent.ParseAdd(AppUserAgent.Value);
}
private void ConfigureHandler(HttpMessageHandlerBuilder builder, HttpClientConfig config)
{
using var scope = _scopeFactory.CreateScope();
@@ -130,6 +146,7 @@ public class DynamicHttpClientConfiguration : IConfigureNamedOptions<HttpClientF
public void Configure(HttpClientFactoryOptions options)
{
// This is called for unnamed clients - we don't need to do anything here
// Never called: the factory prefers the named overload.
// Unnamed clients reach that one with an empty name.
}
}
@@ -50,23 +50,24 @@ public class DynamicHttpClientFactory : IDynamicHttpClientFactory
_configStore.AddConfiguration(clientName, config);
}
public void RegisterRetryClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType)
public void RegisterRetryClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType, bool sendUserAgent)
{
var config = new HttpClientConfig
HttpClientConfig config = new()
{
Name = clientName,
Timeout = timeout,
Type = HttpClientType.WithRetry,
RetryConfig = retryConfig,
CertificateValidationType = certificateType
CertificateValidationType = certificateType,
SendUserAgent = sendUserAgent
};
RegisterConfiguration(clientName, config);
}
public void RegisterDelugeClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType)
public void RegisterDelugeClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType, bool sendUserAgent)
{
var config = new HttpClientConfig
HttpClientConfig config = new()
{
Name = clientName,
Timeout = timeout,
@@ -74,21 +75,23 @@ public class DynamicHttpClientFactory : IDynamicHttpClientFactory
RetryConfig = retryConfig,
AllowAutoRedirect = true,
CertificateValidationType = certificateType,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
SendUserAgent = sendUserAgent
};
RegisterConfiguration(clientName, config);
}
public void RegisterDownloadClient(string clientName, int timeout, HttpClientType clientType, RetryConfig retryConfig, CertificateValidationType certificateType)
public void RegisterDownloadClient(string clientName, int timeout, HttpClientType clientType, RetryConfig retryConfig, CertificateValidationType certificateType, bool sendUserAgent)
{
var config = new HttpClientConfig
HttpClientConfig config = new()
{
Name = clientName,
Timeout = timeout,
Type = clientType,
RetryConfig = retryConfig,
CertificateValidationType = certificateType
CertificateValidationType = certificateType,
SendUserAgent = sendUserAgent
};
// Configure Deluge-specific settings if needed
@@ -97,7 +100,19 @@ public class DynamicHttpClientFactory : IDynamicHttpClientFactory
config.AllowAutoRedirect = true;
config.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
}
RegisterConfiguration(clientName, config);
}
public void RegisterPlainClient(string clientName, bool sendUserAgent)
{
HttpClientConfig config = new()
{
Name = clientName,
Type = HttpClientType.Plain,
SendUserAgent = sendUserAgent
};
RegisterConfiguration(clientName, config);
}
@@ -114,7 +129,7 @@ public class DynamicHttpClientFactory : IDynamicHttpClientFactory
public void UpdateAllClientsFromGeneralConfig(GeneralConfig generalConfig)
{
var allConfigurations = _configStore.GetAllConfigurations().ToList();
if (!allConfigurations.Any())
{
_logger.LogDebug("No HTTP client configurations to update");
@@ -128,7 +143,8 @@ public class DynamicHttpClientFactory : IDynamicHttpClientFactory
// Update timeout and certificate validation for all clients
config.Timeout = generalConfig.HttpTimeout;
config.CertificateValidationType = generalConfig.HttpCertificateValidation;
config.SendUserAgent = generalConfig.HttpSendUserAgent;
// Update retry configuration if it exists
if (config.RetryConfig != null)
{
@@ -12,30 +12,15 @@ public class HttpClientConfig
public int Timeout { get; set; }
public HttpClientType Type { get; set; }
public RetryConfig? RetryConfig { get; set; }
// Deluge-specific settings
public bool AllowAutoRedirect { get; set; } = true;
public DecompressionMethods AutomaticDecompression { get; set; } = DecompressionMethods.GZip | DecompressionMethods.Deflate;
public CertificateValidationType CertificateValidationType { get; set; } = CertificateValidationType.Enabled;
}
/// <summary>
/// Retry configuration for HTTP clients
/// </summary>
public class RetryConfig
{
public int MaxRetries { get; set; }
public bool ExcludeUnauthorized { get; set; } = true;
/// <summary>
/// Whether requests from this client carry a User-Agent header.
/// </summary>
public bool SendUserAgent { get; set; }
}
/// <summary>
/// Types of HTTP clients that can be configured
/// </summary>
public enum HttpClientType
{
Default,
WithRetry,
Deluge,
UTorrent,
}
@@ -47,7 +47,8 @@ public class HttpClientConfigurationService : IHostedService
MaxRetries = config.HttpMaxRetries,
ExcludeUnauthorized = true
},
config.HttpCertificateValidation
config.HttpCertificateValidation,
config.HttpSendUserAgent
);
// Register the Deluge client
@@ -59,9 +60,17 @@ public class HttpClientConfigurationService : IHostedService
MaxRetries = config.HttpMaxRetries,
ExcludeUnauthorized = true
},
config.HttpCertificateValidation
config.HttpCertificateValidation,
config.HttpSendUserAgent
);
// These keep the handler defaults they had before the dynamic system existed.
// Registering them only puts them in reach of the User-Agent setting.
_clientFactory.RegisterPlainClient(Constants.HttpClientPlexAuthName, config.HttpSendUserAgent);
_clientFactory.RegisterPlainClient(Constants.HttpClientOidcAuthName, config.HttpSendUserAgent);
_clientFactory.RegisterPlainClient(Constants.HttpClientConnectivityName, config.HttpSendUserAgent);
_logger.LogInformation("Pre-registered standard HTTP client configurations");
}
catch (Exception ex)
@@ -0,0 +1,18 @@
namespace Cleanuparr.Infrastructure.Http.DynamicHttpClientSystem;
/// <summary>
/// Types of HTTP clients that can be configured
/// </summary>
public enum HttpClientType
{
Default,
WithRetry,
Deluge,
UTorrent,
/// <summary>
/// Carries the User-Agent and nothing else.
/// No primary handler, no timeout override, no retry policy.
/// </summary>
Plain,
}
@@ -26,18 +26,23 @@ public interface IDynamicHttpClientFactory
/// <summary>
/// Registers a retry-enabled HttpClient configuration
/// </summary>
void RegisterRetryClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType);
void RegisterRetryClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType, bool sendUserAgent);
/// <summary>
/// Registers a Deluge-specific HttpClient configuration
/// </summary>
void RegisterDelugeClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType);
void RegisterDelugeClient(string clientName, int timeout, RetryConfig retryConfig, CertificateValidationType certificateType, bool sendUserAgent);
/// <summary>
/// Registers a configuration for a download client
/// </summary>
void RegisterDownloadClient(string clientName, int timeout, HttpClientType clientType, RetryConfig retryConfig, CertificateValidationType certificateType);
void RegisterDownloadClient(string clientName, int timeout, HttpClientType clientType, RetryConfig retryConfig, CertificateValidationType certificateType, bool sendUserAgent);
/// <summary>
/// Registers a client that opts out of every handler setting and only carries the User-Agent.
/// </summary>
void RegisterPlainClient(string clientName, bool sendUserAgent);
/// <summary>
/// Unregisters a configuration
/// </summary>
@@ -0,0 +1,10 @@
namespace Cleanuparr.Infrastructure.Http.DynamicHttpClientSystem;
/// <summary>
/// Retry configuration for HTTP clients
/// </summary>
public class RetryConfig
{
public int MaxRetries { get; set; }
public bool ExcludeUnauthorized { get; set; } = true;
}
@@ -1,5 +1,6 @@
using Cleanuparr.Infrastructure.Services.Interfaces;
using Cleanuparr.Persistence.Models.Configuration.General;
using Cleanuparr.Shared.Helpers;
using Microsoft.Extensions.Logging;
namespace Cleanuparr.Infrastructure.Services;
@@ -24,7 +25,7 @@ public sealed class ConnectivityChecker : IConnectivityChecker
return true;
}
using HttpClient client = _httpClientFactory.CreateClient();
using HttpClient client = _httpClientFactory.CreateClient(Constants.HttpClientConnectivityName);
using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cts.CancelAfter(ProbeTimeout);
@@ -0,0 +1,31 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Cleanuparr.Persistence.Postgres.Migrations.Data
{
/// <inheritdoc />
public partial class AddHttpSendUserAgent : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "http_send_user_agent",
schema: "data",
table: "general_configs",
type: "boolean",
nullable: false,
defaultValue: false);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "http_send_user_agent",
schema: "data",
table: "general_configs");
}
}
}
@@ -727,6 +727,10 @@ namespace Cleanuparr.Persistence.Postgres.Migrations.Data
.HasColumnType("integer")
.HasColumnName("http_max_retries");
b.Property<bool>("HttpSendUserAgent")
.HasColumnType("boolean")
.HasColumnName("http_send_user_agent");
b.Property<int>("HttpTimeout")
.HasColumnType("integer")
.HasColumnName("http_timeout");
@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Cleanuparr.Persistence.Migrations.Data
{
/// <inheritdoc />
public partial class AddHttpSendUserAgent : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "http_send_user_agent",
table: "general_configs",
type: "INTEGER",
nullable: false,
defaultValue: false);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "http_send_user_agent",
table: "general_configs");
}
}
}
@@ -721,6 +721,10 @@ namespace Cleanuparr.Persistence.Migrations.Data
.HasColumnType("INTEGER")
.HasColumnName("http_max_retries");
b.Property<bool>("HttpSendUserAgent")
.HasColumnType("INTEGER")
.HasColumnName("http_send_user_agent");
b.Property<ushort>("HttpTimeout")
.HasColumnType("INTEGER")
.HasColumnName("http_timeout");
@@ -22,6 +22,11 @@ public sealed record GeneralConfig : IConfig
public CertificateValidationType HttpCertificateValidation { get; set; } = CertificateValidationType.Enabled;
/// <summary>
/// Whether outbound requests carry a User-Agent header.
/// </summary>
public bool HttpSendUserAgent { get; set; }
public bool StatusCheckEnabled { get; set; } = true;
@@ -0,0 +1,22 @@
using System.Reflection;
namespace Cleanuparr.Shared.Helpers;
/// <summary>
/// The User-Agent product token, shaped per RFC 9110.
/// </summary>
public static class AppUserAgent
{
private const string ProductName = "Cleanuparr";
/// <summary>
/// The User-Agent value for the running build.
/// Reads "Cleanuparr/1.2.3" on a tagged release.
/// </summary>
public static readonly string Value = Format(Assembly.GetEntryAssembly()?.GetName().Version);
internal static string Format(Version? version) =>
version is null
? ProductName
: $"{ProductName}/{version.Major}.{version.Minor}.{Math.Max(version.Build, 0)}";
}
@@ -14,6 +14,8 @@ public static class Constants
public const string HttpClientOidcAuthName = "OidcAuth";
public const string HttpClientConnectivityName = "Connectivity";
public static readonly MemoryCacheEntryOptions DefaultCacheEntryOptions = new()
{
SlidingExpiration = TimeSpan.FromMinutes(10)
@@ -44,4 +44,5 @@ export const NEW_FEATURES: NewFeature[] = [
{ id: 'ignore-alt-speed' },
{ id: 'sportarr-blocklist' },
{ id: 'lazylibrarian-blocklist' },
{ id: 'http-user-agent' },
];
@@ -50,6 +50,7 @@ export class DocumentationService {
'httpMaxRetries': 'http-max-retries',
'httpTimeout': 'http-timeout',
'httpCertificateValidation': 'http-certificate-validation',
'httpSendUserAgent': 'http-user-agent',
'statusCheckEnabled': 'status-check',
'log.level': 'log-level',
'log.rollingSizeMB': 'rolling-size-mb',
@@ -92,6 +92,12 @@
hint="Enable or disable certificate validation for HTTPS requests"
helpKey="general:httpCertificateValidation" />
</div>
<div class="form-divider"></div>
<app-toggle label="Send User-Agent Header" featureId="http-user-agent" [formField]="genForm.httpSendUserAgent"
hint="When enabled, outgoing requests identify themselves as Cleanuparr/&lt;version&gt;. Turn this on if a reverse proxy or WAF in front of your *arrs or download clients rejects requests that carry no User-Agent."
helpKey="general:httpSendUserAgent" />
</div>
</app-card>
@@ -14,6 +14,7 @@ const CONFIG: GeneralConfig = {
httpMaxRetries: 2,
httpTimeout: 45,
httpCertificateValidation: CertificateValidationType.DisabledForLocalAddresses,
httpSendUserAgent: true,
statusCheckEnabled: false,
strikeInactivityWindowHours: 12,
historyRetentionDays: 90,
@@ -140,6 +141,7 @@ describe('GeneralSettingsComponent', () => {
expect(component.genForm.ignoredDownloads().value()).toEqual([]);
expect(component.genForm.connectivityCheckEnabled().value()).toBe(false);
expect(component.genForm.connectivityCheckUrls().value()).toEqual([]);
expect(component.genForm.httpSendUserAgent().value()).toBe(false);
expect(component.dirty()).toBe(false);
});
@@ -161,6 +163,7 @@ describe('GeneralSettingsComponent', () => {
httpMaxRetries: 2,
httpTimeout: 60,
httpCertificateValidation: CertificateValidationType.DisabledForLocalAddresses,
httpSendUserAgent: true,
statusCheckEnabled: false,
strikeInactivityWindowHours: 12,
historyRetentionDays: 90,
@@ -37,6 +37,7 @@ interface GeneralSettingsFormModel {
httpMaxRetries: number | null;
httpTimeout: number | null;
httpCertificateValidation: CertificateValidationType;
httpSendUserAgent: boolean;
statusCheckEnabled: boolean;
ignoredDownloads: string[];
connectivityCheckEnabled: boolean;
@@ -96,6 +97,7 @@ export class GeneralSettingsComponent implements HasPendingChanges {
httpMaxRetries: 3,
httpTimeout: 30,
httpCertificateValidation: CertificateValidationType.Enabled,
httpSendUserAgent: false,
statusCheckEnabled: true,
ignoredDownloads: [],
connectivityCheckEnabled: false,
@@ -185,6 +187,7 @@ export class GeneralSettingsComponent implements HasPendingChanges {
httpMaxRetries: config.httpMaxRetries,
httpTimeout: config.httpTimeout,
httpCertificateValidation: config.httpCertificateValidation,
httpSendUserAgent: config.httpSendUserAgent ?? false,
statusCheckEnabled: config.statusCheckEnabled,
ignoredDownloads: config.ignoredDownloads ?? [],
connectivityCheckEnabled: config.connectivityCheckEnabled ?? false,
@@ -233,6 +236,7 @@ export class GeneralSettingsComponent implements HasPendingChanges {
httpMaxRetries: m.httpMaxRetries ?? 3,
httpTimeout: m.httpTimeout ?? 30,
httpCertificateValidation: m.httpCertificateValidation as CertificateValidationType,
httpSendUserAgent: m.httpSendUserAgent,
statusCheckEnabled: m.statusCheckEnabled,
strikeInactivityWindowHours: m.strikeInactivityWindowHours ?? 24,
historyRetentionDays: m.historyRetentionDays ?? 365,
@@ -22,6 +22,7 @@ export interface GeneralConfig {
httpMaxRetries: number;
httpTimeout: number;
httpCertificateValidation: CertificateValidationType;
httpSendUserAgent: boolean;
statusCheckEnabled: boolean;
strikeInactivityWindowHours: number;
historyRetentionDays: number;
+10
View File
@@ -138,6 +138,16 @@ SSL certificate validation for HTTPS connections.
</ConfigSection>
<ConfigSection
title="HTTP User-Agent"
>
Cleanuparr sends no `User-Agent` header by default. Enable this setting to identify requests coming from it.
The header contains the name and the version of the app, for example `Cleanuparr/2.0.0`. It applies to every outgoing request.
</ConfigSection>
</div>
<div className={styles.section}>
@@ -0,0 +1,85 @@
import { test, expect, TEST_CONFIG } from '../fixtures/base';
import { buildDownloadClientPayload } from '../helpers/api/download-client';
import { ArrStubs, DownloadClientStubs } from '../helpers/mocks';
import type { LoggedRequest } from '../helpers/mocks';
const USER_AGENT = /^Cleanuparr(\/\d+\.\d+\.\d+)?$/;
function userAgentOf(request: LoggedRequest): string {
const match = Object.entries(request.headers).find(
([name]) => name.toLowerCase() === 'user-agent',
);
if (!match) {
return '';
}
const value = match[1];
return Array.isArray(value) ? value.join(', ') : value;
}
test.describe('HTTP user agent', () => {
test.afterEach(async ({ api }) => {
await api.general.patch({ httpSendUserAgent: false });
});
test('arr requests carry the user agent when the setting is on', async ({ api, mocks }) => {
await api.general.patch({ httpSendUserAgent: true });
await mocks.arr.stub(ArrStubs.arrHealthStub());
await mocks.arr.resetRequests();
const res = await api.arr.testInstance('sonarr', {
name: 'ua-arr',
url: TEST_CONFIG.mocks.arrUrl,
apiKey: 'good-key',
version: 3,
});
expect(res.ok).toBe(true);
const entry = await mocks.arr.waitForRequest({
method: 'GET',
urlPattern: '/api/v[0-9]+/system/status.*',
});
expect(userAgentOf(entry)).toMatch(USER_AGENT);
});
test('download client requests carry the user agent when the setting is on', async ({ api, mocks }) => {
await api.general.patch({ httpSendUserAgent: true });
await mocks.downloadClient.stub(DownloadClientStubs.delugeLoginStub());
await mocks.downloadClient.resetRequests();
const res = await api.downloadClient.test(
buildDownloadClientPayload('deluge', {
name: 'ua-deluge',
host: TEST_CONFIG.mocks.downloadClientUrl,
password: 'admin',
}),
);
expect(res.ok).toBe(true);
const entry = await mocks.downloadClient.waitForRequest({ method: 'POST' });
expect(userAgentOf(entry)).toMatch(USER_AGENT);
});
test('the setting takes effect without restarting the app', async ({ api, mocks }) => {
await mocks.arr.stub(ArrStubs.arrHealthStub());
const instance = {
name: 'ua-toggle',
url: TEST_CONFIG.mocks.arrUrl,
apiKey: 'good-key',
version: 3,
};
await api.general.patch({ httpSendUserAgent: false });
await mocks.arr.resetRequests();
await api.arr.testInstance('sonarr', instance);
const withoutHeader = await mocks.arr.waitForRequest({ method: 'GET' });
expect(userAgentOf(withoutHeader)).not.toMatch(USER_AGENT);
await api.general.patch({ httpSendUserAgent: true });
await mocks.arr.resetRequests();
await api.arr.testInstance('sonarr', instance);
const withHeader = await mocks.arr.waitForRequest({ method: 'GET' });
expect(userAgentOf(withHeader)).toMatch(USER_AGENT);
});
});
+23 -11
View File
@@ -33,16 +33,20 @@ export interface Mapping {
metadata?: Record<string, unknown>;
}
export interface LoggedRequest {
method: string;
url: string;
absoluteUrl: string;
headers: Record<string, string | string[]>;
body?: string;
bodyAsBase64?: string;
/** Epoch milliseconds. */
loggedDate: number;
}
/** One serve event: what `GET /__admin/requests` returns, unlike `/requests/find`. */
export interface RequestLogEntry {
request: {
method: string;
url: string;
headers: Record<string, string | string[]>;
body?: string;
bodyAsBase64?: string;
/** Epoch milliseconds. */
loggedDate: number;
};
request: LoggedRequest;
response: {
status: number;
body?: string;
@@ -80,6 +84,14 @@ export class WireMockClient {
}
}
/** Clears the request journal and leaves the mappings in place. */
async resetRequests(): Promise<void> {
const res = await fetch(`${this.adminUrl}/requests`, { method: 'DELETE' });
if (!res.ok) {
throw new Error(`Failed to reset WireMock requests at ${this.adminUrl}: ${res.status}`);
}
}
async resetAll(): Promise<void> {
const res = await fetch(`${this.adminUrl}/reset`, { method: 'POST' });
if (!res.ok) {
@@ -96,7 +108,7 @@ export class WireMockClient {
return data.requests ?? [];
}
async findRequests(criteria: { method?: string; urlPath?: string; urlPattern?: string }): Promise<RequestLogEntry[]> {
async findRequests(criteria: { method?: string; urlPath?: string; urlPattern?: string }): Promise<LoggedRequest[]> {
const res = await fetch(`${this.adminUrl}/requests/find`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -112,7 +124,7 @@ export class WireMockClient {
async waitForRequest(
criteria: { method?: string; urlPath?: string; urlPattern?: string },
timeoutMs = 30_000,
): Promise<RequestLogEntry> {
): Promise<LoggedRequest> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
const matches = await this.findRequests(criteria);
@@ -168,7 +168,7 @@ async function queueRequestSeen(arr: WireMockClient, timeoutMs: number): Promise
}
async function readDeletes(arr: WireMockClient): Promise<DeleteCall[]> {
const reqs = (await arr.findRequests({ method: 'DELETE', urlPattern: '/api/v3/queue/.*' })) as unknown as Array<{ url: string }>;
const reqs = await arr.findRequests({ method: 'DELETE', urlPattern: '/api/v3/queue/.*' });
return reqs.map((r) => {
const parsed = new URL(`http://stub${r.url}`);
return {
@@ -79,10 +79,10 @@ test.describe.serial('QueueCleaner — fractional sizeleft from Sonarr v3', () =
await expect
.poll(
async () => {
const requests = (await mocks.arr.findRequests({
const requests = await mocks.arr.findRequests({
method: 'DELETE',
urlPattern: '/api/v3/queue/.*',
})) as unknown as Array<{ url: string }>;
});
return requests.some((r) => r.url.includes(`/api/v3/queue/${RECORD_ID}`));
},
{ timeout: 60_000, intervals: [1_000] },