Merge pull request #1996 from aliasvault/1046-feature-request-add-vault-storage-insights-to-web-app-client

This commit is contained in:
Leendert de Borst
2026-05-06 22:16:50 +02:00
committed by GitHub
25 changed files with 1827 additions and 201 deletions

View File

@@ -11,10 +11,31 @@ export class LogoQueries {
WHERE Source = ? AND IsDeleted = 0
LIMIT 1`;
/**
* Look up a logo by source regardless of IsDeleted state. Used by getOrCreate to detect
* soft-deleted rows that still occupy the UNIQUE(Source) slot, so we can refill them
* instead of trying (and failing) to INSERT a duplicate.
*/
public static readonly GET_BY_SOURCE_INCLUDING_DELETED = `
SELECT Id, IsDeleted FROM Logos
WHERE Source = ?
LIMIT 1`;
/**
* Insert new logo.
*/
public static readonly INSERT = `
INSERT INTO Logos (Id, Source, FileData, CreatedAt, UpdatedAt, IsDeleted)
VALUES (?, ?, ?, ?, ?, ?)`;
/**
* Restore a soft-deleted logo and refill its bytes in one statement. Caller passes
* (FileData, UpdatedAt, Id).
*/
public static readonly RESTORE_WITH_FILE_DATA = `
UPDATE Logos
SET IsDeleted = 0,
FileData = ?,
UpdatedAt = ?
WHERE Id = ?`;
}

View File

@@ -41,10 +41,22 @@ export class LogoRepository extends BaseRepository {
* @returns The logo ID (existing or newly created)
*/
public getOrCreate(source: string, logoData: Uint8Array, currentDateTime: string): string {
// Check if a logo for this source already exists
const existingId = this.getIdForSource(source);
if (existingId) {
return existingId;
const existing = this.client.executeQuery<{ Id: string; IsDeleted: number }>(
LogoQueries.GET_BY_SOURCE_INCLUDING_DELETED,
[source]
);
if (existing.length > 0) {
const row = existing[0];
if (row.IsDeleted === 1) {
// Restore a previously soft-deleted record and refill its FileData.
this.client.executeUpdate(LogoQueries.RESTORE_WITH_FILE_DATA, [
logoData,
currentDateTime,
row.Id
]);
}
return row.Id;
}
// Create new logo entry

View File

@@ -549,6 +549,7 @@ class PasskeyRepository(database: VaultDatabase) : BaseRepository(database) {
// Sanity check: restore if soft-deleted
if (isDeleted) {
executeUpdate(LogoQueries.RESTORE, arrayOf(timestamp, existingLogoId))
executeUpdate(LogoQueries.UPDATE_FILE_DATA, arrayOf(logoData, timestamp, existingLogoId))
}
return existingLogoId
}

View File

@@ -486,7 +486,9 @@ public class PasskeyRepository: BaseRepository {
// Sanity check: restore if soft-deleted
if isDeleted {
let logoDataParam = "av-base64-to-blob:\(logoData.base64EncodedString())"
try client.executeUpdate(LogoQueries.restore, params: [now, existingLogoId])
try client.executeUpdate(LogoQueries.updateFileData, params: [logoDataParam, now, existingLogoId])
}
return existingLogoId
}

View File

@@ -14,6 +14,14 @@ const LogoQueries = {
WHERE Source = ? AND IsDeleted = 0
LIMIT 1`,
/**
* Look up a logo by source regardless of IsDeleted state.
*/
GET_BY_SOURCE_INCLUDING_DELETED: `
SELECT Id, IsDeleted FROM Logos
WHERE Source = ?
LIMIT 1`,
/**
* Insert new logo.
*/
@@ -21,6 +29,16 @@ const LogoQueries = {
INSERT INTO Logos (Id, Source, FileData, CreatedAt, UpdatedAt, IsDeleted)
VALUES (?, ?, ?, ?, ?, ?)`,
/**
* Restore a soft-deleted logo and refill its FileData in one statement.
*/
RESTORE_WITH_FILE_DATA: `
UPDATE Logos
SET IsDeleted = 0,
FileData = ?,
UpdatedAt = ?
WHERE Id = ?`,
/**
* Count items using a logo.
*/
@@ -75,10 +93,22 @@ export class LogoRepository extends BaseRepository {
* @returns The logo ID (existing or newly created)
*/
public async getOrCreate(source: string, logoData: Uint8Array, currentDateTime: string): Promise<string> {
// Check if a logo for this source already exists
const existingId = await this.getIdForSource(source);
if (existingId) {
return existingId;
const existing = await this.client.executeQuery<{ Id: string; IsDeleted: number }>(
LogoQueries.GET_BY_SOURCE_INCLUDING_DELETED,
[source]
);
if (existing.length > 0) {
const row = existing[0];
if (row.IsDeleted === 1) {
// Restore a previously soft-deleted record and refill its FileData.
await this.client.executeUpdate(LogoQueries.RESTORE_WITH_FILE_DATA, [
logoData,
currentDateTime,
row.Id
]);
}
return row.Id;
}
// Create new logo entry

View File

@@ -9,6 +9,7 @@ namespace AliasVault.Api.Controllers;
using AliasServerDb;
using AliasVault.Api.Controllers.Abstracts;
using AliasVault.Api.Services;
using AliasVault.Shared.Models.WebApi.Favicon;
using Asp.Versioning;
using Microsoft.AspNetCore.Identity;
@@ -18,15 +19,24 @@ using Microsoft.AspNetCore.Mvc;
/// Controller for retrieving favicons from external websites.
/// </summary>
/// <param name="userManager">UserManager instance.</param>
/// <param name="rateLimitService">In-memory per-user favicon extraction rate limiter.</param>
/// <param name="logger">Logger instance.</param>
[ApiVersion("1")]
public class FaviconController(UserManager<AliasVaultUser> userManager, ILogger<FaviconController> logger) : AuthenticatedRequestController(userManager)
public class FaviconController(
UserManager<AliasVaultUser> userManager,
FaviconRateLimitService rateLimitService,
ILogger<FaviconController> logger) : AuthenticatedRequestController(userManager)
{
/// <summary>
/// Proxies the request to the identity generator to generate a random identity.
/// Maximum number of URLs accepted in a single batch request.
/// </summary>
public const int MaxBatchSize = 10;
/// <summary>
/// Extracts the favicon from a single URL.
/// </summary>
/// <param name="url">URL to extract the favicon from.</param>
/// <returns>Identity model.</returns>
/// <returns>Favicon image bytes, or null if extraction failed.</returns>
[HttpGet("Extract")]
public async Task<IActionResult> Extract(string url)
{
@@ -36,23 +46,84 @@ public class FaviconController(UserManager<AliasVaultUser> userManager, ILogger<
return Unauthorized();
}
// Get the favicon from the URL.
if (!rateLimitService.TryConsume(user.Id, 1))
{
return StatusCode(StatusCodes.Status429TooManyRequests);
}
try
{
var image = await FaviconExtractor.FaviconExtractor.GetFaviconAsync(url);
// Return the favicon as base64 string of image representation.
return Ok(new FaviconExtractModel { Image = image });
}
catch (Exception ex)
{
// Anonymize the URL by replacing all a-Z characters with 'x' before logging.
// This will still allow to see the host structure but not the actual domain.
var anonymizedUrl = new string(url.Select(c => char.IsLetter(c) ? 'x' : c).ToArray());
logger.LogInformation(ex, "Failed to extract favicon from {Url}", anonymizedUrl);
logger.LogInformation(ex, "Failed to extract favicon from {Url}", AnonymizeUrl(url));
}
// Return null if favicon extraction failed.
return Ok(new FaviconExtractModel { Image = null });
}
/// <summary>
/// Extracts favicons for multiple URLs in parallel server-side. Cuts down on round trips when
/// the client needs to fetch many favicons (initial vault import, bulk re-download from the
/// storage insights page).
/// </summary>
/// <param name="request">The batch request payload.</param>
/// <returns>A list of favicon results, one per requested URL, in the same order.</returns>
[HttpPost("ExtractBatch")]
public async Task<IActionResult> ExtractBatch([FromBody] FaviconExtractBatchRequest request)
{
var user = await GetCurrentUserAsync();
if (user == null)
{
return Unauthorized();
}
if (request.Urls.Count == 0)
{
return Ok(new FaviconExtractBatchResponse());
}
if (request.Urls.Count > MaxBatchSize)
{
return BadRequest(new ProblemDetails
{
Title = "Too many URLs",
Detail = $"Batch size is capped at {MaxBatchSize} URLs per request.",
});
}
if (!rateLimitService.TryConsume(user.Id, request.Urls.Count))
{
return StatusCode(StatusCodes.Status429TooManyRequests);
}
var images = await FaviconExtractor.FaviconExtractor.GetFaviconsAsync(request.Urls);
var response = new FaviconExtractBatchResponse
{
Results = new List<FaviconExtractBatchResult>(request.Urls.Count),
};
for (int i = 0; i < request.Urls.Count; i++)
{
response.Results.Add(new FaviconExtractBatchResult
{
Url = request.Urls[i],
Image = images[i],
});
}
return Ok(response);
}
/// <summary>
/// Anonymizes a URL by replacing letters with 'x'. Lets us log host structure without
/// recording the actual domain a user was browsing.
/// </summary>
private static string AnonymizeUrl(string url)
{
return new string(url.Select(c => char.IsLetter(c) ? 'x' : c).ToArray());
}
}

View File

@@ -14,6 +14,7 @@ using AliasServerDb;
using AliasServerDb.Configuration;
using AliasVault.Api;
using AliasVault.Api.Jwt;
using AliasVault.Api.Services;
using AliasVault.Auth;
using AliasVault.Cryptography.Server;
using AliasVault.Logging;
@@ -84,6 +85,7 @@ builder.Services.AddScoped<TimeValidationJwtBearerEvents>();
builder.Services.AddScoped<AuthLoggingService>();
builder.Services.AddScoped<ServerSettingsService>();
builder.Services.AddScoped<RegistrationRateLimitService>();
builder.Services.AddSingleton<FaviconRateLimitService>();
builder.Services.AddHttpContextAccessor();
builder.Services.AddLogging(logging =>

View File

@@ -0,0 +1,80 @@
//-----------------------------------------------------------------------
// <copyright file="FaviconRateLimitService.cs" company="aliasvault">
// Copyright (c) aliasvault. All rights reserved.
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
// </copyright>
//-----------------------------------------------------------------------
namespace AliasVault.Api.Services;
using System;
using System.Collections.Concurrent;
/// <summary>
/// In-memory per-user rate limiter for the favicon extraction API. Each successful extraction
/// (single or one URL inside a batch) consumes one unit; the limit is enforced over a rolling
/// 24-hour window. State is held in process so it is not shared across instances — a deliberate
/// trade-off, since the limit's purpose is to bound the SSRF/DDoS amplification an individual
/// user could cause from a single API instance, not to provide a hard global ceiling.
/// </summary>
public sealed class FaviconRateLimitService
{
/// <summary>
/// Default maximum favicon extractions per user per 24 hours to prevent flood.
/// </summary>
public const int DefaultMaxPer24Hours = 5000;
private static readonly TimeSpan WindowDuration = TimeSpan.FromHours(24);
private readonly ConcurrentDictionary<string, FaviconUserUsage> _usage = new();
private readonly int _maxPer24Hours;
/// <summary>
/// Initializes a new instance of the <see cref="FaviconRateLimitService"/> class.
/// </summary>
public FaviconRateLimitService()
: this(DefaultMaxPer24Hours)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="FaviconRateLimitService"/> class
/// with a custom limit (for tests or future configuration).
/// </summary>
/// <param name="maxPer24Hours">Maximum extractions allowed per user per 24 hours. Set to 0 or less to disable.</param>
public FaviconRateLimitService(int maxPer24Hours)
{
_maxPer24Hours = maxPer24Hours;
}
/// <summary>
/// Attempts to consume <paramref name="count"/> units from the user's allowance. Atomic per user:
/// either the full count is consumed and true is returned, or nothing is consumed and false is returned.
/// </summary>
/// <param name="userId">The authenticated user id.</param>
/// <param name="count">Number of units to consume (one per favicon URL).</param>
/// <returns>True if the request fits in the remaining allowance; false if the limit would be exceeded.</returns>
public bool TryConsume(string userId, int count)
{
if (_maxPer24Hours <= 0 || count <= 0)
{
return true;
}
var usage = _usage.GetOrAdd(userId, _ => new FaviconUserUsage());
lock (usage.SyncRoot)
{
var now = DateTime.UtcNow;
usage.Prune(now - WindowDuration);
if (usage.Total + count > _maxPer24Hours)
{
return false;
}
usage.Add(now, count);
return true;
}
}
}

View File

@@ -0,0 +1,71 @@
//-----------------------------------------------------------------------
// <copyright file="FaviconUserUsage.cs" company="aliasvault">
// Copyright (c) aliasvault. All rights reserved.
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
// </copyright>
//-----------------------------------------------------------------------
namespace AliasVault.Api.Services;
using System;
using System.Collections.Generic;
/// <summary>
/// Per-user usage record for <see cref="FaviconRateLimitService"/>. Tracks recent extractions
/// in per-minute buckets so a power-user re-downloading thousands of favicons in quick succession
/// still produces a bounded number of entries.
/// </summary>
internal sealed class FaviconUserUsage
{
// Bucket entries by the minute they happened in. A power-user re-downloading a 5k-item
// vault produces at most ~minutes-of-batches entries this way, even though the underlying
// unit count can be in the thousands — keeps memory bounded without losing accuracy.
private readonly LinkedList<(DateTime BucketStart, int Count)> _entries = new();
/// <summary>
/// Gets the lock guarding mutations to this usage record. Callers must hold this lock
/// across read-modify-write sequences (Prune + Add) to keep TryConsume atomic.
/// </summary>
public object SyncRoot { get; } = new object();
/// <summary>
/// Gets the total number of units consumed inside the currently retained window.
/// </summary>
public int Total { get; private set; }
/// <summary>
/// Records <paramref name="count"/> consumed units at <paramref name="now"/>, merging into
/// the last bucket when it falls in the same minute.
/// </summary>
/// <param name="now">The timestamp to record under (typically <see cref="DateTime.UtcNow"/>).</param>
/// <param name="count">Number of units consumed.</param>
public void Add(DateTime now, int count)
{
var bucket = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0, DateTimeKind.Utc);
if (_entries.Last is { Value.BucketStart: var lastBucket } lastNode && lastBucket == bucket)
{
_entries.Last.Value = (lastBucket, lastNode.Value.Count + count);
}
else
{
_entries.AddLast((bucket, count));
}
Total += count;
}
/// <summary>
/// Drops buckets older than <paramref name="cutoff"/> from the front of the list, which is
/// where the oldest entries live since Add only appends.
/// </summary>
/// <param name="cutoff">Buckets strictly older than this are removed.</param>
public void Prune(DateTime cutoff)
{
while (_entries.First is { Value.BucketStart: var first } node && first < cutoff)
{
Total -= node.Value.Count;
_entries.RemoveFirst();
}
}
}

View File

@@ -106,6 +106,11 @@ else
{
return "image/png";
}
if (bytes[0] == 0xFF && bytes[1] == 0xD8 && bytes[2] == 0xFF)
{
return "image/jpeg";
}
}
return "image/x-icon";

View File

@@ -69,6 +69,11 @@
@Localizer["SecuritySettingsNav"]
</NavLink>
</li>
<li>
<NavLink href="/settings/storage-insights" class="block py-2 px-4 text-sm hover:bg-gray-100 dark:hover:bg-gray-600 dark:text-gray-400 dark:hover:text-white" ActiveClass="text-primary-700 dark:text-primary-500" Match="NavLinkMatch.All">
@Localizer["StorageInsightsNav"]
</NavLink>
</li>
<li>
<NavLink href="/settings/import-export" class="block py-2 px-4 text-sm hover:bg-gray-100 dark:hover:bg-gray-600 dark:text-gray-400 dark:hover:text-white" ActiveClass="text-primary-700 dark:text-primary-500" Match="NavLinkMatch.All">
@Localizer["ImportExportNav"]

View File

@@ -4,7 +4,7 @@
@inject FolderService FolderService
@inject NavigationManager NavigationManager
@inject GlobalNotificationService GlobalNotificationService
@inject HttpClient HttpClient
@inject FaviconService FaviconService
@inject IStringLocalizerFactory LocalizerFactory
@inject AliasVault.Client.Services.Crypto.AvexCryptoService AvexCrypto
@using Microsoft.Extensions.Localization
@@ -12,7 +12,6 @@
@using AliasVault.ImportExport.Importers
@using AliasVault.ImportExport.Models
@using AliasVault.ImportExport.Models.Exports
@using AliasVault.Shared.Models.WebApi.Favicon
@using AliasClientDb
@using AliasClientDb.Models
@using AliasVault.Client.Auth.Components
@@ -612,8 +611,9 @@
}
/// <summary>
/// Extracts favicons for credentials. Uses domain-based deduplication to avoid
/// fetching the same favicon multiple times for the same domain.
/// Extracts favicons for credentials. Delegates dedup-by-domain and batched server-side
/// fetching to <see cref="FaviconService"/>; this method only handles UI progress reporting
/// and cancellation.
/// </summary>
private async Task ExtractFaviconsForCredentials()
{
@@ -623,59 +623,43 @@
FaviconExtractionCancellation = new CancellationTokenSource();
StateHasChanged();
// Only extract favicons for credentials that don't already have them and have URLs
var credentialsWithUrls = ImportedCredentials
.Where(c => c.FaviconBytes == null && c.ServiceUrls?.FirstOrDefault() != null)
// Only extract favicons for credentials that don't already have them and have URLs.
var urls = ImportedCredentials
.Where(c => c.FaviconBytes == null)
.Select(c => c.ServiceUrls?.FirstOrDefault())
.Where(u => !string.IsNullOrEmpty(u))
.Cast<string>()
.ToList();
// Group credentials by normalized domain to avoid duplicate fetches
var processedDomains = new HashSet<string>();
// The dialog reports progress as "domains processed" rather than "credentials processed",
// since the FaviconService dedupes by domain and that's the actual unit of fetch work.
TotalFaviconsToExtract = urls
.Select(u => FaviconService.TryNormalizeDomain(u, out var d) ? d : null)
.Where(d => d != null)
.Distinct()
.Count();
TotalFaviconsToExtract = credentialsWithUrls.Count();
// Update UI every N items - Task.Delay(1) is required to let browser actually render
// Task.Yield() doesn't work because it doesn't give the browser event loop time to paint
const int updateEveryNItems = 10;
foreach (var credential in credentialsWithUrls)
// Task.Delay(1) inside the progress callback lets Blazor actually paint between batches.
var progress = new Progress<int>(async count =>
{
if (FaviconExtractionCancellation.Token.IsCancellationRequested)
{
break;
}
FaviconExtractionProgress = count;
StateHasChanged();
await Task.Delay(1);
});
// Extract normalized domain for deduplication
try
{
var domain = new Uri(credential.ServiceUrls!.First()).Host.ToLowerInvariant();
if (domain.StartsWith("www."))
{
domain = domain[4..];
}
// Skip if we've already fetched a favicon for this domain
if (!processedDomains.Contains(domain))
{
processedDomains.Add(domain);
await ExtractFaviconForCredential(credential);
}
}
catch
{
// Invalid URL
}
FaviconExtractionProgress++;
// Update UI periodically
if (FaviconExtractionProgress % updateEveryNItems == 0)
{
StateHasChanged();
await Task.Delay(1);
}
var result = await FaviconService.ExtractBulkAsync(urls, progress, FaviconExtractionCancellation.Token);
foreach (var kvp in result.Favicons)
{
ExtractedFavicons[kvp.Key] = kvp.Value;
}
// Final update to show 100% completion before transitioning to next phase
if (result.Status == FaviconService.BulkExtractStatus.RateLimited)
{
// Rate limit error occurred. Silently skip the favicons that couldn't be extracted.
}
// Final update to show 100% completion before transitioning to next phase.
FaviconExtractionProgress = TotalFaviconsToExtract;
StateHasChanged();
await Task.Delay(50);
@@ -683,35 +667,6 @@
StateHasChanged();
}
/// <summary>
/// Extracts a favicon for a credential. Stores by normalized domain for deduplication.
/// </summary>
/// <param name="credential">The credential to extract the favicon for.</param>
private async Task ExtractFaviconForCredential(ImportedCredential credential)
{
try
{
// Extract normalized domain for storage key
var url = credential.ServiceUrls!.First();
var domain = new Uri(url).Host.ToLowerInvariant();
if (domain.StartsWith("www."))
{
domain = domain[4..];
}
var apiReturn = await HttpClient.GetFromJsonAsync<FaviconExtractModel>($"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}");
if (apiReturn?.Image is not null)
{
// Store by normalized domain for deduplication
ExtractedFavicons[domain] = apiReturn.Image;
}
}
catch
{
// Ignore favicon extraction errors
}
}
/// <summary>
/// Imports the items to the database.
/// </summary>
@@ -952,22 +907,34 @@
{
var context = await DbService.GetDbContextAsync();
// Check if logo already exists by source
// Lookup ignores IsDeleted because Logos.Source is UNIQUE — a soft-deleted row
// with the same domain still occupies the slot and would block a new insert.
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
if (existingLogo != null)
if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 })
{
// Healthy existing logo — reuse without overwriting.
item.LogoId = existingLogo.Id;
ImportSessionLogoIds[domain] = existingLogo.Id;
}
else if (existingLogo != null)
{
// Refill a soft-deleted or pruner-emptied row with the imported bytes so the
// new item gets a usable logo, instead of inheriting an empty blob.
var nowUtc = DateTime.UtcNow;
existingLogo.IsDeleted = false;
existingLogo.FileData = importedCredential.FaviconBytes;
existingLogo.FetchedAt = nowUtc;
existingLogo.UpdatedAt = nowUtc;
item.LogoId = existingLogo.Id;
ImportSessionLogoIds[domain] = existingLogo.Id;
}
else
{
// Create new logo from embedded bytes
var logo = new AliasClientDb.Logo
{
Id = Guid.NewGuid(),
Source = domain,
FileData = importedCredential.FaviconBytes,
MimeType = "image/png",
FetchedAt = DateTime.UtcNow,
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow,
@@ -1003,30 +970,38 @@
var context = await DbService.GetDbContextAsync();
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
if (existingLogo != null)
if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 })
{
// Reuse existing logo
item.LogoId = existingLogo.Id;
ImportSessionLogoIds[domain] = existingLogo.Id;
}
else if (ExtractedFavicons.TryGetValue(domain, out var favicon))
{
// Create new logo from extracted favicon
var logo = new AliasClientDb.Logo
var nowUtc = DateTime.UtcNow;
if (existingLogo != null)
{
Id = Guid.NewGuid(),
Source = domain,
FileData = favicon,
MimeType = "image/png",
FetchedAt = DateTime.UtcNow,
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow,
};
context.Logos.Add(logo);
item.LogoId = logo.Id;
// Track this logo for subsequent items with same domain
ImportSessionLogoIds[domain] = logo.Id;
existingLogo.IsDeleted = false;
existingLogo.FileData = favicon;
existingLogo.FetchedAt = nowUtc;
existingLogo.UpdatedAt = nowUtc;
item.LogoId = existingLogo.Id;
ImportSessionLogoIds[domain] = existingLogo.Id;
}
else
{
var logo = new AliasClientDb.Logo
{
Id = Guid.NewGuid(),
Source = domain,
FileData = favicon,
FetchedAt = nowUtc,
CreatedAt = nowUtc,
UpdatedAt = nowUtc,
};
context.Logos.Add(logo);
item.LogoId = logo.Id;
ImportSessionLogoIds[domain] = logo.Id;
}
}
}
}

View File

@@ -90,6 +90,23 @@
</div>
<Button OnClick="@(() => ShowExportConfirmation(ExportType.Csv))">@Localizer["ExportCsvButton"]</Button>
</div>
@* DEBUG-only raw SQLite export. *@
@if (IsDebugBuild)
{
<div class="flex items-center justify-between p-3 border-2 border-dashed border-yellow-400 dark:border-yellow-600 rounded-lg bg-yellow-50 dark:bg-yellow-900/20">
<div class="flex-1 min-w-0 mr-4">
<div class="flex items-center gap-2 mb-1">
<h4 class="text-sm font-semibold text-gray-900 dark:text-white">Export raw SQLite</h4>
<span class="px-1.5 py-0.5 text-xs font-medium text-yellow-800 dark:text-yellow-200 bg-yellow-200 dark:bg-yellow-800 rounded">DEBUG</span>
</div>
<p class="text-xs text-gray-600 dark:text-gray-300">
Downloads the unencrypted SQLite vault file as-is. Anyone with the file can read everything.
</p>
</div>
<Button OnClick="@ExportVaultSqlite">Download .sqlite</Button>
</div>
}
</div>
</div>
@@ -112,6 +129,13 @@
OnClose="@CloseExportPasswordModal" />
@code {
// Assign debug static flag for Razor markup check.
#if DEBUG
private static readonly bool IsDebugBuild = true;
#else
private static readonly bool IsDebugBuild = false;
#endif
private string _username = string.Empty;
private bool _showPasswordConfirmation;
private string _passwordError = string.Empty;
@@ -295,6 +319,27 @@
}
}
// Always defined so the markup binding compiles, only invoked when IsDebugBuild is true.
private async Task ExportVaultSqlite()
{
GlobalLoadingSpinner.Show("Exporting SQLite vault...");
try
{
var base64 = await DbService.ExportSqliteToBase64Async();
var bytes = Convert.FromBase64String(base64);
await JsInteropService.DownloadFileFromStream(await GetExportFileName("sqlite"), bytes);
}
catch (Exception ex)
{
Logger.LogError(ex, "Error exporting raw SQLite vault");
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
}
finally
{
GlobalLoadingSpinner.Hide();
}
}
private async Task ExportVaultAvux()
{
GlobalLoadingSpinner.Show(Localizer["ExportingVaultMessage"]);

View File

@@ -0,0 +1,601 @@
@page "/settings/storage-insights"
@inherits MainBase
@inject AliasVault.Client.Services.FaviconService FaviconService
@inject AliasVault.Client.Services.ItemService ItemService
@inject AliasVault.RazorComponents.Services.ConfirmModalService ConfirmModalService
@using AliasClientDb.Models
@using AliasVault.RazorComponents.Tables
@using Microsoft.EntityFrameworkCore
@using Microsoft.Extensions.Localization
<LayoutPageTitle>@Localizer["PageTitle"]</LayoutPageTitle>
<PageHeader
BreadcrumbItems="@BreadcrumbItems"
Title="@Localizer["PageTitle"]"
Description="@Localizer["PageDescription"]">
</PageHeader>
@if (IsLoading)
{
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
<p class="text-gray-600 dark:text-gray-400">@SharedLocalizer["Loading"]</p>
</div>
}
else
{
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["EstimatedTotalTitle"]</h3>
<p class="text-4xl font-semibold text-gray-900 dark:text-white">@FormatSize(_estimatedTotalBytes)</p>
<p class="mt-2 text-sm text-gray-500 dark:text-gray-400">@Localizer["EstimatedTotalDescription"]</p>
</div>
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
<h3 class="mb-4 text-lg font-medium text-gray-900 dark:text-white">@Localizer["CountsTitle"]</h3>
<div class="grid gap-4 sm:grid-cols-3">
<div class="p-4 border border-gray-200 rounded-lg dark:border-gray-700">
<p class="text-sm text-gray-500 dark:text-gray-400">@Localizer["ItemCountLabel"]</p>
<p class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white">@_totalItems</p>
</div>
<div class="p-4 border border-gray-200 rounded-lg dark:border-gray-700">
<p class="text-sm text-gray-500 dark:text-gray-400">@Localizer["ItemsWithAttachmentsLabel"]</p>
<p class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white">@_itemsWithAttachments</p>
</div>
<div class="p-4 border border-gray-200 rounded-lg dark:border-gray-700">
<p class="text-sm text-gray-500 dark:text-gray-400">@Localizer["ItemsWithLogosLabel"]</p>
<p class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white">@_itemsWithLogos</p>
</div>
</div>
</div>
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["BreakdownTitle"]</h3>
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["BreakdownDescription"]</p>
@if (_estimatedTotalBytes == 0)
{
<p class="text-sm text-gray-500 dark:text-gray-400">-</p>
}
else
{
<div class="flex w-full h-4 overflow-hidden rounded-full bg-gray-200 dark:bg-gray-700">
<div class="bg-gray-400 dark:bg-gray-500" style="width: @(Percent(_baseOverheadBytes, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownBaseOverheadLabel"]"></div>
<div class="bg-blue-500" style="width: @(Percent(_credentialBytes, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownCredentialsLabel"]"></div>
<div class="bg-amber-500" style="width: @(Percent(_attachmentBytesWithOverhead, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownAttachmentsLabel"]"></div>
<div class="bg-emerald-500" style="width: @(Percent(_logoBytesWithOverhead, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownLogosLabel"]"></div>
</div>
<div class="mt-4 grid gap-2 sm:grid-cols-2 lg:grid-cols-4 text-sm">
<div class="flex items-center gap-2">
<span class="inline-block w-3 h-3 rounded-sm bg-gray-400 dark:bg-gray-500"></span>
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownBaseOverheadLabel"]:</span>
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_baseOverheadBytes) (@Percent(_baseOverheadBytes, _estimatedTotalBytes).ToString("F1")%)</span>
</div>
<div class="flex items-center gap-2">
<span class="inline-block w-3 h-3 rounded-sm bg-blue-500"></span>
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownCredentialsLabel"]:</span>
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_credentialBytes) (@Percent(_credentialBytes, _estimatedTotalBytes).ToString("F1")%)</span>
</div>
<div class="flex items-center gap-2">
<span class="inline-block w-3 h-3 rounded-sm bg-amber-500"></span>
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownAttachmentsLabel"]:</span>
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_attachmentBytesWithOverhead) (@Percent(_attachmentBytesWithOverhead, _estimatedTotalBytes).ToString("F1")%)</span>
</div>
<div class="flex items-center gap-2">
<span class="inline-block w-3 h-3 rounded-sm bg-emerald-500"></span>
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownLogosLabel"]:</span>
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_logoBytesWithOverhead) (@Percent(_logoBytesWithOverhead, _estimatedTotalBytes).ToString("F1")%)</span>
</div>
</div>
}
</div>
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["TopAttachmentsTitle"]</h3>
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["TopAttachmentsDescription"]</p>
@if (_topAttachments.Count == 0)
{
<p class="text-sm text-gray-500 dark:text-gray-400">-</p>
}
else
{
<SortableTable Columns="@_attachmentColumns" SortColumn="@string.Empty" SortDirection="SortDirection.Descending" OnSortChanged="(_ => Task.CompletedTask)">
@foreach (var attachment in _topAttachments)
{
<SortableTableRow Class="cursor-pointer" OnClick="@(() => NavigateToItem(attachment.ItemId))">
<SortableTableColumn IsPrimary="true">@attachment.Filename</SortableTableColumn>
<SortableTableColumn>@FormatSize(attachment.SizeBytes)</SortableTableColumn>
<SortableTableColumn>@(string.IsNullOrWhiteSpace(attachment.ItemName) ? "-" : attachment.ItemName)</SortableTableColumn>
<SortableTableColumn>@attachment.CreatedAt.ToLocalTime().ToString("yyyy-MM-dd")</SortableTableColumn>
</SortableTableRow>
}
</SortableTable>
}
</div>
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["TopLogosTitle"]</h3>
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["TopLogosDescription"]</p>
@if (_topLogos.Count == 0)
{
<p class="text-sm text-gray-500 dark:text-gray-400">-</p>
}
else
{
<SortableTable Columns="@_logoColumns" SortColumn="@string.Empty" SortDirection="SortDirection.Descending" OnSortChanged="(_ => Task.CompletedTask)">
@foreach (var logo in _topLogos)
{
<SortableTableRow Class="@(logo.FirstItemId.HasValue ? "cursor-pointer" : string.Empty)" OnClick="@(logo.FirstItemId.HasValue ? EventCallback.Factory.Create(this, () => NavigateToItem(logo.FirstItemId!.Value)) : default)">
<SortableTableColumn IsPrimary="true">@logo.Source</SortableTableColumn>
<SortableTableColumn>@FormatSize(logo.SizeBytes)</SortableTableColumn>
<SortableTableColumn>@logo.ItemCount</SortableTableColumn>
</SortableTableRow>
}
</SortableTable>
}
</div>
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["LogoManagementTitle"]</h3>
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["LogoManagementDescription"]</p>
@if (_isRedownloadingLogos)
{
<div class="mb-4">
<div class="flex justify-between text-sm text-gray-700 dark:text-gray-300 mb-1">
<span>@SharedLocalizer["Loading"]</span>
<span>@_redownloadProgress / @_redownloadTotal</span>
</div>
<div class="w-full bg-gray-200 rounded-full h-2.5 dark:bg-gray-700">
<div class="bg-primary-600 h-2.5 rounded-full transition-all duration-300" style="width: @(RedownloadPercent().ToString("F1", System.Globalization.CultureInfo.InvariantCulture))%"></div>
</div>
</div>
}
<div class="flex flex-col sm:flex-row gap-3">
<Button Color="default" OnClick="RedownloadAllLogos" IsDisabled="@(_isRedownloadingLogos || _isDeletingLogos)">@Localizer["RedownloadAllLogosButton"]</Button>
<Button Color="danger" OnClick="DeleteAllLogos" IsDisabled="@(_isRedownloadingLogos || _isDeletingLogos)">@Localizer["DeleteAllLogosButton"]</Button>
</div>
</div>
}
@code {
// Approximate per-row SQLite overhead in bytes. SQLite uses 4 KiB pages by default;
// small rows pack many per page, while large blobs spill into overflow pages. These
// constants cover row headers and the indexed columns AliasClientDbContext defines.
// They are intentionally rough — attachments and logos dominate real vault size.
private const int PerItemOverheadBytes = 200;
private const int PerFieldValueOverheadBytes = 80;
private const int PerAttachmentOverheadBytes = 120;
private const int PerLogoOverheadBytes = 150;
private const int PerTotpOverheadBytes = 100;
private const int PerPasskeyOverheadBytes = 200;
// Empty AliasVault SQLite file is ~200 KB on disk: file header, page allocation
// for each table/index, sqlite_master schema rows, and __EFMigrationsHistory.
// Constant rather than measured because we don't have access to the file size
// here and the value is stable across vaults of the same schema version.
private const int BaseSqliteOverheadBytes = 200 * 1024;
private readonly List<TableColumn> _attachmentColumns =
[
new TableColumn { Title = "Filename", Sortable = false },
new TableColumn { Title = "Size", Sortable = false },
new TableColumn { Title = "Item", Sortable = false },
new TableColumn { Title = "Created", Sortable = false },
];
private readonly List<TableColumn> _logoColumns =
[
new TableColumn { Title = "Source", Sortable = false },
new TableColumn { Title = "Size", Sortable = false },
new TableColumn { Title = "Used by", Sortable = false },
];
private bool IsLoading { get; set; } = true;
private int _totalItems;
private int _itemsWithAttachments;
private int _itemsWithLogos;
private long _credentialBytes;
private long _attachmentBytesWithOverhead;
private long _logoBytesWithOverhead;
private long _baseOverheadBytes;
private long _estimatedTotalBytes;
private List<AttachmentRow> _topAttachments = [];
private List<LogoRow> _topLogos = [];
private bool _isDeletingLogos;
private bool _isRedownloadingLogos;
private int _redownloadProgress;
private int _redownloadTotal;
private IStringLocalizer Localizer => LocalizerFactory.Create("Pages.Main.Settings.StorageInsights", "AliasVault.Client");
/// <inheritdoc />
protected override async Task OnInitializedAsync()
{
await base.OnInitializedAsync();
BreadcrumbItems.Add(new BreadcrumbItem { DisplayName = Localizer["BreadcrumbTitle"] });
// Localize the table column headers (created at field init time before localizer was available).
_attachmentColumns[0].Title = Localizer["ColumnFilename"];
_attachmentColumns[1].Title = Localizer["ColumnSize"];
_attachmentColumns[2].Title = Localizer["ColumnItem"];
_attachmentColumns[3].Title = Localizer["ColumnCreated"];
_logoColumns[0].Title = Localizer["ColumnWebsiteURL"];
_logoColumns[1].Title = Localizer["ColumnSize"];
_logoColumns[2].Title = Localizer["ColumnItemCount"];
await LoadStatisticsAsync();
}
private static string FormatSize(long bytes)
{
const double Kib = 1024d;
const double Mib = Kib * 1024d;
if (bytes < Kib)
{
return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{bytes} B");
}
if (bytes < Mib)
{
return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{(bytes / Kib):F1} KB");
}
return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{(bytes / Mib):F1} MB");
}
private static double Percent(long part, long total)
{
return total == 0 ? 0d : Math.Round((double)part / total * 100d, 2);
}
private async Task LoadStatisticsAsync()
{
var ctx = await DbService.GetDbContextAsync();
_totalItems = await ctx.Items.CountAsync(i => !i.IsDeleted && i.DeletedAt == null);
_itemsWithAttachments = await ctx.Attachments
.Where(a => !a.IsDeleted)
.Select(a => a.ItemId)
.Distinct()
.CountAsync();
_itemsWithLogos = await ctx.Items.CountAsync(i => !i.IsDeleted && i.DeletedAt == null && i.LogoId != null);
var attachmentRows = await ctx.Attachments
.Where(a => !a.IsDeleted)
.Select(a => new AttachmentRow
{
Id = a.Id,
Filename = a.Filename,
SizeBytes = a.Blob.Length,
ItemId = a.ItemId,
ItemName = a.Item.Name,
CreatedAt = a.CreatedAt,
})
.ToListAsync();
var logoProjections = await ctx.Logos
.Where(l => !l.IsDeleted && l.FileData != null)
.Select(l => new
{
l.Id,
l.Source,
SizeBytes = l.FileData!.Length,
ItemIds = l.Items
.Where(i => !i.IsDeleted && i.DeletedAt == null)
.Select(i => i.Id)
.ToList(),
})
.ToListAsync();
var logoRows = logoProjections
.Select(l => new LogoRow
{
Id = l.Id,
Source = l.Source,
SizeBytes = l.SizeBytes,
ItemCount = l.ItemIds.Count,
FirstItemId = l.ItemIds.Count > 0 ? l.ItemIds[0] : null,
})
.ToList();
var fieldValueLengths = await ctx.FieldValues
.Where(fv => !fv.IsDeleted)
.Select(fv => fv.Value == null ? 0 : fv.Value.Length)
.ToListAsync();
var fieldValueBytes = fieldValueLengths.Sum(x => (long)x);
var fieldValueCount = fieldValueLengths.Count;
var passkeySizes = await ctx.Passkeys
.Where(p => !p.IsDeleted)
.Select(p => new
{
UserHandleLen = p.UserHandle.Length,
PrfKeyLen = p.PrfKey == null ? 0 : p.PrfKey.Length,
AdditionalDataLen = p.AdditionalData == null ? 0 : p.AdditionalData.Length,
PublicKeyLen = p.PublicKey.Length,
PrivateKeyLen = p.PrivateKey.Length,
})
.ToListAsync();
var passkeyBytes = passkeySizes.Sum(p => (long)(p.UserHandleLen + p.PrfKeyLen + p.AdditionalDataLen + p.PublicKeyLen + p.PrivateKeyLen));
var passkeyCount = passkeySizes.Count;
var totpCount = await ctx.TotpCodes.CountAsync(t => !t.IsDeleted);
var attachmentBytes = attachmentRows.Sum(a => (long)a.SizeBytes);
var logoBytes = logoRows.Sum(l => (long)l.SizeBytes);
_attachmentBytesWithOverhead = attachmentBytes + ((long)attachmentRows.Count * PerAttachmentOverheadBytes);
_logoBytesWithOverhead = logoBytes + ((long)logoRows.Count * PerLogoOverheadBytes);
_credentialBytes = fieldValueBytes
+ passkeyBytes
+ ((long)_totalItems * PerItemOverheadBytes)
+ ((long)fieldValueCount * PerFieldValueOverheadBytes)
+ ((long)passkeyCount * PerPasskeyOverheadBytes)
+ ((long)totpCount * PerTotpOverheadBytes);
_baseOverheadBytes = BaseSqliteOverheadBytes;
_estimatedTotalBytes = _credentialBytes
+ _attachmentBytesWithOverhead
+ _logoBytesWithOverhead
+ _baseOverheadBytes;
_topAttachments = attachmentRows
.OrderByDescending(a => a.SizeBytes)
.Take(10)
.ToList();
_topLogos = logoRows
.OrderByDescending(l => l.SizeBytes)
.Take(10)
.ToList();
IsLoading = false;
StateHasChanged();
}
private void NavigateToItem(Guid itemId)
{
NavigationManager.NavigateTo($"/items/{itemId}");
}
private double RedownloadPercent()
{
return _redownloadTotal == 0 ? 0d : Math.Round((double)_redownloadProgress / _redownloadTotal * 100d, 1);
}
/// <summary>
/// Soft-deletes every logo in the vault and clears the LogoId on every item that referenced one.
/// The vault sync that follows propagates these changes to other devices.
/// </summary>
private async Task DeleteAllLogos()
{
var confirmed = await ConfirmModalService.ShowConfirmation(
Localizer["DeleteAllLogosConfirmTitle"],
Localizer["DeleteAllLogosConfirmMessage"],
SharedLocalizer["Confirm"],
SharedLocalizer["Cancel"]);
if (!confirmed)
{
return;
}
_isDeletingLogos = true;
StateHasChanged();
GlobalLoadingSpinner.Show(SharedLocalizer["Loading"]);
try
{
var ctx = await DbService.GetDbContextAsync();
var now = DateTime.UtcNow;
// Soft-delete the row AND clear the binary blob to free up space.
var logos = await ctx.Logos.Where(l => !l.IsDeleted).ToListAsync();
foreach (var logo in logos)
{
logo.IsDeleted = true;
logo.FileData = Array.Empty<byte>();
logo.UpdatedAt = now;
}
// Clear LogoId on items so they no longer reference soft-deleted logos.
var itemsWithLogos = await ctx.Items.Where(i => i.LogoId != null).ToListAsync();
foreach (var item in itemsWithLogos)
{
item.LogoId = null;
item.UpdatedAt = now;
}
await ctx.SaveChangesAsync();
// VACUUM after a bulk blob delete — without it the SQLite pages stay allocated and
// the user wouldn't actually see the size drop they came here for.
await DbService.VacuumDatabaseAsync();
await DbService.SaveDatabaseAsync();
GlobalNotificationService.AddSuccessMessage(SharedLocalizer["Success"], true);
await LoadStatisticsAsync();
}
catch (Exception)
{
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
}
finally
{
GlobalLoadingSpinner.Hide();
_isDeletingLogos = false;
StateHasChanged();
}
}
/// <summary>
/// Re-downloads favicons for every item that has a login URL, overwriting any existing logo.
/// Useful after the favicon extractor's sizing/quality settings change so the vault picks up
/// the new output.
/// </summary>
private async Task RedownloadAllLogos()
{
var confirmed = await ConfirmModalService.ShowConfirmation(Localizer["RedownloadAllLogosConfirmTitle"], Localizer["RedownloadAllLogosConfirmMessage"], SharedLocalizer["Confirm"], SharedLocalizer["Cancel"]);
if (!confirmed)
{
return;
}
_isRedownloadingLogos = true;
_redownloadProgress = 0;
_redownloadTotal = 0;
StateHasChanged();
try
{
var items = await ItemService.LoadAllAsync();
// Group items by normalized domain so each domain is fetched once and assigned to all items.
var itemsByDomain = new Dictionary<string, List<AliasClientDb.Item>>();
var representativeUrls = new Dictionary<string, string>();
foreach (var item in items)
{
var url = ItemService.GetFieldValue(item, FieldKey.LoginUrl);
if (string.IsNullOrEmpty(url) || url == ItemService.DefaultServiceUrl)
{
continue;
}
if (!FaviconService.TryNormalizeDomain(url, out var domain))
{
continue;
}
if (!itemsByDomain.TryGetValue(domain, out var bucket))
{
bucket = new List<AliasClientDb.Item>();
itemsByDomain[domain] = bucket;
representativeUrls[domain] = url;
}
bucket.Add(item);
}
_redownloadTotal = itemsByDomain.Count;
StateHasChanged();
if (_redownloadTotal == 0)
{
GlobalNotificationService.AddSuccessMessage(Localizer["RedownloadAllLogosNoItemsMessage"], true);
return;
}
var progress = new Progress<int>(count =>
{
_redownloadProgress = count;
InvokeAsync(StateHasChanged);
});
var result = await FaviconService.ExtractBulkAsync(representativeUrls.Values, progress);
var ctx = await DbService.GetDbContextAsync();
var now = DateTime.UtcNow;
foreach (var (domain, bucket) in itemsByDomain)
{
if (!result.Favicons.TryGetValue(domain, out var image))
{
continue;
}
// Look up by Source (unique field).
var existingLogo = await ctx.Logos.FirstOrDefaultAsync(l => l.Source == domain);
Guid logoId;
if (existingLogo != null)
{
existingLogo.IsDeleted = false;
existingLogo.FileData = image;
existingLogo.FetchedAt = now;
existingLogo.UpdatedAt = now;
logoId = existingLogo.Id;
}
else
{
var newLogo = new AliasClientDb.Logo
{
Id = Guid.NewGuid(),
Source = domain,
FileData = image,
FetchedAt = now,
CreatedAt = now,
UpdatedAt = now,
};
ctx.Logos.Add(newLogo);
logoId = newLogo.Id;
}
foreach (var item in bucket)
{
if (item.LogoId != logoId)
{
item.LogoId = logoId;
item.UpdatedAt = now;
}
}
}
await ctx.SaveChangesAsync();
await DbService.SaveDatabaseAsync();
if (result.Status == FaviconService.BulkExtractStatus.RateLimited)
{
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
}
else
{
GlobalNotificationService.AddSuccessMessage(SharedLocalizer["Success"], true);
}
await LoadStatisticsAsync();
}
catch (Exception)
{
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
}
finally
{
_isRedownloadingLogos = false;
StateHasChanged();
}
}
private sealed class AttachmentRow
{
public Guid Id { get; init; }
public string Filename { get; init; } = string.Empty;
public int SizeBytes { get; init; }
public Guid ItemId { get; init; }
public string? ItemName { get; init; }
public DateTime CreatedAt { get; init; }
}
private sealed class LogoRow
{
public Guid Id { get; init; }
public string Source { get; init; } = string.Empty;
public int SizeBytes { get; init; }
public int ItemCount { get; init; }
public Guid? FirstItemId { get; init; }
}
}

View File

@@ -87,6 +87,7 @@ builder.Services.AddTransient<AliasVaultApiHandlerService>();
builder.Services.AddScoped<AuthService>();
builder.Services.AddScoped<UserRegistrationService>();
builder.Services.AddScoped<AuthenticationStateProvider, AuthStateProvider>();
builder.Services.AddScoped<FaviconService>();
builder.Services.AddScoped<ItemService>();
builder.Services.AddScoped<FolderService>();
builder.Services.AddScoped<DbService>();

View File

@@ -39,6 +39,10 @@
<value>Security settings</value>
<comment>Navigation link for security settings</comment>
</data>
<data name="StorageInsightsNav">
<value>Storage insights</value>
<comment>Navigation link for vault storage insights page</comment>
</data>
<data name="ImportExportNav">
<value>Import / Export</value>
<comment>Navigation link for import/export settings</comment>

View File

@@ -0,0 +1,174 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:element name="root" msdata:IsDataSet="true">
<xsd:complexType>
<xsd:choice maxOccurs="unbounded">
<xsd:element name="data">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" />
<xsd:element name="comment" type="xsd:string" minOccurs="0" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" />
<xsd:attribute name="type" type="xsd:string" />
<xsd:attribute name="mimetype" type="xsd:string" />
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>
<!-- Page title and description -->
<data name="PageTitle">
<value>Vault storage insights</value>
<comment>Page title for vault storage insights</comment>
</data>
<data name="PageDescription">
<value>See an estimate of your vault size. For best sync performance, it's advised to keep your vault size small and delete items you no longer need.</value>
<comment>Page description for vault storage insights</comment>
</data>
<data name="BreadcrumbTitle">
<value>Storage insights</value>
<comment>Breadcrumb title for storage insights page</comment>
</data>
<!-- Estimated total -->
<data name="EstimatedTotalTitle">
<value>Estimated vault size</value>
<comment>Title for the estimated total vault size card</comment>
</data>
<data name="EstimatedTotalDescription">
<value>This is an approximation calculated from your local data.</value>
<comment>Subtext explaining that the estimate is approximate</comment>
</data>
<!-- Counts -->
<data name="CountsTitle">
<value>Overview</value>
<comment>Title for the counts/overview section</comment>
</data>
<data name="ItemCountLabel">
<value>Items</value>
<comment>Label for total item count card</comment>
</data>
<data name="ItemsWithAttachmentsLabel">
<value>Items with attachments</value>
<comment>Label for count of items that have at least one attachment</comment>
</data>
<data name="ItemsWithLogosLabel">
<value>Items with a logo</value>
<comment>Label for count of items that reference a logo</comment>
</data>
<!-- Breakdown -->
<data name="BreakdownTitle">
<value>Storage usage breakdown</value>
<comment>Title for the breakdown bar section</comment>
</data>
<data name="BreakdownDescription">
<value>Approximate share of vault size by category. Attachments and logos usually take up most of the space.</value>
<comment>Description for the breakdown bar</comment>
</data>
<data name="BreakdownCredentialsLabel">
<value>Credentials</value>
<comment>Legend label for credential data slice</comment>
</data>
<data name="BreakdownAttachmentsLabel">
<value>Attachments</value>
<comment>Legend label for attachments slice</comment>
</data>
<data name="BreakdownLogosLabel">
<value>Logos</value>
<comment>Legend label for logos slice</comment>
</data>
<data name="BreakdownBaseOverheadLabel">
<value>Database overhead</value>
<comment>Legend label for the fixed SQLite base/schema overhead slice</comment>
</data>
<!-- Top attachments table -->
<data name="TopAttachmentsTitle">
<value>Largest attachments</value>
<comment>Title for the top attachments table</comment>
</data>
<data name="TopAttachmentsDescription">
<value>Top 10 largest file attachments. Remove the attachment if you no longer need it.</value>
<comment>Description for the top attachments table</comment>
</data>
<!-- Top logos table -->
<data name="TopLogosTitle">
<value>Largest logos</value>
<comment>Title for the top logos table</comment>
</data>
<data name="TopLogosDescription">
<value>Top 10 largest service logos. Logos are reused for items with the same domain.</value>
<comment>Description for the top logos table</comment>
</data>
<!-- Table column headers -->
<data name="ColumnFilename">
<value>Filename</value>
<comment>Table column header for attachment filename</comment>
</data>
<data name="ColumnSize">
<value>Size</value>
<comment>Table column header for size in KB</comment>
</data>
<data name="ColumnItem">
<value>Item</value>
<comment>Table column header for the parent item name</comment>
</data>
<data name="ColumnCreated">
<value>Created</value>
<comment>Table column header for creation date</comment>
</data>
<data name="ColumnWebsiteURL">
<value>Website URL</value>
<comment>Table column header for logo source domain</comment>
</data>
<data name="ColumnItemCount">
<value>Used by</value>
<comment>Table column header for the number of items using a logo</comment>
</data>
<!-- Logo management -->
<data name="LogoManagementTitle">
<value>Manage logos</value>
<comment>Title for the logo management section</comment>
</data>
<data name="LogoManagementDescription">
<value>Logos are fetched from the favicon of each item's website. You can delete every logo to reclaim space, or re-download them to pick up improved versions.</value>
<comment>Description for the logo management section</comment>
</data>
<data name="DeleteAllLogosButton">
<value>Delete all logos</value>
<comment>Button to delete every logo in the vault</comment>
</data>
<data name="RedownloadAllLogosButton">
<value>Re-download all logos</value>
<comment>Button to re-fetch favicons for every item with a URL</comment>
</data>
<data name="DeleteAllLogosConfirmTitle">
<value>Delete all logos?</value>
<comment>Confirmation modal title for delete-all-logos</comment>
</data>
<data name="DeleteAllLogosConfirmMessage">
<value>This removes every stored logo from your vault and clears the logo on each credential. You can re-download them at any time. Continue?</value>
<comment>Confirmation modal body for delete-all-logos</comment>
</data>
<data name="RedownloadAllLogosConfirmTitle">
<value>Re-download all logos?</value>
<comment>Confirmation modal title for redownload-all-logos</comment>
</data>
<data name="RedownloadAllLogosConfirmMessage">
<value>This re-fetches the favicon for every credential that has a website URL, replacing the existing logo. Depending on your vault size this may take a while. Continue?</value>
<comment>Confirmation modal body for redownload-all-logos</comment>
</data>
<data name="RedownloadAllLogosNoItemsMessage">
<value>No items with a website URL were found, nothing to download.</value>
<comment>Notification shown when redownload was triggered but no items qualify</comment>
</data>
</root>

View File

@@ -0,0 +1,221 @@
//-----------------------------------------------------------------------
// <copyright file="FaviconService.cs" company="aliasvault">
// Copyright (c) aliasvault. All rights reserved.
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
// </copyright>
//-----------------------------------------------------------------------
namespace AliasVault.Client.Services;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using AliasVault.Shared.Models.WebApi.Favicon;
/// <summary>
/// Wraps calls to the server-side favicon API. Centralizes the single-URL and batched
/// extraction paths so item add/edit, vault import, and bulk re-download all share one
/// implementation (and one place to evolve, e.g. when adjusting batch size).
/// </summary>
public sealed class FaviconService(HttpClient httpClient)
{
/// <summary>
/// Number of URLs sent per batch request. Mirrors <c>FaviconController.MaxBatchSize</c> on the server.
/// </summary>
public const int BatchSize = 10;
/// <summary>
/// Outcome of a bulk extraction.
/// </summary>
public enum BulkExtractStatus
{
/// <summary>All requested URLs were processed.</summary>
Completed,
/// <summary>The user cancelled the operation before all URLs were processed.</summary>
Cancelled,
/// <summary>Server returned 429 — the per-user 24h rate limit was hit. Partial results are returned.</summary>
RateLimited,
}
/// <summary>
/// Normalizes a URL to a lowercase host without leading "www.". Returns false for URLs that
/// can't be parsed; callers should skip those rather than fall back to a different key.
/// </summary>
/// <param name="url">The URL to normalize.</param>
/// <param name="domain">The normalized domain.</param>
/// <returns>True if the URL was parseable.</returns>
public static bool TryNormalizeDomain(string url, out string domain)
{
domain = string.Empty;
if (string.IsNullOrWhiteSpace(url))
{
return false;
}
try
{
var host = new Uri(url).Host.ToLowerInvariant();
if (host.StartsWith("www.", StringComparison.Ordinal))
{
host = host[4..];
}
if (string.IsNullOrEmpty(host))
{
return false;
}
domain = host;
return true;
}
catch
{
return false;
}
}
/// <summary>
/// Extracts the favicon for a single URL.
/// </summary>
/// <param name="url">The URL to fetch the favicon for.</param>
/// <returns>Favicon bytes, or null if extraction failed or the rate limit was hit.</returns>
public async Task<byte[]?> ExtractAsync(string url)
{
try
{
var apiReturn = await httpClient.GetFromJsonAsync<FaviconExtractModel>(
$"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}");
return apiReturn?.Image;
}
catch
{
return null;
}
}
/// <summary>
/// Extracts favicons for many URLs in batches, deduplicating by normalized domain so the
/// same domain is only fetched once even if many items reference it.
/// </summary>
/// <param name="urls">URLs to extract favicons for. Duplicates by domain are collapsed.</param>
/// <param name="progress">
/// Optional progress callback. Receives the number of unique domains processed so far
/// (regardless of fetch success). The total reported is the count of unique domains, not
/// the count of input URLs.
/// </param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The favicon dictionary plus the operation status.</returns>
public async Task<BulkExtractResult> ExtractBulkAsync(IEnumerable<string> urls, IProgress<int>? progress = null, CancellationToken cancellationToken = default)
{
var favicons = new Dictionary<string, byte[]>();
// Deduplicate by domain up front so a 5000-item vault that all uses 200 unique domains
// makes 200 fetches, not 5000.
var domainToUrl = new Dictionary<string, string>();
foreach (var url in urls)
{
if (string.IsNullOrWhiteSpace(url))
{
continue;
}
if (TryNormalizeDomain(url, out var domain) && !domainToUrl.ContainsKey(domain))
{
domainToUrl[domain] = url;
}
}
if (domainToUrl.Count == 0)
{
return new BulkExtractResult(favicons, BulkExtractStatus.Completed);
}
var entries = domainToUrl.ToList();
int processed = 0;
for (int i = 0; i < entries.Count; i += BatchSize)
{
if (cancellationToken.IsCancellationRequested)
{
return new BulkExtractResult(favicons, BulkExtractStatus.Cancelled);
}
var chunk = entries.Skip(i).Take(BatchSize).ToList();
var request = new FaviconExtractBatchRequest
{
Urls = chunk.Select(e => e.Value).ToList(),
};
HttpResponseMessage response;
try
{
response = await httpClient.PostAsJsonAsync("v1/Favicon/ExtractBatch", request, cancellationToken);
}
catch
{
// Network or other transport error — treat the whole chunk as failed but keep going.
processed += chunk.Count;
progress?.Report(processed);
continue;
}
if (response.StatusCode == HttpStatusCode.TooManyRequests)
{
return new BulkExtractResult(favicons, BulkExtractStatus.RateLimited);
}
if (!response.IsSuccessStatusCode)
{
processed += chunk.Count;
progress?.Report(processed);
continue;
}
FaviconExtractBatchResponse? body;
try
{
body = await response.Content.ReadFromJsonAsync<FaviconExtractBatchResponse>(cancellationToken: cancellationToken);
}
catch
{
processed += chunk.Count;
progress?.Report(processed);
continue;
}
if (body?.Results != null)
{
// Match results back to the domains we asked for. Server echoes the URL, but matching
// by index is robust against any normalization differences.
for (int j = 0; j < body.Results.Count && j < chunk.Count; j++)
{
var image = body.Results[j].Image;
if (image != null)
{
favicons[chunk[j].Key] = image;
}
}
}
processed += chunk.Count;
progress?.Report(processed);
}
return new BulkExtractResult(favicons, BulkExtractStatus.Completed);
}
/// <summary>
/// Result of a bulk extraction call. Favicons are keyed by normalized domain so callers
/// can reuse a single fetched favicon across all items that share that domain.
/// </summary>
/// <param name="Favicons">Dictionary of normalized domain to favicon bytes.</param>
/// <param name="Status">Outcome of the operation.</param>
public record BulkExtractResult(Dictionary<string, byte[]> Favicons, BulkExtractStatus Status);
}

View File

@@ -18,14 +18,12 @@ using System.Threading.Tasks;
using AliasClientDb;
using AliasClientDb.Models;
using AliasVault.Client.Main.Models;
using AliasVault.Client.Utilities;
using AliasVault.Shared.Models.WebApi.Favicon;
using Microsoft.EntityFrameworkCore;
/// <summary>
/// Service class for Item operations.
/// </summary>
public sealed class ItemService(HttpClient httpClient, DbService dbService, Config config, JsInteropService jsInteropService)
public sealed class ItemService(HttpClient httpClient, DbService dbService, Config config, JsInteropService jsInteropService, FaviconService faviconService)
{
/// <summary>
/// The default service URL used as placeholder in forms. When this value is set, the URL field is considered empty
@@ -1492,59 +1490,68 @@ public sealed class ItemService(HttpClient httpClient, DbService dbService, Conf
/// <returns>Task.</returns>
private async Task ExtractFaviconAsync(Item item)
{
// Try to extract favicon from service URL
var url = GetFieldValue(item, FieldKey.LoginUrl);
if (url != null && !string.IsNullOrEmpty(url) && url != DefaultServiceUrl)
{
try
{
// Extract and normalize domain for deduplication
var domain = new Uri(url).Host.ToLowerInvariant();
if (domain.StartsWith("www."))
{
domain = domain[4..];
}
var context = await dbService.GetDbContextAsync();
// Check if logo already exists for this source (deduplication)
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
if (existingLogo != null)
{
// Reuse existing logo - no need to fetch
item.LogoId = existingLogo.Id;
return;
}
// No existing logo - fetch from API
var apiReturn = await httpClient.GetFromJsonAsync<FaviconExtractModel>($"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}");
if (apiReturn?.Image is not null)
{
// Create new logo
var newLogo = new Logo
{
Id = Guid.NewGuid(),
Source = domain,
FileData = apiReturn.Image,
MimeType = "image/png",
FetchedAt = DateTime.UtcNow,
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow,
};
context.Logos.Add(newLogo);
item.LogoId = newLogo.Id;
}
}
catch
{
// Ignore favicon extraction errors
}
}
else
if (url == null || string.IsNullOrEmpty(url) || url == DefaultServiceUrl)
{
// URL is empty or just the placeholder - clear any existing logo
item.LogoId = null;
return;
}
if (!FaviconService.TryNormalizeDomain(url, out var domain))
{
return;
}
try
{
var context = await dbService.GetDbContextAsync();
// Look up by Source regardless of IsDeleted — Logos.Source has a UNIQUE index, so a
// soft-deleted row with the same domain still occupies the slot and we'd hit a UNIQUE
// violation on insert. Filter against IsDeleted/empty FileData when deciding whether
// it's safe to reuse without re-fetching.
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 })
{
item.LogoId = existingLogo.Id;
return;
}
var image = await faviconService.ExtractAsync(url);
if (image is null)
{
return;
}
var now = DateTime.UtcNow;
if (existingLogo != null)
{
// Restore (or refill) the existing row.
existingLogo.IsDeleted = false;
existingLogo.FileData = image;
existingLogo.FetchedAt = now;
existingLogo.UpdatedAt = now;
item.LogoId = existingLogo.Id;
}
else
{
var newLogo = new Logo
{
Id = Guid.NewGuid(),
Source = domain,
FileData = image,
FetchedAt = now,
CreatedAt = now,
UpdatedAt = now,
};
context.Logos.Add(newLogo);
item.LogoId = newLogo.Id;
}
}
catch
{
// Ignore favicon extraction errors
}
}

View File

@@ -1638,6 +1638,10 @@ video {
border-radius: 0.375rem;
}
.rounded-sm {
border-radius: 0.125rem;
}
.rounded-b-lg {
border-bottom-right-radius: 0.5rem;
border-bottom-left-radius: 0.5rem;
@@ -1809,6 +1813,11 @@ video {
border-color: rgb(254 240 138 / var(--tw-border-opacity));
}
.border-yellow-400 {
--tw-border-opacity: 1;
border-color: rgb(250 204 21 / var(--tw-border-opacity));
}
.bg-amber-100 {
--tw-bg-opacity: 1;
background-color: rgb(254 243 199 / var(--tw-bg-opacity));
@@ -2018,6 +2027,21 @@ video {
background-color: rgb(234 179 8 / var(--tw-bg-opacity));
}
.bg-amber-500 {
--tw-bg-opacity: 1;
background-color: rgb(245 158 11 / var(--tw-bg-opacity));
}
.bg-emerald-500 {
--tw-bg-opacity: 1;
background-color: rgb(16 185 129 / var(--tw-bg-opacity));
}
.bg-yellow-200 {
--tw-bg-opacity: 1;
background-color: rgb(254 240 138 / var(--tw-bg-opacity));
}
.bg-opacity-50 {
--tw-bg-opacity: 0.5;
}
@@ -3294,6 +3318,11 @@ video {
border-color: rgb(133 77 14 / var(--tw-border-opacity));
}
.dark\:border-yellow-600:is(.dark *) {
--tw-border-opacity: 1;
border-color: rgb(202 138 4 / var(--tw-border-opacity));
}
.dark\:bg-amber-800\/30:is(.dark *) {
background-color: rgb(146 64 14 / 0.3);
}
@@ -4269,6 +4298,10 @@ video {
grid-template-columns: repeat(3, minmax(0, 1fr));
}
.lg\:grid-cols-4 {
grid-template-columns: repeat(4, minmax(0, 1fr));
}
.lg\:flex-row {
flex-direction: row;
}

View File

@@ -0,0 +1,22 @@
//-----------------------------------------------------------------------
// <copyright file="FaviconExtractBatchRequest.cs" company="aliasvault">
// Copyright (c) aliasvault. All rights reserved.
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
// </copyright>
//-----------------------------------------------------------------------
namespace AliasVault.Shared.Models.WebApi.Favicon;
using System.Collections.Generic;
/// <summary>
/// Request payload for batch favicon extraction.
/// </summary>
public class FaviconExtractBatchRequest
{
/// <summary>
/// Gets or sets the URLs to extract favicons for. The server caps the number of URLs
/// it processes per request; callers should chunk larger inputs into multiple requests.
/// </summary>
public List<string> Urls { get; set; } = new();
}

View File

@@ -0,0 +1,22 @@
//-----------------------------------------------------------------------
// <copyright file="FaviconExtractBatchResponse.cs" company="aliasvault">
// Copyright (c) aliasvault. All rights reserved.
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
// </copyright>
//-----------------------------------------------------------------------
namespace AliasVault.Shared.Models.WebApi.Favicon;
using System.Collections.Generic;
/// <summary>
/// Response payload for batch favicon extraction. Each result lines up with the URL at the
/// same index in the request, with a null Image when extraction failed for that URL.
/// </summary>
public class FaviconExtractBatchResponse
{
/// <summary>
/// Gets or sets the per-URL extraction results.
/// </summary>
public List<FaviconExtractBatchResult> Results { get; set; } = new();
}

View File

@@ -0,0 +1,24 @@
//-----------------------------------------------------------------------
// <copyright file="FaviconExtractBatchResult.cs" company="aliasvault">
// Copyright (c) aliasvault. All rights reserved.
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
// </copyright>
//-----------------------------------------------------------------------
namespace AliasVault.Shared.Models.WebApi.Favicon;
/// <summary>
/// Per-URL favicon extraction result.
/// </summary>
public class FaviconExtractBatchResult
{
/// <summary>
/// Gets or sets the URL the favicon was requested for (echoed back so the client can correlate).
/// </summary>
public string Url { get; set; } = string.Empty;
/// <summary>
/// Gets or sets the favicon image bytes, or null if extraction failed.
/// </summary>
public byte[]? Image { get; set; }
}

View File

@@ -8,6 +8,7 @@
namespace AliasVault.FaviconExtractor;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
@@ -21,9 +22,10 @@ using SkiaSharp;
/// </summary>
public static class FaviconExtractor
{
private const int MaxSizeBytes = 50 * 1024; // 50KB max size before resizing
private const int TargetWidth = 96; // Resize target width
private static readonly string[] _allowedSchemes = { "http", "https" };
private const int MaxSizeBytes = 20 * 1024; // 20KB max size; images above this are resized/re-encoded.
private static readonly int[] _resizeWidths = [96, 64, 48, 32];
private static readonly int[] _jpegFallbackQualities = [80, 65, 50];
private static readonly string[] _allowedSchemes = ["http", "https"];
/// <summary>
/// Extracts the favicon from a URL with enhanced browser like behavior.
@@ -56,6 +58,43 @@ public static class FaviconExtractor
return null;
}
/// <summary>
/// Extracts favicons for multiple URLs in parallel. Each URL is processed independently;
/// individual failures are returned as null entries rather than throwing. The returned
/// list lines up index-for-index with the input.
/// </summary>
/// <param name="urls">The URLs to extract favicons for.</param>
/// <returns>A list of favicon byte arrays, in the same order as the input urls.</returns>
public static async Task<IReadOnlyList<byte[]?>> GetFaviconsAsync(IReadOnlyList<string> urls)
{
if (urls.Count == 0)
{
return Array.Empty<byte[]?>();
}
var tasks = new Task<byte[]?>[urls.Count];
for (int i = 0; i < urls.Count; i++)
{
// Wrap each call in a try/catch so one bad URL doesn't fail the whole batch.
var url = urls[i];
tasks[i] = SafeGetFaviconAsync(url);
}
return await Task.WhenAll(tasks);
}
private static async Task<byte[]?> SafeGetFaviconAsync(string url)
{
try
{
return await GetFaviconAsync(url);
}
catch
{
return null;
}
}
/// <summary>
/// Tries to get the favicon from the URL.
/// </summary>
@@ -184,18 +223,13 @@ public static class FaviconExtractor
return null;
}
// If image is too large, attempt to resize
// If image is too large, attempt to resize/re-encode it down under the cap.
if (imageBytes.Length > MaxSizeBytes)
{
var resizedBytes = ResizeImageAsync(imageBytes, contentType);
if (resizedBytes != null)
{
imageBytes = resizedBytes;
}
return ResizeImage(imageBytes, contentType);
}
// Return only if within size limits
return imageBytes.Length <= MaxSizeBytes ? imageBytes : null;
return imageBytes;
}
catch
{
@@ -377,13 +411,22 @@ public static class FaviconExtractor
}
/// <summary>
/// Resizes the image to the target width.
/// Iteratively shrinks the image until it fits in the size cap. Tries
/// progressively smaller widths in PNG (lossless after downscale handles most cases), then
/// falls back to JPEG at decreasing quality if PNG at the smallest width is still too large.
/// </summary>
/// <param name="imageBytes">The image bytes to resize.</param>
/// <param name="contentType">The content type of the image.</param>
/// <returns>The resized image bytes.</returns>
private static byte[]? ResizeImageAsync(byte[] imageBytes, string contentType)
/// <returns>The resized image bytes, or null if it could not be brought under the size cap.</returns>
private static byte[]? ResizeImage(byte[] imageBytes, string contentType)
{
// SVG is a text format; we can't usefully raster-resize it here. Caller already rejects
// anything over the cap, so just bail and let it return null.
if (contentType == "image/svg+xml")
{
return null;
}
try
{
using var original = SKBitmap.Decode(imageBytes);
@@ -392,30 +435,59 @@ public static class FaviconExtractor
return null;
}
var scale = (float)TargetWidth / original.Width;
var targetHeight = (int)(original.Height * scale);
using var resized = original.Resize(new SKImageInfo(TargetWidth, targetHeight), new SKSamplingOptions(SKFilterMode.Linear));
if (resized == null)
// Pass 1: PNG at progressively smaller widths. Preserves transparency.
foreach (var width in _resizeWidths)
{
return null;
var encoded = EncodeAtWidth(original, width, SKEncodedImageFormat.Png, 100);
if (encoded != null && encoded.Length <= MaxSizeBytes)
{
return encoded;
}
}
using var image = SKImage.FromBitmap(resized);
var format = contentType switch
// Pass 2: JPEG at the smallest width with decreasing quality. Loses transparency but
// gives much smaller files for photographic favicons that resist PNG compression.
var fallbackWidth = _resizeWidths[^1];
foreach (var quality in _jpegFallbackQualities)
{
"image/png" => SKEncodedImageFormat.Png,
"image/jpeg" => SKEncodedImageFormat.Jpeg,
"image/gif" => SKEncodedImageFormat.Gif,
_ => SKEncodedImageFormat.Png,
};
var encoded = EncodeAtWidth(original, fallbackWidth, SKEncodedImageFormat.Jpeg, quality);
if (encoded != null && encoded.Length <= MaxSizeBytes)
{
return encoded;
}
}
var data = image.Encode(format, 90);
return data?.ToArray();
return null;
}
catch
{
return null;
}
}
/// <summary>
/// Resizes the bitmap to the target width (preserving aspect ratio, never upscaling) and
/// encodes it in the given format.
/// </summary>
/// <param name="original">The decoded source bitmap.</param>
/// <param name="targetWidth">Desired output width in pixels.</param>
/// <param name="format">Encode format.</param>
/// <param name="quality">Encoder quality (only meaningful for lossy formats).</param>
/// <returns>Encoded bytes, or null if encoding failed.</returns>
private static byte[]? EncodeAtWidth(SKBitmap original, int targetWidth, SKEncodedImageFormat format, int quality)
{
var width = Math.Min(targetWidth, original.Width);
var scale = (float)width / original.Width;
var height = Math.Max(1, (int)(original.Height * scale));
using var resized = original.Resize(new SKImageInfo(width, height), new SKSamplingOptions(SKFilterMode.Linear));
if (resized == null)
{
return null;
}
using var image = SKImage.FromBitmap(resized);
using var data = image.Encode(format, quality);
return data?.ToArray();
}
}

View File

@@ -72,6 +72,9 @@ pub struct PruneStats {
/// Number of tombstoned attachments whose blob bytes were cleared.
#[serde(default)]
pub attachment_blobs_cleared: u32,
/// Number of tombstoned logos whose FileData bytes were cleared.
#[serde(default)]
pub logo_blobs_cleared: u32,
}
/// Output of the prune operation.
@@ -263,7 +266,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
if let Some(logo_id) = logo.get("Id").and_then(|v| v.as_str()) {
if !referenced_logo_ids.contains(logo_id) {
statements.push(SqlStatement {
sql: "UPDATE Logos SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ?".to_string(),
sql: "UPDATE Logos SET IsDeleted = 1, FileData = X'', UpdatedAt = ? WHERE Id = ?".to_string(),
params: vec![
serde_json::json!(now_str),
serde_json::json!(logo_id),
@@ -307,6 +310,35 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
}
}
// Pass 4 — sweep tombstoned logos that still carry FileData bytes. Logos that have
// been soft-deleted but still have FileData bytes take up space for no reason.
// This behaviour could have occurred in older clients (before 0.29.x).
if let Some(logos_table) = input.tables.iter().find(|t| t.name == "Logos") {
for logo in &logos_table.records {
let is_deleted = logo.get("IsDeleted")
.map(|v| v.as_i64() == Some(1) || v.as_bool() == Some(true))
.unwrap_or(false);
if !is_deleted {
continue;
}
if !logo_has_file_data_bytes(logo) {
continue;
}
if let Some(logo_id) = logo.get("Id").and_then(|v| v.as_str()) {
statements.push(SqlStatement {
sql: "UPDATE Logos SET FileData = X'', UpdatedAt = ? WHERE Id = ?".to_string(),
params: vec![
serde_json::json!(now_str),
serde_json::json!(logo_id),
],
});
stats.logo_blobs_cleared += 1;
}
}
}
Ok(PruneOutput {
success: true,
statements,
@@ -325,6 +357,17 @@ fn attachment_has_blob_bytes(attachment: &Record) -> bool {
}
}
/// True if the logo's FileData field is present and non-empty.
fn logo_has_file_data_bytes(logo: &Record) -> bool {
match logo.get("FileData") {
None => false,
Some(serde_json::Value::Null) => false,
Some(serde_json::Value::String(s)) => !s.is_empty(),
Some(serde_json::Value::Array(a)) => !a.is_empty(),
Some(_) => true,
}
}
/// Prune vault using JSON strings.
/// Convenience function for FFI.
pub fn prune_vault_json(input_json: &str) -> VaultResult<String> {
@@ -433,6 +476,12 @@ mod tests {
record
}
fn make_logo_record_with_blob(id: &str, is_deleted: bool, blob: serde_json::Value) -> Record {
let mut record = make_logo_record(id, is_deleted);
record.insert("FileData".to_string(), blob);
record
}
#[test]
fn test_prune_expired_items() {
let now = Utc::now();
@@ -699,6 +748,82 @@ mod tests {
assert_eq!(output.stats.logos_pruned, 0);
}
#[test]
fn test_orphan_logo_pruning_emits_filedata_clear_in_same_statement() {
// Pass 2 must clear FileData when it tombstones a logo, otherwise the
// encrypted vault keeps the blob bytes even after the row is "deleted".
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
let input = PruneInput {
tables: vec![
TableData {
name: "Items".to_string(),
records: vec![],
},
TableData {
name: "Logos".to_string(),
records: vec![make_logo_record_with_blob("logo-orphan", false, serde_json::json!("aGVsbG8="))],
},
],
retention_days: 30,
current_time: now_str,
};
let output = prune_vault(input).unwrap();
assert_eq!(output.stats.logos_pruned, 1);
assert!(output.statements.iter().any(|s| s.sql.contains("FileData = X''")));
}
#[test]
fn test_tombstoned_logo_with_blob_bytes_is_swept() {
// Pass 4 must catch historical logos that are IsDeleted=1 but still carry FileData
// (e.g. tombstoned by an older client before the FileData=X'' fix landed).
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
let input = PruneInput {
tables: vec![
TableData {
name: "Items".to_string(),
records: vec![],
},
TableData {
name: "Logos".to_string(),
records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::json!("aGVsbG8="))],
},
],
retention_days: 30,
current_time: now_str,
};
let output = prune_vault(input).unwrap();
assert_eq!(output.stats.logo_blobs_cleared, 1);
assert_eq!(output.stats.logos_pruned, 0);
}
#[test]
fn test_tombstoned_logo_without_blob_is_not_touched() {
// Logos already cleared shouldn't generate redundant updates.
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
let input = PruneInput {
tables: vec![
TableData {
name: "Items".to_string(),
records: vec![],
},
TableData {
name: "Logos".to_string(),
records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::Value::Null)],
},
],
retention_days: 30,
current_time: now_str,
};
let output = prune_vault(input).unwrap();
assert_eq!(output.stats.logo_blobs_cleared, 0);
}
#[test]
fn test_already_soft_deleted_logo_is_not_re_pruned() {
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();