diff --git a/apps/browser-extension/src/utils/db/queries/LogoQueries.ts b/apps/browser-extension/src/utils/db/queries/LogoQueries.ts index 86c55ad5b..8b91cd73b 100644 --- a/apps/browser-extension/src/utils/db/queries/LogoQueries.ts +++ b/apps/browser-extension/src/utils/db/queries/LogoQueries.ts @@ -11,10 +11,31 @@ export class LogoQueries { WHERE Source = ? AND IsDeleted = 0 LIMIT 1`; + /** + * Look up a logo by source regardless of IsDeleted state. Used by getOrCreate to detect + * soft-deleted rows that still occupy the UNIQUE(Source) slot, so we can refill them + * instead of trying (and failing) to INSERT a duplicate. + */ + public static readonly GET_BY_SOURCE_INCLUDING_DELETED = ` + SELECT Id, IsDeleted FROM Logos + WHERE Source = ? + LIMIT 1`; + /** * Insert new logo. */ public static readonly INSERT = ` INSERT INTO Logos (Id, Source, FileData, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, ?, ?)`; + + /** + * Restore a soft-deleted logo and refill its bytes in one statement. Caller passes + * (FileData, UpdatedAt, Id). + */ + public static readonly RESTORE_WITH_FILE_DATA = ` + UPDATE Logos + SET IsDeleted = 0, + FileData = ?, + UpdatedAt = ? + WHERE Id = ?`; } diff --git a/apps/browser-extension/src/utils/db/repositories/LogoRepository.ts b/apps/browser-extension/src/utils/db/repositories/LogoRepository.ts index 91ad9e39c..b755a9d3d 100644 --- a/apps/browser-extension/src/utils/db/repositories/LogoRepository.ts +++ b/apps/browser-extension/src/utils/db/repositories/LogoRepository.ts @@ -41,10 +41,22 @@ export class LogoRepository extends BaseRepository { * @returns The logo ID (existing or newly created) */ public getOrCreate(source: string, logoData: Uint8Array, currentDateTime: string): string { - // Check if a logo for this source already exists - const existingId = this.getIdForSource(source); - if (existingId) { - return existingId; + const existing = this.client.executeQuery<{ Id: string; IsDeleted: number }>( + LogoQueries.GET_BY_SOURCE_INCLUDING_DELETED, + [source] + ); + + if (existing.length > 0) { + const row = existing[0]; + if (row.IsDeleted === 1) { + // Restore a previously soft-deleted record and refill its FileData. + this.client.executeUpdate(LogoQueries.RESTORE_WITH_FILE_DATA, [ + logoData, + currentDateTime, + row.Id + ]); + } + return row.Id; } // Create new logo entry diff --git a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/repositories/PasskeyRepository.kt b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/repositories/PasskeyRepository.kt index 3e22a9860..8ecf97361 100644 --- a/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/repositories/PasskeyRepository.kt +++ b/apps/mobile-app/android/app/src/main/java/net/aliasvault/app/vaultstore/repositories/PasskeyRepository.kt @@ -549,6 +549,7 @@ class PasskeyRepository(database: VaultDatabase) : BaseRepository(database) { // Sanity check: restore if soft-deleted if (isDeleted) { executeUpdate(LogoQueries.RESTORE, arrayOf(timestamp, existingLogoId)) + executeUpdate(LogoQueries.UPDATE_FILE_DATA, arrayOf(logoData, timestamp, existingLogoId)) } return existingLogoId } diff --git a/apps/mobile-app/ios/VaultStoreKit/Database/Repositories/PasskeyRepository.swift b/apps/mobile-app/ios/VaultStoreKit/Database/Repositories/PasskeyRepository.swift index f0209c370..2d4d2beee 100644 --- a/apps/mobile-app/ios/VaultStoreKit/Database/Repositories/PasskeyRepository.swift +++ b/apps/mobile-app/ios/VaultStoreKit/Database/Repositories/PasskeyRepository.swift @@ -486,7 +486,9 @@ public class PasskeyRepository: BaseRepository { // Sanity check: restore if soft-deleted if isDeleted { + let logoDataParam = "av-base64-to-blob:\(logoData.base64EncodedString())" try client.executeUpdate(LogoQueries.restore, params: [now, existingLogoId]) + try client.executeUpdate(LogoQueries.updateFileData, params: [logoDataParam, now, existingLogoId]) } return existingLogoId } diff --git a/apps/mobile-app/utils/db/repositories/LogoRepository.ts b/apps/mobile-app/utils/db/repositories/LogoRepository.ts index 20f437931..4b7f7d32e 100644 --- a/apps/mobile-app/utils/db/repositories/LogoRepository.ts +++ b/apps/mobile-app/utils/db/repositories/LogoRepository.ts @@ -14,6 +14,14 @@ const LogoQueries = { WHERE Source = ? AND IsDeleted = 0 LIMIT 1`, + /** + * Look up a logo by source regardless of IsDeleted state. + */ + GET_BY_SOURCE_INCLUDING_DELETED: ` + SELECT Id, IsDeleted FROM Logos + WHERE Source = ? + LIMIT 1`, + /** * Insert new logo. */ @@ -21,6 +29,16 @@ const LogoQueries = { INSERT INTO Logos (Id, Source, FileData, CreatedAt, UpdatedAt, IsDeleted) VALUES (?, ?, ?, ?, ?, ?)`, + /** + * Restore a soft-deleted logo and refill its FileData in one statement. + */ + RESTORE_WITH_FILE_DATA: ` + UPDATE Logos + SET IsDeleted = 0, + FileData = ?, + UpdatedAt = ? + WHERE Id = ?`, + /** * Count items using a logo. */ @@ -75,10 +93,22 @@ export class LogoRepository extends BaseRepository { * @returns The logo ID (existing or newly created) */ public async getOrCreate(source: string, logoData: Uint8Array, currentDateTime: string): Promise { - // Check if a logo for this source already exists - const existingId = await this.getIdForSource(source); - if (existingId) { - return existingId; + const existing = await this.client.executeQuery<{ Id: string; IsDeleted: number }>( + LogoQueries.GET_BY_SOURCE_INCLUDING_DELETED, + [source] + ); + + if (existing.length > 0) { + const row = existing[0]; + if (row.IsDeleted === 1) { + // Restore a previously soft-deleted record and refill its FileData. + await this.client.executeUpdate(LogoQueries.RESTORE_WITH_FILE_DATA, [ + logoData, + currentDateTime, + row.Id + ]); + } + return row.Id; } // Create new logo entry diff --git a/apps/server/AliasVault.Api/Controllers/FaviconController.cs b/apps/server/AliasVault.Api/Controllers/FaviconController.cs index 5bed8101f..7ad6f2d52 100644 --- a/apps/server/AliasVault.Api/Controllers/FaviconController.cs +++ b/apps/server/AliasVault.Api/Controllers/FaviconController.cs @@ -9,6 +9,7 @@ namespace AliasVault.Api.Controllers; using AliasServerDb; using AliasVault.Api.Controllers.Abstracts; +using AliasVault.Api.Services; using AliasVault.Shared.Models.WebApi.Favicon; using Asp.Versioning; using Microsoft.AspNetCore.Identity; @@ -18,15 +19,24 @@ using Microsoft.AspNetCore.Mvc; /// Controller for retrieving favicons from external websites. /// /// UserManager instance. +/// In-memory per-user favicon extraction rate limiter. /// Logger instance. [ApiVersion("1")] -public class FaviconController(UserManager userManager, ILogger logger) : AuthenticatedRequestController(userManager) +public class FaviconController( + UserManager userManager, + FaviconRateLimitService rateLimitService, + ILogger logger) : AuthenticatedRequestController(userManager) { /// - /// Proxies the request to the identity generator to generate a random identity. + /// Maximum number of URLs accepted in a single batch request. + /// + public const int MaxBatchSize = 10; + + /// + /// Extracts the favicon from a single URL. /// /// URL to extract the favicon from. - /// Identity model. + /// Favicon image bytes, or null if extraction failed. [HttpGet("Extract")] public async Task Extract(string url) { @@ -36,23 +46,84 @@ public class FaviconController(UserManager userManager, ILogger< return Unauthorized(); } - // Get the favicon from the URL. + if (!rateLimitService.TryConsume(user.Id, 1)) + { + return StatusCode(StatusCodes.Status429TooManyRequests); + } + try { var image = await FaviconExtractor.FaviconExtractor.GetFaviconAsync(url); - - // Return the favicon as base64 string of image representation. return Ok(new FaviconExtractModel { Image = image }); } catch (Exception ex) { - // Anonymize the URL by replacing all a-Z characters with 'x' before logging. - // This will still allow to see the host structure but not the actual domain. - var anonymizedUrl = new string(url.Select(c => char.IsLetter(c) ? 'x' : c).ToArray()); - logger.LogInformation(ex, "Failed to extract favicon from {Url}", anonymizedUrl); + logger.LogInformation(ex, "Failed to extract favicon from {Url}", AnonymizeUrl(url)); } - // Return null if favicon extraction failed. return Ok(new FaviconExtractModel { Image = null }); } + + /// + /// Extracts favicons for multiple URLs in parallel server-side. Cuts down on round trips when + /// the client needs to fetch many favicons (initial vault import, bulk re-download from the + /// storage insights page). + /// + /// The batch request payload. + /// A list of favicon results, one per requested URL, in the same order. + [HttpPost("ExtractBatch")] + public async Task ExtractBatch([FromBody] FaviconExtractBatchRequest request) + { + var user = await GetCurrentUserAsync(); + if (user == null) + { + return Unauthorized(); + } + + if (request.Urls.Count == 0) + { + return Ok(new FaviconExtractBatchResponse()); + } + + if (request.Urls.Count > MaxBatchSize) + { + return BadRequest(new ProblemDetails + { + Title = "Too many URLs", + Detail = $"Batch size is capped at {MaxBatchSize} URLs per request.", + }); + } + + if (!rateLimitService.TryConsume(user.Id, request.Urls.Count)) + { + return StatusCode(StatusCodes.Status429TooManyRequests); + } + + var images = await FaviconExtractor.FaviconExtractor.GetFaviconsAsync(request.Urls); + + var response = new FaviconExtractBatchResponse + { + Results = new List(request.Urls.Count), + }; + + for (int i = 0; i < request.Urls.Count; i++) + { + response.Results.Add(new FaviconExtractBatchResult + { + Url = request.Urls[i], + Image = images[i], + }); + } + + return Ok(response); + } + + /// + /// Anonymizes a URL by replacing letters with 'x'. Lets us log host structure without + /// recording the actual domain a user was browsing. + /// + private static string AnonymizeUrl(string url) + { + return new string(url.Select(c => char.IsLetter(c) ? 'x' : c).ToArray()); + } } diff --git a/apps/server/AliasVault.Api/Program.cs b/apps/server/AliasVault.Api/Program.cs index 0024b0b21..cc89a7182 100644 --- a/apps/server/AliasVault.Api/Program.cs +++ b/apps/server/AliasVault.Api/Program.cs @@ -14,6 +14,7 @@ using AliasServerDb; using AliasServerDb.Configuration; using AliasVault.Api; using AliasVault.Api.Jwt; +using AliasVault.Api.Services; using AliasVault.Auth; using AliasVault.Cryptography.Server; using AliasVault.Logging; @@ -84,6 +85,7 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddSingleton(); builder.Services.AddHttpContextAccessor(); builder.Services.AddLogging(logging => diff --git a/apps/server/AliasVault.Api/Services/FaviconRateLimitService.cs b/apps/server/AliasVault.Api/Services/FaviconRateLimitService.cs new file mode 100644 index 000000000..f09deb30a --- /dev/null +++ b/apps/server/AliasVault.Api/Services/FaviconRateLimitService.cs @@ -0,0 +1,80 @@ +//----------------------------------------------------------------------- +// +// Copyright (c) aliasvault. All rights reserved. +// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information. +// +//----------------------------------------------------------------------- + +namespace AliasVault.Api.Services; + +using System; +using System.Collections.Concurrent; + +/// +/// In-memory per-user rate limiter for the favicon extraction API. Each successful extraction +/// (single or one URL inside a batch) consumes one unit; the limit is enforced over a rolling +/// 24-hour window. State is held in process so it is not shared across instances — a deliberate +/// trade-off, since the limit's purpose is to bound the SSRF/DDoS amplification an individual +/// user could cause from a single API instance, not to provide a hard global ceiling. +/// +public sealed class FaviconRateLimitService +{ + /// + /// Default maximum favicon extractions per user per 24 hours to prevent flood. + /// + public const int DefaultMaxPer24Hours = 5000; + + private static readonly TimeSpan WindowDuration = TimeSpan.FromHours(24); + + private readonly ConcurrentDictionary _usage = new(); + private readonly int _maxPer24Hours; + + /// + /// Initializes a new instance of the class. + /// + public FaviconRateLimitService() + : this(DefaultMaxPer24Hours) + { + } + + /// + /// Initializes a new instance of the class + /// with a custom limit (for tests or future configuration). + /// + /// Maximum extractions allowed per user per 24 hours. Set to 0 or less to disable. + public FaviconRateLimitService(int maxPer24Hours) + { + _maxPer24Hours = maxPer24Hours; + } + + /// + /// Attempts to consume units from the user's allowance. Atomic per user: + /// either the full count is consumed and true is returned, or nothing is consumed and false is returned. + /// + /// The authenticated user id. + /// Number of units to consume (one per favicon URL). + /// True if the request fits in the remaining allowance; false if the limit would be exceeded. + public bool TryConsume(string userId, int count) + { + if (_maxPer24Hours <= 0 || count <= 0) + { + return true; + } + + var usage = _usage.GetOrAdd(userId, _ => new FaviconUserUsage()); + + lock (usage.SyncRoot) + { + var now = DateTime.UtcNow; + usage.Prune(now - WindowDuration); + + if (usage.Total + count > _maxPer24Hours) + { + return false; + } + + usage.Add(now, count); + return true; + } + } +} diff --git a/apps/server/AliasVault.Api/Services/FaviconUserUsage.cs b/apps/server/AliasVault.Api/Services/FaviconUserUsage.cs new file mode 100644 index 000000000..e10442b12 --- /dev/null +++ b/apps/server/AliasVault.Api/Services/FaviconUserUsage.cs @@ -0,0 +1,71 @@ +//----------------------------------------------------------------------- +// +// Copyright (c) aliasvault. All rights reserved. +// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information. +// +//----------------------------------------------------------------------- + +namespace AliasVault.Api.Services; + +using System; +using System.Collections.Generic; + +/// +/// Per-user usage record for . Tracks recent extractions +/// in per-minute buckets so a power-user re-downloading thousands of favicons in quick succession +/// still produces a bounded number of entries. +/// +internal sealed class FaviconUserUsage +{ + // Bucket entries by the minute they happened in. A power-user re-downloading a 5k-item + // vault produces at most ~minutes-of-batches entries this way, even though the underlying + // unit count can be in the thousands — keeps memory bounded without losing accuracy. + private readonly LinkedList<(DateTime BucketStart, int Count)> _entries = new(); + + /// + /// Gets the lock guarding mutations to this usage record. Callers must hold this lock + /// across read-modify-write sequences (Prune + Add) to keep TryConsume atomic. + /// + public object SyncRoot { get; } = new object(); + + /// + /// Gets the total number of units consumed inside the currently retained window. + /// + public int Total { get; private set; } + + /// + /// Records consumed units at , merging into + /// the last bucket when it falls in the same minute. + /// + /// The timestamp to record under (typically ). + /// Number of units consumed. + public void Add(DateTime now, int count) + { + var bucket = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0, DateTimeKind.Utc); + + if (_entries.Last is { Value.BucketStart: var lastBucket } lastNode && lastBucket == bucket) + { + _entries.Last.Value = (lastBucket, lastNode.Value.Count + count); + } + else + { + _entries.AddLast((bucket, count)); + } + + Total += count; + } + + /// + /// Drops buckets older than from the front of the list, which is + /// where the oldest entries live since Add only appends. + /// + /// Buckets strictly older than this are removed. + public void Prune(DateTime cutoff) + { + while (_entries.First is { Value.BucketStart: var first } node && first < cutoff) + { + Total -= node.Value.Count; + _entries.RemoveFirst(); + } + } +} diff --git a/apps/server/AliasVault.Client/Main/Components/Items/ItemIcon.razor b/apps/server/AliasVault.Client/Main/Components/Items/ItemIcon.razor index d8470e920..bfb530248 100644 --- a/apps/server/AliasVault.Client/Main/Components/Items/ItemIcon.razor +++ b/apps/server/AliasVault.Client/Main/Components/Items/ItemIcon.razor @@ -106,6 +106,11 @@ else { return "image/png"; } + + if (bytes[0] == 0xFF && bytes[1] == 0xD8 && bytes[2] == 0xFF) + { + return "image/jpeg"; + } } return "image/x-icon"; diff --git a/apps/server/AliasVault.Client/Main/Layout/TopMenu.razor b/apps/server/AliasVault.Client/Main/Layout/TopMenu.razor index 41e929304..e7302f711 100644 --- a/apps/server/AliasVault.Client/Main/Layout/TopMenu.razor +++ b/apps/server/AliasVault.Client/Main/Layout/TopMenu.razor @@ -69,6 +69,11 @@ @Localizer["SecuritySettingsNav"] +
  • + + @Localizer["StorageInsightsNav"] + +
  • @Localizer["ImportExportNav"] diff --git a/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/Components/ImportServiceCard.razor b/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/Components/ImportServiceCard.razor index 71ed45d5f..e881220d7 100644 --- a/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/Components/ImportServiceCard.razor +++ b/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/Components/ImportServiceCard.razor @@ -4,7 +4,7 @@ @inject FolderService FolderService @inject NavigationManager NavigationManager @inject GlobalNotificationService GlobalNotificationService -@inject HttpClient HttpClient +@inject FaviconService FaviconService @inject IStringLocalizerFactory LocalizerFactory @inject AliasVault.Client.Services.Crypto.AvexCryptoService AvexCrypto @using Microsoft.Extensions.Localization @@ -12,7 +12,6 @@ @using AliasVault.ImportExport.Importers @using AliasVault.ImportExport.Models @using AliasVault.ImportExport.Models.Exports -@using AliasVault.Shared.Models.WebApi.Favicon @using AliasClientDb @using AliasClientDb.Models @using AliasVault.Client.Auth.Components @@ -612,8 +611,9 @@ } /// - /// Extracts favicons for credentials. Uses domain-based deduplication to avoid - /// fetching the same favicon multiple times for the same domain. + /// Extracts favicons for credentials. Delegates dedup-by-domain and batched server-side + /// fetching to ; this method only handles UI progress reporting + /// and cancellation. /// private async Task ExtractFaviconsForCredentials() { @@ -623,59 +623,43 @@ FaviconExtractionCancellation = new CancellationTokenSource(); StateHasChanged(); - // Only extract favicons for credentials that don't already have them and have URLs - var credentialsWithUrls = ImportedCredentials - .Where(c => c.FaviconBytes == null && c.ServiceUrls?.FirstOrDefault() != null) + // Only extract favicons for credentials that don't already have them and have URLs. + var urls = ImportedCredentials + .Where(c => c.FaviconBytes == null) + .Select(c => c.ServiceUrls?.FirstOrDefault()) + .Where(u => !string.IsNullOrEmpty(u)) + .Cast() .ToList(); - // Group credentials by normalized domain to avoid duplicate fetches - var processedDomains = new HashSet(); + // The dialog reports progress as "domains processed" rather than "credentials processed", + // since the FaviconService dedupes by domain and that's the actual unit of fetch work. + TotalFaviconsToExtract = urls + .Select(u => FaviconService.TryNormalizeDomain(u, out var d) ? d : null) + .Where(d => d != null) + .Distinct() + .Count(); - TotalFaviconsToExtract = credentialsWithUrls.Count(); - - // Update UI every N items - Task.Delay(1) is required to let browser actually render - // Task.Yield() doesn't work because it doesn't give the browser event loop time to paint - const int updateEveryNItems = 10; - - foreach (var credential in credentialsWithUrls) + // Task.Delay(1) inside the progress callback lets Blazor actually paint between batches. + var progress = new Progress(async count => { - if (FaviconExtractionCancellation.Token.IsCancellationRequested) - { - break; - } + FaviconExtractionProgress = count; + StateHasChanged(); + await Task.Delay(1); + }); - // Extract normalized domain for deduplication - try - { - var domain = new Uri(credential.ServiceUrls!.First()).Host.ToLowerInvariant(); - if (domain.StartsWith("www.")) - { - domain = domain[4..]; - } - - // Skip if we've already fetched a favicon for this domain - if (!processedDomains.Contains(domain)) - { - processedDomains.Add(domain); - await ExtractFaviconForCredential(credential); - } - } - catch - { - // Invalid URL - } - - FaviconExtractionProgress++; - - // Update UI periodically - if (FaviconExtractionProgress % updateEveryNItems == 0) - { - StateHasChanged(); - await Task.Delay(1); - } + var result = await FaviconService.ExtractBulkAsync(urls, progress, FaviconExtractionCancellation.Token); + foreach (var kvp in result.Favicons) + { + ExtractedFavicons[kvp.Key] = kvp.Value; } - // Final update to show 100% completion before transitioning to next phase + if (result.Status == FaviconService.BulkExtractStatus.RateLimited) + { + // Rate limit error occurred. Silently skip the favicons that couldn't be extracted. + } + + // Final update to show 100% completion before transitioning to next phase. + FaviconExtractionProgress = TotalFaviconsToExtract; StateHasChanged(); await Task.Delay(50); @@ -683,35 +667,6 @@ StateHasChanged(); } - /// - /// Extracts a favicon for a credential. Stores by normalized domain for deduplication. - /// - /// The credential to extract the favicon for. - private async Task ExtractFaviconForCredential(ImportedCredential credential) - { - try - { - // Extract normalized domain for storage key - var url = credential.ServiceUrls!.First(); - var domain = new Uri(url).Host.ToLowerInvariant(); - if (domain.StartsWith("www.")) - { - domain = domain[4..]; - } - - var apiReturn = await HttpClient.GetFromJsonAsync($"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}"); - if (apiReturn?.Image is not null) - { - // Store by normalized domain for deduplication - ExtractedFavicons[domain] = apiReturn.Image; - } - } - catch - { - // Ignore favicon extraction errors - } - } - /// /// Imports the items to the database. /// @@ -952,22 +907,34 @@ { var context = await DbService.GetDbContextAsync(); - // Check if logo already exists by source + // Lookup ignores IsDeleted because Logos.Source is UNIQUE — a soft-deleted row + // with the same domain still occupies the slot and would block a new insert. var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain); - if (existingLogo != null) + if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 }) { + // Healthy existing logo — reuse without overwriting. + item.LogoId = existingLogo.Id; + ImportSessionLogoIds[domain] = existingLogo.Id; + } + else if (existingLogo != null) + { + // Refill a soft-deleted or pruner-emptied row with the imported bytes so the + // new item gets a usable logo, instead of inheriting an empty blob. + var nowUtc = DateTime.UtcNow; + existingLogo.IsDeleted = false; + existingLogo.FileData = importedCredential.FaviconBytes; + existingLogo.FetchedAt = nowUtc; + existingLogo.UpdatedAt = nowUtc; item.LogoId = existingLogo.Id; ImportSessionLogoIds[domain] = existingLogo.Id; } else { - // Create new logo from embedded bytes var logo = new AliasClientDb.Logo { Id = Guid.NewGuid(), Source = domain, FileData = importedCredential.FaviconBytes, - MimeType = "image/png", FetchedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow, UpdatedAt = DateTime.UtcNow, @@ -1003,30 +970,38 @@ var context = await DbService.GetDbContextAsync(); var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain); - if (existingLogo != null) + if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 }) { - // Reuse existing logo item.LogoId = existingLogo.Id; ImportSessionLogoIds[domain] = existingLogo.Id; } else if (ExtractedFavicons.TryGetValue(domain, out var favicon)) { - // Create new logo from extracted favicon - var logo = new AliasClientDb.Logo + var nowUtc = DateTime.UtcNow; + if (existingLogo != null) { - Id = Guid.NewGuid(), - Source = domain, - FileData = favicon, - MimeType = "image/png", - FetchedAt = DateTime.UtcNow, - CreatedAt = DateTime.UtcNow, - UpdatedAt = DateTime.UtcNow, - }; - context.Logos.Add(logo); - item.LogoId = logo.Id; - - // Track this logo for subsequent items with same domain - ImportSessionLogoIds[domain] = logo.Id; + existingLogo.IsDeleted = false; + existingLogo.FileData = favicon; + existingLogo.FetchedAt = nowUtc; + existingLogo.UpdatedAt = nowUtc; + item.LogoId = existingLogo.Id; + ImportSessionLogoIds[domain] = existingLogo.Id; + } + else + { + var logo = new AliasClientDb.Logo + { + Id = Guid.NewGuid(), + Source = domain, + FileData = favicon, + FetchedAt = nowUtc, + CreatedAt = nowUtc, + UpdatedAt = nowUtc, + }; + context.Logos.Add(logo); + item.LogoId = logo.Id; + ImportSessionLogoIds[domain] = logo.Id; + } } } } diff --git a/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/ImportExport.razor b/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/ImportExport.razor index 13d01d413..0b35fa698 100644 --- a/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/ImportExport.razor +++ b/apps/server/AliasVault.Client/Main/Pages/Settings/ImportExport/ImportExport.razor @@ -90,6 +90,23 @@ + +@* DEBUG-only raw SQLite export. *@ +@if (IsDebugBuild) +{ +
    +
    +
    +

    Export raw SQLite

    + DEBUG +
    +

    + Downloads the unencrypted SQLite vault file as-is. Anyone with the file can read everything. +

    +
    + +
    +} @@ -112,6 +129,13 @@ OnClose="@CloseExportPasswordModal" /> @code { + // Assign debug static flag for Razor markup check. +#if DEBUG + private static readonly bool IsDebugBuild = true; +#else + private static readonly bool IsDebugBuild = false; +#endif + private string _username = string.Empty; private bool _showPasswordConfirmation; private string _passwordError = string.Empty; @@ -295,6 +319,27 @@ } } + // Always defined so the markup binding compiles, only invoked when IsDebugBuild is true. + private async Task ExportVaultSqlite() + { + GlobalLoadingSpinner.Show("Exporting SQLite vault..."); + try + { + var base64 = await DbService.ExportSqliteToBase64Async(); + var bytes = Convert.FromBase64String(base64); + await JsInteropService.DownloadFileFromStream(await GetExportFileName("sqlite"), bytes); + } + catch (Exception ex) + { + Logger.LogError(ex, "Error exporting raw SQLite vault"); + GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true); + } + finally + { + GlobalLoadingSpinner.Hide(); + } + } + private async Task ExportVaultAvux() { GlobalLoadingSpinner.Show(Localizer["ExportingVaultMessage"]); diff --git a/apps/server/AliasVault.Client/Main/Pages/Settings/StorageInsights.razor b/apps/server/AliasVault.Client/Main/Pages/Settings/StorageInsights.razor new file mode 100644 index 000000000..7c43d788a --- /dev/null +++ b/apps/server/AliasVault.Client/Main/Pages/Settings/StorageInsights.razor @@ -0,0 +1,601 @@ +@page "/settings/storage-insights" +@inherits MainBase +@inject AliasVault.Client.Services.FaviconService FaviconService +@inject AliasVault.Client.Services.ItemService ItemService +@inject AliasVault.RazorComponents.Services.ConfirmModalService ConfirmModalService +@using AliasClientDb.Models +@using AliasVault.RazorComponents.Tables +@using Microsoft.EntityFrameworkCore +@using Microsoft.Extensions.Localization + +@Localizer["PageTitle"] + + + + +@if (IsLoading) +{ +
    +

    @SharedLocalizer["Loading"]

    +
    +} +else +{ +
    +

    @Localizer["EstimatedTotalTitle"]

    +

    @FormatSize(_estimatedTotalBytes)

    +

    @Localizer["EstimatedTotalDescription"]

    +
    + +
    +

    @Localizer["CountsTitle"]

    +
    +
    +

    @Localizer["ItemCountLabel"]

    +

    @_totalItems

    +
    +
    +

    @Localizer["ItemsWithAttachmentsLabel"]

    +

    @_itemsWithAttachments

    +
    +
    +

    @Localizer["ItemsWithLogosLabel"]

    +

    @_itemsWithLogos

    +
    +
    +
    + +
    +

    @Localizer["BreakdownTitle"]

    +

    @Localizer["BreakdownDescription"]

    + + @if (_estimatedTotalBytes == 0) + { +

    -

    + } + else + { +
    +
    +
    +
    +
    +
    +
    +
    + + @Localizer["BreakdownBaseOverheadLabel"]: + @FormatSize(_baseOverheadBytes) (@Percent(_baseOverheadBytes, _estimatedTotalBytes).ToString("F1")%) +
    +
    + + @Localizer["BreakdownCredentialsLabel"]: + @FormatSize(_credentialBytes) (@Percent(_credentialBytes, _estimatedTotalBytes).ToString("F1")%) +
    +
    + + @Localizer["BreakdownAttachmentsLabel"]: + @FormatSize(_attachmentBytesWithOverhead) (@Percent(_attachmentBytesWithOverhead, _estimatedTotalBytes).ToString("F1")%) +
    +
    + + @Localizer["BreakdownLogosLabel"]: + @FormatSize(_logoBytesWithOverhead) (@Percent(_logoBytesWithOverhead, _estimatedTotalBytes).ToString("F1")%) +
    +
    + } +
    + +
    +

    @Localizer["TopAttachmentsTitle"]

    +

    @Localizer["TopAttachmentsDescription"]

    + + @if (_topAttachments.Count == 0) + { +

    -

    + } + else + { + + @foreach (var attachment in _topAttachments) + { + + @attachment.Filename + @FormatSize(attachment.SizeBytes) + @(string.IsNullOrWhiteSpace(attachment.ItemName) ? "-" : attachment.ItemName) + @attachment.CreatedAt.ToLocalTime().ToString("yyyy-MM-dd") + + } + + } +
    + +
    +

    @Localizer["TopLogosTitle"]

    +

    @Localizer["TopLogosDescription"]

    + + @if (_topLogos.Count == 0) + { +

    -

    + } + else + { + + @foreach (var logo in _topLogos) + { + + @logo.Source + @FormatSize(logo.SizeBytes) + @logo.ItemCount + + } + + } +
    + +
    +

    @Localizer["LogoManagementTitle"]

    +

    @Localizer["LogoManagementDescription"]

    + + @if (_isRedownloadingLogos) + { +
    +
    + @SharedLocalizer["Loading"] + @_redownloadProgress / @_redownloadTotal +
    +
    +
    +
    +
    + } + +
    + + +
    +
    +} + +@code { + // Approximate per-row SQLite overhead in bytes. SQLite uses 4 KiB pages by default; + // small rows pack many per page, while large blobs spill into overflow pages. These + // constants cover row headers and the indexed columns AliasClientDbContext defines. + // They are intentionally rough — attachments and logos dominate real vault size. + private const int PerItemOverheadBytes = 200; + private const int PerFieldValueOverheadBytes = 80; + private const int PerAttachmentOverheadBytes = 120; + private const int PerLogoOverheadBytes = 150; + private const int PerTotpOverheadBytes = 100; + private const int PerPasskeyOverheadBytes = 200; + + // Empty AliasVault SQLite file is ~200 KB on disk: file header, page allocation + // for each table/index, sqlite_master schema rows, and __EFMigrationsHistory. + // Constant rather than measured because we don't have access to the file size + // here and the value is stable across vaults of the same schema version. + private const int BaseSqliteOverheadBytes = 200 * 1024; + + private readonly List _attachmentColumns = + [ + new TableColumn { Title = "Filename", Sortable = false }, + new TableColumn { Title = "Size", Sortable = false }, + new TableColumn { Title = "Item", Sortable = false }, + new TableColumn { Title = "Created", Sortable = false }, + ]; + + private readonly List _logoColumns = + [ + new TableColumn { Title = "Source", Sortable = false }, + new TableColumn { Title = "Size", Sortable = false }, + new TableColumn { Title = "Used by", Sortable = false }, + ]; + + private bool IsLoading { get; set; } = true; + + private int _totalItems; + private int _itemsWithAttachments; + private int _itemsWithLogos; + private long _credentialBytes; + private long _attachmentBytesWithOverhead; + private long _logoBytesWithOverhead; + private long _baseOverheadBytes; + private long _estimatedTotalBytes; + private List _topAttachments = []; + private List _topLogos = []; + + private bool _isDeletingLogos; + private bool _isRedownloadingLogos; + private int _redownloadProgress; + private int _redownloadTotal; + + private IStringLocalizer Localizer => LocalizerFactory.Create("Pages.Main.Settings.StorageInsights", "AliasVault.Client"); + + /// + protected override async Task OnInitializedAsync() + { + await base.OnInitializedAsync(); + BreadcrumbItems.Add(new BreadcrumbItem { DisplayName = Localizer["BreadcrumbTitle"] }); + + // Localize the table column headers (created at field init time before localizer was available). + _attachmentColumns[0].Title = Localizer["ColumnFilename"]; + _attachmentColumns[1].Title = Localizer["ColumnSize"]; + _attachmentColumns[2].Title = Localizer["ColumnItem"]; + _attachmentColumns[3].Title = Localizer["ColumnCreated"]; + _logoColumns[0].Title = Localizer["ColumnWebsiteURL"]; + _logoColumns[1].Title = Localizer["ColumnSize"]; + _logoColumns[2].Title = Localizer["ColumnItemCount"]; + + await LoadStatisticsAsync(); + } + + private static string FormatSize(long bytes) + { + const double Kib = 1024d; + const double Mib = Kib * 1024d; + + if (bytes < Kib) + { + return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{bytes} B"); + } + + if (bytes < Mib) + { + return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{(bytes / Kib):F1} KB"); + } + + return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{(bytes / Mib):F1} MB"); + } + + private static double Percent(long part, long total) + { + return total == 0 ? 0d : Math.Round((double)part / total * 100d, 2); + } + + private async Task LoadStatisticsAsync() + { + var ctx = await DbService.GetDbContextAsync(); + + _totalItems = await ctx.Items.CountAsync(i => !i.IsDeleted && i.DeletedAt == null); + _itemsWithAttachments = await ctx.Attachments + .Where(a => !a.IsDeleted) + .Select(a => a.ItemId) + .Distinct() + .CountAsync(); + _itemsWithLogos = await ctx.Items.CountAsync(i => !i.IsDeleted && i.DeletedAt == null && i.LogoId != null); + + var attachmentRows = await ctx.Attachments + .Where(a => !a.IsDeleted) + .Select(a => new AttachmentRow + { + Id = a.Id, + Filename = a.Filename, + SizeBytes = a.Blob.Length, + ItemId = a.ItemId, + ItemName = a.Item.Name, + CreatedAt = a.CreatedAt, + }) + .ToListAsync(); + + var logoProjections = await ctx.Logos + .Where(l => !l.IsDeleted && l.FileData != null) + .Select(l => new + { + l.Id, + l.Source, + SizeBytes = l.FileData!.Length, + ItemIds = l.Items + .Where(i => !i.IsDeleted && i.DeletedAt == null) + .Select(i => i.Id) + .ToList(), + }) + .ToListAsync(); + + var logoRows = logoProjections + .Select(l => new LogoRow + { + Id = l.Id, + Source = l.Source, + SizeBytes = l.SizeBytes, + ItemCount = l.ItemIds.Count, + FirstItemId = l.ItemIds.Count > 0 ? l.ItemIds[0] : null, + }) + .ToList(); + + var fieldValueLengths = await ctx.FieldValues + .Where(fv => !fv.IsDeleted) + .Select(fv => fv.Value == null ? 0 : fv.Value.Length) + .ToListAsync(); + var fieldValueBytes = fieldValueLengths.Sum(x => (long)x); + var fieldValueCount = fieldValueLengths.Count; + + var passkeySizes = await ctx.Passkeys + .Where(p => !p.IsDeleted) + .Select(p => new + { + UserHandleLen = p.UserHandle.Length, + PrfKeyLen = p.PrfKey == null ? 0 : p.PrfKey.Length, + AdditionalDataLen = p.AdditionalData == null ? 0 : p.AdditionalData.Length, + PublicKeyLen = p.PublicKey.Length, + PrivateKeyLen = p.PrivateKey.Length, + }) + .ToListAsync(); + var passkeyBytes = passkeySizes.Sum(p => (long)(p.UserHandleLen + p.PrfKeyLen + p.AdditionalDataLen + p.PublicKeyLen + p.PrivateKeyLen)); + var passkeyCount = passkeySizes.Count; + + var totpCount = await ctx.TotpCodes.CountAsync(t => !t.IsDeleted); + + var attachmentBytes = attachmentRows.Sum(a => (long)a.SizeBytes); + var logoBytes = logoRows.Sum(l => (long)l.SizeBytes); + + _attachmentBytesWithOverhead = attachmentBytes + ((long)attachmentRows.Count * PerAttachmentOverheadBytes); + _logoBytesWithOverhead = logoBytes + ((long)logoRows.Count * PerLogoOverheadBytes); + + _credentialBytes = fieldValueBytes + + passkeyBytes + + ((long)_totalItems * PerItemOverheadBytes) + + ((long)fieldValueCount * PerFieldValueOverheadBytes) + + ((long)passkeyCount * PerPasskeyOverheadBytes) + + ((long)totpCount * PerTotpOverheadBytes); + + _baseOverheadBytes = BaseSqliteOverheadBytes; + + _estimatedTotalBytes = _credentialBytes + + _attachmentBytesWithOverhead + + _logoBytesWithOverhead + + _baseOverheadBytes; + + _topAttachments = attachmentRows + .OrderByDescending(a => a.SizeBytes) + .Take(10) + .ToList(); + + _topLogos = logoRows + .OrderByDescending(l => l.SizeBytes) + .Take(10) + .ToList(); + + IsLoading = false; + StateHasChanged(); + } + + private void NavigateToItem(Guid itemId) + { + NavigationManager.NavigateTo($"/items/{itemId}"); + } + + private double RedownloadPercent() + { + return _redownloadTotal == 0 ? 0d : Math.Round((double)_redownloadProgress / _redownloadTotal * 100d, 1); + } + + /// + /// Soft-deletes every logo in the vault and clears the LogoId on every item that referenced one. + /// The vault sync that follows propagates these changes to other devices. + /// + private async Task DeleteAllLogos() + { + var confirmed = await ConfirmModalService.ShowConfirmation( + Localizer["DeleteAllLogosConfirmTitle"], + Localizer["DeleteAllLogosConfirmMessage"], + SharedLocalizer["Confirm"], + SharedLocalizer["Cancel"]); + if (!confirmed) + { + return; + } + + _isDeletingLogos = true; + StateHasChanged(); + GlobalLoadingSpinner.Show(SharedLocalizer["Loading"]); + + try + { + var ctx = await DbService.GetDbContextAsync(); + var now = DateTime.UtcNow; + + // Soft-delete the row AND clear the binary blob to free up space. + var logos = await ctx.Logos.Where(l => !l.IsDeleted).ToListAsync(); + foreach (var logo in logos) + { + logo.IsDeleted = true; + logo.FileData = Array.Empty(); + logo.UpdatedAt = now; + } + + // Clear LogoId on items so they no longer reference soft-deleted logos. + var itemsWithLogos = await ctx.Items.Where(i => i.LogoId != null).ToListAsync(); + foreach (var item in itemsWithLogos) + { + item.LogoId = null; + item.UpdatedAt = now; + } + + await ctx.SaveChangesAsync(); + + // VACUUM after a bulk blob delete — without it the SQLite pages stay allocated and + // the user wouldn't actually see the size drop they came here for. + await DbService.VacuumDatabaseAsync(); + await DbService.SaveDatabaseAsync(); + + GlobalNotificationService.AddSuccessMessage(SharedLocalizer["Success"], true); + await LoadStatisticsAsync(); + } + catch (Exception) + { + GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true); + } + finally + { + GlobalLoadingSpinner.Hide(); + _isDeletingLogos = false; + StateHasChanged(); + } + } + + /// + /// Re-downloads favicons for every item that has a login URL, overwriting any existing logo. + /// Useful after the favicon extractor's sizing/quality settings change so the vault picks up + /// the new output. + /// + private async Task RedownloadAllLogos() + { + var confirmed = await ConfirmModalService.ShowConfirmation(Localizer["RedownloadAllLogosConfirmTitle"], Localizer["RedownloadAllLogosConfirmMessage"], SharedLocalizer["Confirm"], SharedLocalizer["Cancel"]); + if (!confirmed) + { + return; + } + + _isRedownloadingLogos = true; + _redownloadProgress = 0; + _redownloadTotal = 0; + StateHasChanged(); + + try + { + var items = await ItemService.LoadAllAsync(); + + // Group items by normalized domain so each domain is fetched once and assigned to all items. + var itemsByDomain = new Dictionary>(); + var representativeUrls = new Dictionary(); + foreach (var item in items) + { + var url = ItemService.GetFieldValue(item, FieldKey.LoginUrl); + if (string.IsNullOrEmpty(url) || url == ItemService.DefaultServiceUrl) + { + continue; + } + + if (!FaviconService.TryNormalizeDomain(url, out var domain)) + { + continue; + } + + if (!itemsByDomain.TryGetValue(domain, out var bucket)) + { + bucket = new List(); + itemsByDomain[domain] = bucket; + representativeUrls[domain] = url; + } + + bucket.Add(item); + } + + _redownloadTotal = itemsByDomain.Count; + StateHasChanged(); + + if (_redownloadTotal == 0) + { + GlobalNotificationService.AddSuccessMessage(Localizer["RedownloadAllLogosNoItemsMessage"], true); + return; + } + + var progress = new Progress(count => + { + _redownloadProgress = count; + InvokeAsync(StateHasChanged); + }); + + var result = await FaviconService.ExtractBulkAsync(representativeUrls.Values, progress); + + var ctx = await DbService.GetDbContextAsync(); + var now = DateTime.UtcNow; + + foreach (var (domain, bucket) in itemsByDomain) + { + if (!result.Favicons.TryGetValue(domain, out var image)) + { + continue; + } + + // Look up by Source (unique field). + var existingLogo = await ctx.Logos.FirstOrDefaultAsync(l => l.Source == domain); + Guid logoId; + if (existingLogo != null) + { + existingLogo.IsDeleted = false; + existingLogo.FileData = image; + existingLogo.FetchedAt = now; + existingLogo.UpdatedAt = now; + logoId = existingLogo.Id; + } + else + { + var newLogo = new AliasClientDb.Logo + { + Id = Guid.NewGuid(), + Source = domain, + FileData = image, + FetchedAt = now, + CreatedAt = now, + UpdatedAt = now, + }; + ctx.Logos.Add(newLogo); + logoId = newLogo.Id; + } + + foreach (var item in bucket) + { + if (item.LogoId != logoId) + { + item.LogoId = logoId; + item.UpdatedAt = now; + } + } + } + + await ctx.SaveChangesAsync(); + await DbService.SaveDatabaseAsync(); + + if (result.Status == FaviconService.BulkExtractStatus.RateLimited) + { + GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true); + } + else + { + GlobalNotificationService.AddSuccessMessage(SharedLocalizer["Success"], true); + } + + await LoadStatisticsAsync(); + } + catch (Exception) + { + GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true); + } + finally + { + _isRedownloadingLogos = false; + StateHasChanged(); + } + } + + private sealed class AttachmentRow + { + public Guid Id { get; init; } + + public string Filename { get; init; } = string.Empty; + + public int SizeBytes { get; init; } + + public Guid ItemId { get; init; } + + public string? ItemName { get; init; } + + public DateTime CreatedAt { get; init; } + } + + private sealed class LogoRow + { + public Guid Id { get; init; } + + public string Source { get; init; } = string.Empty; + + public int SizeBytes { get; init; } + + public int ItemCount { get; init; } + + public Guid? FirstItemId { get; init; } + } +} diff --git a/apps/server/AliasVault.Client/Program.cs b/apps/server/AliasVault.Client/Program.cs index f1459d5aa..936b3468f 100644 --- a/apps/server/AliasVault.Client/Program.cs +++ b/apps/server/AliasVault.Client/Program.cs @@ -87,6 +87,7 @@ builder.Services.AddTransient(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/apps/server/AliasVault.Client/Resources/Layout/TopMenu.en.resx b/apps/server/AliasVault.Client/Resources/Layout/TopMenu.en.resx index 091b2e8b6..e19e7858c 100644 --- a/apps/server/AliasVault.Client/Resources/Layout/TopMenu.en.resx +++ b/apps/server/AliasVault.Client/Resources/Layout/TopMenu.en.resx @@ -39,6 +39,10 @@ Security settings Navigation link for security settings + + Storage insights + Navigation link for vault storage insights page + Import / Export Navigation link for import/export settings diff --git a/apps/server/AliasVault.Client/Resources/Pages/Main/Settings/StorageInsights.en.resx b/apps/server/AliasVault.Client/Resources/Pages/Main/Settings/StorageInsights.en.resx new file mode 100644 index 000000000..5b9984255 --- /dev/null +++ b/apps/server/AliasVault.Client/Resources/Pages/Main/Settings/StorageInsights.en.resx @@ -0,0 +1,174 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Vault storage insights + Page title for vault storage insights + + + See an estimate of your vault size. For best sync performance, it's advised to keep your vault size small and delete items you no longer need. + Page description for vault storage insights + + + Storage insights + Breadcrumb title for storage insights page + + + + + Estimated vault size + Title for the estimated total vault size card + + + This is an approximation calculated from your local data. + Subtext explaining that the estimate is approximate + + + + + Overview + Title for the counts/overview section + + + Items + Label for total item count card + + + Items with attachments + Label for count of items that have at least one attachment + + + Items with a logo + Label for count of items that reference a logo + + + + + Storage usage breakdown + Title for the breakdown bar section + + + Approximate share of vault size by category. Attachments and logos usually take up most of the space. + Description for the breakdown bar + + + Credentials + Legend label for credential data slice + + + Attachments + Legend label for attachments slice + + + Logos + Legend label for logos slice + + + Database overhead + Legend label for the fixed SQLite base/schema overhead slice + + + + + Largest attachments + Title for the top attachments table + + + Top 10 largest file attachments. Remove the attachment if you no longer need it. + Description for the top attachments table + + + + + Largest logos + Title for the top logos table + + + Top 10 largest service logos. Logos are reused for items with the same domain. + Description for the top logos table + + + + + Filename + Table column header for attachment filename + + + Size + Table column header for size in KB + + + Item + Table column header for the parent item name + + + Created + Table column header for creation date + + + Website URL + Table column header for logo source domain + + + Used by + Table column header for the number of items using a logo + + + + + Manage logos + Title for the logo management section + + + Logos are fetched from the favicon of each item's website. You can delete every logo to reclaim space, or re-download them to pick up improved versions. + Description for the logo management section + + + Delete all logos + Button to delete every logo in the vault + + + Re-download all logos + Button to re-fetch favicons for every item with a URL + + + Delete all logos? + Confirmation modal title for delete-all-logos + + + This removes every stored logo from your vault and clears the logo on each credential. You can re-download them at any time. Continue? + Confirmation modal body for delete-all-logos + + + Re-download all logos? + Confirmation modal title for redownload-all-logos + + + This re-fetches the favicon for every credential that has a website URL, replacing the existing logo. Depending on your vault size this may take a while. Continue? + Confirmation modal body for redownload-all-logos + + + No items with a website URL were found, nothing to download. + Notification shown when redownload was triggered but no items qualify + + diff --git a/apps/server/AliasVault.Client/Services/FaviconService.cs b/apps/server/AliasVault.Client/Services/FaviconService.cs new file mode 100644 index 000000000..afd903ea6 --- /dev/null +++ b/apps/server/AliasVault.Client/Services/FaviconService.cs @@ -0,0 +1,221 @@ +//----------------------------------------------------------------------- +// +// Copyright (c) aliasvault. All rights reserved. +// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information. +// +//----------------------------------------------------------------------- + +namespace AliasVault.Client.Services; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using AliasVault.Shared.Models.WebApi.Favicon; + +/// +/// Wraps calls to the server-side favicon API. Centralizes the single-URL and batched +/// extraction paths so item add/edit, vault import, and bulk re-download all share one +/// implementation (and one place to evolve, e.g. when adjusting batch size). +/// +public sealed class FaviconService(HttpClient httpClient) +{ + /// + /// Number of URLs sent per batch request. Mirrors FaviconController.MaxBatchSize on the server. + /// + public const int BatchSize = 10; + + /// + /// Outcome of a bulk extraction. + /// + public enum BulkExtractStatus + { + /// All requested URLs were processed. + Completed, + + /// The user cancelled the operation before all URLs were processed. + Cancelled, + + /// Server returned 429 — the per-user 24h rate limit was hit. Partial results are returned. + RateLimited, + } + + /// + /// Normalizes a URL to a lowercase host without leading "www.". Returns false for URLs that + /// can't be parsed; callers should skip those rather than fall back to a different key. + /// + /// The URL to normalize. + /// The normalized domain. + /// True if the URL was parseable. + public static bool TryNormalizeDomain(string url, out string domain) + { + domain = string.Empty; + if (string.IsNullOrWhiteSpace(url)) + { + return false; + } + + try + { + var host = new Uri(url).Host.ToLowerInvariant(); + if (host.StartsWith("www.", StringComparison.Ordinal)) + { + host = host[4..]; + } + + if (string.IsNullOrEmpty(host)) + { + return false; + } + + domain = host; + return true; + } + catch + { + return false; + } + } + + /// + /// Extracts the favicon for a single URL. + /// + /// The URL to fetch the favicon for. + /// Favicon bytes, or null if extraction failed or the rate limit was hit. + public async Task ExtractAsync(string url) + { + try + { + var apiReturn = await httpClient.GetFromJsonAsync( + $"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}"); + return apiReturn?.Image; + } + catch + { + return null; + } + } + + /// + /// Extracts favicons for many URLs in batches, deduplicating by normalized domain so the + /// same domain is only fetched once even if many items reference it. + /// + /// URLs to extract favicons for. Duplicates by domain are collapsed. + /// + /// Optional progress callback. Receives the number of unique domains processed so far + /// (regardless of fetch success). The total reported is the count of unique domains, not + /// the count of input URLs. + /// + /// Cancellation token. + /// The favicon dictionary plus the operation status. + public async Task ExtractBulkAsync(IEnumerable urls, IProgress? progress = null, CancellationToken cancellationToken = default) + { + var favicons = new Dictionary(); + + // Deduplicate by domain up front so a 5000-item vault that all uses 200 unique domains + // makes 200 fetches, not 5000. + var domainToUrl = new Dictionary(); + foreach (var url in urls) + { + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + if (TryNormalizeDomain(url, out var domain) && !domainToUrl.ContainsKey(domain)) + { + domainToUrl[domain] = url; + } + } + + if (domainToUrl.Count == 0) + { + return new BulkExtractResult(favicons, BulkExtractStatus.Completed); + } + + var entries = domainToUrl.ToList(); + int processed = 0; + + for (int i = 0; i < entries.Count; i += BatchSize) + { + if (cancellationToken.IsCancellationRequested) + { + return new BulkExtractResult(favicons, BulkExtractStatus.Cancelled); + } + + var chunk = entries.Skip(i).Take(BatchSize).ToList(); + var request = new FaviconExtractBatchRequest + { + Urls = chunk.Select(e => e.Value).ToList(), + }; + + HttpResponseMessage response; + try + { + response = await httpClient.PostAsJsonAsync("v1/Favicon/ExtractBatch", request, cancellationToken); + } + catch + { + // Network or other transport error — treat the whole chunk as failed but keep going. + processed += chunk.Count; + progress?.Report(processed); + continue; + } + + if (response.StatusCode == HttpStatusCode.TooManyRequests) + { + return new BulkExtractResult(favicons, BulkExtractStatus.RateLimited); + } + + if (!response.IsSuccessStatusCode) + { + processed += chunk.Count; + progress?.Report(processed); + continue; + } + + FaviconExtractBatchResponse? body; + try + { + body = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); + } + catch + { + processed += chunk.Count; + progress?.Report(processed); + continue; + } + + if (body?.Results != null) + { + // Match results back to the domains we asked for. Server echoes the URL, but matching + // by index is robust against any normalization differences. + for (int j = 0; j < body.Results.Count && j < chunk.Count; j++) + { + var image = body.Results[j].Image; + if (image != null) + { + favicons[chunk[j].Key] = image; + } + } + } + + processed += chunk.Count; + progress?.Report(processed); + } + + return new BulkExtractResult(favicons, BulkExtractStatus.Completed); + } + + /// + /// Result of a bulk extraction call. Favicons are keyed by normalized domain so callers + /// can reuse a single fetched favicon across all items that share that domain. + /// + /// Dictionary of normalized domain to favicon bytes. + /// Outcome of the operation. + public record BulkExtractResult(Dictionary Favicons, BulkExtractStatus Status); +} diff --git a/apps/server/AliasVault.Client/Services/ItemService.cs b/apps/server/AliasVault.Client/Services/ItemService.cs index a1777581b..6c1bae855 100644 --- a/apps/server/AliasVault.Client/Services/ItemService.cs +++ b/apps/server/AliasVault.Client/Services/ItemService.cs @@ -18,14 +18,12 @@ using System.Threading.Tasks; using AliasClientDb; using AliasClientDb.Models; using AliasVault.Client.Main.Models; -using AliasVault.Client.Utilities; -using AliasVault.Shared.Models.WebApi.Favicon; using Microsoft.EntityFrameworkCore; /// /// Service class for Item operations. /// -public sealed class ItemService(HttpClient httpClient, DbService dbService, Config config, JsInteropService jsInteropService) +public sealed class ItemService(HttpClient httpClient, DbService dbService, Config config, JsInteropService jsInteropService, FaviconService faviconService) { /// /// The default service URL used as placeholder in forms. When this value is set, the URL field is considered empty @@ -1492,59 +1490,68 @@ public sealed class ItemService(HttpClient httpClient, DbService dbService, Conf /// Task. private async Task ExtractFaviconAsync(Item item) { - // Try to extract favicon from service URL var url = GetFieldValue(item, FieldKey.LoginUrl); - if (url != null && !string.IsNullOrEmpty(url) && url != DefaultServiceUrl) - { - try - { - // Extract and normalize domain for deduplication - var domain = new Uri(url).Host.ToLowerInvariant(); - if (domain.StartsWith("www.")) - { - domain = domain[4..]; - } - - var context = await dbService.GetDbContextAsync(); - - // Check if logo already exists for this source (deduplication) - var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain); - - if (existingLogo != null) - { - // Reuse existing logo - no need to fetch - item.LogoId = existingLogo.Id; - return; - } - - // No existing logo - fetch from API - var apiReturn = await httpClient.GetFromJsonAsync($"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}"); - if (apiReturn?.Image is not null) - { - // Create new logo - var newLogo = new Logo - { - Id = Guid.NewGuid(), - Source = domain, - FileData = apiReturn.Image, - MimeType = "image/png", - FetchedAt = DateTime.UtcNow, - CreatedAt = DateTime.UtcNow, - UpdatedAt = DateTime.UtcNow, - }; - context.Logos.Add(newLogo); - item.LogoId = newLogo.Id; - } - } - catch - { - // Ignore favicon extraction errors - } - } - else + if (url == null || string.IsNullOrEmpty(url) || url == DefaultServiceUrl) { // URL is empty or just the placeholder - clear any existing logo item.LogoId = null; + return; + } + + if (!FaviconService.TryNormalizeDomain(url, out var domain)) + { + return; + } + + try + { + var context = await dbService.GetDbContextAsync(); + + // Look up by Source regardless of IsDeleted — Logos.Source has a UNIQUE index, so a + // soft-deleted row with the same domain still occupies the slot and we'd hit a UNIQUE + // violation on insert. Filter against IsDeleted/empty FileData when deciding whether + // it's safe to reuse without re-fetching. + var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain); + if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 }) + { + item.LogoId = existingLogo.Id; + return; + } + + var image = await faviconService.ExtractAsync(url); + if (image is null) + { + return; + } + + var now = DateTime.UtcNow; + if (existingLogo != null) + { + // Restore (or refill) the existing row. + existingLogo.IsDeleted = false; + existingLogo.FileData = image; + existingLogo.FetchedAt = now; + existingLogo.UpdatedAt = now; + item.LogoId = existingLogo.Id; + } + else + { + var newLogo = new Logo + { + Id = Guid.NewGuid(), + Source = domain, + FileData = image, + FetchedAt = now, + CreatedAt = now, + UpdatedAt = now, + }; + context.Logos.Add(newLogo); + item.LogoId = newLogo.Id; + } + } + catch + { + // Ignore favicon extraction errors } } diff --git a/apps/server/AliasVault.Client/wwwroot/css/tailwind.css b/apps/server/AliasVault.Client/wwwroot/css/tailwind.css index 4a8ad1a82..2cdcda33b 100644 --- a/apps/server/AliasVault.Client/wwwroot/css/tailwind.css +++ b/apps/server/AliasVault.Client/wwwroot/css/tailwind.css @@ -1638,6 +1638,10 @@ video { border-radius: 0.375rem; } +.rounded-sm { + border-radius: 0.125rem; +} + .rounded-b-lg { border-bottom-right-radius: 0.5rem; border-bottom-left-radius: 0.5rem; @@ -1809,6 +1813,11 @@ video { border-color: rgb(254 240 138 / var(--tw-border-opacity)); } +.border-yellow-400 { + --tw-border-opacity: 1; + border-color: rgb(250 204 21 / var(--tw-border-opacity)); +} + .bg-amber-100 { --tw-bg-opacity: 1; background-color: rgb(254 243 199 / var(--tw-bg-opacity)); @@ -2018,6 +2027,21 @@ video { background-color: rgb(234 179 8 / var(--tw-bg-opacity)); } +.bg-amber-500 { + --tw-bg-opacity: 1; + background-color: rgb(245 158 11 / var(--tw-bg-opacity)); +} + +.bg-emerald-500 { + --tw-bg-opacity: 1; + background-color: rgb(16 185 129 / var(--tw-bg-opacity)); +} + +.bg-yellow-200 { + --tw-bg-opacity: 1; + background-color: rgb(254 240 138 / var(--tw-bg-opacity)); +} + .bg-opacity-50 { --tw-bg-opacity: 0.5; } @@ -3294,6 +3318,11 @@ video { border-color: rgb(133 77 14 / var(--tw-border-opacity)); } +.dark\:border-yellow-600:is(.dark *) { + --tw-border-opacity: 1; + border-color: rgb(202 138 4 / var(--tw-border-opacity)); +} + .dark\:bg-amber-800\/30:is(.dark *) { background-color: rgb(146 64 14 / 0.3); } @@ -4269,6 +4298,10 @@ video { grid-template-columns: repeat(3, minmax(0, 1fr)); } + .lg\:grid-cols-4 { + grid-template-columns: repeat(4, minmax(0, 1fr)); + } + .lg\:flex-row { flex-direction: row; } diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchRequest.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchRequest.cs new file mode 100644 index 000000000..a8062eb97 --- /dev/null +++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchRequest.cs @@ -0,0 +1,22 @@ +//----------------------------------------------------------------------- +// +// Copyright (c) aliasvault. All rights reserved. +// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information. +// +//----------------------------------------------------------------------- + +namespace AliasVault.Shared.Models.WebApi.Favicon; + +using System.Collections.Generic; + +/// +/// Request payload for batch favicon extraction. +/// +public class FaviconExtractBatchRequest +{ + /// + /// Gets or sets the URLs to extract favicons for. The server caps the number of URLs + /// it processes per request; callers should chunk larger inputs into multiple requests. + /// + public List Urls { get; set; } = new(); +} diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchResponse.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchResponse.cs new file mode 100644 index 000000000..238356ce5 --- /dev/null +++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchResponse.cs @@ -0,0 +1,22 @@ +//----------------------------------------------------------------------- +// +// Copyright (c) aliasvault. All rights reserved. +// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information. +// +//----------------------------------------------------------------------- + +namespace AliasVault.Shared.Models.WebApi.Favicon; + +using System.Collections.Generic; + +/// +/// Response payload for batch favicon extraction. Each result lines up with the URL at the +/// same index in the request, with a null Image when extraction failed for that URL. +/// +public class FaviconExtractBatchResponse +{ + /// + /// Gets or sets the per-URL extraction results. + /// + public List Results { get; set; } = new(); +} diff --git a/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchResult.cs b/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchResult.cs new file mode 100644 index 000000000..c54d6cd1d --- /dev/null +++ b/apps/server/Shared/AliasVault.Shared/Models/WebApi/Favicon/FaviconExtractBatchResult.cs @@ -0,0 +1,24 @@ +//----------------------------------------------------------------------- +// +// Copyright (c) aliasvault. All rights reserved. +// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information. +// +//----------------------------------------------------------------------- + +namespace AliasVault.Shared.Models.WebApi.Favicon; + +/// +/// Per-URL favicon extraction result. +/// +public class FaviconExtractBatchResult +{ + /// + /// Gets or sets the URL the favicon was requested for (echoed back so the client can correlate). + /// + public string Url { get; set; } = string.Empty; + + /// + /// Gets or sets the favicon image bytes, or null if extraction failed. + /// + public byte[]? Image { get; set; } +} diff --git a/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs b/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs index 9176d281b..02d847d44 100644 --- a/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs +++ b/apps/server/Utilities/AliasVault.FaviconExtractor/FaviconExtractor.cs @@ -8,6 +8,7 @@ namespace AliasVault.FaviconExtractor; using System; +using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; @@ -21,9 +22,10 @@ using SkiaSharp; /// public static class FaviconExtractor { - private const int MaxSizeBytes = 50 * 1024; // 50KB max size before resizing - private const int TargetWidth = 96; // Resize target width - private static readonly string[] _allowedSchemes = { "http", "https" }; + private const int MaxSizeBytes = 20 * 1024; // 20KB max size; images above this are resized/re-encoded. + private static readonly int[] _resizeWidths = [96, 64, 48, 32]; + private static readonly int[] _jpegFallbackQualities = [80, 65, 50]; + private static readonly string[] _allowedSchemes = ["http", "https"]; /// /// Extracts the favicon from a URL with enhanced browser like behavior. @@ -56,6 +58,43 @@ public static class FaviconExtractor return null; } + /// + /// Extracts favicons for multiple URLs in parallel. Each URL is processed independently; + /// individual failures are returned as null entries rather than throwing. The returned + /// list lines up index-for-index with the input. + /// + /// The URLs to extract favicons for. + /// A list of favicon byte arrays, in the same order as the input urls. + public static async Task> GetFaviconsAsync(IReadOnlyList urls) + { + if (urls.Count == 0) + { + return Array.Empty(); + } + + var tasks = new Task[urls.Count]; + for (int i = 0; i < urls.Count; i++) + { + // Wrap each call in a try/catch so one bad URL doesn't fail the whole batch. + var url = urls[i]; + tasks[i] = SafeGetFaviconAsync(url); + } + + return await Task.WhenAll(tasks); + } + + private static async Task SafeGetFaviconAsync(string url) + { + try + { + return await GetFaviconAsync(url); + } + catch + { + return null; + } + } + /// /// Tries to get the favicon from the URL. /// @@ -184,18 +223,13 @@ public static class FaviconExtractor return null; } - // If image is too large, attempt to resize + // If image is too large, attempt to resize/re-encode it down under the cap. if (imageBytes.Length > MaxSizeBytes) { - var resizedBytes = ResizeImageAsync(imageBytes, contentType); - if (resizedBytes != null) - { - imageBytes = resizedBytes; - } + return ResizeImage(imageBytes, contentType); } - // Return only if within size limits - return imageBytes.Length <= MaxSizeBytes ? imageBytes : null; + return imageBytes; } catch { @@ -377,13 +411,22 @@ public static class FaviconExtractor } /// - /// Resizes the image to the target width. + /// Iteratively shrinks the image until it fits in the size cap. Tries + /// progressively smaller widths in PNG (lossless after downscale handles most cases), then + /// falls back to JPEG at decreasing quality if PNG at the smallest width is still too large. /// /// The image bytes to resize. /// The content type of the image. - /// The resized image bytes. - private static byte[]? ResizeImageAsync(byte[] imageBytes, string contentType) + /// The resized image bytes, or null if it could not be brought under the size cap. + private static byte[]? ResizeImage(byte[] imageBytes, string contentType) { + // SVG is a text format; we can't usefully raster-resize it here. Caller already rejects + // anything over the cap, so just bail and let it return null. + if (contentType == "image/svg+xml") + { + return null; + } + try { using var original = SKBitmap.Decode(imageBytes); @@ -392,30 +435,59 @@ public static class FaviconExtractor return null; } - var scale = (float)TargetWidth / original.Width; - var targetHeight = (int)(original.Height * scale); - - using var resized = original.Resize(new SKImageInfo(TargetWidth, targetHeight), new SKSamplingOptions(SKFilterMode.Linear)); - if (resized == null) + // Pass 1: PNG at progressively smaller widths. Preserves transparency. + foreach (var width in _resizeWidths) { - return null; + var encoded = EncodeAtWidth(original, width, SKEncodedImageFormat.Png, 100); + if (encoded != null && encoded.Length <= MaxSizeBytes) + { + return encoded; + } } - using var image = SKImage.FromBitmap(resized); - var format = contentType switch + // Pass 2: JPEG at the smallest width with decreasing quality. Loses transparency but + // gives much smaller files for photographic favicons that resist PNG compression. + var fallbackWidth = _resizeWidths[^1]; + foreach (var quality in _jpegFallbackQualities) { - "image/png" => SKEncodedImageFormat.Png, - "image/jpeg" => SKEncodedImageFormat.Jpeg, - "image/gif" => SKEncodedImageFormat.Gif, - _ => SKEncodedImageFormat.Png, - }; + var encoded = EncodeAtWidth(original, fallbackWidth, SKEncodedImageFormat.Jpeg, quality); + if (encoded != null && encoded.Length <= MaxSizeBytes) + { + return encoded; + } + } - var data = image.Encode(format, 90); - return data?.ToArray(); + return null; } catch { return null; } } + + /// + /// Resizes the bitmap to the target width (preserving aspect ratio, never upscaling) and + /// encodes it in the given format. + /// + /// The decoded source bitmap. + /// Desired output width in pixels. + /// Encode format. + /// Encoder quality (only meaningful for lossy formats). + /// Encoded bytes, or null if encoding failed. + private static byte[]? EncodeAtWidth(SKBitmap original, int targetWidth, SKEncodedImageFormat format, int quality) + { + var width = Math.Min(targetWidth, original.Width); + var scale = (float)width / original.Width; + var height = Math.Max(1, (int)(original.Height * scale)); + + using var resized = original.Resize(new SKImageInfo(width, height), new SKSamplingOptions(SKFilterMode.Linear)); + if (resized == null) + { + return null; + } + + using var image = SKImage.FromBitmap(resized); + using var data = image.Encode(format, quality); + return data?.ToArray(); + } } diff --git a/core/rust/src/vault_pruner/mod.rs b/core/rust/src/vault_pruner/mod.rs index fd8f3901b..53d125d1f 100644 --- a/core/rust/src/vault_pruner/mod.rs +++ b/core/rust/src/vault_pruner/mod.rs @@ -72,6 +72,9 @@ pub struct PruneStats { /// Number of tombstoned attachments whose blob bytes were cleared. #[serde(default)] pub attachment_blobs_cleared: u32, + /// Number of tombstoned logos whose FileData bytes were cleared. + #[serde(default)] + pub logo_blobs_cleared: u32, } /// Output of the prune operation. @@ -263,7 +266,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult { if let Some(logo_id) = logo.get("Id").and_then(|v| v.as_str()) { if !referenced_logo_ids.contains(logo_id) { statements.push(SqlStatement { - sql: "UPDATE Logos SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ?".to_string(), + sql: "UPDATE Logos SET IsDeleted = 1, FileData = X'', UpdatedAt = ? WHERE Id = ?".to_string(), params: vec![ serde_json::json!(now_str), serde_json::json!(logo_id), @@ -307,6 +310,35 @@ pub fn prune_vault(input: PruneInput) -> VaultResult { } } + // Pass 4 — sweep tombstoned logos that still carry FileData bytes. Logos that have + // been soft-deleted but still have FileData bytes take up space for no reason. + // This behaviour could have occurred in older clients (before 0.29.x). + if let Some(logos_table) = input.tables.iter().find(|t| t.name == "Logos") { + for logo in &logos_table.records { + let is_deleted = logo.get("IsDeleted") + .map(|v| v.as_i64() == Some(1) || v.as_bool() == Some(true)) + .unwrap_or(false); + if !is_deleted { + continue; + } + + if !logo_has_file_data_bytes(logo) { + continue; + } + + if let Some(logo_id) = logo.get("Id").and_then(|v| v.as_str()) { + statements.push(SqlStatement { + sql: "UPDATE Logos SET FileData = X'', UpdatedAt = ? WHERE Id = ?".to_string(), + params: vec![ + serde_json::json!(now_str), + serde_json::json!(logo_id), + ], + }); + stats.logo_blobs_cleared += 1; + } + } + } + Ok(PruneOutput { success: true, statements, @@ -325,6 +357,17 @@ fn attachment_has_blob_bytes(attachment: &Record) -> bool { } } +/// True if the logo's FileData field is present and non-empty. +fn logo_has_file_data_bytes(logo: &Record) -> bool { + match logo.get("FileData") { + None => false, + Some(serde_json::Value::Null) => false, + Some(serde_json::Value::String(s)) => !s.is_empty(), + Some(serde_json::Value::Array(a)) => !a.is_empty(), + Some(_) => true, + } +} + /// Prune vault using JSON strings. /// Convenience function for FFI. pub fn prune_vault_json(input_json: &str) -> VaultResult { @@ -433,6 +476,12 @@ mod tests { record } + fn make_logo_record_with_blob(id: &str, is_deleted: bool, blob: serde_json::Value) -> Record { + let mut record = make_logo_record(id, is_deleted); + record.insert("FileData".to_string(), blob); + record + } + #[test] fn test_prune_expired_items() { let now = Utc::now(); @@ -699,6 +748,82 @@ mod tests { assert_eq!(output.stats.logos_pruned, 0); } + #[test] + fn test_orphan_logo_pruning_emits_filedata_clear_in_same_statement() { + // Pass 2 must clear FileData when it tombstones a logo, otherwise the + // encrypted vault keeps the blob bytes even after the row is "deleted". + let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let input = PruneInput { + tables: vec![ + TableData { + name: "Items".to_string(), + records: vec![], + }, + TableData { + name: "Logos".to_string(), + records: vec![make_logo_record_with_blob("logo-orphan", false, serde_json::json!("aGVsbG8="))], + }, + ], + retention_days: 30, + current_time: now_str, + }; + + let output = prune_vault(input).unwrap(); + + assert_eq!(output.stats.logos_pruned, 1); + assert!(output.statements.iter().any(|s| s.sql.contains("FileData = X''"))); + } + + #[test] + fn test_tombstoned_logo_with_blob_bytes_is_swept() { + // Pass 4 must catch historical logos that are IsDeleted=1 but still carry FileData + // (e.g. tombstoned by an older client before the FileData=X'' fix landed). + let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let input = PruneInput { + tables: vec![ + TableData { + name: "Items".to_string(), + records: vec![], + }, + TableData { + name: "Logos".to_string(), + records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::json!("aGVsbG8="))], + }, + ], + retention_days: 30, + current_time: now_str, + }; + + let output = prune_vault(input).unwrap(); + + assert_eq!(output.stats.logo_blobs_cleared, 1); + assert_eq!(output.stats.logos_pruned, 0); + } + + #[test] + fn test_tombstoned_logo_without_blob_is_not_touched() { + // Logos already cleared shouldn't generate redundant updates. + let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(); + let input = PruneInput { + tables: vec![ + TableData { + name: "Items".to_string(), + records: vec![], + }, + TableData { + name: "Logos".to_string(), + records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::Value::Null)], + }, + ], + retention_days: 30, + current_time: now_str, + }; + + let output = prune_vault(input).unwrap(); + + assert_eq!(output.stats.logo_blobs_cleared, 0); + } + #[test] fn test_already_soft_deleted_logo_is_not_re_pruned() { let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();