mirror of
https://github.com/aliasvault/aliasvault.git
synced 2026-05-09 07:46:13 -04:00
Merge pull request #1996 from aliasvault/1046-feature-request-add-vault-storage-insights-to-web-app-client
This commit is contained in:
@@ -11,10 +11,31 @@ export class LogoQueries {
|
||||
WHERE Source = ? AND IsDeleted = 0
|
||||
LIMIT 1`;
|
||||
|
||||
/**
|
||||
* Look up a logo by source regardless of IsDeleted state. Used by getOrCreate to detect
|
||||
* soft-deleted rows that still occupy the UNIQUE(Source) slot, so we can refill them
|
||||
* instead of trying (and failing) to INSERT a duplicate.
|
||||
*/
|
||||
public static readonly GET_BY_SOURCE_INCLUDING_DELETED = `
|
||||
SELECT Id, IsDeleted FROM Logos
|
||||
WHERE Source = ?
|
||||
LIMIT 1`;
|
||||
|
||||
/**
|
||||
* Insert new logo.
|
||||
*/
|
||||
public static readonly INSERT = `
|
||||
INSERT INTO Logos (Id, Source, FileData, CreatedAt, UpdatedAt, IsDeleted)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`;
|
||||
|
||||
/**
|
||||
* Restore a soft-deleted logo and refill its bytes in one statement. Caller passes
|
||||
* (FileData, UpdatedAt, Id).
|
||||
*/
|
||||
public static readonly RESTORE_WITH_FILE_DATA = `
|
||||
UPDATE Logos
|
||||
SET IsDeleted = 0,
|
||||
FileData = ?,
|
||||
UpdatedAt = ?
|
||||
WHERE Id = ?`;
|
||||
}
|
||||
|
||||
@@ -41,10 +41,22 @@ export class LogoRepository extends BaseRepository {
|
||||
* @returns The logo ID (existing or newly created)
|
||||
*/
|
||||
public getOrCreate(source: string, logoData: Uint8Array, currentDateTime: string): string {
|
||||
// Check if a logo for this source already exists
|
||||
const existingId = this.getIdForSource(source);
|
||||
if (existingId) {
|
||||
return existingId;
|
||||
const existing = this.client.executeQuery<{ Id: string; IsDeleted: number }>(
|
||||
LogoQueries.GET_BY_SOURCE_INCLUDING_DELETED,
|
||||
[source]
|
||||
);
|
||||
|
||||
if (existing.length > 0) {
|
||||
const row = existing[0];
|
||||
if (row.IsDeleted === 1) {
|
||||
// Restore a previously soft-deleted record and refill its FileData.
|
||||
this.client.executeUpdate(LogoQueries.RESTORE_WITH_FILE_DATA, [
|
||||
logoData,
|
||||
currentDateTime,
|
||||
row.Id
|
||||
]);
|
||||
}
|
||||
return row.Id;
|
||||
}
|
||||
|
||||
// Create new logo entry
|
||||
|
||||
@@ -549,6 +549,7 @@ class PasskeyRepository(database: VaultDatabase) : BaseRepository(database) {
|
||||
// Sanity check: restore if soft-deleted
|
||||
if (isDeleted) {
|
||||
executeUpdate(LogoQueries.RESTORE, arrayOf(timestamp, existingLogoId))
|
||||
executeUpdate(LogoQueries.UPDATE_FILE_DATA, arrayOf(logoData, timestamp, existingLogoId))
|
||||
}
|
||||
return existingLogoId
|
||||
}
|
||||
|
||||
@@ -486,7 +486,9 @@ public class PasskeyRepository: BaseRepository {
|
||||
|
||||
// Sanity check: restore if soft-deleted
|
||||
if isDeleted {
|
||||
let logoDataParam = "av-base64-to-blob:\(logoData.base64EncodedString())"
|
||||
try client.executeUpdate(LogoQueries.restore, params: [now, existingLogoId])
|
||||
try client.executeUpdate(LogoQueries.updateFileData, params: [logoDataParam, now, existingLogoId])
|
||||
}
|
||||
return existingLogoId
|
||||
}
|
||||
|
||||
@@ -14,6 +14,14 @@ const LogoQueries = {
|
||||
WHERE Source = ? AND IsDeleted = 0
|
||||
LIMIT 1`,
|
||||
|
||||
/**
|
||||
* Look up a logo by source regardless of IsDeleted state.
|
||||
*/
|
||||
GET_BY_SOURCE_INCLUDING_DELETED: `
|
||||
SELECT Id, IsDeleted FROM Logos
|
||||
WHERE Source = ?
|
||||
LIMIT 1`,
|
||||
|
||||
/**
|
||||
* Insert new logo.
|
||||
*/
|
||||
@@ -21,6 +29,16 @@ const LogoQueries = {
|
||||
INSERT INTO Logos (Id, Source, FileData, CreatedAt, UpdatedAt, IsDeleted)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
|
||||
/**
|
||||
* Restore a soft-deleted logo and refill its FileData in one statement.
|
||||
*/
|
||||
RESTORE_WITH_FILE_DATA: `
|
||||
UPDATE Logos
|
||||
SET IsDeleted = 0,
|
||||
FileData = ?,
|
||||
UpdatedAt = ?
|
||||
WHERE Id = ?`,
|
||||
|
||||
/**
|
||||
* Count items using a logo.
|
||||
*/
|
||||
@@ -75,10 +93,22 @@ export class LogoRepository extends BaseRepository {
|
||||
* @returns The logo ID (existing or newly created)
|
||||
*/
|
||||
public async getOrCreate(source: string, logoData: Uint8Array, currentDateTime: string): Promise<string> {
|
||||
// Check if a logo for this source already exists
|
||||
const existingId = await this.getIdForSource(source);
|
||||
if (existingId) {
|
||||
return existingId;
|
||||
const existing = await this.client.executeQuery<{ Id: string; IsDeleted: number }>(
|
||||
LogoQueries.GET_BY_SOURCE_INCLUDING_DELETED,
|
||||
[source]
|
||||
);
|
||||
|
||||
if (existing.length > 0) {
|
||||
const row = existing[0];
|
||||
if (row.IsDeleted === 1) {
|
||||
// Restore a previously soft-deleted record and refill its FileData.
|
||||
await this.client.executeUpdate(LogoQueries.RESTORE_WITH_FILE_DATA, [
|
||||
logoData,
|
||||
currentDateTime,
|
||||
row.Id
|
||||
]);
|
||||
}
|
||||
return row.Id;
|
||||
}
|
||||
|
||||
// Create new logo entry
|
||||
|
||||
@@ -9,6 +9,7 @@ namespace AliasVault.Api.Controllers;
|
||||
|
||||
using AliasServerDb;
|
||||
using AliasVault.Api.Controllers.Abstracts;
|
||||
using AliasVault.Api.Services;
|
||||
using AliasVault.Shared.Models.WebApi.Favicon;
|
||||
using Asp.Versioning;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
@@ -18,15 +19,24 @@ using Microsoft.AspNetCore.Mvc;
|
||||
/// Controller for retrieving favicons from external websites.
|
||||
/// </summary>
|
||||
/// <param name="userManager">UserManager instance.</param>
|
||||
/// <param name="rateLimitService">In-memory per-user favicon extraction rate limiter.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
[ApiVersion("1")]
|
||||
public class FaviconController(UserManager<AliasVaultUser> userManager, ILogger<FaviconController> logger) : AuthenticatedRequestController(userManager)
|
||||
public class FaviconController(
|
||||
UserManager<AliasVaultUser> userManager,
|
||||
FaviconRateLimitService rateLimitService,
|
||||
ILogger<FaviconController> logger) : AuthenticatedRequestController(userManager)
|
||||
{
|
||||
/// <summary>
|
||||
/// Proxies the request to the identity generator to generate a random identity.
|
||||
/// Maximum number of URLs accepted in a single batch request.
|
||||
/// </summary>
|
||||
public const int MaxBatchSize = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the favicon from a single URL.
|
||||
/// </summary>
|
||||
/// <param name="url">URL to extract the favicon from.</param>
|
||||
/// <returns>Identity model.</returns>
|
||||
/// <returns>Favicon image bytes, or null if extraction failed.</returns>
|
||||
[HttpGet("Extract")]
|
||||
public async Task<IActionResult> Extract(string url)
|
||||
{
|
||||
@@ -36,23 +46,84 @@ public class FaviconController(UserManager<AliasVaultUser> userManager, ILogger<
|
||||
return Unauthorized();
|
||||
}
|
||||
|
||||
// Get the favicon from the URL.
|
||||
if (!rateLimitService.TryConsume(user.Id, 1))
|
||||
{
|
||||
return StatusCode(StatusCodes.Status429TooManyRequests);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var image = await FaviconExtractor.FaviconExtractor.GetFaviconAsync(url);
|
||||
|
||||
// Return the favicon as base64 string of image representation.
|
||||
return Ok(new FaviconExtractModel { Image = image });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Anonymize the URL by replacing all a-Z characters with 'x' before logging.
|
||||
// This will still allow to see the host structure but not the actual domain.
|
||||
var anonymizedUrl = new string(url.Select(c => char.IsLetter(c) ? 'x' : c).ToArray());
|
||||
logger.LogInformation(ex, "Failed to extract favicon from {Url}", anonymizedUrl);
|
||||
logger.LogInformation(ex, "Failed to extract favicon from {Url}", AnonymizeUrl(url));
|
||||
}
|
||||
|
||||
// Return null if favicon extraction failed.
|
||||
return Ok(new FaviconExtractModel { Image = null });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts favicons for multiple URLs in parallel server-side. Cuts down on round trips when
|
||||
/// the client needs to fetch many favicons (initial vault import, bulk re-download from the
|
||||
/// storage insights page).
|
||||
/// </summary>
|
||||
/// <param name="request">The batch request payload.</param>
|
||||
/// <returns>A list of favicon results, one per requested URL, in the same order.</returns>
|
||||
[HttpPost("ExtractBatch")]
|
||||
public async Task<IActionResult> ExtractBatch([FromBody] FaviconExtractBatchRequest request)
|
||||
{
|
||||
var user = await GetCurrentUserAsync();
|
||||
if (user == null)
|
||||
{
|
||||
return Unauthorized();
|
||||
}
|
||||
|
||||
if (request.Urls.Count == 0)
|
||||
{
|
||||
return Ok(new FaviconExtractBatchResponse());
|
||||
}
|
||||
|
||||
if (request.Urls.Count > MaxBatchSize)
|
||||
{
|
||||
return BadRequest(new ProblemDetails
|
||||
{
|
||||
Title = "Too many URLs",
|
||||
Detail = $"Batch size is capped at {MaxBatchSize} URLs per request.",
|
||||
});
|
||||
}
|
||||
|
||||
if (!rateLimitService.TryConsume(user.Id, request.Urls.Count))
|
||||
{
|
||||
return StatusCode(StatusCodes.Status429TooManyRequests);
|
||||
}
|
||||
|
||||
var images = await FaviconExtractor.FaviconExtractor.GetFaviconsAsync(request.Urls);
|
||||
|
||||
var response = new FaviconExtractBatchResponse
|
||||
{
|
||||
Results = new List<FaviconExtractBatchResult>(request.Urls.Count),
|
||||
};
|
||||
|
||||
for (int i = 0; i < request.Urls.Count; i++)
|
||||
{
|
||||
response.Results.Add(new FaviconExtractBatchResult
|
||||
{
|
||||
Url = request.Urls[i],
|
||||
Image = images[i],
|
||||
});
|
||||
}
|
||||
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Anonymizes a URL by replacing letters with 'x'. Lets us log host structure without
|
||||
/// recording the actual domain a user was browsing.
|
||||
/// </summary>
|
||||
private static string AnonymizeUrl(string url)
|
||||
{
|
||||
return new string(url.Select(c => char.IsLetter(c) ? 'x' : c).ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ using AliasServerDb;
|
||||
using AliasServerDb.Configuration;
|
||||
using AliasVault.Api;
|
||||
using AliasVault.Api.Jwt;
|
||||
using AliasVault.Api.Services;
|
||||
using AliasVault.Auth;
|
||||
using AliasVault.Cryptography.Server;
|
||||
using AliasVault.Logging;
|
||||
@@ -84,6 +85,7 @@ builder.Services.AddScoped<TimeValidationJwtBearerEvents>();
|
||||
builder.Services.AddScoped<AuthLoggingService>();
|
||||
builder.Services.AddScoped<ServerSettingsService>();
|
||||
builder.Services.AddScoped<RegistrationRateLimitService>();
|
||||
builder.Services.AddSingleton<FaviconRateLimitService>();
|
||||
builder.Services.AddHttpContextAccessor();
|
||||
|
||||
builder.Services.AddLogging(logging =>
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// <copyright file="FaviconRateLimitService.cs" company="aliasvault">
|
||||
// Copyright (c) aliasvault. All rights reserved.
|
||||
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
|
||||
// </copyright>
|
||||
//-----------------------------------------------------------------------
|
||||
|
||||
namespace AliasVault.Api.Services;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory per-user rate limiter for the favicon extraction API. Each successful extraction
|
||||
/// (single or one URL inside a batch) consumes one unit; the limit is enforced over a rolling
|
||||
/// 24-hour window. State is held in process so it is not shared across instances — a deliberate
|
||||
/// trade-off, since the limit's purpose is to bound the SSRF/DDoS amplification an individual
|
||||
/// user could cause from a single API instance, not to provide a hard global ceiling.
|
||||
/// </summary>
|
||||
public sealed class FaviconRateLimitService
|
||||
{
|
||||
/// <summary>
|
||||
/// Default maximum favicon extractions per user per 24 hours to prevent flood.
|
||||
/// </summary>
|
||||
public const int DefaultMaxPer24Hours = 5000;
|
||||
|
||||
private static readonly TimeSpan WindowDuration = TimeSpan.FromHours(24);
|
||||
|
||||
private readonly ConcurrentDictionary<string, FaviconUserUsage> _usage = new();
|
||||
private readonly int _maxPer24Hours;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FaviconRateLimitService"/> class.
|
||||
/// </summary>
|
||||
public FaviconRateLimitService()
|
||||
: this(DefaultMaxPer24Hours)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="FaviconRateLimitService"/> class
|
||||
/// with a custom limit (for tests or future configuration).
|
||||
/// </summary>
|
||||
/// <param name="maxPer24Hours">Maximum extractions allowed per user per 24 hours. Set to 0 or less to disable.</param>
|
||||
public FaviconRateLimitService(int maxPer24Hours)
|
||||
{
|
||||
_maxPer24Hours = maxPer24Hours;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to consume <paramref name="count"/> units from the user's allowance. Atomic per user:
|
||||
/// either the full count is consumed and true is returned, or nothing is consumed and false is returned.
|
||||
/// </summary>
|
||||
/// <param name="userId">The authenticated user id.</param>
|
||||
/// <param name="count">Number of units to consume (one per favicon URL).</param>
|
||||
/// <returns>True if the request fits in the remaining allowance; false if the limit would be exceeded.</returns>
|
||||
public bool TryConsume(string userId, int count)
|
||||
{
|
||||
if (_maxPer24Hours <= 0 || count <= 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var usage = _usage.GetOrAdd(userId, _ => new FaviconUserUsage());
|
||||
|
||||
lock (usage.SyncRoot)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
usage.Prune(now - WindowDuration);
|
||||
|
||||
if (usage.Total + count > _maxPer24Hours)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
usage.Add(now, count);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
71
apps/server/AliasVault.Api/Services/FaviconUserUsage.cs
Normal file
71
apps/server/AliasVault.Api/Services/FaviconUserUsage.cs
Normal file
@@ -0,0 +1,71 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// <copyright file="FaviconUserUsage.cs" company="aliasvault">
|
||||
// Copyright (c) aliasvault. All rights reserved.
|
||||
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
|
||||
// </copyright>
|
||||
//-----------------------------------------------------------------------
|
||||
|
||||
namespace AliasVault.Api.Services;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
/// <summary>
|
||||
/// Per-user usage record for <see cref="FaviconRateLimitService"/>. Tracks recent extractions
|
||||
/// in per-minute buckets so a power-user re-downloading thousands of favicons in quick succession
|
||||
/// still produces a bounded number of entries.
|
||||
/// </summary>
|
||||
internal sealed class FaviconUserUsage
|
||||
{
|
||||
// Bucket entries by the minute they happened in. A power-user re-downloading a 5k-item
|
||||
// vault produces at most ~minutes-of-batches entries this way, even though the underlying
|
||||
// unit count can be in the thousands — keeps memory bounded without losing accuracy.
|
||||
private readonly LinkedList<(DateTime BucketStart, int Count)> _entries = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the lock guarding mutations to this usage record. Callers must hold this lock
|
||||
/// across read-modify-write sequences (Prune + Add) to keep TryConsume atomic.
|
||||
/// </summary>
|
||||
public object SyncRoot { get; } = new object();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the total number of units consumed inside the currently retained window.
|
||||
/// </summary>
|
||||
public int Total { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Records <paramref name="count"/> consumed units at <paramref name="now"/>, merging into
|
||||
/// the last bucket when it falls in the same minute.
|
||||
/// </summary>
|
||||
/// <param name="now">The timestamp to record under (typically <see cref="DateTime.UtcNow"/>).</param>
|
||||
/// <param name="count">Number of units consumed.</param>
|
||||
public void Add(DateTime now, int count)
|
||||
{
|
||||
var bucket = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0, DateTimeKind.Utc);
|
||||
|
||||
if (_entries.Last is { Value.BucketStart: var lastBucket } lastNode && lastBucket == bucket)
|
||||
{
|
||||
_entries.Last.Value = (lastBucket, lastNode.Value.Count + count);
|
||||
}
|
||||
else
|
||||
{
|
||||
_entries.AddLast((bucket, count));
|
||||
}
|
||||
|
||||
Total += count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drops buckets older than <paramref name="cutoff"/> from the front of the list, which is
|
||||
/// where the oldest entries live since Add only appends.
|
||||
/// </summary>
|
||||
/// <param name="cutoff">Buckets strictly older than this are removed.</param>
|
||||
public void Prune(DateTime cutoff)
|
||||
{
|
||||
while (_entries.First is { Value.BucketStart: var first } node && first < cutoff)
|
||||
{
|
||||
Total -= node.Value.Count;
|
||||
_entries.RemoveFirst();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -106,6 +106,11 @@ else
|
||||
{
|
||||
return "image/png";
|
||||
}
|
||||
|
||||
if (bytes[0] == 0xFF && bytes[1] == 0xD8 && bytes[2] == 0xFF)
|
||||
{
|
||||
return "image/jpeg";
|
||||
}
|
||||
}
|
||||
|
||||
return "image/x-icon";
|
||||
|
||||
@@ -69,6 +69,11 @@
|
||||
@Localizer["SecuritySettingsNav"]
|
||||
</NavLink>
|
||||
</li>
|
||||
<li>
|
||||
<NavLink href="/settings/storage-insights" class="block py-2 px-4 text-sm hover:bg-gray-100 dark:hover:bg-gray-600 dark:text-gray-400 dark:hover:text-white" ActiveClass="text-primary-700 dark:text-primary-500" Match="NavLinkMatch.All">
|
||||
@Localizer["StorageInsightsNav"]
|
||||
</NavLink>
|
||||
</li>
|
||||
<li>
|
||||
<NavLink href="/settings/import-export" class="block py-2 px-4 text-sm hover:bg-gray-100 dark:hover:bg-gray-600 dark:text-gray-400 dark:hover:text-white" ActiveClass="text-primary-700 dark:text-primary-500" Match="NavLinkMatch.All">
|
||||
@Localizer["ImportExportNav"]
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
@inject FolderService FolderService
|
||||
@inject NavigationManager NavigationManager
|
||||
@inject GlobalNotificationService GlobalNotificationService
|
||||
@inject HttpClient HttpClient
|
||||
@inject FaviconService FaviconService
|
||||
@inject IStringLocalizerFactory LocalizerFactory
|
||||
@inject AliasVault.Client.Services.Crypto.AvexCryptoService AvexCrypto
|
||||
@using Microsoft.Extensions.Localization
|
||||
@@ -12,7 +12,6 @@
|
||||
@using AliasVault.ImportExport.Importers
|
||||
@using AliasVault.ImportExport.Models
|
||||
@using AliasVault.ImportExport.Models.Exports
|
||||
@using AliasVault.Shared.Models.WebApi.Favicon
|
||||
@using AliasClientDb
|
||||
@using AliasClientDb.Models
|
||||
@using AliasVault.Client.Auth.Components
|
||||
@@ -612,8 +611,9 @@
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts favicons for credentials. Uses domain-based deduplication to avoid
|
||||
/// fetching the same favicon multiple times for the same domain.
|
||||
/// Extracts favicons for credentials. Delegates dedup-by-domain and batched server-side
|
||||
/// fetching to <see cref="FaviconService"/>; this method only handles UI progress reporting
|
||||
/// and cancellation.
|
||||
/// </summary>
|
||||
private async Task ExtractFaviconsForCredentials()
|
||||
{
|
||||
@@ -623,59 +623,43 @@
|
||||
FaviconExtractionCancellation = new CancellationTokenSource();
|
||||
StateHasChanged();
|
||||
|
||||
// Only extract favicons for credentials that don't already have them and have URLs
|
||||
var credentialsWithUrls = ImportedCredentials
|
||||
.Where(c => c.FaviconBytes == null && c.ServiceUrls?.FirstOrDefault() != null)
|
||||
// Only extract favicons for credentials that don't already have them and have URLs.
|
||||
var urls = ImportedCredentials
|
||||
.Where(c => c.FaviconBytes == null)
|
||||
.Select(c => c.ServiceUrls?.FirstOrDefault())
|
||||
.Where(u => !string.IsNullOrEmpty(u))
|
||||
.Cast<string>()
|
||||
.ToList();
|
||||
|
||||
// Group credentials by normalized domain to avoid duplicate fetches
|
||||
var processedDomains = new HashSet<string>();
|
||||
// The dialog reports progress as "domains processed" rather than "credentials processed",
|
||||
// since the FaviconService dedupes by domain and that's the actual unit of fetch work.
|
||||
TotalFaviconsToExtract = urls
|
||||
.Select(u => FaviconService.TryNormalizeDomain(u, out var d) ? d : null)
|
||||
.Where(d => d != null)
|
||||
.Distinct()
|
||||
.Count();
|
||||
|
||||
TotalFaviconsToExtract = credentialsWithUrls.Count();
|
||||
|
||||
// Update UI every N items - Task.Delay(1) is required to let browser actually render
|
||||
// Task.Yield() doesn't work because it doesn't give the browser event loop time to paint
|
||||
const int updateEveryNItems = 10;
|
||||
|
||||
foreach (var credential in credentialsWithUrls)
|
||||
// Task.Delay(1) inside the progress callback lets Blazor actually paint between batches.
|
||||
var progress = new Progress<int>(async count =>
|
||||
{
|
||||
if (FaviconExtractionCancellation.Token.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
FaviconExtractionProgress = count;
|
||||
StateHasChanged();
|
||||
await Task.Delay(1);
|
||||
});
|
||||
|
||||
// Extract normalized domain for deduplication
|
||||
try
|
||||
{
|
||||
var domain = new Uri(credential.ServiceUrls!.First()).Host.ToLowerInvariant();
|
||||
if (domain.StartsWith("www."))
|
||||
{
|
||||
domain = domain[4..];
|
||||
}
|
||||
|
||||
// Skip if we've already fetched a favicon for this domain
|
||||
if (!processedDomains.Contains(domain))
|
||||
{
|
||||
processedDomains.Add(domain);
|
||||
await ExtractFaviconForCredential(credential);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Invalid URL
|
||||
}
|
||||
|
||||
FaviconExtractionProgress++;
|
||||
|
||||
// Update UI periodically
|
||||
if (FaviconExtractionProgress % updateEveryNItems == 0)
|
||||
{
|
||||
StateHasChanged();
|
||||
await Task.Delay(1);
|
||||
}
|
||||
var result = await FaviconService.ExtractBulkAsync(urls, progress, FaviconExtractionCancellation.Token);
|
||||
foreach (var kvp in result.Favicons)
|
||||
{
|
||||
ExtractedFavicons[kvp.Key] = kvp.Value;
|
||||
}
|
||||
|
||||
// Final update to show 100% completion before transitioning to next phase
|
||||
if (result.Status == FaviconService.BulkExtractStatus.RateLimited)
|
||||
{
|
||||
// Rate limit error occurred. Silently skip the favicons that couldn't be extracted.
|
||||
}
|
||||
|
||||
// Final update to show 100% completion before transitioning to next phase.
|
||||
FaviconExtractionProgress = TotalFaviconsToExtract;
|
||||
StateHasChanged();
|
||||
await Task.Delay(50);
|
||||
|
||||
@@ -683,35 +667,6 @@
|
||||
StateHasChanged();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts a favicon for a credential. Stores by normalized domain for deduplication.
|
||||
/// </summary>
|
||||
/// <param name="credential">The credential to extract the favicon for.</param>
|
||||
private async Task ExtractFaviconForCredential(ImportedCredential credential)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Extract normalized domain for storage key
|
||||
var url = credential.ServiceUrls!.First();
|
||||
var domain = new Uri(url).Host.ToLowerInvariant();
|
||||
if (domain.StartsWith("www."))
|
||||
{
|
||||
domain = domain[4..];
|
||||
}
|
||||
|
||||
var apiReturn = await HttpClient.GetFromJsonAsync<FaviconExtractModel>($"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}");
|
||||
if (apiReturn?.Image is not null)
|
||||
{
|
||||
// Store by normalized domain for deduplication
|
||||
ExtractedFavicons[domain] = apiReturn.Image;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore favicon extraction errors
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports the items to the database.
|
||||
/// </summary>
|
||||
@@ -952,22 +907,34 @@
|
||||
{
|
||||
var context = await DbService.GetDbContextAsync();
|
||||
|
||||
// Check if logo already exists by source
|
||||
// Lookup ignores IsDeleted because Logos.Source is UNIQUE — a soft-deleted row
|
||||
// with the same domain still occupies the slot and would block a new insert.
|
||||
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
|
||||
if (existingLogo != null)
|
||||
if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 })
|
||||
{
|
||||
// Healthy existing logo — reuse without overwriting.
|
||||
item.LogoId = existingLogo.Id;
|
||||
ImportSessionLogoIds[domain] = existingLogo.Id;
|
||||
}
|
||||
else if (existingLogo != null)
|
||||
{
|
||||
// Refill a soft-deleted or pruner-emptied row with the imported bytes so the
|
||||
// new item gets a usable logo, instead of inheriting an empty blob.
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
existingLogo.IsDeleted = false;
|
||||
existingLogo.FileData = importedCredential.FaviconBytes;
|
||||
existingLogo.FetchedAt = nowUtc;
|
||||
existingLogo.UpdatedAt = nowUtc;
|
||||
item.LogoId = existingLogo.Id;
|
||||
ImportSessionLogoIds[domain] = existingLogo.Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Create new logo from embedded bytes
|
||||
var logo = new AliasClientDb.Logo
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = domain,
|
||||
FileData = importedCredential.FaviconBytes,
|
||||
MimeType = "image/png",
|
||||
FetchedAt = DateTime.UtcNow,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
UpdatedAt = DateTime.UtcNow,
|
||||
@@ -1003,30 +970,38 @@
|
||||
var context = await DbService.GetDbContextAsync();
|
||||
|
||||
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
|
||||
if (existingLogo != null)
|
||||
if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 })
|
||||
{
|
||||
// Reuse existing logo
|
||||
item.LogoId = existingLogo.Id;
|
||||
ImportSessionLogoIds[domain] = existingLogo.Id;
|
||||
}
|
||||
else if (ExtractedFavicons.TryGetValue(domain, out var favicon))
|
||||
{
|
||||
// Create new logo from extracted favicon
|
||||
var logo = new AliasClientDb.Logo
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
if (existingLogo != null)
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = domain,
|
||||
FileData = favicon,
|
||||
MimeType = "image/png",
|
||||
FetchedAt = DateTime.UtcNow,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
UpdatedAt = DateTime.UtcNow,
|
||||
};
|
||||
context.Logos.Add(logo);
|
||||
item.LogoId = logo.Id;
|
||||
|
||||
// Track this logo for subsequent items with same domain
|
||||
ImportSessionLogoIds[domain] = logo.Id;
|
||||
existingLogo.IsDeleted = false;
|
||||
existingLogo.FileData = favicon;
|
||||
existingLogo.FetchedAt = nowUtc;
|
||||
existingLogo.UpdatedAt = nowUtc;
|
||||
item.LogoId = existingLogo.Id;
|
||||
ImportSessionLogoIds[domain] = existingLogo.Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
var logo = new AliasClientDb.Logo
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = domain,
|
||||
FileData = favicon,
|
||||
FetchedAt = nowUtc,
|
||||
CreatedAt = nowUtc,
|
||||
UpdatedAt = nowUtc,
|
||||
};
|
||||
context.Logos.Add(logo);
|
||||
item.LogoId = logo.Id;
|
||||
ImportSessionLogoIds[domain] = logo.Id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,23 @@
|
||||
</div>
|
||||
<Button OnClick="@(() => ShowExportConfirmation(ExportType.Csv))">@Localizer["ExportCsvButton"]</Button>
|
||||
</div>
|
||||
|
||||
@* DEBUG-only raw SQLite export. *@
|
||||
@if (IsDebugBuild)
|
||||
{
|
||||
<div class="flex items-center justify-between p-3 border-2 border-dashed border-yellow-400 dark:border-yellow-600 rounded-lg bg-yellow-50 dark:bg-yellow-900/20">
|
||||
<div class="flex-1 min-w-0 mr-4">
|
||||
<div class="flex items-center gap-2 mb-1">
|
||||
<h4 class="text-sm font-semibold text-gray-900 dark:text-white">Export raw SQLite</h4>
|
||||
<span class="px-1.5 py-0.5 text-xs font-medium text-yellow-800 dark:text-yellow-200 bg-yellow-200 dark:bg-yellow-800 rounded">DEBUG</span>
|
||||
</div>
|
||||
<p class="text-xs text-gray-600 dark:text-gray-300">
|
||||
Downloads the unencrypted SQLite vault file as-is. Anyone with the file can read everything.
|
||||
</p>
|
||||
</div>
|
||||
<Button OnClick="@ExportVaultSqlite">Download .sqlite</Button>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -112,6 +129,13 @@
|
||||
OnClose="@CloseExportPasswordModal" />
|
||||
|
||||
@code {
|
||||
// Assign debug static flag for Razor markup check.
|
||||
#if DEBUG
|
||||
private static readonly bool IsDebugBuild = true;
|
||||
#else
|
||||
private static readonly bool IsDebugBuild = false;
|
||||
#endif
|
||||
|
||||
private string _username = string.Empty;
|
||||
private bool _showPasswordConfirmation;
|
||||
private string _passwordError = string.Empty;
|
||||
@@ -295,6 +319,27 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Always defined so the markup binding compiles, only invoked when IsDebugBuild is true.
|
||||
private async Task ExportVaultSqlite()
|
||||
{
|
||||
GlobalLoadingSpinner.Show("Exporting SQLite vault...");
|
||||
try
|
||||
{
|
||||
var base64 = await DbService.ExportSqliteToBase64Async();
|
||||
var bytes = Convert.FromBase64String(base64);
|
||||
await JsInteropService.DownloadFileFromStream(await GetExportFileName("sqlite"), bytes);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "Error exporting raw SQLite vault");
|
||||
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
GlobalLoadingSpinner.Hide();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ExportVaultAvux()
|
||||
{
|
||||
GlobalLoadingSpinner.Show(Localizer["ExportingVaultMessage"]);
|
||||
|
||||
@@ -0,0 +1,601 @@
|
||||
@page "/settings/storage-insights"
|
||||
@inherits MainBase
|
||||
@inject AliasVault.Client.Services.FaviconService FaviconService
|
||||
@inject AliasVault.Client.Services.ItemService ItemService
|
||||
@inject AliasVault.RazorComponents.Services.ConfirmModalService ConfirmModalService
|
||||
@using AliasClientDb.Models
|
||||
@using AliasVault.RazorComponents.Tables
|
||||
@using Microsoft.EntityFrameworkCore
|
||||
@using Microsoft.Extensions.Localization
|
||||
|
||||
<LayoutPageTitle>@Localizer["PageTitle"]</LayoutPageTitle>
|
||||
|
||||
<PageHeader
|
||||
BreadcrumbItems="@BreadcrumbItems"
|
||||
Title="@Localizer["PageTitle"]"
|
||||
Description="@Localizer["PageDescription"]">
|
||||
</PageHeader>
|
||||
|
||||
@if (IsLoading)
|
||||
{
|
||||
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
|
||||
<p class="text-gray-600 dark:text-gray-400">@SharedLocalizer["Loading"]</p>
|
||||
</div>
|
||||
}
|
||||
else
|
||||
{
|
||||
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
|
||||
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["EstimatedTotalTitle"]</h3>
|
||||
<p class="text-4xl font-semibold text-gray-900 dark:text-white">@FormatSize(_estimatedTotalBytes)</p>
|
||||
<p class="mt-2 text-sm text-gray-500 dark:text-gray-400">@Localizer["EstimatedTotalDescription"]</p>
|
||||
</div>
|
||||
|
||||
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
|
||||
<h3 class="mb-4 text-lg font-medium text-gray-900 dark:text-white">@Localizer["CountsTitle"]</h3>
|
||||
<div class="grid gap-4 sm:grid-cols-3">
|
||||
<div class="p-4 border border-gray-200 rounded-lg dark:border-gray-700">
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">@Localizer["ItemCountLabel"]</p>
|
||||
<p class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white">@_totalItems</p>
|
||||
</div>
|
||||
<div class="p-4 border border-gray-200 rounded-lg dark:border-gray-700">
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">@Localizer["ItemsWithAttachmentsLabel"]</p>
|
||||
<p class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white">@_itemsWithAttachments</p>
|
||||
</div>
|
||||
<div class="p-4 border border-gray-200 rounded-lg dark:border-gray-700">
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">@Localizer["ItemsWithLogosLabel"]</p>
|
||||
<p class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white">@_itemsWithLogos</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
|
||||
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["BreakdownTitle"]</h3>
|
||||
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["BreakdownDescription"]</p>
|
||||
|
||||
@if (_estimatedTotalBytes == 0)
|
||||
{
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">-</p>
|
||||
}
|
||||
else
|
||||
{
|
||||
<div class="flex w-full h-4 overflow-hidden rounded-full bg-gray-200 dark:bg-gray-700">
|
||||
<div class="bg-gray-400 dark:bg-gray-500" style="width: @(Percent(_baseOverheadBytes, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownBaseOverheadLabel"]"></div>
|
||||
<div class="bg-blue-500" style="width: @(Percent(_credentialBytes, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownCredentialsLabel"]"></div>
|
||||
<div class="bg-amber-500" style="width: @(Percent(_attachmentBytesWithOverhead, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownAttachmentsLabel"]"></div>
|
||||
<div class="bg-emerald-500" style="width: @(Percent(_logoBytesWithOverhead, _estimatedTotalBytes).ToString("F2", System.Globalization.CultureInfo.InvariantCulture))%;" title="@Localizer["BreakdownLogosLabel"]"></div>
|
||||
</div>
|
||||
<div class="mt-4 grid gap-2 sm:grid-cols-2 lg:grid-cols-4 text-sm">
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="inline-block w-3 h-3 rounded-sm bg-gray-400 dark:bg-gray-500"></span>
|
||||
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownBaseOverheadLabel"]:</span>
|
||||
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_baseOverheadBytes) (@Percent(_baseOverheadBytes, _estimatedTotalBytes).ToString("F1")%)</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="inline-block w-3 h-3 rounded-sm bg-blue-500"></span>
|
||||
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownCredentialsLabel"]:</span>
|
||||
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_credentialBytes) (@Percent(_credentialBytes, _estimatedTotalBytes).ToString("F1")%)</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="inline-block w-3 h-3 rounded-sm bg-amber-500"></span>
|
||||
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownAttachmentsLabel"]:</span>
|
||||
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_attachmentBytesWithOverhead) (@Percent(_attachmentBytesWithOverhead, _estimatedTotalBytes).ToString("F1")%)</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="inline-block w-3 h-3 rounded-sm bg-emerald-500"></span>
|
||||
<span class="text-gray-700 dark:text-gray-300">@Localizer["BreakdownLogosLabel"]:</span>
|
||||
<span class="font-medium text-gray-900 dark:text-white">@FormatSize(_logoBytesWithOverhead) (@Percent(_logoBytesWithOverhead, _estimatedTotalBytes).ToString("F1")%)</span>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
|
||||
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
|
||||
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["TopAttachmentsTitle"]</h3>
|
||||
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["TopAttachmentsDescription"]</p>
|
||||
|
||||
@if (_topAttachments.Count == 0)
|
||||
{
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">-</p>
|
||||
}
|
||||
else
|
||||
{
|
||||
<SortableTable Columns="@_attachmentColumns" SortColumn="@string.Empty" SortDirection="SortDirection.Descending" OnSortChanged="(_ => Task.CompletedTask)">
|
||||
@foreach (var attachment in _topAttachments)
|
||||
{
|
||||
<SortableTableRow Class="cursor-pointer" OnClick="@(() => NavigateToItem(attachment.ItemId))">
|
||||
<SortableTableColumn IsPrimary="true">@attachment.Filename</SortableTableColumn>
|
||||
<SortableTableColumn>@FormatSize(attachment.SizeBytes)</SortableTableColumn>
|
||||
<SortableTableColumn>@(string.IsNullOrWhiteSpace(attachment.ItemName) ? "-" : attachment.ItemName)</SortableTableColumn>
|
||||
<SortableTableColumn>@attachment.CreatedAt.ToLocalTime().ToString("yyyy-MM-dd")</SortableTableColumn>
|
||||
</SortableTableRow>
|
||||
}
|
||||
</SortableTable>
|
||||
}
|
||||
</div>
|
||||
|
||||
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
|
||||
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["TopLogosTitle"]</h3>
|
||||
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["TopLogosDescription"]</p>
|
||||
|
||||
@if (_topLogos.Count == 0)
|
||||
{
|
||||
<p class="text-sm text-gray-500 dark:text-gray-400">-</p>
|
||||
}
|
||||
else
|
||||
{
|
||||
<SortableTable Columns="@_logoColumns" SortColumn="@string.Empty" SortDirection="SortDirection.Descending" OnSortChanged="(_ => Task.CompletedTask)">
|
||||
@foreach (var logo in _topLogos)
|
||||
{
|
||||
<SortableTableRow Class="@(logo.FirstItemId.HasValue ? "cursor-pointer" : string.Empty)" OnClick="@(logo.FirstItemId.HasValue ? EventCallback.Factory.Create(this, () => NavigateToItem(logo.FirstItemId!.Value)) : default)">
|
||||
<SortableTableColumn IsPrimary="true">@logo.Source</SortableTableColumn>
|
||||
<SortableTableColumn>@FormatSize(logo.SizeBytes)</SortableTableColumn>
|
||||
<SortableTableColumn>@logo.ItemCount</SortableTableColumn>
|
||||
</SortableTableRow>
|
||||
}
|
||||
</SortableTable>
|
||||
}
|
||||
</div>
|
||||
|
||||
<div class="p-4 mb-4 mx-4 bg-white border border-gray-200 rounded-lg shadow-sm dark:border-gray-700 sm:p-6 dark:bg-gray-800">
|
||||
<h3 class="mb-2 text-lg font-medium text-gray-900 dark:text-white">@Localizer["LogoManagementTitle"]</h3>
|
||||
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">@Localizer["LogoManagementDescription"]</p>
|
||||
|
||||
@if (_isRedownloadingLogos)
|
||||
{
|
||||
<div class="mb-4">
|
||||
<div class="flex justify-between text-sm text-gray-700 dark:text-gray-300 mb-1">
|
||||
<span>@SharedLocalizer["Loading"]</span>
|
||||
<span>@_redownloadProgress / @_redownloadTotal</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 rounded-full h-2.5 dark:bg-gray-700">
|
||||
<div class="bg-primary-600 h-2.5 rounded-full transition-all duration-300" style="width: @(RedownloadPercent().ToString("F1", System.Globalization.CultureInfo.InvariantCulture))%"></div>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
|
||||
<div class="flex flex-col sm:flex-row gap-3">
|
||||
<Button Color="default" OnClick="RedownloadAllLogos" IsDisabled="@(_isRedownloadingLogos || _isDeletingLogos)">@Localizer["RedownloadAllLogosButton"]</Button>
|
||||
<Button Color="danger" OnClick="DeleteAllLogos" IsDisabled="@(_isRedownloadingLogos || _isDeletingLogos)">@Localizer["DeleteAllLogosButton"]</Button>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
|
||||
@code {
|
||||
// Approximate per-row SQLite overhead in bytes. SQLite uses 4 KiB pages by default;
|
||||
// small rows pack many per page, while large blobs spill into overflow pages. These
|
||||
// constants cover row headers and the indexed columns AliasClientDbContext defines.
|
||||
// They are intentionally rough — attachments and logos dominate real vault size.
|
||||
private const int PerItemOverheadBytes = 200;
|
||||
private const int PerFieldValueOverheadBytes = 80;
|
||||
private const int PerAttachmentOverheadBytes = 120;
|
||||
private const int PerLogoOverheadBytes = 150;
|
||||
private const int PerTotpOverheadBytes = 100;
|
||||
private const int PerPasskeyOverheadBytes = 200;
|
||||
|
||||
// Empty AliasVault SQLite file is ~200 KB on disk: file header, page allocation
|
||||
// for each table/index, sqlite_master schema rows, and __EFMigrationsHistory.
|
||||
// Constant rather than measured because we don't have access to the file size
|
||||
// here and the value is stable across vaults of the same schema version.
|
||||
private const int BaseSqliteOverheadBytes = 200 * 1024;
|
||||
|
||||
private readonly List<TableColumn> _attachmentColumns =
|
||||
[
|
||||
new TableColumn { Title = "Filename", Sortable = false },
|
||||
new TableColumn { Title = "Size", Sortable = false },
|
||||
new TableColumn { Title = "Item", Sortable = false },
|
||||
new TableColumn { Title = "Created", Sortable = false },
|
||||
];
|
||||
|
||||
private readonly List<TableColumn> _logoColumns =
|
||||
[
|
||||
new TableColumn { Title = "Source", Sortable = false },
|
||||
new TableColumn { Title = "Size", Sortable = false },
|
||||
new TableColumn { Title = "Used by", Sortable = false },
|
||||
];
|
||||
|
||||
private bool IsLoading { get; set; } = true;
|
||||
|
||||
private int _totalItems;
|
||||
private int _itemsWithAttachments;
|
||||
private int _itemsWithLogos;
|
||||
private long _credentialBytes;
|
||||
private long _attachmentBytesWithOverhead;
|
||||
private long _logoBytesWithOverhead;
|
||||
private long _baseOverheadBytes;
|
||||
private long _estimatedTotalBytes;
|
||||
private List<AttachmentRow> _topAttachments = [];
|
||||
private List<LogoRow> _topLogos = [];
|
||||
|
||||
private bool _isDeletingLogos;
|
||||
private bool _isRedownloadingLogos;
|
||||
private int _redownloadProgress;
|
||||
private int _redownloadTotal;
|
||||
|
||||
private IStringLocalizer Localizer => LocalizerFactory.Create("Pages.Main.Settings.StorageInsights", "AliasVault.Client");
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await base.OnInitializedAsync();
|
||||
BreadcrumbItems.Add(new BreadcrumbItem { DisplayName = Localizer["BreadcrumbTitle"] });
|
||||
|
||||
// Localize the table column headers (created at field init time before localizer was available).
|
||||
_attachmentColumns[0].Title = Localizer["ColumnFilename"];
|
||||
_attachmentColumns[1].Title = Localizer["ColumnSize"];
|
||||
_attachmentColumns[2].Title = Localizer["ColumnItem"];
|
||||
_attachmentColumns[3].Title = Localizer["ColumnCreated"];
|
||||
_logoColumns[0].Title = Localizer["ColumnWebsiteURL"];
|
||||
_logoColumns[1].Title = Localizer["ColumnSize"];
|
||||
_logoColumns[2].Title = Localizer["ColumnItemCount"];
|
||||
|
||||
await LoadStatisticsAsync();
|
||||
}
|
||||
|
||||
private static string FormatSize(long bytes)
|
||||
{
|
||||
const double Kib = 1024d;
|
||||
const double Mib = Kib * 1024d;
|
||||
|
||||
if (bytes < Kib)
|
||||
{
|
||||
return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{bytes} B");
|
||||
}
|
||||
|
||||
if (bytes < Mib)
|
||||
{
|
||||
return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{(bytes / Kib):F1} KB");
|
||||
}
|
||||
|
||||
return string.Create(System.Globalization.CultureInfo.CurrentCulture, $"{(bytes / Mib):F1} MB");
|
||||
}
|
||||
|
||||
private static double Percent(long part, long total)
|
||||
{
|
||||
return total == 0 ? 0d : Math.Round((double)part / total * 100d, 2);
|
||||
}
|
||||
|
||||
private async Task LoadStatisticsAsync()
|
||||
{
|
||||
var ctx = await DbService.GetDbContextAsync();
|
||||
|
||||
_totalItems = await ctx.Items.CountAsync(i => !i.IsDeleted && i.DeletedAt == null);
|
||||
_itemsWithAttachments = await ctx.Attachments
|
||||
.Where(a => !a.IsDeleted)
|
||||
.Select(a => a.ItemId)
|
||||
.Distinct()
|
||||
.CountAsync();
|
||||
_itemsWithLogos = await ctx.Items.CountAsync(i => !i.IsDeleted && i.DeletedAt == null && i.LogoId != null);
|
||||
|
||||
var attachmentRows = await ctx.Attachments
|
||||
.Where(a => !a.IsDeleted)
|
||||
.Select(a => new AttachmentRow
|
||||
{
|
||||
Id = a.Id,
|
||||
Filename = a.Filename,
|
||||
SizeBytes = a.Blob.Length,
|
||||
ItemId = a.ItemId,
|
||||
ItemName = a.Item.Name,
|
||||
CreatedAt = a.CreatedAt,
|
||||
})
|
||||
.ToListAsync();
|
||||
|
||||
var logoProjections = await ctx.Logos
|
||||
.Where(l => !l.IsDeleted && l.FileData != null)
|
||||
.Select(l => new
|
||||
{
|
||||
l.Id,
|
||||
l.Source,
|
||||
SizeBytes = l.FileData!.Length,
|
||||
ItemIds = l.Items
|
||||
.Where(i => !i.IsDeleted && i.DeletedAt == null)
|
||||
.Select(i => i.Id)
|
||||
.ToList(),
|
||||
})
|
||||
.ToListAsync();
|
||||
|
||||
var logoRows = logoProjections
|
||||
.Select(l => new LogoRow
|
||||
{
|
||||
Id = l.Id,
|
||||
Source = l.Source,
|
||||
SizeBytes = l.SizeBytes,
|
||||
ItemCount = l.ItemIds.Count,
|
||||
FirstItemId = l.ItemIds.Count > 0 ? l.ItemIds[0] : null,
|
||||
})
|
||||
.ToList();
|
||||
|
||||
var fieldValueLengths = await ctx.FieldValues
|
||||
.Where(fv => !fv.IsDeleted)
|
||||
.Select(fv => fv.Value == null ? 0 : fv.Value.Length)
|
||||
.ToListAsync();
|
||||
var fieldValueBytes = fieldValueLengths.Sum(x => (long)x);
|
||||
var fieldValueCount = fieldValueLengths.Count;
|
||||
|
||||
var passkeySizes = await ctx.Passkeys
|
||||
.Where(p => !p.IsDeleted)
|
||||
.Select(p => new
|
||||
{
|
||||
UserHandleLen = p.UserHandle.Length,
|
||||
PrfKeyLen = p.PrfKey == null ? 0 : p.PrfKey.Length,
|
||||
AdditionalDataLen = p.AdditionalData == null ? 0 : p.AdditionalData.Length,
|
||||
PublicKeyLen = p.PublicKey.Length,
|
||||
PrivateKeyLen = p.PrivateKey.Length,
|
||||
})
|
||||
.ToListAsync();
|
||||
var passkeyBytes = passkeySizes.Sum(p => (long)(p.UserHandleLen + p.PrfKeyLen + p.AdditionalDataLen + p.PublicKeyLen + p.PrivateKeyLen));
|
||||
var passkeyCount = passkeySizes.Count;
|
||||
|
||||
var totpCount = await ctx.TotpCodes.CountAsync(t => !t.IsDeleted);
|
||||
|
||||
var attachmentBytes = attachmentRows.Sum(a => (long)a.SizeBytes);
|
||||
var logoBytes = logoRows.Sum(l => (long)l.SizeBytes);
|
||||
|
||||
_attachmentBytesWithOverhead = attachmentBytes + ((long)attachmentRows.Count * PerAttachmentOverheadBytes);
|
||||
_logoBytesWithOverhead = logoBytes + ((long)logoRows.Count * PerLogoOverheadBytes);
|
||||
|
||||
_credentialBytes = fieldValueBytes
|
||||
+ passkeyBytes
|
||||
+ ((long)_totalItems * PerItemOverheadBytes)
|
||||
+ ((long)fieldValueCount * PerFieldValueOverheadBytes)
|
||||
+ ((long)passkeyCount * PerPasskeyOverheadBytes)
|
||||
+ ((long)totpCount * PerTotpOverheadBytes);
|
||||
|
||||
_baseOverheadBytes = BaseSqliteOverheadBytes;
|
||||
|
||||
_estimatedTotalBytes = _credentialBytes
|
||||
+ _attachmentBytesWithOverhead
|
||||
+ _logoBytesWithOverhead
|
||||
+ _baseOverheadBytes;
|
||||
|
||||
_topAttachments = attachmentRows
|
||||
.OrderByDescending(a => a.SizeBytes)
|
||||
.Take(10)
|
||||
.ToList();
|
||||
|
||||
_topLogos = logoRows
|
||||
.OrderByDescending(l => l.SizeBytes)
|
||||
.Take(10)
|
||||
.ToList();
|
||||
|
||||
IsLoading = false;
|
||||
StateHasChanged();
|
||||
}
|
||||
|
||||
private void NavigateToItem(Guid itemId)
|
||||
{
|
||||
NavigationManager.NavigateTo($"/items/{itemId}");
|
||||
}
|
||||
|
||||
private double RedownloadPercent()
|
||||
{
|
||||
return _redownloadTotal == 0 ? 0d : Math.Round((double)_redownloadProgress / _redownloadTotal * 100d, 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Soft-deletes every logo in the vault and clears the LogoId on every item that referenced one.
|
||||
/// The vault sync that follows propagates these changes to other devices.
|
||||
/// </summary>
|
||||
private async Task DeleteAllLogos()
|
||||
{
|
||||
var confirmed = await ConfirmModalService.ShowConfirmation(
|
||||
Localizer["DeleteAllLogosConfirmTitle"],
|
||||
Localizer["DeleteAllLogosConfirmMessage"],
|
||||
SharedLocalizer["Confirm"],
|
||||
SharedLocalizer["Cancel"]);
|
||||
if (!confirmed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_isDeletingLogos = true;
|
||||
StateHasChanged();
|
||||
GlobalLoadingSpinner.Show(SharedLocalizer["Loading"]);
|
||||
|
||||
try
|
||||
{
|
||||
var ctx = await DbService.GetDbContextAsync();
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
// Soft-delete the row AND clear the binary blob to free up space.
|
||||
var logos = await ctx.Logos.Where(l => !l.IsDeleted).ToListAsync();
|
||||
foreach (var logo in logos)
|
||||
{
|
||||
logo.IsDeleted = true;
|
||||
logo.FileData = Array.Empty<byte>();
|
||||
logo.UpdatedAt = now;
|
||||
}
|
||||
|
||||
// Clear LogoId on items so they no longer reference soft-deleted logos.
|
||||
var itemsWithLogos = await ctx.Items.Where(i => i.LogoId != null).ToListAsync();
|
||||
foreach (var item in itemsWithLogos)
|
||||
{
|
||||
item.LogoId = null;
|
||||
item.UpdatedAt = now;
|
||||
}
|
||||
|
||||
await ctx.SaveChangesAsync();
|
||||
|
||||
// VACUUM after a bulk blob delete — without it the SQLite pages stay allocated and
|
||||
// the user wouldn't actually see the size drop they came here for.
|
||||
await DbService.VacuumDatabaseAsync();
|
||||
await DbService.SaveDatabaseAsync();
|
||||
|
||||
GlobalNotificationService.AddSuccessMessage(SharedLocalizer["Success"], true);
|
||||
await LoadStatisticsAsync();
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
GlobalLoadingSpinner.Hide();
|
||||
_isDeletingLogos = false;
|
||||
StateHasChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-downloads favicons for every item that has a login URL, overwriting any existing logo.
|
||||
/// Useful after the favicon extractor's sizing/quality settings change so the vault picks up
|
||||
/// the new output.
|
||||
/// </summary>
|
||||
private async Task RedownloadAllLogos()
|
||||
{
|
||||
var confirmed = await ConfirmModalService.ShowConfirmation(Localizer["RedownloadAllLogosConfirmTitle"], Localizer["RedownloadAllLogosConfirmMessage"], SharedLocalizer["Confirm"], SharedLocalizer["Cancel"]);
|
||||
if (!confirmed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_isRedownloadingLogos = true;
|
||||
_redownloadProgress = 0;
|
||||
_redownloadTotal = 0;
|
||||
StateHasChanged();
|
||||
|
||||
try
|
||||
{
|
||||
var items = await ItemService.LoadAllAsync();
|
||||
|
||||
// Group items by normalized domain so each domain is fetched once and assigned to all items.
|
||||
var itemsByDomain = new Dictionary<string, List<AliasClientDb.Item>>();
|
||||
var representativeUrls = new Dictionary<string, string>();
|
||||
foreach (var item in items)
|
||||
{
|
||||
var url = ItemService.GetFieldValue(item, FieldKey.LoginUrl);
|
||||
if (string.IsNullOrEmpty(url) || url == ItemService.DefaultServiceUrl)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!FaviconService.TryNormalizeDomain(url, out var domain))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!itemsByDomain.TryGetValue(domain, out var bucket))
|
||||
{
|
||||
bucket = new List<AliasClientDb.Item>();
|
||||
itemsByDomain[domain] = bucket;
|
||||
representativeUrls[domain] = url;
|
||||
}
|
||||
|
||||
bucket.Add(item);
|
||||
}
|
||||
|
||||
_redownloadTotal = itemsByDomain.Count;
|
||||
StateHasChanged();
|
||||
|
||||
if (_redownloadTotal == 0)
|
||||
{
|
||||
GlobalNotificationService.AddSuccessMessage(Localizer["RedownloadAllLogosNoItemsMessage"], true);
|
||||
return;
|
||||
}
|
||||
|
||||
var progress = new Progress<int>(count =>
|
||||
{
|
||||
_redownloadProgress = count;
|
||||
InvokeAsync(StateHasChanged);
|
||||
});
|
||||
|
||||
var result = await FaviconService.ExtractBulkAsync(representativeUrls.Values, progress);
|
||||
|
||||
var ctx = await DbService.GetDbContextAsync();
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
foreach (var (domain, bucket) in itemsByDomain)
|
||||
{
|
||||
if (!result.Favicons.TryGetValue(domain, out var image))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look up by Source (unique field).
|
||||
var existingLogo = await ctx.Logos.FirstOrDefaultAsync(l => l.Source == domain);
|
||||
Guid logoId;
|
||||
if (existingLogo != null)
|
||||
{
|
||||
existingLogo.IsDeleted = false;
|
||||
existingLogo.FileData = image;
|
||||
existingLogo.FetchedAt = now;
|
||||
existingLogo.UpdatedAt = now;
|
||||
logoId = existingLogo.Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
var newLogo = new AliasClientDb.Logo
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = domain,
|
||||
FileData = image,
|
||||
FetchedAt = now,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
ctx.Logos.Add(newLogo);
|
||||
logoId = newLogo.Id;
|
||||
}
|
||||
|
||||
foreach (var item in bucket)
|
||||
{
|
||||
if (item.LogoId != logoId)
|
||||
{
|
||||
item.LogoId = logoId;
|
||||
item.UpdatedAt = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.SaveChangesAsync();
|
||||
await DbService.SaveDatabaseAsync();
|
||||
|
||||
if (result.Status == FaviconService.BulkExtractStatus.RateLimited)
|
||||
{
|
||||
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
|
||||
}
|
||||
else
|
||||
{
|
||||
GlobalNotificationService.AddSuccessMessage(SharedLocalizer["Success"], true);
|
||||
}
|
||||
|
||||
await LoadStatisticsAsync();
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
GlobalNotificationService.AddErrorMessage(SharedLocalizer["ErrorGeneric"], true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_isRedownloadingLogos = false;
|
||||
StateHasChanged();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class AttachmentRow
|
||||
{
|
||||
public Guid Id { get; init; }
|
||||
|
||||
public string Filename { get; init; } = string.Empty;
|
||||
|
||||
public int SizeBytes { get; init; }
|
||||
|
||||
public Guid ItemId { get; init; }
|
||||
|
||||
public string? ItemName { get; init; }
|
||||
|
||||
public DateTime CreatedAt { get; init; }
|
||||
}
|
||||
|
||||
private sealed class LogoRow
|
||||
{
|
||||
public Guid Id { get; init; }
|
||||
|
||||
public string Source { get; init; } = string.Empty;
|
||||
|
||||
public int SizeBytes { get; init; }
|
||||
|
||||
public int ItemCount { get; init; }
|
||||
|
||||
public Guid? FirstItemId { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -87,6 +87,7 @@ builder.Services.AddTransient<AliasVaultApiHandlerService>();
|
||||
builder.Services.AddScoped<AuthService>();
|
||||
builder.Services.AddScoped<UserRegistrationService>();
|
||||
builder.Services.AddScoped<AuthenticationStateProvider, AuthStateProvider>();
|
||||
builder.Services.AddScoped<FaviconService>();
|
||||
builder.Services.AddScoped<ItemService>();
|
||||
builder.Services.AddScoped<FolderService>();
|
||||
builder.Services.AddScoped<DbService>();
|
||||
|
||||
@@ -39,6 +39,10 @@
|
||||
<value>Security settings</value>
|
||||
<comment>Navigation link for security settings</comment>
|
||||
</data>
|
||||
<data name="StorageInsightsNav">
|
||||
<value>Storage insights</value>
|
||||
<comment>Navigation link for vault storage insights page</comment>
|
||||
</data>
|
||||
<data name="ImportExportNav">
|
||||
<value>Import / Export</value>
|
||||
<comment>Navigation link for import/export settings</comment>
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<root>
|
||||
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
|
||||
<xsd:element name="root" msdata:IsDataSet="true">
|
||||
<xsd:complexType>
|
||||
<xsd:choice maxOccurs="unbounded">
|
||||
<xsd:element name="data">
|
||||
<xsd:complexType>
|
||||
<xsd:sequence>
|
||||
<xsd:element name="value" type="xsd:string" minOccurs="0" />
|
||||
<xsd:element name="comment" type="xsd:string" minOccurs="0" />
|
||||
</xsd:sequence>
|
||||
<xsd:attribute name="name" type="xsd:string" />
|
||||
<xsd:attribute name="type" type="xsd:string" />
|
||||
<xsd:attribute name="mimetype" type="xsd:string" />
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
</xsd:choice>
|
||||
</xsd:complexType>
|
||||
</xsd:element>
|
||||
</xsd:schema>
|
||||
|
||||
<!-- Page title and description -->
|
||||
<data name="PageTitle">
|
||||
<value>Vault storage insights</value>
|
||||
<comment>Page title for vault storage insights</comment>
|
||||
</data>
|
||||
<data name="PageDescription">
|
||||
<value>See an estimate of your vault size. For best sync performance, it's advised to keep your vault size small and delete items you no longer need.</value>
|
||||
<comment>Page description for vault storage insights</comment>
|
||||
</data>
|
||||
<data name="BreadcrumbTitle">
|
||||
<value>Storage insights</value>
|
||||
<comment>Breadcrumb title for storage insights page</comment>
|
||||
</data>
|
||||
|
||||
<!-- Estimated total -->
|
||||
<data name="EstimatedTotalTitle">
|
||||
<value>Estimated vault size</value>
|
||||
<comment>Title for the estimated total vault size card</comment>
|
||||
</data>
|
||||
<data name="EstimatedTotalDescription">
|
||||
<value>This is an approximation calculated from your local data.</value>
|
||||
<comment>Subtext explaining that the estimate is approximate</comment>
|
||||
</data>
|
||||
|
||||
<!-- Counts -->
|
||||
<data name="CountsTitle">
|
||||
<value>Overview</value>
|
||||
<comment>Title for the counts/overview section</comment>
|
||||
</data>
|
||||
<data name="ItemCountLabel">
|
||||
<value>Items</value>
|
||||
<comment>Label for total item count card</comment>
|
||||
</data>
|
||||
<data name="ItemsWithAttachmentsLabel">
|
||||
<value>Items with attachments</value>
|
||||
<comment>Label for count of items that have at least one attachment</comment>
|
||||
</data>
|
||||
<data name="ItemsWithLogosLabel">
|
||||
<value>Items with a logo</value>
|
||||
<comment>Label for count of items that reference a logo</comment>
|
||||
</data>
|
||||
|
||||
<!-- Breakdown -->
|
||||
<data name="BreakdownTitle">
|
||||
<value>Storage usage breakdown</value>
|
||||
<comment>Title for the breakdown bar section</comment>
|
||||
</data>
|
||||
<data name="BreakdownDescription">
|
||||
<value>Approximate share of vault size by category. Attachments and logos usually take up most of the space.</value>
|
||||
<comment>Description for the breakdown bar</comment>
|
||||
</data>
|
||||
<data name="BreakdownCredentialsLabel">
|
||||
<value>Credentials</value>
|
||||
<comment>Legend label for credential data slice</comment>
|
||||
</data>
|
||||
<data name="BreakdownAttachmentsLabel">
|
||||
<value>Attachments</value>
|
||||
<comment>Legend label for attachments slice</comment>
|
||||
</data>
|
||||
<data name="BreakdownLogosLabel">
|
||||
<value>Logos</value>
|
||||
<comment>Legend label for logos slice</comment>
|
||||
</data>
|
||||
<data name="BreakdownBaseOverheadLabel">
|
||||
<value>Database overhead</value>
|
||||
<comment>Legend label for the fixed SQLite base/schema overhead slice</comment>
|
||||
</data>
|
||||
|
||||
<!-- Top attachments table -->
|
||||
<data name="TopAttachmentsTitle">
|
||||
<value>Largest attachments</value>
|
||||
<comment>Title for the top attachments table</comment>
|
||||
</data>
|
||||
<data name="TopAttachmentsDescription">
|
||||
<value>Top 10 largest file attachments. Remove the attachment if you no longer need it.</value>
|
||||
<comment>Description for the top attachments table</comment>
|
||||
</data>
|
||||
|
||||
<!-- Top logos table -->
|
||||
<data name="TopLogosTitle">
|
||||
<value>Largest logos</value>
|
||||
<comment>Title for the top logos table</comment>
|
||||
</data>
|
||||
<data name="TopLogosDescription">
|
||||
<value>Top 10 largest service logos. Logos are reused for items with the same domain.</value>
|
||||
<comment>Description for the top logos table</comment>
|
||||
</data>
|
||||
|
||||
<!-- Table column headers -->
|
||||
<data name="ColumnFilename">
|
||||
<value>Filename</value>
|
||||
<comment>Table column header for attachment filename</comment>
|
||||
</data>
|
||||
<data name="ColumnSize">
|
||||
<value>Size</value>
|
||||
<comment>Table column header for size in KB</comment>
|
||||
</data>
|
||||
<data name="ColumnItem">
|
||||
<value>Item</value>
|
||||
<comment>Table column header for the parent item name</comment>
|
||||
</data>
|
||||
<data name="ColumnCreated">
|
||||
<value>Created</value>
|
||||
<comment>Table column header for creation date</comment>
|
||||
</data>
|
||||
<data name="ColumnWebsiteURL">
|
||||
<value>Website URL</value>
|
||||
<comment>Table column header for logo source domain</comment>
|
||||
</data>
|
||||
<data name="ColumnItemCount">
|
||||
<value>Used by</value>
|
||||
<comment>Table column header for the number of items using a logo</comment>
|
||||
</data>
|
||||
|
||||
<!-- Logo management -->
|
||||
<data name="LogoManagementTitle">
|
||||
<value>Manage logos</value>
|
||||
<comment>Title for the logo management section</comment>
|
||||
</data>
|
||||
<data name="LogoManagementDescription">
|
||||
<value>Logos are fetched from the favicon of each item's website. You can delete every logo to reclaim space, or re-download them to pick up improved versions.</value>
|
||||
<comment>Description for the logo management section</comment>
|
||||
</data>
|
||||
<data name="DeleteAllLogosButton">
|
||||
<value>Delete all logos</value>
|
||||
<comment>Button to delete every logo in the vault</comment>
|
||||
</data>
|
||||
<data name="RedownloadAllLogosButton">
|
||||
<value>Re-download all logos</value>
|
||||
<comment>Button to re-fetch favicons for every item with a URL</comment>
|
||||
</data>
|
||||
<data name="DeleteAllLogosConfirmTitle">
|
||||
<value>Delete all logos?</value>
|
||||
<comment>Confirmation modal title for delete-all-logos</comment>
|
||||
</data>
|
||||
<data name="DeleteAllLogosConfirmMessage">
|
||||
<value>This removes every stored logo from your vault and clears the logo on each credential. You can re-download them at any time. Continue?</value>
|
||||
<comment>Confirmation modal body for delete-all-logos</comment>
|
||||
</data>
|
||||
<data name="RedownloadAllLogosConfirmTitle">
|
||||
<value>Re-download all logos?</value>
|
||||
<comment>Confirmation modal title for redownload-all-logos</comment>
|
||||
</data>
|
||||
<data name="RedownloadAllLogosConfirmMessage">
|
||||
<value>This re-fetches the favicon for every credential that has a website URL, replacing the existing logo. Depending on your vault size this may take a while. Continue?</value>
|
||||
<comment>Confirmation modal body for redownload-all-logos</comment>
|
||||
</data>
|
||||
<data name="RedownloadAllLogosNoItemsMessage">
|
||||
<value>No items with a website URL were found, nothing to download.</value>
|
||||
<comment>Notification shown when redownload was triggered but no items qualify</comment>
|
||||
</data>
|
||||
</root>
|
||||
221
apps/server/AliasVault.Client/Services/FaviconService.cs
Normal file
221
apps/server/AliasVault.Client/Services/FaviconService.cs
Normal file
@@ -0,0 +1,221 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// <copyright file="FaviconService.cs" company="aliasvault">
|
||||
// Copyright (c) aliasvault. All rights reserved.
|
||||
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
|
||||
// </copyright>
|
||||
//-----------------------------------------------------------------------
|
||||
|
||||
namespace AliasVault.Client.Services;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AliasVault.Shared.Models.WebApi.Favicon;
|
||||
|
||||
/// <summary>
|
||||
/// Wraps calls to the server-side favicon API. Centralizes the single-URL and batched
|
||||
/// extraction paths so item add/edit, vault import, and bulk re-download all share one
|
||||
/// implementation (and one place to evolve, e.g. when adjusting batch size).
|
||||
/// </summary>
|
||||
public sealed class FaviconService(HttpClient httpClient)
|
||||
{
|
||||
/// <summary>
|
||||
/// Number of URLs sent per batch request. Mirrors <c>FaviconController.MaxBatchSize</c> on the server.
|
||||
/// </summary>
|
||||
public const int BatchSize = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of a bulk extraction.
|
||||
/// </summary>
|
||||
public enum BulkExtractStatus
|
||||
{
|
||||
/// <summary>All requested URLs were processed.</summary>
|
||||
Completed,
|
||||
|
||||
/// <summary>The user cancelled the operation before all URLs were processed.</summary>
|
||||
Cancelled,
|
||||
|
||||
/// <summary>Server returned 429 — the per-user 24h rate limit was hit. Partial results are returned.</summary>
|
||||
RateLimited,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes a URL to a lowercase host without leading "www.". Returns false for URLs that
|
||||
/// can't be parsed; callers should skip those rather than fall back to a different key.
|
||||
/// </summary>
|
||||
/// <param name="url">The URL to normalize.</param>
|
||||
/// <param name="domain">The normalized domain.</param>
|
||||
/// <returns>True if the URL was parseable.</returns>
|
||||
public static bool TryNormalizeDomain(string url, out string domain)
|
||||
{
|
||||
domain = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var host = new Uri(url).Host.ToLowerInvariant();
|
||||
if (host.StartsWith("www.", StringComparison.Ordinal))
|
||||
{
|
||||
host = host[4..];
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(host))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
domain = host;
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the favicon for a single URL.
|
||||
/// </summary>
|
||||
/// <param name="url">The URL to fetch the favicon for.</param>
|
||||
/// <returns>Favicon bytes, or null if extraction failed or the rate limit was hit.</returns>
|
||||
public async Task<byte[]?> ExtractAsync(string url)
|
||||
{
|
||||
try
|
||||
{
|
||||
var apiReturn = await httpClient.GetFromJsonAsync<FaviconExtractModel>(
|
||||
$"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}");
|
||||
return apiReturn?.Image;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts favicons for many URLs in batches, deduplicating by normalized domain so the
|
||||
/// same domain is only fetched once even if many items reference it.
|
||||
/// </summary>
|
||||
/// <param name="urls">URLs to extract favicons for. Duplicates by domain are collapsed.</param>
|
||||
/// <param name="progress">
|
||||
/// Optional progress callback. Receives the number of unique domains processed so far
|
||||
/// (regardless of fetch success). The total reported is the count of unique domains, not
|
||||
/// the count of input URLs.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The favicon dictionary plus the operation status.</returns>
|
||||
public async Task<BulkExtractResult> ExtractBulkAsync(IEnumerable<string> urls, IProgress<int>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var favicons = new Dictionary<string, byte[]>();
|
||||
|
||||
// Deduplicate by domain up front so a 5000-item vault that all uses 200 unique domains
|
||||
// makes 200 fetches, not 5000.
|
||||
var domainToUrl = new Dictionary<string, string>();
|
||||
foreach (var url in urls)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryNormalizeDomain(url, out var domain) && !domainToUrl.ContainsKey(domain))
|
||||
{
|
||||
domainToUrl[domain] = url;
|
||||
}
|
||||
}
|
||||
|
||||
if (domainToUrl.Count == 0)
|
||||
{
|
||||
return new BulkExtractResult(favicons, BulkExtractStatus.Completed);
|
||||
}
|
||||
|
||||
var entries = domainToUrl.ToList();
|
||||
int processed = 0;
|
||||
|
||||
for (int i = 0; i < entries.Count; i += BatchSize)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return new BulkExtractResult(favicons, BulkExtractStatus.Cancelled);
|
||||
}
|
||||
|
||||
var chunk = entries.Skip(i).Take(BatchSize).ToList();
|
||||
var request = new FaviconExtractBatchRequest
|
||||
{
|
||||
Urls = chunk.Select(e => e.Value).ToList(),
|
||||
};
|
||||
|
||||
HttpResponseMessage response;
|
||||
try
|
||||
{
|
||||
response = await httpClient.PostAsJsonAsync("v1/Favicon/ExtractBatch", request, cancellationToken);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Network or other transport error — treat the whole chunk as failed but keep going.
|
||||
processed += chunk.Count;
|
||||
progress?.Report(processed);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (response.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
return new BulkExtractResult(favicons, BulkExtractStatus.RateLimited);
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
processed += chunk.Count;
|
||||
progress?.Report(processed);
|
||||
continue;
|
||||
}
|
||||
|
||||
FaviconExtractBatchResponse? body;
|
||||
try
|
||||
{
|
||||
body = await response.Content.ReadFromJsonAsync<FaviconExtractBatchResponse>(cancellationToken: cancellationToken);
|
||||
}
|
||||
catch
|
||||
{
|
||||
processed += chunk.Count;
|
||||
progress?.Report(processed);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (body?.Results != null)
|
||||
{
|
||||
// Match results back to the domains we asked for. Server echoes the URL, but matching
|
||||
// by index is robust against any normalization differences.
|
||||
for (int j = 0; j < body.Results.Count && j < chunk.Count; j++)
|
||||
{
|
||||
var image = body.Results[j].Image;
|
||||
if (image != null)
|
||||
{
|
||||
favicons[chunk[j].Key] = image;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
processed += chunk.Count;
|
||||
progress?.Report(processed);
|
||||
}
|
||||
|
||||
return new BulkExtractResult(favicons, BulkExtractStatus.Completed);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a bulk extraction call. Favicons are keyed by normalized domain so callers
|
||||
/// can reuse a single fetched favicon across all items that share that domain.
|
||||
/// </summary>
|
||||
/// <param name="Favicons">Dictionary of normalized domain to favicon bytes.</param>
|
||||
/// <param name="Status">Outcome of the operation.</param>
|
||||
public record BulkExtractResult(Dictionary<string, byte[]> Favicons, BulkExtractStatus Status);
|
||||
}
|
||||
@@ -18,14 +18,12 @@ using System.Threading.Tasks;
|
||||
using AliasClientDb;
|
||||
using AliasClientDb.Models;
|
||||
using AliasVault.Client.Main.Models;
|
||||
using AliasVault.Client.Utilities;
|
||||
using AliasVault.Shared.Models.WebApi.Favicon;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
/// <summary>
|
||||
/// Service class for Item operations.
|
||||
/// </summary>
|
||||
public sealed class ItemService(HttpClient httpClient, DbService dbService, Config config, JsInteropService jsInteropService)
|
||||
public sealed class ItemService(HttpClient httpClient, DbService dbService, Config config, JsInteropService jsInteropService, FaviconService faviconService)
|
||||
{
|
||||
/// <summary>
|
||||
/// The default service URL used as placeholder in forms. When this value is set, the URL field is considered empty
|
||||
@@ -1492,59 +1490,68 @@ public sealed class ItemService(HttpClient httpClient, DbService dbService, Conf
|
||||
/// <returns>Task.</returns>
|
||||
private async Task ExtractFaviconAsync(Item item)
|
||||
{
|
||||
// Try to extract favicon from service URL
|
||||
var url = GetFieldValue(item, FieldKey.LoginUrl);
|
||||
if (url != null && !string.IsNullOrEmpty(url) && url != DefaultServiceUrl)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Extract and normalize domain for deduplication
|
||||
var domain = new Uri(url).Host.ToLowerInvariant();
|
||||
if (domain.StartsWith("www."))
|
||||
{
|
||||
domain = domain[4..];
|
||||
}
|
||||
|
||||
var context = await dbService.GetDbContextAsync();
|
||||
|
||||
// Check if logo already exists for this source (deduplication)
|
||||
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
|
||||
|
||||
if (existingLogo != null)
|
||||
{
|
||||
// Reuse existing logo - no need to fetch
|
||||
item.LogoId = existingLogo.Id;
|
||||
return;
|
||||
}
|
||||
|
||||
// No existing logo - fetch from API
|
||||
var apiReturn = await httpClient.GetFromJsonAsync<FaviconExtractModel>($"v1/Favicon/Extract?url={Uri.EscapeDataString(url)}");
|
||||
if (apiReturn?.Image is not null)
|
||||
{
|
||||
// Create new logo
|
||||
var newLogo = new Logo
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = domain,
|
||||
FileData = apiReturn.Image,
|
||||
MimeType = "image/png",
|
||||
FetchedAt = DateTime.UtcNow,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
UpdatedAt = DateTime.UtcNow,
|
||||
};
|
||||
context.Logos.Add(newLogo);
|
||||
item.LogoId = newLogo.Id;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore favicon extraction errors
|
||||
}
|
||||
}
|
||||
else
|
||||
if (url == null || string.IsNullOrEmpty(url) || url == DefaultServiceUrl)
|
||||
{
|
||||
// URL is empty or just the placeholder - clear any existing logo
|
||||
item.LogoId = null;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!FaviconService.TryNormalizeDomain(url, out var domain))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var context = await dbService.GetDbContextAsync();
|
||||
|
||||
// Look up by Source regardless of IsDeleted — Logos.Source has a UNIQUE index, so a
|
||||
// soft-deleted row with the same domain still occupies the slot and we'd hit a UNIQUE
|
||||
// violation on insert. Filter against IsDeleted/empty FileData when deciding whether
|
||||
// it's safe to reuse without re-fetching.
|
||||
var existingLogo = await context.Logos.FirstOrDefaultAsync(l => l.Source == domain);
|
||||
if (existingLogo != null && !existingLogo.IsDeleted && existingLogo.FileData is { Length: > 0 })
|
||||
{
|
||||
item.LogoId = existingLogo.Id;
|
||||
return;
|
||||
}
|
||||
|
||||
var image = await faviconService.ExtractAsync(url);
|
||||
if (image is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
if (existingLogo != null)
|
||||
{
|
||||
// Restore (or refill) the existing row.
|
||||
existingLogo.IsDeleted = false;
|
||||
existingLogo.FileData = image;
|
||||
existingLogo.FetchedAt = now;
|
||||
existingLogo.UpdatedAt = now;
|
||||
item.LogoId = existingLogo.Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
var newLogo = new Logo
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Source = domain,
|
||||
FileData = image,
|
||||
FetchedAt = now,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
context.Logos.Add(newLogo);
|
||||
item.LogoId = newLogo.Id;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore favicon extraction errors
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1638,6 +1638,10 @@ video {
|
||||
border-radius: 0.375rem;
|
||||
}
|
||||
|
||||
.rounded-sm {
|
||||
border-radius: 0.125rem;
|
||||
}
|
||||
|
||||
.rounded-b-lg {
|
||||
border-bottom-right-radius: 0.5rem;
|
||||
border-bottom-left-radius: 0.5rem;
|
||||
@@ -1809,6 +1813,11 @@ video {
|
||||
border-color: rgb(254 240 138 / var(--tw-border-opacity));
|
||||
}
|
||||
|
||||
.border-yellow-400 {
|
||||
--tw-border-opacity: 1;
|
||||
border-color: rgb(250 204 21 / var(--tw-border-opacity));
|
||||
}
|
||||
|
||||
.bg-amber-100 {
|
||||
--tw-bg-opacity: 1;
|
||||
background-color: rgb(254 243 199 / var(--tw-bg-opacity));
|
||||
@@ -2018,6 +2027,21 @@ video {
|
||||
background-color: rgb(234 179 8 / var(--tw-bg-opacity));
|
||||
}
|
||||
|
||||
.bg-amber-500 {
|
||||
--tw-bg-opacity: 1;
|
||||
background-color: rgb(245 158 11 / var(--tw-bg-opacity));
|
||||
}
|
||||
|
||||
.bg-emerald-500 {
|
||||
--tw-bg-opacity: 1;
|
||||
background-color: rgb(16 185 129 / var(--tw-bg-opacity));
|
||||
}
|
||||
|
||||
.bg-yellow-200 {
|
||||
--tw-bg-opacity: 1;
|
||||
background-color: rgb(254 240 138 / var(--tw-bg-opacity));
|
||||
}
|
||||
|
||||
.bg-opacity-50 {
|
||||
--tw-bg-opacity: 0.5;
|
||||
}
|
||||
@@ -3294,6 +3318,11 @@ video {
|
||||
border-color: rgb(133 77 14 / var(--tw-border-opacity));
|
||||
}
|
||||
|
||||
.dark\:border-yellow-600:is(.dark *) {
|
||||
--tw-border-opacity: 1;
|
||||
border-color: rgb(202 138 4 / var(--tw-border-opacity));
|
||||
}
|
||||
|
||||
.dark\:bg-amber-800\/30:is(.dark *) {
|
||||
background-color: rgb(146 64 14 / 0.3);
|
||||
}
|
||||
@@ -4269,6 +4298,10 @@ video {
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.lg\:grid-cols-4 {
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.lg\:flex-row {
|
||||
flex-direction: row;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// <copyright file="FaviconExtractBatchRequest.cs" company="aliasvault">
|
||||
// Copyright (c) aliasvault. All rights reserved.
|
||||
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
|
||||
// </copyright>
|
||||
//-----------------------------------------------------------------------
|
||||
|
||||
namespace AliasVault.Shared.Models.WebApi.Favicon;
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
/// <summary>
|
||||
/// Request payload for batch favicon extraction.
|
||||
/// </summary>
|
||||
public class FaviconExtractBatchRequest
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the URLs to extract favicons for. The server caps the number of URLs
|
||||
/// it processes per request; callers should chunk larger inputs into multiple requests.
|
||||
/// </summary>
|
||||
public List<string> Urls { get; set; } = new();
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// <copyright file="FaviconExtractBatchResponse.cs" company="aliasvault">
|
||||
// Copyright (c) aliasvault. All rights reserved.
|
||||
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
|
||||
// </copyright>
|
||||
//-----------------------------------------------------------------------
|
||||
|
||||
namespace AliasVault.Shared.Models.WebApi.Favicon;
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
/// <summary>
|
||||
/// Response payload for batch favicon extraction. Each result lines up with the URL at the
|
||||
/// same index in the request, with a null Image when extraction failed for that URL.
|
||||
/// </summary>
|
||||
public class FaviconExtractBatchResponse
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the per-URL extraction results.
|
||||
/// </summary>
|
||||
public List<FaviconExtractBatchResult> Results { get; set; } = new();
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// <copyright file="FaviconExtractBatchResult.cs" company="aliasvault">
|
||||
// Copyright (c) aliasvault. All rights reserved.
|
||||
// Licensed under the AGPLv3 license. See LICENSE.md file in the project root for full license information.
|
||||
// </copyright>
|
||||
//-----------------------------------------------------------------------
|
||||
|
||||
namespace AliasVault.Shared.Models.WebApi.Favicon;
|
||||
|
||||
/// <summary>
|
||||
/// Per-URL favicon extraction result.
|
||||
/// </summary>
|
||||
public class FaviconExtractBatchResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the URL the favicon was requested for (echoed back so the client can correlate).
|
||||
/// </summary>
|
||||
public string Url { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the favicon image bytes, or null if extraction failed.
|
||||
/// </summary>
|
||||
public byte[]? Image { get; set; }
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
namespace AliasVault.FaviconExtractor;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
@@ -21,9 +22,10 @@ using SkiaSharp;
|
||||
/// </summary>
|
||||
public static class FaviconExtractor
|
||||
{
|
||||
private const int MaxSizeBytes = 50 * 1024; // 50KB max size before resizing
|
||||
private const int TargetWidth = 96; // Resize target width
|
||||
private static readonly string[] _allowedSchemes = { "http", "https" };
|
||||
private const int MaxSizeBytes = 20 * 1024; // 20KB max size; images above this are resized/re-encoded.
|
||||
private static readonly int[] _resizeWidths = [96, 64, 48, 32];
|
||||
private static readonly int[] _jpegFallbackQualities = [80, 65, 50];
|
||||
private static readonly string[] _allowedSchemes = ["http", "https"];
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the favicon from a URL with enhanced browser like behavior.
|
||||
@@ -56,6 +58,43 @@ public static class FaviconExtractor
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts favicons for multiple URLs in parallel. Each URL is processed independently;
|
||||
/// individual failures are returned as null entries rather than throwing. The returned
|
||||
/// list lines up index-for-index with the input.
|
||||
/// </summary>
|
||||
/// <param name="urls">The URLs to extract favicons for.</param>
|
||||
/// <returns>A list of favicon byte arrays, in the same order as the input urls.</returns>
|
||||
public static async Task<IReadOnlyList<byte[]?>> GetFaviconsAsync(IReadOnlyList<string> urls)
|
||||
{
|
||||
if (urls.Count == 0)
|
||||
{
|
||||
return Array.Empty<byte[]?>();
|
||||
}
|
||||
|
||||
var tasks = new Task<byte[]?>[urls.Count];
|
||||
for (int i = 0; i < urls.Count; i++)
|
||||
{
|
||||
// Wrap each call in a try/catch so one bad URL doesn't fail the whole batch.
|
||||
var url = urls[i];
|
||||
tasks[i] = SafeGetFaviconAsync(url);
|
||||
}
|
||||
|
||||
return await Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
private static async Task<byte[]?> SafeGetFaviconAsync(string url)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await GetFaviconAsync(url);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries to get the favicon from the URL.
|
||||
/// </summary>
|
||||
@@ -184,18 +223,13 @@ public static class FaviconExtractor
|
||||
return null;
|
||||
}
|
||||
|
||||
// If image is too large, attempt to resize
|
||||
// If image is too large, attempt to resize/re-encode it down under the cap.
|
||||
if (imageBytes.Length > MaxSizeBytes)
|
||||
{
|
||||
var resizedBytes = ResizeImageAsync(imageBytes, contentType);
|
||||
if (resizedBytes != null)
|
||||
{
|
||||
imageBytes = resizedBytes;
|
||||
}
|
||||
return ResizeImage(imageBytes, contentType);
|
||||
}
|
||||
|
||||
// Return only if within size limits
|
||||
return imageBytes.Length <= MaxSizeBytes ? imageBytes : null;
|
||||
return imageBytes;
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -377,13 +411,22 @@ public static class FaviconExtractor
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resizes the image to the target width.
|
||||
/// Iteratively shrinks the image until it fits in the size cap. Tries
|
||||
/// progressively smaller widths in PNG (lossless after downscale handles most cases), then
|
||||
/// falls back to JPEG at decreasing quality if PNG at the smallest width is still too large.
|
||||
/// </summary>
|
||||
/// <param name="imageBytes">The image bytes to resize.</param>
|
||||
/// <param name="contentType">The content type of the image.</param>
|
||||
/// <returns>The resized image bytes.</returns>
|
||||
private static byte[]? ResizeImageAsync(byte[] imageBytes, string contentType)
|
||||
/// <returns>The resized image bytes, or null if it could not be brought under the size cap.</returns>
|
||||
private static byte[]? ResizeImage(byte[] imageBytes, string contentType)
|
||||
{
|
||||
// SVG is a text format; we can't usefully raster-resize it here. Caller already rejects
|
||||
// anything over the cap, so just bail and let it return null.
|
||||
if (contentType == "image/svg+xml")
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var original = SKBitmap.Decode(imageBytes);
|
||||
@@ -392,30 +435,59 @@ public static class FaviconExtractor
|
||||
return null;
|
||||
}
|
||||
|
||||
var scale = (float)TargetWidth / original.Width;
|
||||
var targetHeight = (int)(original.Height * scale);
|
||||
|
||||
using var resized = original.Resize(new SKImageInfo(TargetWidth, targetHeight), new SKSamplingOptions(SKFilterMode.Linear));
|
||||
if (resized == null)
|
||||
// Pass 1: PNG at progressively smaller widths. Preserves transparency.
|
||||
foreach (var width in _resizeWidths)
|
||||
{
|
||||
return null;
|
||||
var encoded = EncodeAtWidth(original, width, SKEncodedImageFormat.Png, 100);
|
||||
if (encoded != null && encoded.Length <= MaxSizeBytes)
|
||||
{
|
||||
return encoded;
|
||||
}
|
||||
}
|
||||
|
||||
using var image = SKImage.FromBitmap(resized);
|
||||
var format = contentType switch
|
||||
// Pass 2: JPEG at the smallest width with decreasing quality. Loses transparency but
|
||||
// gives much smaller files for photographic favicons that resist PNG compression.
|
||||
var fallbackWidth = _resizeWidths[^1];
|
||||
foreach (var quality in _jpegFallbackQualities)
|
||||
{
|
||||
"image/png" => SKEncodedImageFormat.Png,
|
||||
"image/jpeg" => SKEncodedImageFormat.Jpeg,
|
||||
"image/gif" => SKEncodedImageFormat.Gif,
|
||||
_ => SKEncodedImageFormat.Png,
|
||||
};
|
||||
var encoded = EncodeAtWidth(original, fallbackWidth, SKEncodedImageFormat.Jpeg, quality);
|
||||
if (encoded != null && encoded.Length <= MaxSizeBytes)
|
||||
{
|
||||
return encoded;
|
||||
}
|
||||
}
|
||||
|
||||
var data = image.Encode(format, 90);
|
||||
return data?.ToArray();
|
||||
return null;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resizes the bitmap to the target width (preserving aspect ratio, never upscaling) and
|
||||
/// encodes it in the given format.
|
||||
/// </summary>
|
||||
/// <param name="original">The decoded source bitmap.</param>
|
||||
/// <param name="targetWidth">Desired output width in pixels.</param>
|
||||
/// <param name="format">Encode format.</param>
|
||||
/// <param name="quality">Encoder quality (only meaningful for lossy formats).</param>
|
||||
/// <returns>Encoded bytes, or null if encoding failed.</returns>
|
||||
private static byte[]? EncodeAtWidth(SKBitmap original, int targetWidth, SKEncodedImageFormat format, int quality)
|
||||
{
|
||||
var width = Math.Min(targetWidth, original.Width);
|
||||
var scale = (float)width / original.Width;
|
||||
var height = Math.Max(1, (int)(original.Height * scale));
|
||||
|
||||
using var resized = original.Resize(new SKImageInfo(width, height), new SKSamplingOptions(SKFilterMode.Linear));
|
||||
if (resized == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
using var image = SKImage.FromBitmap(resized);
|
||||
using var data = image.Encode(format, quality);
|
||||
return data?.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +72,9 @@ pub struct PruneStats {
|
||||
/// Number of tombstoned attachments whose blob bytes were cleared.
|
||||
#[serde(default)]
|
||||
pub attachment_blobs_cleared: u32,
|
||||
/// Number of tombstoned logos whose FileData bytes were cleared.
|
||||
#[serde(default)]
|
||||
pub logo_blobs_cleared: u32,
|
||||
}
|
||||
|
||||
/// Output of the prune operation.
|
||||
@@ -263,7 +266,7 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
|
||||
if let Some(logo_id) = logo.get("Id").and_then(|v| v.as_str()) {
|
||||
if !referenced_logo_ids.contains(logo_id) {
|
||||
statements.push(SqlStatement {
|
||||
sql: "UPDATE Logos SET IsDeleted = 1, UpdatedAt = ? WHERE Id = ?".to_string(),
|
||||
sql: "UPDATE Logos SET IsDeleted = 1, FileData = X'', UpdatedAt = ? WHERE Id = ?".to_string(),
|
||||
params: vec![
|
||||
serde_json::json!(now_str),
|
||||
serde_json::json!(logo_id),
|
||||
@@ -307,6 +310,35 @@ pub fn prune_vault(input: PruneInput) -> VaultResult<PruneOutput> {
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 4 — sweep tombstoned logos that still carry FileData bytes. Logos that have
|
||||
// been soft-deleted but still have FileData bytes take up space for no reason.
|
||||
// This behaviour could have occurred in older clients (before 0.29.x).
|
||||
if let Some(logos_table) = input.tables.iter().find(|t| t.name == "Logos") {
|
||||
for logo in &logos_table.records {
|
||||
let is_deleted = logo.get("IsDeleted")
|
||||
.map(|v| v.as_i64() == Some(1) || v.as_bool() == Some(true))
|
||||
.unwrap_or(false);
|
||||
if !is_deleted {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !logo_has_file_data_bytes(logo) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(logo_id) = logo.get("Id").and_then(|v| v.as_str()) {
|
||||
statements.push(SqlStatement {
|
||||
sql: "UPDATE Logos SET FileData = X'', UpdatedAt = ? WHERE Id = ?".to_string(),
|
||||
params: vec![
|
||||
serde_json::json!(now_str),
|
||||
serde_json::json!(logo_id),
|
||||
],
|
||||
});
|
||||
stats.logo_blobs_cleared += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(PruneOutput {
|
||||
success: true,
|
||||
statements,
|
||||
@@ -325,6 +357,17 @@ fn attachment_has_blob_bytes(attachment: &Record) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// True if the logo's FileData field is present and non-empty.
|
||||
fn logo_has_file_data_bytes(logo: &Record) -> bool {
|
||||
match logo.get("FileData") {
|
||||
None => false,
|
||||
Some(serde_json::Value::Null) => false,
|
||||
Some(serde_json::Value::String(s)) => !s.is_empty(),
|
||||
Some(serde_json::Value::Array(a)) => !a.is_empty(),
|
||||
Some(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Prune vault using JSON strings.
|
||||
/// Convenience function for FFI.
|
||||
pub fn prune_vault_json(input_json: &str) -> VaultResult<String> {
|
||||
@@ -433,6 +476,12 @@ mod tests {
|
||||
record
|
||||
}
|
||||
|
||||
fn make_logo_record_with_blob(id: &str, is_deleted: bool, blob: serde_json::Value) -> Record {
|
||||
let mut record = make_logo_record(id, is_deleted);
|
||||
record.insert("FileData".to_string(), blob);
|
||||
record
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_prune_expired_items() {
|
||||
let now = Utc::now();
|
||||
@@ -699,6 +748,82 @@ mod tests {
|
||||
assert_eq!(output.stats.logos_pruned, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_orphan_logo_pruning_emits_filedata_clear_in_same_statement() {
|
||||
// Pass 2 must clear FileData when it tombstones a logo, otherwise the
|
||||
// encrypted vault keeps the blob bytes even after the row is "deleted".
|
||||
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
|
||||
let input = PruneInput {
|
||||
tables: vec![
|
||||
TableData {
|
||||
name: "Items".to_string(),
|
||||
records: vec![],
|
||||
},
|
||||
TableData {
|
||||
name: "Logos".to_string(),
|
||||
records: vec![make_logo_record_with_blob("logo-orphan", false, serde_json::json!("aGVsbG8="))],
|
||||
},
|
||||
],
|
||||
retention_days: 30,
|
||||
current_time: now_str,
|
||||
};
|
||||
|
||||
let output = prune_vault(input).unwrap();
|
||||
|
||||
assert_eq!(output.stats.logos_pruned, 1);
|
||||
assert!(output.statements.iter().any(|s| s.sql.contains("FileData = X''")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tombstoned_logo_with_blob_bytes_is_swept() {
|
||||
// Pass 4 must catch historical logos that are IsDeleted=1 but still carry FileData
|
||||
// (e.g. tombstoned by an older client before the FileData=X'' fix landed).
|
||||
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
|
||||
let input = PruneInput {
|
||||
tables: vec![
|
||||
TableData {
|
||||
name: "Items".to_string(),
|
||||
records: vec![],
|
||||
},
|
||||
TableData {
|
||||
name: "Logos".to_string(),
|
||||
records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::json!("aGVsbG8="))],
|
||||
},
|
||||
],
|
||||
retention_days: 30,
|
||||
current_time: now_str,
|
||||
};
|
||||
|
||||
let output = prune_vault(input).unwrap();
|
||||
|
||||
assert_eq!(output.stats.logo_blobs_cleared, 1);
|
||||
assert_eq!(output.stats.logos_pruned, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tombstoned_logo_without_blob_is_not_touched() {
|
||||
// Logos already cleared shouldn't generate redundant updates.
|
||||
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
|
||||
let input = PruneInput {
|
||||
tables: vec![
|
||||
TableData {
|
||||
name: "Items".to_string(),
|
||||
records: vec![],
|
||||
},
|
||||
TableData {
|
||||
name: "Logos".to_string(),
|
||||
records: vec![make_logo_record_with_blob("logo-tombstoned", true, serde_json::Value::Null)],
|
||||
},
|
||||
],
|
||||
retention_days: 30,
|
||||
current_time: now_str,
|
||||
};
|
||||
|
||||
let output = prune_vault(input).unwrap();
|
||||
|
||||
assert_eq!(output.stats.logo_blobs_cleared, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_already_soft_deleted_logo_is_not_re_pruned() {
|
||||
let now_str = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
|
||||
|
||||
Reference in New Issue
Block a user