From d2f65716fea906f7645edb8a0acae926ca759352 Mon Sep 17 00:00:00 2001 From: Eric Hess <92155191+eric-hess@users.noreply.github.com> Date: Thu, 16 Apr 2026 21:28:48 +0200 Subject: [PATCH] feat(mirrors): add selective snapshot sync to mirror repositories (#755) * feat(mirrors): add selective snapshot sync to mirror repositories Allow users to sync missing snapshots from the source repository to a mirror. A new sync button per mirror opens a dialog showing which snapshots are missing, with checkboxes to select which ones to copy. - Add GET /:shortId/mirrors/:mirrorShortId/status endpoint to compare snapshots between source and mirror repositories - Add POST /:shortId/mirrors/:mirrorShortId/sync endpoint accepting selected snapshotIds in the request body - Extend restic copy command to accept multiple snapshotIds - Add sync preview dialog with snapshot selection to the frontend * refactor: stylistic changes --------- Co-authored-by: Nicolas Meienberger --- .../api-client/@tanstack/react-query.gen.ts | 39 ++- app/client/api-client/index.ts | 9 + app/client/api-client/sdk.gen.ts | 19 +- app/client/api-client/types.gen.ts | 57 ++++ .../components/schedule-mirrors-config.tsx | 194 +++++++++++++- .../__tests__/backups.controller.test.ts | 2 + .../__tests__/backups.mirror-sync.test.ts | 248 ++++++++++++++++++ .../modules/backups/backups.controller.ts | 24 +- app/server/modules/backups/backups.dto.ts | 59 +++++ app/server/modules/backups/backups.queries.ts | 7 + app/server/modules/backups/backups.service.ts | 80 +++++- .../backups/helpers/backup-maintenance.ts | 108 ++++++++ e2e/0004-mirror-sync.spec.ts | 139 ++++++++++ .../restic/commands/__tests__/copy.test.ts | 40 ++- packages/core/src/restic/commands/copy.ts | 8 +- 15 files changed, 1011 insertions(+), 22 deletions(-) create mode 100644 app/server/modules/backups/__tests__/backups.mirror-sync.test.ts create mode 100644 e2e/0004-mirror-sync.spec.ts diff --git a/app/client/api-client/@tanstack/react-query.gen.ts b/app/client/api-client/@tanstack/react-query.gen.ts index d6510ffe..8b99be1a 100644 --- a/app/client/api-client/@tanstack/react-query.gen.ts +++ b/app/client/api-client/@tanstack/react-query.gen.ts @@ -4,8 +4,8 @@ import { type DefaultError, type InfiniteData, infiniteQueryOptions, queryOptions, type UseMutationOptions } from '@tanstack/react-query'; import { client } from '../client.gen'; -import { browseFilesystem, cancelDoctor, createBackupSchedule, createNotificationDestination, createRepository, createVolume, deleteBackupSchedule, deleteNotificationDestination, deleteRepository, deleteSnapshot, deleteSnapshots, deleteSsoInvitation, deleteSsoProvider, deleteUserAccount, deleteVolume, downloadResticPassword, dumpSnapshot, getAdminUsers, getBackupProgress, getBackupSchedule, getBackupScheduleForVolume, getDevPanel, getMirrorCompatibility, getNotificationDestination, getOrgMembers, getPublicSsoProviders, getRegistrationStatus, getRepository, getRepositoryStats, getScheduleMirrors, getScheduleNotifications, getSnapshotDetails, getSsoSettings, getStatus, getSystemInfo, getUpdates, getUserDeletionImpact, getVolume, healthCheckVolume, listBackupSchedules, listFiles, listNotificationDestinations, listRcloneRemotes, listRepositories, listSnapshotFiles, listSnapshots, listVolumes, mountVolume, type Options, refreshRepositoryStats, refreshSnapshots, removeOrgMember, reorderBackupSchedules, restoreSnapshot, runBackupNow, runForget, setRegistrationStatus, startDoctor, stopBackup, tagSnapshots, testConnection, testNotificationDestination, unlockRepository, unmountVolume, updateBackupSchedule, updateMemberRole, updateNotificationDestination, updateRepository, updateScheduleMirrors, updateScheduleNotifications, updateSsoProviderAutoLinking, updateVolume } from '../sdk.gen'; -import type { BrowseFilesystemData, BrowseFilesystemResponse, CancelDoctorData, CancelDoctorResponse, CreateBackupScheduleData, CreateBackupScheduleResponse, CreateNotificationDestinationData, CreateNotificationDestinationResponse, CreateRepositoryData, CreateRepositoryResponse, CreateVolumeData, CreateVolumeResponse, DeleteBackupScheduleData, DeleteBackupScheduleResponse, DeleteNotificationDestinationData, DeleteNotificationDestinationResponse, DeleteRepositoryData, DeleteRepositoryResponse, DeleteSnapshotData, DeleteSnapshotResponse, DeleteSnapshotsData, DeleteSnapshotsResponse, DeleteSsoInvitationData, DeleteSsoProviderData, DeleteUserAccountData, DeleteVolumeData, DeleteVolumeResponse, DownloadResticPasswordData, DownloadResticPasswordResponse, DumpSnapshotData, DumpSnapshotResponse, GetAdminUsersData, GetAdminUsersResponse, GetBackupProgressData, GetBackupProgressResponse, GetBackupScheduleData, GetBackupScheduleForVolumeData, GetBackupScheduleForVolumeResponse, GetBackupScheduleResponse, GetDevPanelData, GetDevPanelResponse, GetMirrorCompatibilityData, GetMirrorCompatibilityResponse, GetNotificationDestinationData, GetNotificationDestinationResponse, GetOrgMembersData, GetOrgMembersResponse, GetPublicSsoProvidersData, GetPublicSsoProvidersResponse, GetRegistrationStatusData, GetRegistrationStatusResponse, GetRepositoryData, GetRepositoryResponse, GetRepositoryStatsData, GetRepositoryStatsResponse, GetScheduleMirrorsData, GetScheduleMirrorsResponse, GetScheduleNotificationsData, GetScheduleNotificationsResponse, GetSnapshotDetailsData, GetSnapshotDetailsResponse, GetSsoSettingsData, GetSsoSettingsResponse, GetStatusData, GetStatusResponse, GetSystemInfoData, GetSystemInfoResponse, GetUpdatesData, GetUpdatesResponse, GetUserDeletionImpactData, GetUserDeletionImpactResponse, GetVolumeData, GetVolumeResponse, HealthCheckVolumeData, HealthCheckVolumeResponse, ListBackupSchedulesData, ListBackupSchedulesResponse, ListFilesData, ListFilesResponse, ListNotificationDestinationsData, ListNotificationDestinationsResponse, ListRcloneRemotesData, ListRcloneRemotesResponse, ListRepositoriesData, ListRepositoriesResponse, ListSnapshotFilesData, ListSnapshotFilesResponse, ListSnapshotsData, ListSnapshotsResponse, ListVolumesData, ListVolumesResponse, MountVolumeData, MountVolumeResponse, RefreshRepositoryStatsData, RefreshRepositoryStatsResponse, RefreshSnapshotsData, RefreshSnapshotsResponse, RemoveOrgMemberData, ReorderBackupSchedulesData, ReorderBackupSchedulesResponse, RestoreSnapshotData, RestoreSnapshotResponse, RunBackupNowData, RunBackupNowResponse, RunForgetData, RunForgetResponse, SetRegistrationStatusData, SetRegistrationStatusResponse, StartDoctorData, StartDoctorResponse, StopBackupData, StopBackupResponse, TagSnapshotsData, TagSnapshotsResponse, TestConnectionData, TestConnectionResponse, TestNotificationDestinationData, TestNotificationDestinationResponse, UnlockRepositoryData, UnlockRepositoryResponse, UnmountVolumeData, UnmountVolumeResponse, UpdateBackupScheduleData, UpdateBackupScheduleResponse, UpdateMemberRoleData, UpdateNotificationDestinationData, UpdateNotificationDestinationResponse, UpdateRepositoryData, UpdateRepositoryResponse, UpdateScheduleMirrorsData, UpdateScheduleMirrorsResponse, UpdateScheduleNotificationsData, UpdateScheduleNotificationsResponse, UpdateSsoProviderAutoLinkingData, UpdateVolumeData, UpdateVolumeResponse } from '../types.gen'; +import { browseFilesystem, cancelDoctor, createBackupSchedule, createNotificationDestination, createRepository, createVolume, deleteBackupSchedule, deleteNotificationDestination, deleteRepository, deleteSnapshot, deleteSnapshots, deleteSsoInvitation, deleteSsoProvider, deleteUserAccount, deleteVolume, downloadResticPassword, dumpSnapshot, getAdminUsers, getBackupProgress, getBackupSchedule, getBackupScheduleForVolume, getDevPanel, getMirrorCompatibility, getMirrorSyncStatus, getNotificationDestination, getOrgMembers, getPublicSsoProviders, getRegistrationStatus, getRepository, getRepositoryStats, getScheduleMirrors, getScheduleNotifications, getSnapshotDetails, getSsoSettings, getStatus, getSystemInfo, getUpdates, getUserDeletionImpact, getVolume, healthCheckVolume, listBackupSchedules, listFiles, listNotificationDestinations, listRcloneRemotes, listRepositories, listSnapshotFiles, listSnapshots, listVolumes, mountVolume, type Options, refreshRepositoryStats, refreshSnapshots, removeOrgMember, reorderBackupSchedules, restoreSnapshot, runBackupNow, runForget, setRegistrationStatus, startDoctor, stopBackup, syncMirror, tagSnapshots, testConnection, testNotificationDestination, unlockRepository, unmountVolume, updateBackupSchedule, updateMemberRole, updateNotificationDestination, updateRepository, updateScheduleMirrors, updateScheduleNotifications, updateSsoProviderAutoLinking, updateVolume } from '../sdk.gen'; +import type { BrowseFilesystemData, BrowseFilesystemResponse, CancelDoctorData, CancelDoctorResponse, CreateBackupScheduleData, CreateBackupScheduleResponse, CreateNotificationDestinationData, CreateNotificationDestinationResponse, CreateRepositoryData, CreateRepositoryResponse, CreateVolumeData, CreateVolumeResponse, DeleteBackupScheduleData, DeleteBackupScheduleResponse, DeleteNotificationDestinationData, DeleteNotificationDestinationResponse, DeleteRepositoryData, DeleteRepositoryResponse, DeleteSnapshotData, DeleteSnapshotResponse, DeleteSnapshotsData, DeleteSnapshotsResponse, DeleteSsoInvitationData, DeleteSsoProviderData, DeleteUserAccountData, DeleteVolumeData, DeleteVolumeResponse, DownloadResticPasswordData, DownloadResticPasswordResponse, DumpSnapshotData, DumpSnapshotResponse, GetAdminUsersData, GetAdminUsersResponse, GetBackupProgressData, GetBackupProgressResponse, GetBackupScheduleData, GetBackupScheduleForVolumeData, GetBackupScheduleForVolumeResponse, GetBackupScheduleResponse, GetDevPanelData, GetDevPanelResponse, GetMirrorCompatibilityData, GetMirrorCompatibilityResponse, GetMirrorSyncStatusData, GetMirrorSyncStatusResponse, GetNotificationDestinationData, GetNotificationDestinationResponse, GetOrgMembersData, GetOrgMembersResponse, GetPublicSsoProvidersData, GetPublicSsoProvidersResponse, GetRegistrationStatusData, GetRegistrationStatusResponse, GetRepositoryData, GetRepositoryResponse, GetRepositoryStatsData, GetRepositoryStatsResponse, GetScheduleMirrorsData, GetScheduleMirrorsResponse, GetScheduleNotificationsData, GetScheduleNotificationsResponse, GetSnapshotDetailsData, GetSnapshotDetailsResponse, GetSsoSettingsData, GetSsoSettingsResponse, GetStatusData, GetStatusResponse, GetSystemInfoData, GetSystemInfoResponse, GetUpdatesData, GetUpdatesResponse, GetUserDeletionImpactData, GetUserDeletionImpactResponse, GetVolumeData, GetVolumeResponse, HealthCheckVolumeData, HealthCheckVolumeResponse, ListBackupSchedulesData, ListBackupSchedulesResponse, ListFilesData, ListFilesResponse, ListNotificationDestinationsData, ListNotificationDestinationsResponse, ListRcloneRemotesData, ListRcloneRemotesResponse, ListRepositoriesData, ListRepositoriesResponse, ListSnapshotFilesData, ListSnapshotFilesResponse, ListSnapshotsData, ListSnapshotsResponse, ListVolumesData, ListVolumesResponse, MountVolumeData, MountVolumeResponse, RefreshRepositoryStatsData, RefreshRepositoryStatsResponse, RefreshSnapshotsData, RefreshSnapshotsResponse, RemoveOrgMemberData, ReorderBackupSchedulesData, ReorderBackupSchedulesResponse, RestoreSnapshotData, RestoreSnapshotResponse, RunBackupNowData, RunBackupNowResponse, RunForgetData, RunForgetResponse, SetRegistrationStatusData, SetRegistrationStatusResponse, StartDoctorData, StartDoctorResponse, StopBackupData, StopBackupResponse, SyncMirrorData, SyncMirrorResponse, TagSnapshotsData, TagSnapshotsResponse, TestConnectionData, TestConnectionResponse, TestNotificationDestinationData, TestNotificationDestinationResponse, UnlockRepositoryData, UnlockRepositoryResponse, UnmountVolumeData, UnmountVolumeResponse, UpdateBackupScheduleData, UpdateBackupScheduleResponse, UpdateMemberRoleData, UpdateNotificationDestinationData, UpdateNotificationDestinationResponse, UpdateRepositoryData, UpdateRepositoryResponse, UpdateScheduleMirrorsData, UpdateScheduleMirrorsResponse, UpdateScheduleNotificationsData, UpdateScheduleNotificationsResponse, UpdateSsoProviderAutoLinkingData, UpdateVolumeData, UpdateVolumeResponse } from '../types.gen'; export type QueryKey = [ Pick & { @@ -1098,6 +1098,41 @@ export const updateScheduleMirrorsMutation = (options?: Partial) => createQueryKey('getMirrorSyncStatus', options); + +/** + * Get sync status for a specific mirror, including missing snapshots + */ +export const getMirrorSyncStatusOptions = (options: Options) => queryOptions>({ + queryFn: async ({ queryKey, signal }) => { + const { data } = await getMirrorSyncStatus({ + ...options, + ...queryKey[0], + signal, + throwOnError: true + }); + return data; + }, + queryKey: getMirrorSyncStatusQueryKey(options) +}); + +/** + * Sync selected snapshots to a specific mirror repository + */ +export const syncMirrorMutation = (options?: Partial>): UseMutationOptions> => { + const mutationOptions: UseMutationOptions> = { + mutationFn: async (fnOptions) => { + const { data } = await syncMirror({ + ...options, + ...fnOptions, + throwOnError: true + }); + return data; + } + }; + return mutationOptions; +}; + export const getMirrorCompatibilityQueryKey = (options: Options) => createQueryKey('getMirrorCompatibility', options); /** diff --git a/app/client/api-client/index.ts b/app/client/api-client/index.ts index 58bbc410..8be7f36f 100644 --- a/app/client/api-client/index.ts +++ b/app/client/api-client/index.ts @@ -26,6 +26,7 @@ export { getBackupScheduleForVolume, getDevPanel, getMirrorCompatibility, + getMirrorSyncStatus, getNotificationDestination, getOrgMembers, getPublicSsoProviders, @@ -62,6 +63,7 @@ export { setRegistrationStatus, startDoctor, stopBackup, + syncMirror, tagSnapshots, testConnection, testNotificationDestination, @@ -153,6 +155,9 @@ export type { GetMirrorCompatibilityData, GetMirrorCompatibilityResponse, GetMirrorCompatibilityResponses, + GetMirrorSyncStatusData, + GetMirrorSyncStatusResponse, + GetMirrorSyncStatusResponses, GetNotificationDestinationData, GetNotificationDestinationErrors, GetNotificationDestinationResponse, @@ -263,6 +268,10 @@ export type { StopBackupErrors, StopBackupResponse, StopBackupResponses, + SyncMirrorData, + SyncMirrorErrors, + SyncMirrorResponse, + SyncMirrorResponses, TagSnapshotsData, TagSnapshotsResponse, TagSnapshotsResponses, diff --git a/app/client/api-client/sdk.gen.ts b/app/client/api-client/sdk.gen.ts index de0e764f..c2aedc4a 100644 --- a/app/client/api-client/sdk.gen.ts +++ b/app/client/api-client/sdk.gen.ts @@ -3,7 +3,7 @@ import type { Client, Options as Options2, TDataShape } from './client'; import { client } from './client.gen'; -import type { BrowseFilesystemData, BrowseFilesystemResponses, CancelDoctorData, CancelDoctorErrors, CancelDoctorResponses, CreateBackupScheduleData, CreateBackupScheduleResponses, CreateNotificationDestinationData, CreateNotificationDestinationResponses, CreateRepositoryData, CreateRepositoryResponses, CreateVolumeData, CreateVolumeResponses, DeleteBackupScheduleData, DeleteBackupScheduleResponses, DeleteNotificationDestinationData, DeleteNotificationDestinationErrors, DeleteNotificationDestinationResponses, DeleteRepositoryData, DeleteRepositoryResponses, DeleteSnapshotData, DeleteSnapshotResponses, DeleteSnapshotsData, DeleteSnapshotsResponses, DeleteSsoInvitationData, DeleteSsoInvitationErrors, DeleteSsoInvitationResponses, DeleteSsoProviderData, DeleteSsoProviderErrors, DeleteSsoProviderResponses, DeleteUserAccountData, DeleteUserAccountErrors, DeleteUserAccountResponses, DeleteVolumeData, DeleteVolumeResponses, DevPanelExecData, DevPanelExecErrors, DevPanelExecResponse, DevPanelExecResponses, DownloadResticPasswordData, DownloadResticPasswordResponses, DumpSnapshotData, DumpSnapshotResponses, GetAdminUsersData, GetAdminUsersResponses, GetBackupProgressData, GetBackupProgressResponses, GetBackupScheduleData, GetBackupScheduleForVolumeData, GetBackupScheduleForVolumeResponses, GetBackupScheduleResponses, GetDevPanelData, GetDevPanelResponses, GetMirrorCompatibilityData, GetMirrorCompatibilityResponses, GetNotificationDestinationData, GetNotificationDestinationErrors, GetNotificationDestinationResponses, GetOrgMembersData, GetOrgMembersResponses, GetPublicSsoProvidersData, GetPublicSsoProvidersResponses, GetRegistrationStatusData, GetRegistrationStatusResponses, GetRepositoryData, GetRepositoryResponses, GetRepositoryStatsData, GetRepositoryStatsResponses, GetScheduleMirrorsData, GetScheduleMirrorsResponses, GetScheduleNotificationsData, GetScheduleNotificationsResponses, GetSnapshotDetailsData, GetSnapshotDetailsResponses, GetSsoSettingsData, GetSsoSettingsResponses, GetStatusData, GetStatusResponses, GetSystemInfoData, GetSystemInfoResponses, GetUpdatesData, GetUpdatesResponses, GetUserDeletionImpactData, GetUserDeletionImpactResponses, GetVolumeData, GetVolumeErrors, GetVolumeResponses, HealthCheckVolumeData, HealthCheckVolumeErrors, HealthCheckVolumeResponses, ListBackupSchedulesData, ListBackupSchedulesResponses, ListFilesData, ListFilesResponses, ListNotificationDestinationsData, ListNotificationDestinationsResponses, ListRcloneRemotesData, ListRcloneRemotesResponses, ListRepositoriesData, ListRepositoriesResponses, ListSnapshotFilesData, ListSnapshotFilesResponses, ListSnapshotsData, ListSnapshotsResponses, ListVolumesData, ListVolumesResponses, MountVolumeData, MountVolumeResponses, RefreshRepositoryStatsData, RefreshRepositoryStatsResponses, RefreshSnapshotsData, RefreshSnapshotsResponses, RemoveOrgMemberData, RemoveOrgMemberErrors, RemoveOrgMemberResponses, ReorderBackupSchedulesData, ReorderBackupSchedulesResponses, RestoreSnapshotData, RestoreSnapshotResponses, RunBackupNowData, RunBackupNowResponses, RunForgetData, RunForgetResponses, SetRegistrationStatusData, SetRegistrationStatusResponses, StartDoctorData, StartDoctorErrors, StartDoctorResponses, StopBackupData, StopBackupErrors, StopBackupResponses, TagSnapshotsData, TagSnapshotsResponses, TestConnectionData, TestConnectionResponses, TestNotificationDestinationData, TestNotificationDestinationErrors, TestNotificationDestinationResponses, UnlockRepositoryData, UnlockRepositoryResponses, UnmountVolumeData, UnmountVolumeResponses, UpdateBackupScheduleData, UpdateBackupScheduleResponses, UpdateMemberRoleData, UpdateMemberRoleErrors, UpdateMemberRoleResponses, UpdateNotificationDestinationData, UpdateNotificationDestinationErrors, UpdateNotificationDestinationResponses, UpdateRepositoryData, UpdateRepositoryErrors, UpdateRepositoryResponses, UpdateScheduleMirrorsData, UpdateScheduleMirrorsResponses, UpdateScheduleNotificationsData, UpdateScheduleNotificationsResponses, UpdateSsoProviderAutoLinkingData, UpdateSsoProviderAutoLinkingErrors, UpdateSsoProviderAutoLinkingResponses, UpdateVolumeData, UpdateVolumeErrors, UpdateVolumeResponses } from './types.gen'; +import type { BrowseFilesystemData, BrowseFilesystemResponses, CancelDoctorData, CancelDoctorErrors, CancelDoctorResponses, CreateBackupScheduleData, CreateBackupScheduleResponses, CreateNotificationDestinationData, CreateNotificationDestinationResponses, CreateRepositoryData, CreateRepositoryResponses, CreateVolumeData, CreateVolumeResponses, DeleteBackupScheduleData, DeleteBackupScheduleResponses, DeleteNotificationDestinationData, DeleteNotificationDestinationErrors, DeleteNotificationDestinationResponses, DeleteRepositoryData, DeleteRepositoryResponses, DeleteSnapshotData, DeleteSnapshotResponses, DeleteSnapshotsData, DeleteSnapshotsResponses, DeleteSsoInvitationData, DeleteSsoInvitationErrors, DeleteSsoInvitationResponses, DeleteSsoProviderData, DeleteSsoProviderErrors, DeleteSsoProviderResponses, DeleteUserAccountData, DeleteUserAccountErrors, DeleteUserAccountResponses, DeleteVolumeData, DeleteVolumeResponses, DevPanelExecData, DevPanelExecErrors, DevPanelExecResponse, DevPanelExecResponses, DownloadResticPasswordData, DownloadResticPasswordResponses, DumpSnapshotData, DumpSnapshotResponses, GetAdminUsersData, GetAdminUsersResponses, GetBackupProgressData, GetBackupProgressResponses, GetBackupScheduleData, GetBackupScheduleForVolumeData, GetBackupScheduleForVolumeResponses, GetBackupScheduleResponses, GetDevPanelData, GetDevPanelResponses, GetMirrorCompatibilityData, GetMirrorCompatibilityResponses, GetMirrorSyncStatusData, GetMirrorSyncStatusResponses, GetNotificationDestinationData, GetNotificationDestinationErrors, GetNotificationDestinationResponses, GetOrgMembersData, GetOrgMembersResponses, GetPublicSsoProvidersData, GetPublicSsoProvidersResponses, GetRegistrationStatusData, GetRegistrationStatusResponses, GetRepositoryData, GetRepositoryResponses, GetRepositoryStatsData, GetRepositoryStatsResponses, GetScheduleMirrorsData, GetScheduleMirrorsResponses, GetScheduleNotificationsData, GetScheduleNotificationsResponses, GetSnapshotDetailsData, GetSnapshotDetailsResponses, GetSsoSettingsData, GetSsoSettingsResponses, GetStatusData, GetStatusResponses, GetSystemInfoData, GetSystemInfoResponses, GetUpdatesData, GetUpdatesResponses, GetUserDeletionImpactData, GetUserDeletionImpactResponses, GetVolumeData, GetVolumeErrors, GetVolumeResponses, HealthCheckVolumeData, HealthCheckVolumeErrors, HealthCheckVolumeResponses, ListBackupSchedulesData, ListBackupSchedulesResponses, ListFilesData, ListFilesResponses, ListNotificationDestinationsData, ListNotificationDestinationsResponses, ListRcloneRemotesData, ListRcloneRemotesResponses, ListRepositoriesData, ListRepositoriesResponses, ListSnapshotFilesData, ListSnapshotFilesResponses, ListSnapshotsData, ListSnapshotsResponses, ListVolumesData, ListVolumesResponses, MountVolumeData, MountVolumeResponses, RefreshRepositoryStatsData, RefreshRepositoryStatsResponses, RefreshSnapshotsData, RefreshSnapshotsResponses, RemoveOrgMemberData, RemoveOrgMemberErrors, RemoveOrgMemberResponses, ReorderBackupSchedulesData, ReorderBackupSchedulesResponses, RestoreSnapshotData, RestoreSnapshotResponses, RunBackupNowData, RunBackupNowResponses, RunForgetData, RunForgetResponses, SetRegistrationStatusData, SetRegistrationStatusResponses, StartDoctorData, StartDoctorErrors, StartDoctorResponses, StopBackupData, StopBackupErrors, StopBackupResponses, SyncMirrorData, SyncMirrorErrors, SyncMirrorResponses, TagSnapshotsData, TagSnapshotsResponses, TestConnectionData, TestConnectionResponses, TestNotificationDestinationData, TestNotificationDestinationErrors, TestNotificationDestinationResponses, UnlockRepositoryData, UnlockRepositoryResponses, UnmountVolumeData, UnmountVolumeResponses, UpdateBackupScheduleData, UpdateBackupScheduleResponses, UpdateMemberRoleData, UpdateMemberRoleErrors, UpdateMemberRoleResponses, UpdateNotificationDestinationData, UpdateNotificationDestinationErrors, UpdateNotificationDestinationResponses, UpdateRepositoryData, UpdateRepositoryErrors, UpdateRepositoryResponses, UpdateScheduleMirrorsData, UpdateScheduleMirrorsResponses, UpdateScheduleNotificationsData, UpdateScheduleNotificationsResponses, UpdateSsoProviderAutoLinkingData, UpdateSsoProviderAutoLinkingErrors, UpdateSsoProviderAutoLinkingResponses, UpdateVolumeData, UpdateVolumeErrors, UpdateVolumeResponses } from './types.gen'; export type Options = Options2 & { /** @@ -409,6 +409,23 @@ export const updateScheduleMirrors = (opti } }); +/** + * Get sync status for a specific mirror, including missing snapshots + */ +export const getMirrorSyncStatus = (options: Options) => (options.client ?? client).get({ url: '/api/v1/backups/{shortId}/mirrors/{mirrorShortId}/status', ...options }); + +/** + * Sync selected snapshots to a specific mirror repository + */ +export const syncMirror = (options: Options) => (options.client ?? client).post({ + url: '/api/v1/backups/{shortId}/mirrors/{mirrorShortId}/sync', + ...options, + headers: { + 'Content-Type': 'application/json', + ...options.headers + } +}); + /** * Get mirror compatibility info for all repositories relative to a backup schedule's primary repository */ diff --git a/app/client/api-client/types.gen.ts b/app/client/api-client/types.gen.ts index 3d7be795..ddf355f3 100644 --- a/app/client/api-client/types.gen.ts +++ b/app/client/api-client/types.gen.ts @@ -4114,6 +4114,63 @@ export type UpdateScheduleMirrorsResponses = { export type UpdateScheduleMirrorsResponse = UpdateScheduleMirrorsResponses[keyof UpdateScheduleMirrorsResponses]; +export type GetMirrorSyncStatusData = { + body?: never; + path: { + shortId: string; + mirrorShortId: string; + }; + query?: never; + url: '/api/v1/backups/{shortId}/mirrors/{mirrorShortId}/status'; +}; + +export type GetMirrorSyncStatusResponses = { + /** + * Mirror sync status with missing snapshots + */ + 200: { + sourceCount: number; + mirrorCount: number; + missingSnapshots: Array<{ + short_id: string; + time: string; + size: number; + }>; + }; +}; + +export type GetMirrorSyncStatusResponse = GetMirrorSyncStatusResponses[keyof GetMirrorSyncStatusResponses]; + +export type SyncMirrorData = { + body: { + snapshotIds?: Array; + }; + path: { + shortId: string; + mirrorShortId: string; + }; + query?: never; + url: '/api/v1/backups/{shortId}/mirrors/{mirrorShortId}/sync'; +}; + +export type SyncMirrorErrors = { + /** + * Mirror is already syncing + */ + 409: unknown; +}; + +export type SyncMirrorResponses = { + /** + * Mirror sync started successfully + */ + 200: { + success: boolean; + }; +}; + +export type SyncMirrorResponse = SyncMirrorResponses[keyof SyncMirrorResponses]; + export type GetMirrorCompatibilityData = { body?: never; path: { diff --git a/app/client/modules/backups/components/schedule-mirrors-config.tsx b/app/client/modules/backups/components/schedule-mirrors-config.tsx index 5800ae78..f4c82653 100644 --- a/app/client/modules/backups/components/schedule-mirrors-config.tsx +++ b/app/client/modules/backups/components/schedule-mirrors-config.tsx @@ -1,5 +1,5 @@ import { useMutation, useQuery, useSuspenseQuery } from "@tanstack/react-query"; -import { Copy, Plus, Trash2 } from "lucide-react"; +import { Copy, Plus, RefreshCw, Trash2 } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { toast } from "sonner"; import { Button } from "~/client/components/ui/button"; @@ -8,21 +8,33 @@ import { Switch } from "~/client/components/ui/switch"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "~/client/components/ui/select"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "~/client/components/ui/table"; import { Badge } from "~/client/components/ui/badge"; +import { Checkbox } from "~/client/components/ui/checkbox"; import { Tooltip, TooltipContent, TooltipTrigger } from "~/client/components/ui/tooltip"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "~/client/components/ui/dialog"; import { getScheduleMirrorsOptions, getMirrorCompatibilityOptions, + getMirrorSyncStatusOptions, updateScheduleMirrorsMutation, + syncMirrorMutation, } from "~/client/api-client/@tanstack/react-query.gen"; import { parseError } from "~/client/lib/errors"; import type { Repository } from "~/client/lib/types"; import { RepositoryIcon } from "~/client/components/repository-icon"; import { StatusDot } from "~/client/components/status-dot"; +import { ByteSize } from "~/client/components/bytes-size"; import { useServerEvents } from "~/client/hooks/use-server-events"; -import { formatDistanceToNow } from "date-fns"; import { cn } from "~/client/lib/utils"; import type { GetScheduleMirrorsResponse } from "~/client/api-client"; import { Link } from "@tanstack/react-router"; +import { useTimeFormat } from "~/client/lib/datetime"; type Props = { scheduleShortId: string; @@ -56,11 +68,29 @@ const buildAssignments = (mirrors: GetScheduleMirrorsResponse) => ); export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, repositories, initialData }: Props) => { + const { formatDateTime, formatTimeAgo } = useTimeFormat(); const [assignments, setAssignments] = useState>(() => buildAssignments(initialData)); const [hasChanges, setHasChanges] = useState(false); const [isAddingNew, setIsAddingNew] = useState(false); + const [syncDialogMirrorId, setSyncDialogMirrorId] = useState(null); + const [selectedSnapshotIds, setSelectedSnapshotIds] = useState>(new Set()); + const [syncDialogOpen, setSyncDialogOpen] = useState(false); const { addEventListener } = useServerEvents(); + const closeSyncDialog = () => { + setSyncDialogOpen(false); + setTimeout(() => { + setSyncDialogMirrorId(null); + setSelectedSnapshotIds(new Set()); + }, 300); + }; + + const openSyncDialog = (mirrorShortId: string) => { + setSyncDialogOpen(true); + setSelectedSnapshotIds(new Set()); + setSyncDialogMirrorId(mirrorShortId); + }; + const { data: currentMirrors } = useSuspenseQuery({ ...getScheduleMirrorsOptions({ path: { shortId: scheduleShortId } }), }); @@ -88,6 +118,47 @@ export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, re }, }); + const { data: syncStatus, isLoading: isSyncStatusLoading } = useQuery({ + ...getMirrorSyncStatusOptions({ + path: { shortId: scheduleShortId, mirrorShortId: syncDialogMirrorId ?? "" }, + }), + enabled: syncDialogMirrorId !== null, + }); + + const toggleSnapshotSelection = (shortId: string) => { + setSelectedSnapshotIds((prev) => { + const next = new Set(prev); + if (next.has(shortId)) { + next.delete(shortId); + } else { + next.add(shortId); + } + return next; + }); + }; + + const toggleAllSnapshots = () => { + if (!syncStatus) return; + if (selectedSnapshotIds.size === syncStatus.missingSnapshots.length) { + setSelectedSnapshotIds(new Set()); + } else { + setSelectedSnapshotIds(new Set(syncStatus.missingSnapshots.map((s) => s.short_id))); + } + }; + + const triggerSync = useMutation({ + ...syncMirrorMutation(), + onSuccess: () => { + toast.success("Full sync started"); + closeSyncDialog(); + }, + onError: (error) => { + toast.error("Failed to start sync", { + description: parseError(error)?.message, + }); + }, + }); + const compatibilityMap = useMemo(() => { const map = new Map(); if (compatibility) { @@ -221,7 +292,7 @@ export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, re return "Syncing..."; } if (assignment.lastCopyAt) { - return formatDistanceToNow(new Date(assignment.lastCopyAt), { addSuffix: true }); + return formatTimeAgo(assignment.lastCopyAt); } return "Never"; }; @@ -239,6 +310,8 @@ export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, re return "Never copied"; }; + const selectedMirrorRepo = repositories.find((r) => r.shortId === syncDialogMirrorId); + return ( @@ -369,14 +442,30 @@ export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, re - +
+ + + + + Sync more snapshots + + +
); @@ -396,6 +485,89 @@ export const ScheduleMirrorsConfig = ({ scheduleShortId, primaryRepositoryId, re )} + + !open && closeSyncDialog()}> + + + Sync snapshots + + {`Sync missing snapshots to ${selectedMirrorRepo?.name || "mirror repository"}.`} + + + + {isSyncStatusLoading && !syncStatus ? ( +
Loading snapshot status...
+ ) : syncStatus && syncStatus.missingSnapshots.length === 0 ? ( +
+ All {syncStatus.sourceCount} snapshots are already synced to this mirror. +
+ ) : syncStatus ? ( +
+

+ {syncStatus.missingSnapshots.length} of {syncStatus.sourceCount} snapshots are missing in this mirror. +

+
+ + + + + + + ID + Date + Size + + + + {syncStatus.missingSnapshots.map((snapshot) => ( + toggleSnapshotSelection(snapshot.short_id)} + > + e.stopPropagation()}> + toggleSnapshotSelection(snapshot.short_id)} + /> + + {snapshot.short_id} + {formatDateTime(new Date(snapshot.time))} + + + + + ))} + +
+
+
+ ) : null} + + + + + + + ); diff --git a/app/server/modules/backups/__tests__/backups.controller.test.ts b/app/server/modules/backups/__tests__/backups.controller.test.ts index 01f69427..c102b829 100644 --- a/app/server/modules/backups/__tests__/backups.controller.test.ts +++ b/app/server/modules/backups/__tests__/backups.controller.test.ts @@ -54,6 +54,8 @@ describe("backups security", () => { { method: "GET", path: "/api/v1/backups/1/mirrors" }, { method: "PUT", path: "/api/v1/backups/1/mirrors" }, { method: "GET", path: "/api/v1/backups/1/mirrors/compatibility" }, + { method: "GET", path: "/api/v1/backups/1/mirrors/abc/status" }, + { method: "POST", path: "/api/v1/backups/1/mirrors/abc/sync" }, { method: "POST", path: "/api/v1/backups/reorder" }, { method: "GET", path: "/api/v1/backups/1/progress" }, ]; diff --git a/app/server/modules/backups/__tests__/backups.mirror-sync.test.ts b/app/server/modules/backups/__tests__/backups.mirror-sync.test.ts new file mode 100644 index 00000000..c81cf4fd --- /dev/null +++ b/app/server/modules/backups/__tests__/backups.mirror-sync.test.ts @@ -0,0 +1,248 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import waitForExpect from "wait-for-expect"; +import { backupsService } from "../backups.service"; +import { createTestVolume } from "~/test/helpers/volume"; +import { createTestBackupSchedule } from "~/test/helpers/backup"; +import { createTestRepository } from "~/test/helpers/repository"; +import { createTestBackupScheduleMirror } from "~/test/helpers/backup-mirror"; +import { TEST_ORG_ID } from "~/test/helpers/organization"; +import * as context from "~/server/core/request-context"; +import * as resticModule from "~/server/core/restic"; +import * as spawnModule from "@zerobyte/core/node"; +import type { ShortId } from "~/server/utils/branded"; + +const setup = () => { + vi.spyOn(context, "getOrganizationId").mockReturnValue(TEST_ORG_ID); + vi.spyOn(spawnModule, "safeSpawn").mockImplementation(() => Promise.resolve({ exitCode: 0, summary: "", error: "" })); + + return { + mockSnapshots: (sourceSnapshots: unknown[], mirrorSnapshots: unknown[]) => { + let callCount = 0; + vi.spyOn(resticModule.restic, "snapshots").mockImplementation(() => { + callCount++; + if (callCount === 1) return Promise.resolve(sourceSnapshots as never); + return Promise.resolve(mirrorSnapshots as never); + }); + }, + mockCopy: () => { + const copyMock = vi + .spyOn(resticModule.restic, "copy") + .mockImplementation(() => Promise.resolve({ success: true, output: "" })); + return copyMock; + }, + }; +}; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("getMirrorSyncStatus", () => { + test("should return missing snapshots based on time comparison", async () => { + const { mockSnapshots } = setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const mirrorRepository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id); + + mockSnapshots( + [ + { + id: "aaa", + short_id: "aaa", + time: "2025-01-01T10:00:00Z", + paths: ["/data"], + summary: { total_bytes_processed: 100 }, + }, + { + id: "bbb", + short_id: "bbb", + time: "2025-01-02T10:00:00Z", + paths: ["/data"], + summary: { total_bytes_processed: 200 }, + }, + { + id: "ccc", + short_id: "ccc", + time: "2025-01-03T10:00:00Z", + paths: ["/data"], + summary: { total_bytes_processed: 300 }, + }, + ], + [ + { + id: "xxx", + short_id: "xxx", + time: "2025-01-01T10:00:00Z", + paths: ["/data"], + summary: { total_bytes_processed: 100 }, + }, + ], + ); + + const status = await backupsService.getMirrorSyncStatus(schedule.shortId, mirrorRepository.shortId as ShortId); + + expect(status.sourceCount).toBe(3); + expect(status.mirrorCount).toBe(1); + expect(status.missingSnapshots).toHaveLength(2); + expect(status.missingSnapshots.map((s) => s.short_id)).toEqual(["bbb", "ccc"]); + }); + + test("should return empty missing list when all snapshots are synced", async () => { + const { mockSnapshots } = setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const mirrorRepository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id); + + mockSnapshots( + [ + { + id: "aaa", + short_id: "aaa", + time: "2025-01-01T10:00:00Z", + paths: ["/data"], + summary: { total_bytes_processed: 100 }, + }, + ], + [ + { + id: "xxx", + short_id: "xxx", + time: "2025-01-01T10:00:00Z", + paths: ["/data"], + summary: { total_bytes_processed: 100 }, + }, + ], + ); + + const status = await backupsService.getMirrorSyncStatus(schedule.shortId, mirrorRepository.shortId as ShortId); + + expect(status.sourceCount).toBe(1); + expect(status.mirrorCount).toBe(1); + expect(status.missingSnapshots).toHaveLength(0); + }); + + test("should throw if mirror is not configured for the schedule", async () => { + setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const unrelatedRepository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + + await expect( + backupsService.getMirrorSyncStatus(schedule.shortId, unrelatedRepository.shortId as ShortId), + ).rejects.toThrow("Mirror not found for this schedule"); + }); +}); + +describe("syncMirror", () => { + test("should trigger sync and return success", async () => { + setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const mirrorRepository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id); + + const result = await backupsService.syncMirror(schedule.shortId, mirrorRepository.shortId as ShortId, [ + "snap1", + "snap2", + ]); + + expect(result.success).toBe(true); + }); + + test("should reject if mirror is already syncing", async () => { + setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const mirrorRepository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id, { + lastCopyStatus: "in_progress", + }); + + await expect( + backupsService.syncMirror(schedule.shortId, mirrorRepository.shortId as ShortId, ["snap1"]), + ).rejects.toThrow("Mirror is already syncing"); + }); + + test("should reject concurrent sync requests once a sync has started", async () => { + const { mockCopy } = setup(); + const copyMock = mockCopy(); + let releaseCopy: (() => void) | undefined; + const copyStarted = new Promise((resolve) => { + copyMock.mockImplementation( + () => + new Promise((copyResolve) => { + releaseCopy = () => copyResolve({ success: true, output: "" }); + resolve(); + }), + ); + }); + + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const mirrorRepository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id); + + await expect( + backupsService.syncMirror(schedule.shortId, mirrorRepository.shortId as ShortId, ["snap1"]), + ).resolves.toEqual({ success: true }); + + await copyStarted; + + await waitForExpect(async () => { + const mirrors = await backupsService.getMirrors(schedule.shortId); + expect(mirrors[0]?.lastCopyStatus).toBe("in_progress"); + }); + + await expect( + backupsService.syncMirror(schedule.shortId, mirrorRepository.shortId as ShortId, ["snap1"]), + ).rejects.toThrow("Mirror is already syncing"); + + releaseCopy?.(); + + await waitForExpect(async () => { + const mirrors = await backupsService.getMirrors(schedule.shortId); + expect(mirrors[0]?.lastCopyStatus).toBe("success"); + }); + }); + + test("should throw if mirror is not configured for the schedule", async () => { + setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const unrelatedRepository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + + await expect( + backupsService.syncMirror(schedule.shortId, unrelatedRepository.shortId as ShortId, ["snap1"]), + ).rejects.toThrow("Mirror not found for this schedule"); + }); +}); diff --git a/app/server/modules/backups/backups.controller.ts b/app/server/modules/backups/backups.controller.ts index ffcf2bae..f7a31a71 100644 --- a/app/server/modules/backups/backups.controller.ts +++ b/app/server/modules/backups/backups.controller.ts @@ -5,8 +5,10 @@ import { createBackupScheduleDto, deleteBackupScheduleDto, getBackupScheduleDto, + getBackupScheduleResponse, getBackupScheduleForVolumeDto, listBackupSchedulesDto, + listBackupSchedulesResponse, runBackupNowDto, runForgetDto, stopBackupDto, @@ -16,9 +18,12 @@ import { updateScheduleMirrorsDto, updateScheduleMirrorsBody, getMirrorCompatibilityDto, + getMirrorSyncStatusDto, reorderBackupSchedulesDto, reorderBackupSchedulesBody, getBackupProgressDto, + syncMirrorBody, + syncMirrorDto, type CreateBackupScheduleDto, type DeleteBackupScheduleDto, type GetBackupScheduleDto, @@ -33,8 +38,8 @@ import { type GetMirrorCompatibilityDto, type ReorderBackupSchedulesDto, type GetBackupProgressDto, - listBackupSchedulesResponse, - getBackupScheduleResponse, + type GetMirrorSyncStatusDto, + type SyncMirrorDto, } from "./backups.dto"; import { backupsService } from "./backups.service"; import { @@ -155,6 +160,21 @@ export const backupScheduleController = new Hono() return c.json(mirrors, 200); }) + .get("/:shortId/mirrors/:mirrorShortId/status", getMirrorSyncStatusDto, async (c) => { + const shortId = asShortId(c.req.param("shortId")); + const mirrorShortId = asShortId(c.req.param("mirrorShortId")); + const status = await backupsService.getMirrorSyncStatus(shortId, mirrorShortId); + + return c.json(status, 200); + }) + .post("/:shortId/mirrors/:mirrorShortId/sync", syncMirrorDto, validator("json", syncMirrorBody), async (c) => { + const shortId = asShortId(c.req.param("shortId")); + const mirrorShortId = asShortId(c.req.param("mirrorShortId")); + const body = c.req.valid("json"); + const result = await backupsService.syncMirror(shortId, mirrorShortId, body.snapshotIds); + + return c.json(result, 200); + }) .get("/:shortId/mirrors/compatibility", getMirrorCompatibilityDto, async (c) => { const shortId = asShortId(c.req.param("shortId")); const compatibility = await backupsService.getMirrorCompatibility(shortId); diff --git a/app/server/modules/backups/backups.dto.ts b/app/server/modules/backups/backups.dto.ts index 7b1c7c9c..555c3e48 100644 --- a/app/server/modules/backups/backups.dto.ts +++ b/app/server/modules/backups/backups.dto.ts @@ -400,6 +400,65 @@ export const reorderBackupSchedulesDto = describeRoute({ }, }); +const missingSnapshotSchema = z.object({ + short_id: z.string(), + time: z.string(), + size: z.number(), +}); + +export const getMirrorSyncStatusResponse = z.object({ + sourceCount: z.number(), + mirrorCount: z.number(), + missingSnapshots: missingSnapshotSchema.array(), +}); +export type GetMirrorSyncStatusDto = z.infer; + +export const getMirrorSyncStatusDto = describeRoute({ + description: "Get sync status for a specific mirror, including missing snapshots", + operationId: "getMirrorSyncStatus", + tags: ["Backups"], + responses: { + 200: { + description: "Mirror sync status with missing snapshots", + content: { + "application/json": { + schema: resolver(getMirrorSyncStatusResponse), + }, + }, + }, + }, +}); + +export const syncMirrorBody = z.object({ + snapshotIds: z.array(z.string()).optional(), +}); + +export type SyncMirrorBody = z.infer; + +export const syncMirrorResponse = z.object({ + success: z.boolean(), +}); +export type SyncMirrorDto = z.infer; + +export const syncMirrorDto = describeRoute({ + description: "Sync selected snapshots to a specific mirror repository", + operationId: "syncMirror", + tags: ["Backups"], + responses: { + 200: { + description: "Mirror sync started successfully", + content: { + "application/json": { + schema: resolver(syncMirrorResponse), + }, + }, + }, + 409: { + description: "Mirror is already syncing", + }, + }, +}); + const getBackupProgressResponse = backupProgressEventSchema.nullable(); export type GetBackupProgressDto = z.infer; diff --git a/app/server/modules/backups/backups.queries.ts b/app/server/modules/backups/backups.queries.ts index 1d9ef3f2..37d411db 100644 --- a/app/server/modules/backups/backups.queries.ts +++ b/app/server/modules/backups/backups.queries.ts @@ -60,6 +60,13 @@ export const mirrorQueries = { }); }, + findByScheduleAndRepository: async (scheduleId: number, repositoryId: string) => { + return db.query.backupScheduleMirrorsTable.findFirst({ + where: { scheduleId, repositoryId }, + with: { repository: true }, + }); + }, + updateStatus: async (scheduleId: number, repositoryId: string, status: MirrorStatusUpdate) => { return db .update(backupScheduleMirrorsTable) diff --git a/app/server/modules/backups/backups.service.ts b/app/server/modules/backups/backups.service.ts index d39479f0..04b15896 100644 --- a/app/server/modules/backups/backups.service.ts +++ b/app/server/modules/backups/backups.service.ts @@ -25,7 +25,10 @@ import { validateBackupExecution, } from "./helpers/backup-lifecycle"; import { getScheduleByIdOrShortId } from "./helpers/backup-schedule-lookups"; -import { copyToMirrors, runForget } from "./helpers/backup-maintenance"; +import { copyToMirrors, runForget, syncSnapshotsToMirror } from "./helpers/backup-maintenance"; +import { restic } from "../../core/restic"; +import { mirrorQueries } from "./backups.queries"; +import { toMessage } from "../../utils/errors"; const listSchedules = async () => { const organizationId = getOrganizationId(); @@ -469,6 +472,79 @@ const stopBackup = async (scheduleId: number) => { } }; +const getMirrorSyncStatus = async (scheduleIdOrShortId: number | string, mirrorShortId: ShortId) => { + const organizationId = getOrganizationId(); + const schedule = await getScheduleByIdOrShortId(scheduleIdOrShortId); + + const mirrorRepo = await db.query.repositoriesTable.findFirst({ + where: { + AND: [{ shortId: { eq: mirrorShortId } }, { organizationId }], + }, + }); + + if (!mirrorRepo) { + throw new NotFoundError("Mirror repository not found"); + } + + const mirror = await mirrorQueries.findByScheduleAndRepository(schedule.id, mirrorRepo.id); + + if (!mirror) { + throw new NotFoundError("Mirror not found for this schedule"); + } + + const [sourceSnapshots, mirrorSnapshots] = await Promise.all([ + restic.snapshots(schedule.repository.config, { tags: [schedule.shortId], organizationId }), + restic.snapshots(mirrorRepo.config, { tags: [schedule.shortId], organizationId }), + ]); + + const mirrorSnapshotTimes = new Set(mirrorSnapshots.map((s) => s.time)); + + const missingSnapshots = sourceSnapshots + .filter((s) => !mirrorSnapshotTimes.has(s.time)) + .map((s) => ({ + short_id: s.short_id, + time: s.time, + size: s.summary?.total_bytes_processed ?? 0, + })); + + return { + sourceCount: sourceSnapshots.length, + mirrorCount: mirrorSnapshots.length, + missingSnapshots, + }; +}; + +const syncMirror = async (scheduleIdOrShortId: number | string, mirrorShortId: ShortId, snapshotIds?: string[]) => { + const organizationId = getOrganizationId(); + const schedule = await getScheduleByIdOrShortId(scheduleIdOrShortId); + + const mirrorRepo = await db.query.repositoriesTable.findFirst({ + where: { + AND: [{ shortId: { eq: mirrorShortId } }, { organizationId }], + }, + }); + + if (!mirrorRepo) { + throw new NotFoundError("Mirror repository not found"); + } + + const mirror = await mirrorQueries.findByScheduleAndRepository(schedule.id, mirrorRepo.id); + + if (!mirror) { + throw new NotFoundError("Mirror not found for this schedule"); + } + + if (mirror.lastCopyStatus === "in_progress") { + throw new ConflictError("Mirror is already syncing"); + } + + syncSnapshotsToMirror(schedule.id, mirrorRepo.id, organizationId, snapshotIds).catch((error) => { + logger.error(`Error syncing all snapshots to mirror ${mirrorRepo.name}: ${toMessage(error)}`); + }); + + return { success: true }; +}; + export const backupsService = { listSchedules, createSchedule, @@ -487,4 +563,6 @@ export const backupsService = { stopBackup, runForget, copyToMirrors, + getMirrorSyncStatus, + syncMirror, }; diff --git a/app/server/modules/backups/helpers/backup-maintenance.ts b/app/server/modules/backups/helpers/backup-maintenance.ts index 18054eb7..0b4ccd6f 100644 --- a/app/server/modules/backups/helpers/backup-maintenance.ts +++ b/app/server/modules/backups/helpers/backup-maintenance.ts @@ -66,6 +66,114 @@ export async function copyToMirrors( } } +export async function syncSnapshotsToMirror( + scheduleId: number, + mirrorRepositoryId: string, + organizationIdOverride?: string, + snapshotIds?: string[], +) { + const organizationId = organizationIdOverride ?? getOrganizationId(); + const schedule = await scheduleQueries.findById(scheduleId, organizationId); + + if (!schedule) { + throw new NotFoundError("Backup schedule not found"); + } + + const mirror = await mirrorQueries.findByScheduleAndRepository(scheduleId, mirrorRepositoryId); + + if (!mirror) { + throw new NotFoundError("Mirror not found for this schedule"); + } + + const sourceRepository = await repositoryQueries.findById(schedule.repositoryId, organizationId); + + if (!sourceRepository) { + throw new NotFoundError("Source repository not found"); + } + + const mirrorRepository = await repositoryQueries.findById(mirrorRepositoryId, organizationId); + + if (!mirrorRepository) { + throw new NotFoundError("Mirror repository not found"); + } + + try { + logger.info(`[Background] Syncing all snapshots to mirror repository: ${mirrorRepository.name}`); + + serverEvents.emit("mirror:started", { + organizationId, + scheduleId: schedule.shortId, + repositoryId: mirrorRepository.shortId, + repositoryName: mirrorRepository.name, + }); + + await mirrorQueries.updateStatus(scheduleId, mirrorRepositoryId, { + lastCopyStatus: "in_progress", + lastCopyError: null, + }); + + const releaseLocks = await repoMutex.acquireMany([ + { repositoryId: sourceRepository.id, type: "shared", operation: `mirror_sync_source:${scheduleId}` }, + { repositoryId: mirrorRepository.id, type: "exclusive", operation: `mirror_sync:${scheduleId}` }, + ]); + + try { + await restic.copy(sourceRepository.config, mirrorRepository.config, { + tag: schedule.shortId, + organizationId, + snapshotIds, + }); + cache.delByPrefix(cacheKeys.repository.all(mirrorRepository.id)); + } finally { + releaseLocks(); + } + + if (schedule.retentionPolicy) { + void runForget(scheduleId, mirrorRepository.id, organizationId).catch((error) => { + logger.error( + `Failed to run retention policy for mirror repository ${mirrorRepository.name}: ${toMessage(error)}`, + ); + }); + } + + await mirrorQueries.updateStatus(scheduleId, mirrorRepositoryId, { + lastCopyAt: Date.now(), + lastCopyStatus: "success", + lastCopyError: null, + }); + + logger.info(`[Background] Successfully synced all snapshots to mirror repository: ${mirrorRepository.name}`); + + serverEvents.emit("mirror:completed", { + organizationId, + scheduleId: schedule.shortId, + repositoryId: mirrorRepository.shortId, + repositoryName: mirrorRepository.name, + status: "success", + }); + } catch (error) { + const errorMessage = toMessage(error); + logger.error( + `[Background] Failed to sync all snapshots to mirror repository ${mirrorRepository.name}: ${errorMessage}`, + ); + + await mirrorQueries.updateStatus(scheduleId, mirrorRepositoryId, { + lastCopyAt: Date.now(), + lastCopyStatus: "error", + lastCopyError: errorMessage, + }); + + serverEvents.emit("mirror:completed", { + organizationId, + scheduleId: schedule.shortId, + repositoryId: mirrorRepository.shortId, + repositoryName: mirrorRepository.name, + status: "error", + error: errorMessage, + }); + } +} + async function copyToSingleMirror( scheduleId: number, schedule: BackupSchedule, diff --git a/e2e/0004-mirror-sync.spec.ts b/e2e/0004-mirror-sync.spec.ts new file mode 100644 index 00000000..57c53808 --- /dev/null +++ b/e2e/0004-mirror-sync.spec.ts @@ -0,0 +1,139 @@ +import fs from "node:fs"; +import { randomUUID } from "node:crypto"; +import path from "node:path"; +import { type Page, type TestInfo } from "@playwright/test"; +import { expect, test } from "./test"; +import { gotoAndWaitForAppReady } from "./helpers/page"; + +const testDataPath = path.join(process.cwd(), "playwright", "temp"); + +function getRunId(testInfo: TestInfo) { + return `${testInfo.parallelIndex}-${testInfo.retry}-${randomUUID().slice(0, 8)}`; +} + +function getWorkerTestDataPath() { + fs.mkdirSync(testDataPath, { recursive: true }); + return testDataPath; +} + +async function createRepository(page: Page, name: string) { + await gotoAndWaitForAppReady(page, "/repositories"); + await page.getByRole("button", { name: "Create repository" }).click(); + await page.getByRole("textbox", { name: "Name" }).fill(name); + await page.getByRole("combobox", { name: "Backend" }).click(); + await page.getByRole("option", { name: "Local" }).click(); + await page.getByRole("button", { name: "Create repository" }).click(); + await expect(page.getByText("Repository created successfully")).toBeVisible({ timeout: 30000 }); +} + +async function createVolume(page: Page, name: string) { + await gotoAndWaitForAppReady(page, "/volumes"); + const volumeNameInput = page.getByRole("textbox", { name: "Name" }); + await expect(async () => { + await page.getByRole("button", { name: "Create Volume" }).click(); + await expect(volumeNameInput).toBeVisible(); + }).toPass({ timeout: 10000 }); + await volumeNameInput.fill(name); + await page.getByRole("button", { name: "test-data" }).click(); + await page.getByRole("button", { name: "Create Volume" }).click(); + await expect(page.getByText("Volume created successfully")).toBeVisible(); +} + +async function createBackupJob(page: Page, backupName: string, volumeName: string, repositoryName: string) { + await gotoAndWaitForAppReady(page, "/backups"); + const createBackupButton = page.getByRole("button", { name: "Create a backup job" }).first(); + if (await createBackupButton.isVisible()) { + await createBackupButton.click(); + } else { + await page.getByRole("link", { name: "Create a backup job" }).first().click(); + } + const volumeSelect = page.getByRole("combobox").filter({ hasText: "Choose a volume to backup" }); + const volumeOption = page.getByRole("option", { name: volumeName }); + await expect(async () => { + await volumeSelect.click(); + await expect(volumeOption).toBeVisible(); + }).toPass({ timeout: 10000 }); + await volumeOption.click(); + await page.getByRole("textbox", { name: "Backup name" }).fill(backupName); + await page.getByRole("combobox").filter({ hasText: "Select a repository" }).click(); + await page.getByRole("option", { name: repositoryName }).click(); + await page.getByRole("combobox").filter({ hasText: "Select frequency" }).click(); + await page.getByRole("option", { name: "Daily" }).click(); + await page.getByRole("textbox", { name: "Execution time" }).fill("00:00"); + await page.getByRole("button", { name: "Create" }).click(); + await expect(page.getByText("Backup job created successfully")).toBeVisible(); +} + +test("can sync missing snapshots to a mirror repository", async ({ page }, testInfo) => { + const runId = getRunId(testInfo); + const volumeName = `Volume-${runId}`; + const primaryRepoName = `Primary-${runId}`; + const mirrorRepoName = `Mirror-${runId}`; + const backupName = `Backup-${runId}`; + + const workerTestDataPath = getWorkerTestDataPath(); + const runPath = path.join(workerTestDataPath, runId); + fs.mkdirSync(runPath, { recursive: true }); + fs.writeFileSync(path.join(runPath, "test.json"), JSON.stringify({ data: "mirror sync test" })); + + await gotoAndWaitForAppReady(page, "/"); + await expect(page).toHaveURL("/volumes"); + + // Create volume, two repositories, and a backup job + await createVolume(page, volumeName); + await createRepository(page, primaryRepoName); + await createRepository(page, mirrorRepoName); + await createBackupJob(page, backupName, volumeName, primaryRepoName); + + // Run a backup to create a snapshot + await page.getByRole("button", { name: "Backup now" }).click(); + await expect(page.getByText("Backup started successfully")).toBeVisible(); + await expect(page.getByText(/Success|Warning/).first()).toBeVisible({ timeout: 30000 }); + + // Add mirror repository + await page.getByRole("button", { name: "Add mirror" }).click(); + const mirrorSelect = page.getByRole("combobox").filter({ hasText: "Select a repository to mirror to..." }); + await mirrorSelect.click(); + await page.getByRole("option", { name: mirrorRepoName }).click(); + await page.getByRole("button", { name: "Save changes" }).click(); + await expect(page.getByText("Mirror settings saved successfully")).toBeVisible(); + + // Click sync button on the mirror row (first icon button in the actions cell) + const mirrorRow = page.getByRole("row").filter({ hasText: mirrorRepoName }); + await mirrorRow.getByRole("button").first().click(); + + // Verify the sync dialog shows missing snapshots + await expect(page.getByRole("heading", { name: "Sync snapshots" })).toBeVisible(); + await expect(page.getByText(/1 of 1 snapshots are missing/)).toBeVisible({ timeout: 15000 }); + + // Verify there is a checkbox and a snapshot row + const snapshotCheckbox = page.getByRole("dialog").getByRole("checkbox").first(); + await expect(snapshotCheckbox).toBeChecked(); + + // Click sync button + await page.getByRole("button", { name: "Sync 1 snapshots" }).click(); + await expect(page.getByText("Full sync started")).toBeVisible(); + + // Wait for sync to complete + await expect(page.getByText("Syncing...")).toBeVisible({ timeout: 10000 }); + await expect(page.getByText("Syncing...")).not.toBeVisible({ timeout: 30000 }); + + // Open sync dialog again and verify all snapshots are synced + await mirrorRow.getByRole("button").first().click(); + await expect(page.getByRole("heading", { name: "Sync snapshots" })).toBeVisible(); + await expect(page.getByText(/All 1 snapshots are already synced/)).toBeVisible({ timeout: 15000 }); + await page.getByRole("button", { name: "Cancel" }).click(); + + // Verify snapshot appears in the mirror repository's snapshots tab + const response = await page.request.get("/api/v1/repositories"); + expect(response.ok()).toBe(true); + const repositories = (await response.json()) as Array<{ name: string; shortId: string }>; + const mirrorRepo = repositories.find((r) => r.name === mirrorRepoName); + expect(mirrorRepo).toBeDefined(); + + await gotoAndWaitForAppReady(page, `/repositories/${mirrorRepo!.shortId}`); + await page.getByRole("tab", { name: "Snapshots" }).click(); + await expect(page.getByText("Backup snapshots stored in this repository.")).toBeVisible(); + await page.getByRole("button", { name: "Refresh" }).click(); + await expect(page.getByRole("checkbox", { name: /Select snapshot/ })).toHaveCount(1, { timeout: 15000 }); +}); diff --git a/packages/core/src/restic/commands/__tests__/copy.test.ts b/packages/core/src/restic/commands/__tests__/copy.test.ts index b7fbde5a..54d984d2 100644 --- a/packages/core/src/restic/commands/__tests__/copy.test.ts +++ b/packages/core/src/restic/commands/__tests__/copy.test.ts @@ -48,12 +48,46 @@ afterEach(() => { describe("copy command", () => { test("treats flag-like snapshot IDs as positional args", async () => { const { getArgs } = setup(); - const snapshotId = "--help"; - await copy(sourceConfig, destConfig, { organizationId: "org-1", snapshotId, tag: "daily" }, mockDeps); + await copy(sourceConfig, destConfig, { organizationId: "org-1", snapshotIds: ["--help"], tag: "daily" }, mockDeps); const separatorIndex = getArgs().indexOf("--"); expect(separatorIndex).toBeGreaterThan(-1); - expect(getArgs().slice(separatorIndex + 1)).toEqual([snapshotId]); + expect(getArgs().slice(separatorIndex + 1)).toEqual(["--help"]); + }); + + test("defaults to 'latest' when no snapshotIds are provided", async () => { + const { getArgs } = setup(); + + await copy(sourceConfig, destConfig, { organizationId: "org-1", tag: "daily" }, mockDeps); + + const separatorIndex = getArgs().indexOf("--"); + expect(separatorIndex).toBeGreaterThan(-1); + expect(getArgs().slice(separatorIndex + 1)).toEqual(["latest"]); + }); + + test("passes multiple snapshot IDs after separator", async () => { + const { getArgs } = setup(); + + await copy( + sourceConfig, + destConfig, + { organizationId: "org-1", snapshotIds: ["abc123", "def456", "ghi789"], tag: "daily" }, + mockDeps, + ); + + const separatorIndex = getArgs().indexOf("--"); + expect(separatorIndex).toBeGreaterThan(-1); + expect(getArgs().slice(separatorIndex + 1)).toEqual(["abc123", "def456", "ghi789"]); + }); + + test("defaults to 'latest' when snapshotIds is empty array", async () => { + const { getArgs } = setup(); + + await copy(sourceConfig, destConfig, { organizationId: "org-1", snapshotIds: [], tag: "daily" }, mockDeps); + + const separatorIndex = getArgs().indexOf("--"); + expect(separatorIndex).toBeGreaterThan(-1); + expect(getArgs().slice(separatorIndex + 1)).toEqual(["latest"]); }); }); diff --git a/packages/core/src/restic/commands/copy.ts b/packages/core/src/restic/commands/copy.ts index 3ff8ef60..66f22302 100644 --- a/packages/core/src/restic/commands/copy.ts +++ b/packages/core/src/restic/commands/copy.ts @@ -11,7 +11,7 @@ import type { ResticDeps } from "../types"; export const copy = async ( sourceConfig: RepositoryConfig, destConfig: RepositoryConfig, - options: { organizationId: string; tag?: string; snapshotId?: string }, + options: { organizationId: string; tag?: string; snapshotIds?: string[] }, deps: ResticDeps, ) => { const sourceRepoUrl = buildRepoUrl(sourceConfig); @@ -45,7 +45,11 @@ export const copy = async ( args.push("--limit-upload", destUploadLimit); } - args.push("--", options.snapshotId ?? "latest"); + if (options.snapshotIds && options.snapshotIds.length > 0) { + args.push("--", ...options.snapshotIds); + } else { + args.push("--", "latest"); + } logger.info(`Copying snapshots from ${sourceRepoUrl} to ${destRepoUrl}...`); logger.debug(`Executing: restic ${args.join(" ")}`);