diff --git a/app/server/jobs/healthchecks.ts b/app/server/jobs/healthchecks.ts index f119d520..b6d220b4 100644 --- a/app/server/jobs/healthchecks.ts +++ b/app/server/jobs/healthchecks.ts @@ -17,10 +17,7 @@ export class VolumeHealthCheckJob extends Job { for (const volume of volumes) { try { await withContext({ organizationId: volume.organizationId }, async () => { - const { status } = await volumeService.checkHealth(volume.shortId); - if (status === "error" && volume.autoRemount) { - await volumeService.mountVolume(volume.shortId); - } + await volumeService.ensureHealthyVolume(volume.shortId); }); } catch (error) { logger.error(`Health check failed for volume ${volume.name}:`, error); diff --git a/app/server/modules/backups/__tests__/backups.service.execution.test.ts b/app/server/modules/backups/__tests__/backups.service.execution.test.ts index cdc4bd4d..9dedf873 100644 --- a/app/server/modules/backups/__tests__/backups.service.execution.test.ts +++ b/app/server/modules/backups/__tests__/backups.service.execution.test.ts @@ -21,6 +21,8 @@ import { notificationsService } from "~/server/modules/notifications/notificatio import { agentManager } from "~/server/modules/agents/agents-manager"; import { createAgentBackupMocks } from "~/test/helpers/agent-mock"; import { getScheduleByIdOrShortId } from "../helpers/backup-schedule-lookups"; +import { volumeService } from "~/server/modules/volumes/volume.service"; +import { db } from "~/server/db/db"; const setup = () => { const resticBackupMock = vi.fn((_: SafeSpawnParams) => @@ -47,6 +49,31 @@ const setup = () => { vi.spyOn(agentManager, "runBackup").mockImplementation(runBackupMock); vi.spyOn(agentManager, "cancelBackup").mockImplementation(cancelBackupMock); vi.spyOn(context, "getOrganizationId").mockReturnValue(TEST_ORG_ID); + const ensureHealthyVolumeMock = vi.spyOn(volumeService, "ensureHealthyVolume").mockImplementation(async (shortId) => { + const volume = await db.query.volumesTable.findFirst({ + where: { + AND: [{ shortId: { eq: shortId } }, { organizationId: TEST_ORG_ID }], + }, + }); + + if (!volume) { + throw new NotFoundError("Volume not found"); + } + + if (volume.status !== "mounted") { + return { + ready: false as const, + volume, + reason: "Volume is not mounted", + }; + } + + return { + ready: true as const, + volume, + remounted: false, + }; + }); return { resticBackupMock, @@ -55,6 +82,7 @@ const setup = () => { runBackupMock, cancelBackupMock, refreshStatsMock, + ensureHealthyVolumeMock, }; }; @@ -85,6 +113,50 @@ describe("backup execution - validation failures", () => { expect(resticBackupMock).not.toHaveBeenCalled(); }); + test("runs a preflight volume health check before starting a backup", async () => { + setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + const ensureHealthyVolumeSpy = vi.spyOn(volumeService, "ensureHealthyVolume").mockResolvedValue({ + ready: true, + volume, + remounted: false, + }); + + const result = await backupsService.validateBackupExecution(schedule.id); + + expect(result.type).toBe("success"); + expect(ensureHealthyVolumeSpy).toHaveBeenCalledWith(volume.shortId); + }); + + test("fails validation when the preflight health check cannot recover the volume", async () => { + const { resticBackupMock } = setup(); + const volume = await createTestVolume(); + const repository = await createTestRepository(); + const schedule = await createTestBackupSchedule({ + volumeId: volume.id, + repositoryId: repository.id, + }); + vi.spyOn(volumeService, "ensureHealthyVolume").mockResolvedValue({ + ready: false, + volume: { ...volume, status: "error", lastError: "stale mount" }, + reason: "stale mount", + }); + + const result = await backupsService.validateBackupExecution(schedule.id); + + expect(result.type).toBe("failure"); + if (result.type === "failure") { + expect(result.error).toBeInstanceOf(BadRequestError); + expect(result.error.message).toBe("stale mount"); + } + expect(resticBackupMock).not.toHaveBeenCalled(); + }); + test("should fail backup when volume does not exist", async () => { // arrange setup(); diff --git a/app/server/modules/backups/__tests__/backups.service.test.ts b/app/server/modules/backups/__tests__/backups.service.test.ts index 6d6c0185..8b927e44 100644 --- a/app/server/modules/backups/__tests__/backups.service.test.ts +++ b/app/server/modules/backups/__tests__/backups.service.test.ts @@ -17,6 +17,8 @@ import { repositoriesService } from "~/server/modules/repositories/repositories. import { agentManager } from "~/server/modules/agents/agents-manager"; import { createAgentBackupMocks } from "~/test/helpers/agent-mock"; import { getScheduleByIdOrShortId } from "../helpers/backup-schedule-lookups"; +import { volumeService } from "~/server/modules/volumes/volume.service"; +import { NotFoundError } from "http-errors-enhanced"; const setup = () => { const resticBackupMock = vi.fn((_: SafeSpawnParams) => Promise.resolve({ exitCode: 0, summary: "", error: "" })); @@ -36,6 +38,31 @@ const setup = () => { vi.spyOn(agentManager, "runBackup").mockImplementation(runBackupMock); vi.spyOn(agentManager, "cancelBackup").mockImplementation(cancelBackupMock); vi.spyOn(context, "getOrganizationId").mockReturnValue(TEST_ORG_ID); + vi.spyOn(volumeService, "ensureHealthyVolume").mockImplementation(async (shortId) => { + const volume = await db.query.volumesTable.findFirst({ + where: { + AND: [{ shortId: { eq: shortId } }, { organizationId: TEST_ORG_ID }], + }, + }); + + if (!volume) { + throw new NotFoundError("Volume not found"); + } + + if (volume.status !== "mounted") { + return { + ready: false as const, + volume, + reason: "Volume is not mounted", + }; + } + + return { + ready: true as const, + volume, + remounted: false, + }; + }); return { resticBackupMock, diff --git a/app/server/modules/backups/helpers/backup-lifecycle.ts b/app/server/modules/backups/helpers/backup-lifecycle.ts index acc887f1..82525e06 100644 --- a/app/server/modules/backups/helpers/backup-lifecycle.ts +++ b/app/server/modules/backups/helpers/backup-lifecycle.ts @@ -12,6 +12,7 @@ import { calculateNextRun } from "../backup.helpers"; import { scheduleQueries } from "../backups.queries"; import type { BackupExecutionProgress } from "../../agents/agents-manager"; import { repositoriesService } from "../../repositories/repositories.service"; +import { volumeService } from "../../volumes/volume.service"; import { copyToMirrors, runForget } from "./backup-maintenance"; export interface BackupContext { @@ -71,17 +72,19 @@ export async function validateBackupExecution(scheduleId: number, manual = false return { type: "failure", error: new NotFoundError("Repository not found"), partialContext: { schedule, volume } }; } - if (volume.status !== "mounted") { + const volumeReadiness = await volumeService.ensureHealthyVolume(volume.shortId); + + if (!volumeReadiness.ready) { return { type: "failure", - error: new BadRequestError("Volume is not mounted"), - partialContext: { schedule, volume, repository }, + error: new BadRequestError(volumeReadiness.reason), + partialContext: { schedule, volume: volumeReadiness.volume, repository }, }; } return { type: "success", - context: { schedule, volume, repository, organizationId }, + context: { schedule, volume: volumeReadiness.volume, repository, organizationId }, }; } diff --git a/app/server/modules/volumes/__tests__/volumes.service.test.ts b/app/server/modules/volumes/__tests__/volumes.service.test.ts index 0fa2f6dc..1aa09f42 100644 --- a/app/server/modules/volumes/__tests__/volumes.service.test.ts +++ b/app/server/modules/volumes/__tests__/volumes.service.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { volumeService } from "../volume.service"; import { db } from "~/server/db/db"; import { volumesTable } from "~/server/db/schema"; @@ -9,6 +9,12 @@ import path from "node:path"; import { createTestSession } from "~/test/helpers/auth"; import { withContext } from "~/server/core/request-context"; import { asShortId } from "~/server/utils/branded"; +import { createTestVolume } from "~/test/helpers/volume"; +import * as backendModule from "../../backends/backend"; + +afterEach(() => { + vi.restoreAllMocks(); +}); describe("volumeService.getVolume", () => { test("should find volume by shortId", async () => { @@ -124,3 +130,84 @@ describe("volumeService.listFiles security", () => { } }); }); + +describe("volumeService.ensureHealthyVolume", () => { + test("returns ready when the mounted volume passes its health check", async () => { + const { organizationId, user } = await createTestSession(); + const volume = await createTestVolume({ organizationId, status: "mounted" }); + const mount = vi.fn().mockResolvedValue({ status: "mounted" }); + const checkHealth = vi.fn().mockResolvedValue({ status: "mounted" }); + + vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({ + mount, + unmount: vi.fn().mockResolvedValue({ status: "unmounted" }), + checkHealth, + })); + + await withContext({ organizationId, userId: user.id }, async () => { + const result = await volumeService.ensureHealthyVolume(volume.shortId); + + expect(result).toEqual({ + ready: true, + volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }), + remounted: false, + }); + expect(checkHealth).toHaveBeenCalledOnce(); + expect(mount).not.toHaveBeenCalled(); + }); + }); + + test("auto-remounts when the mounted volume fails its health check", async () => { + const { organizationId, user } = await createTestSession(); + const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: true }); + const mount = vi.fn().mockResolvedValue({ status: "mounted" }); + const checkHealth = vi.fn().mockResolvedValue({ status: "error", error: "stale mount" }); + + vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({ + mount, + unmount: vi.fn().mockResolvedValue({ status: "unmounted" }), + checkHealth, + })); + + await withContext({ organizationId, userId: user.id }, async () => { + const result = await volumeService.ensureHealthyVolume(volume.shortId); + + expect(result).toEqual({ + ready: true, + volume: expect.objectContaining({ id: volume.id, status: "mounted", lastError: null }), + remounted: true, + }); + expect(checkHealth).toHaveBeenCalledOnce(); + expect(mount).toHaveBeenCalledOnce(); + + const updatedVolume = await db.query.volumesTable.findFirst({ where: { id: volume.id } }); + expect(updatedVolume?.status).toBe("mounted"); + expect(updatedVolume?.lastError).toBeNull(); + }); + }); + + test("returns not ready when the health check fails and auto-remount is disabled", async () => { + const { organizationId, user } = await createTestSession(); + const volume = await createTestVolume({ organizationId, status: "mounted", autoRemount: false }); + const mount = vi.fn().mockResolvedValue({ status: "mounted" }); + const checkHealth = vi.fn().mockResolvedValue({ status: "error", error: "stale mount" }); + + vi.spyOn(backendModule, "createVolumeBackend").mockImplementation(() => ({ + mount, + unmount: vi.fn().mockResolvedValue({ status: "unmounted" }), + checkHealth, + })); + + await withContext({ organizationId, userId: user.id }, async () => { + const result = await volumeService.ensureHealthyVolume(volume.shortId); + + expect(result).toEqual({ + ready: false, + volume: expect.objectContaining({ id: volume.id, status: "error", lastError: "stale mount" }), + reason: "stale mount", + }); + expect(checkHealth).toHaveBeenCalledOnce(); + expect(mount).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/app/server/modules/volumes/volume.service.ts b/app/server/modules/volumes/volume.service.ts index fe38ef90..c4b73952 100644 --- a/app/server/modules/volumes/volume.service.ts +++ b/app/server/modules/volumes/volume.service.ts @@ -14,12 +14,25 @@ import type { UpdateVolumeBody } from "./volume.dto"; import { getVolumePath } from "./helpers"; import { logger } from "@zerobyte/core/node"; import { serverEvents } from "../../core/events"; +import type { Volume } from "../../db/schema"; import { volumeConfigSchema, type BackendConfig } from "~/schemas/volumes"; import { getOrganizationId } from "~/server/core/request-context"; import { isNodeJSErrnoException } from "~/server/utils/fs"; import { asShortId, type ShortId } from "~/server/utils/branded"; import { encryptVolumeConfig } from "./volume-config-secrets"; +type EnsureHealthyVolumeResult = + | { + ready: true; + volume: Volume; + remounted: boolean; + } + | { + ready: false; + volume: Volume; + reason: string; + }; + const listVolumes = async () => { const organizationId = getOrganizationId(); const volumes = await db.query.volumesTable.findMany({ @@ -274,6 +287,59 @@ const checkHealth = async (shortId: ShortId) => { return { status, error }; }; +const ensureHealthyVolume = async (shortId: ShortId): Promise => { + const volume = await findVolume(shortId); + + if (!volume) { + throw new NotFoundError("Volume not found"); + } + + if (volume.status === "unmounted") { + return { ready: false, volume, reason: volume.lastError ?? "Volume is not mounted" }; + } + + let failureReason = volume.lastError ?? "Volume health check failed"; + let failedVolume = volume; + + if (volume.status !== "error") { + const health = await checkHealth(shortId); + + if (health.status === "mounted") { + return { + ready: true, + volume: { ...volume, status: "mounted", lastError: null }, + remounted: false, + }; + } + + failureReason = health.error ?? failureReason; + failedVolume = { ...volume, status: "error", lastError: health.error ?? null }; + } + + if (!volume.autoRemount) { + return { ready: false, volume: failedVolume, reason: failureReason }; + } + + logger.warn( + `${volume.name} is not healthy. Auto-remount is enabled, attempting to remount. Reason: ${failureReason}`, + ); + const remount = await mountVolume(shortId); + + if (remount.status !== "mounted") { + return { + ready: false, + volume: { ...volume, status: remount.status, lastError: remount.error ?? null }, + reason: remount.error ?? failureReason, + }; + } + + return { + ready: true, + volume: { ...volume, status: "mounted", lastError: null }, + remounted: true, + }; +}; + const DEFAULT_PAGE_SIZE = 500; const MAX_PAGE_SIZE = 500; @@ -407,6 +473,7 @@ export const volumeService = { testConnection, unmountVolume, checkHealth, + ensureHealthyVolume, listFiles, browseFilesystem, };