mirror of
https://github.com/nicotsx/zerobyte.git
synced 2026-04-18 22:09:30 -04:00
526 lines
17 KiB
TypeScript
526 lines
17 KiB
TypeScript
import waitForExpect from "wait-for-expect";
|
|
import { test, describe, mock, expect, afterEach, spyOn } from "bun:test";
|
|
import { backupsService } from "../backups.service";
|
|
import { backupsExecutionService } from "../backups.execution";
|
|
import { createTestVolume } from "~/test/helpers/volume";
|
|
import { createTestBackupSchedule } from "~/test/helpers/backup";
|
|
import { createTestRepository } from "~/test/helpers/repository";
|
|
import { createTestBackupScheduleMirror } from "~/test/helpers/backup-mirror";
|
|
import { generateBackupOutput } from "~/test/helpers/restic";
|
|
import { TEST_ORG_ID } from "~/test/helpers/organization";
|
|
import * as context from "~/server/core/request-context";
|
|
import * as spawnModule from "@zerobyte/core/node";
|
|
import type { SafeSpawnParams } from "@zerobyte/core/node";
|
|
import { restic } from "~/server/core/restic";
|
|
import { NotFoundError, BadRequestError } from "http-errors-enhanced";
|
|
import { scheduleQueries } from "../backups.queries";
|
|
import { fromAny } from "@total-typescript/shoehorn";
|
|
import { repositoriesService } from "~/server/modules/repositories/repositories.service";
|
|
|
|
const setup = () => {
|
|
const resticBackupMock = mock((_: SafeSpawnParams) =>
|
|
Promise.resolve({ exitCode: 0, summary: generateBackupOutput(), error: "" }),
|
|
);
|
|
const resticForgetMock = mock(() => Promise.resolve({ success: true, data: null }));
|
|
const resticCopyMock = mock(() => Promise.resolve({ success: true, output: "" }));
|
|
const refreshStatsMock = mock(() =>
|
|
Promise.resolve({
|
|
total_size: 0,
|
|
total_uncompressed_size: 0,
|
|
compression_ratio: 0,
|
|
compression_progress: 0,
|
|
compression_space_saving: 0,
|
|
snapshots_count: 0,
|
|
}),
|
|
);
|
|
|
|
spyOn(spawnModule, "safeSpawn").mockImplementation(resticBackupMock);
|
|
spyOn(restic, "forget").mockImplementation(resticForgetMock);
|
|
spyOn(restic, "copy").mockImplementation(resticCopyMock);
|
|
spyOn(repositoriesService, "refreshRepositoryStats").mockImplementation(refreshStatsMock);
|
|
spyOn(context, "getOrganizationId").mockReturnValue(TEST_ORG_ID);
|
|
|
|
return {
|
|
resticBackupMock,
|
|
resticForgetMock,
|
|
resticCopyMock,
|
|
refreshStatsMock,
|
|
};
|
|
};
|
|
|
|
afterEach(() => {
|
|
mock.restore();
|
|
});
|
|
|
|
describe("backup execution - validation failures", () => {
|
|
test("should fail backup when volume is not mounted", async () => {
|
|
// arrange
|
|
const { resticBackupMock } = setup();
|
|
const volume = await createTestVolume({ status: "unmounted" });
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: repository.id,
|
|
});
|
|
|
|
// act
|
|
const result = await backupsExecutionService.validateBackupExecution(schedule.id);
|
|
|
|
// assert
|
|
expect(result.type).toBe("failure");
|
|
if (result.type === "failure") {
|
|
expect(result.error).toBeInstanceOf(BadRequestError);
|
|
expect(result.error.message).toBe("Volume is not mounted");
|
|
}
|
|
expect(resticBackupMock).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("should fail backup when volume does not exist", async () => {
|
|
// arrange
|
|
setup();
|
|
const volume = await createTestVolume();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: repository.id,
|
|
});
|
|
|
|
const hydratedSchedule = await scheduleQueries.findById(schedule.id, TEST_ORG_ID);
|
|
expect(hydratedSchedule).toBeDefined();
|
|
const scheduleWithoutVolume = {
|
|
...hydratedSchedule,
|
|
volume: null,
|
|
};
|
|
spyOn(scheduleQueries, "findById").mockResolvedValueOnce(fromAny(scheduleWithoutVolume));
|
|
|
|
// act
|
|
const result = await backupsExecutionService.validateBackupExecution(schedule.id);
|
|
|
|
// assert
|
|
expect(result.type).toBe("failure");
|
|
if (result.type === "failure") {
|
|
expect(result.error).toBeInstanceOf(NotFoundError);
|
|
expect(result.error.message).toBe("Volume not found");
|
|
expect(result.partialContext?.schedule).toBeDefined();
|
|
}
|
|
});
|
|
|
|
test("should fail backup when repository does not exist", async () => {
|
|
// arrange
|
|
setup();
|
|
const volume = await createTestVolume();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: repository.id,
|
|
});
|
|
|
|
const hydratedSchedule = await scheduleQueries.findById(schedule.id, TEST_ORG_ID);
|
|
expect(hydratedSchedule).toBeDefined();
|
|
const scheduleWithoutRepository = {
|
|
...hydratedSchedule,
|
|
repository: null,
|
|
};
|
|
spyOn(scheduleQueries, "findById").mockResolvedValueOnce(fromAny(scheduleWithoutRepository));
|
|
|
|
// act
|
|
const result = await backupsExecutionService.validateBackupExecution(schedule.id);
|
|
|
|
// assert
|
|
expect(result.type).toBe("failure");
|
|
if (result.type === "failure") {
|
|
expect(result.error).toBeInstanceOf(NotFoundError);
|
|
expect(result.error.message).toBe("Repository not found");
|
|
expect(result.partialContext?.schedule).toBeDefined();
|
|
expect(result.partialContext?.volume).toBeDefined();
|
|
}
|
|
});
|
|
|
|
test("should fail backup when schedule does not exist", async () => {
|
|
setup();
|
|
// act
|
|
const result = await backupsExecutionService.validateBackupExecution(99999);
|
|
|
|
// assert
|
|
expect(result.type).toBe("failure");
|
|
if (result.type === "failure") {
|
|
expect(result.error).toBeInstanceOf(NotFoundError);
|
|
expect(result.error.message).toBe("Backup schedule not found");
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("stop backup", () => {
|
|
test("should keep restic warning details when backup completes with read errors", async () => {
|
|
const { resticBackupMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: repository.id,
|
|
});
|
|
|
|
resticBackupMock.mockImplementationOnce((params: SafeSpawnParams) => {
|
|
params.onStderr?.("error: open /mnt/data/private.db: permission denied");
|
|
|
|
return Promise.resolve({
|
|
exitCode: 3,
|
|
summary: generateBackupOutput(),
|
|
error: "Warning: at least one source file could not be read",
|
|
});
|
|
});
|
|
|
|
await backupsExecutionService.executeBackup(schedule.id);
|
|
|
|
const updatedSchedule = await backupsService.getScheduleById(schedule.id);
|
|
expect(updatedSchedule.lastBackupStatus).toBe("warning");
|
|
expect(updatedSchedule.lastBackupError).toBe("error: open /mnt/data/private.db: permission denied");
|
|
});
|
|
|
|
test("should store restic diagnostic details instead of the generic summary on hard failure", async () => {
|
|
const { resticBackupMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: repository.id,
|
|
});
|
|
|
|
resticBackupMock.mockImplementationOnce((params: SafeSpawnParams) => {
|
|
params.onStderr?.("Permissions 0755 for '/tmp/zerobyte-ssh-key' are too open.");
|
|
params.onStderr?.("This private key will be ignored.");
|
|
|
|
return Promise.resolve({
|
|
exitCode: 1,
|
|
summary: "",
|
|
error: "ssh command exited",
|
|
stderr: "Permissions 0755 for '/tmp/zerobyte-ssh-key' are too open.\nThis private key will be ignored.",
|
|
});
|
|
});
|
|
|
|
await backupsExecutionService.executeBackup(schedule.id);
|
|
|
|
const updatedSchedule = await backupsService.getScheduleById(schedule.id);
|
|
expect(updatedSchedule.lastBackupStatus).toBe("error");
|
|
expect(updatedSchedule.lastBackupError).toBe(
|
|
"Permissions 0755 for '/tmp/zerobyte-ssh-key' are too open.\nThis private key will be ignored.",
|
|
);
|
|
});
|
|
|
|
test("should stop a running backup", async () => {
|
|
// arrange
|
|
const { resticBackupMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: repository.id,
|
|
});
|
|
|
|
resticBackupMock.mockImplementation(async () => {
|
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
return { exitCode: 0, summary: generateBackupOutput(), error: "" };
|
|
});
|
|
|
|
void backupsExecutionService.executeBackup(schedule.id);
|
|
|
|
await waitForExpect(async () => {
|
|
const runningSchedule = await backupsService.getScheduleById(schedule.id);
|
|
expect(runningSchedule.lastBackupStatus).toBe("in_progress");
|
|
});
|
|
|
|
// act
|
|
await backupsExecutionService.stopBackup(schedule.id);
|
|
|
|
// assert
|
|
const updatedSchedule = await backupsService.getScheduleById(schedule.id);
|
|
expect(updatedSchedule.lastBackupStatus).toBe("warning");
|
|
expect(updatedSchedule.lastBackupError).toBe("Backup was stopped by the user");
|
|
});
|
|
|
|
test("should throw ConflictError when trying to stop non-running backup", async () => {
|
|
// arrange
|
|
setup();
|
|
const volume = await createTestVolume();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: repository.id,
|
|
});
|
|
|
|
// act & assert
|
|
await expect(backupsExecutionService.stopBackup(schedule.id)).rejects.toThrow(
|
|
"No backup is currently running for this schedule",
|
|
);
|
|
});
|
|
|
|
test("should throw NotFoundError when schedule does not exist", async () => {
|
|
setup();
|
|
// act & assert
|
|
await expect(backupsExecutionService.stopBackup(99999)).rejects.toThrow("Backup schedule not found");
|
|
});
|
|
});
|
|
|
|
describe("retention policy - runForget", () => {
|
|
test("should execute forget with retention policy", async () => {
|
|
// arrange
|
|
const { resticForgetMock } = setup();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
repositoryId: repository.id,
|
|
retentionPolicy: {
|
|
keepHourly: 24,
|
|
keepDaily: 7,
|
|
keepWeekly: 4,
|
|
keepMonthly: 12,
|
|
keepYearly: 3,
|
|
},
|
|
});
|
|
|
|
// act
|
|
await backupsExecutionService.runForget(schedule.id);
|
|
|
|
// assert
|
|
expect(resticForgetMock).toHaveBeenCalledWith(
|
|
repository.config,
|
|
expect.objectContaining({
|
|
keepHourly: 24,
|
|
keepDaily: 7,
|
|
keepWeekly: 4,
|
|
keepMonthly: 12,
|
|
keepYearly: 3,
|
|
}),
|
|
expect.objectContaining({
|
|
tag: schedule.shortId,
|
|
organizationId: TEST_ORG_ID,
|
|
}),
|
|
);
|
|
});
|
|
|
|
test("should throw BadRequestError if no retention policy configured", async () => {
|
|
// arrange
|
|
setup();
|
|
const repository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
repositoryId: repository.id,
|
|
retentionPolicy: undefined,
|
|
});
|
|
|
|
// act & assert
|
|
await expect(backupsExecutionService.runForget(schedule.id)).rejects.toThrow(
|
|
"No retention policy configured for this schedule",
|
|
);
|
|
});
|
|
|
|
test("should throw NotFoundError when schedule does not exist", async () => {
|
|
setup();
|
|
// act & assert
|
|
await expect(backupsExecutionService.runForget(99999)).rejects.toThrow("Backup schedule not found");
|
|
});
|
|
|
|
test("should throw NotFoundError when repository does not exist", async () => {
|
|
// arrange
|
|
setup();
|
|
const schedule = await createTestBackupSchedule({
|
|
retentionPolicy: {
|
|
keepHourly: 24,
|
|
},
|
|
});
|
|
|
|
// act & assert
|
|
await expect(backupsExecutionService.runForget(schedule.id, "non-existent-repo")).rejects.toThrow(
|
|
"Repository not found",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("mirror operations", () => {
|
|
test("should copy snapshots to mirror repositories", async () => {
|
|
// arrange
|
|
const { resticCopyMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const sourceRepository = await createTestRepository();
|
|
const mirrorRepository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: sourceRepository.id,
|
|
});
|
|
|
|
await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id);
|
|
|
|
// act
|
|
await backupsExecutionService.copyToMirrors(schedule.id, sourceRepository, null);
|
|
|
|
// assert
|
|
expect(resticCopyMock).toHaveBeenCalledWith(
|
|
sourceRepository.config,
|
|
mirrorRepository.config,
|
|
expect.objectContaining({
|
|
tag: schedule.shortId,
|
|
organizationId: TEST_ORG_ID,
|
|
}),
|
|
);
|
|
});
|
|
|
|
test("should skip disabled mirrors", async () => {
|
|
// arrange
|
|
const { resticCopyMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const sourceRepository = await createTestRepository();
|
|
const mirrorRepository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: sourceRepository.id,
|
|
});
|
|
|
|
await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id, { enabled: false });
|
|
|
|
// act
|
|
await backupsExecutionService.copyToMirrors(schedule.id, sourceRepository, null);
|
|
|
|
// assert
|
|
expect(resticCopyMock).not.toHaveBeenCalled();
|
|
});
|
|
|
|
test("should update mirror status on success", async () => {
|
|
// arrange
|
|
setup();
|
|
const volume = await createTestVolume();
|
|
const sourceRepository = await createTestRepository();
|
|
const mirrorRepository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: sourceRepository.id,
|
|
});
|
|
|
|
const mirror = await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id);
|
|
|
|
// act
|
|
await backupsExecutionService.copyToMirrors(schedule.id, sourceRepository, null);
|
|
|
|
// assert
|
|
const mirrors = await backupsService.getMirrors(schedule.id);
|
|
const updatedMirror = mirrors.find((m) => m.id === mirror.id);
|
|
expect(updatedMirror?.lastCopyStatus).toBe("success");
|
|
expect(updatedMirror?.lastCopyError).toBeNull();
|
|
expect(updatedMirror?.lastCopyAt).not.toBeNull();
|
|
});
|
|
|
|
test("should finalize mirror status when mirror settings are updated during copy", async () => {
|
|
// arrange
|
|
const { resticCopyMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const sourceRepository = await createTestRepository();
|
|
const mirrorRepository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: sourceRepository.id,
|
|
});
|
|
|
|
const originalMirror = await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id);
|
|
|
|
resticCopyMock.mockImplementationOnce(async () => {
|
|
await backupsService.updateMirrors(schedule.id, {
|
|
mirrors: [{ repositoryId: mirrorRepository.id, enabled: true }],
|
|
});
|
|
return { success: true, output: "" };
|
|
});
|
|
|
|
// act
|
|
await backupsExecutionService.copyToMirrors(schedule.id, sourceRepository, null);
|
|
|
|
// assert
|
|
const mirrors = await backupsService.getMirrors(schedule.id);
|
|
expect(mirrors).toHaveLength(1);
|
|
expect(mirrors[0]?.id).not.toBe(originalMirror.id);
|
|
expect(mirrors[0]?.lastCopyStatus).toBe("success");
|
|
expect(mirrors[0]?.lastCopyError).toBeNull();
|
|
expect(mirrors[0]?.lastCopyAt).not.toBeNull();
|
|
});
|
|
|
|
test("should update mirror status on failure", async () => {
|
|
// arrange
|
|
const { resticCopyMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const sourceRepository = await createTestRepository();
|
|
const mirrorRepository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: sourceRepository.id,
|
|
});
|
|
|
|
const mirror = await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id);
|
|
|
|
resticCopyMock.mockImplementationOnce(() => Promise.reject(new Error("Copy failed")));
|
|
|
|
// act
|
|
await backupsExecutionService.copyToMirrors(schedule.id, sourceRepository, null);
|
|
|
|
// assert
|
|
const mirrors = await backupsService.getMirrors(schedule.id);
|
|
const updatedMirror = mirrors.find((m) => m.id === mirror.id);
|
|
expect(updatedMirror?.lastCopyStatus).toBe("error");
|
|
expect(updatedMirror?.lastCopyError).toBe("Copy failed");
|
|
expect(updatedMirror?.lastCopyAt).not.toBeNull();
|
|
});
|
|
|
|
test("should run forget on mirror after successful copy when retention policy exists", async () => {
|
|
// arrange
|
|
const { resticCopyMock, resticForgetMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const sourceRepository = await createTestRepository();
|
|
const mirrorRepository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: sourceRepository.id,
|
|
retentionPolicy: { keepHourly: 24, keepDaily: 7 },
|
|
});
|
|
|
|
await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id);
|
|
|
|
resticCopyMock.mockClear();
|
|
resticCopyMock.mockImplementation(() => Promise.resolve({ success: true, output: "" }));
|
|
|
|
// act
|
|
await backupsExecutionService.copyToMirrors(schedule.id, sourceRepository, schedule.retentionPolicy);
|
|
|
|
await waitForExpect(() => {
|
|
expect(resticCopyMock).toHaveBeenCalled();
|
|
});
|
|
|
|
// assert
|
|
expect(resticForgetMock).toHaveBeenCalledWith(
|
|
mirrorRepository.config,
|
|
expect.objectContaining({ keepHourly: 24, keepDaily: 7 }),
|
|
expect.objectContaining({ tag: schedule.shortId, organizationId: TEST_ORG_ID }),
|
|
);
|
|
});
|
|
|
|
test("should not run forget on mirror when no retention policy", async () => {
|
|
// arrange
|
|
const { resticCopyMock, resticForgetMock } = setup();
|
|
const volume = await createTestVolume();
|
|
const sourceRepository = await createTestRepository();
|
|
const mirrorRepository = await createTestRepository();
|
|
const schedule = await createTestBackupSchedule({
|
|
volumeId: volume.id,
|
|
repositoryId: sourceRepository.id,
|
|
retentionPolicy: undefined,
|
|
});
|
|
|
|
await createTestBackupScheduleMirror(schedule.id, mirrorRepository.id);
|
|
|
|
resticForgetMock.mockClear();
|
|
|
|
// act
|
|
await backupsExecutionService.copyToMirrors(schedule.id, sourceRepository, schedule.retentionPolicy);
|
|
|
|
await waitForExpect(() => {
|
|
expect(resticCopyMock).toHaveBeenCalled();
|
|
});
|
|
|
|
// assert
|
|
expect(resticForgetMock).not.toHaveBeenCalled();
|
|
});
|
|
});
|