test: automate SFTP integration coverage

This commit is contained in:
Nicolas Meienberger
2026-06-02 17:25:10 +02:00
parent 648ccae5fc
commit 756ecbddcd
14 changed files with 457 additions and 174 deletions

View File

@@ -24,3 +24,4 @@
node_modules/**
dist/**
.output/**
app/test/integration/artifacts/**

View File

@@ -44,6 +44,15 @@ RUN bzip2 -d restic.bz2 && chmod +x restic
RUN mv rclone-v*-linux-*/rclone /deps/rclone && chmod +x /deps/rclone
RUN tar -xzf shoutrrr.tar.gz && chmod +x shoutrrr
# ------------------------------
# RUNTIME TOOLS
# ------------------------------
FROM base AS runtime-tools
COPY --from=deps /deps/restic /usr/local/bin/restic
COPY --from=deps /deps/rclone /usr/local/bin/rclone
COPY --from=deps /deps/shoutrrr /usr/local/bin/shoutrrr
# ------------------------------
# DEVELOPMENT
# ------------------------------

View File

@@ -8,14 +8,12 @@ FIXTURE_UID="1000"
FIXTURE_GID="1000"
ARTIFACTS_DIR="$SCRIPT_DIR/artifacts/$TARGET_HOST"
KEY_PATH="$ARTIFACTS_DIR/zerobyte-sftp-ed25519"
KNOWN_HOSTS_PATH="$ARTIFACTS_DIR/known_hosts"
CONFIG_PATH="$ARTIFACTS_DIR/config.generated.json"
SMB_PASSWORD_FILE="$ARTIFACTS_DIR/smb-password.txt"
SFTP_PASSWORD_FILE="$ARTIFACTS_DIR/sftp-password.txt"
WEBDAV_PASSWORD_FILE="$ARTIFACTS_DIR/webdav-password.txt"
RESTIC_PASSWORD_FILE="$ARTIFACTS_DIR/restic-password.txt"
read_or_create_secret() {
local file_path="$1"
@@ -35,16 +33,8 @@ chmod 700 "$ARTIFACTS_DIR"
SMB_PASSWORD="$(read_or_create_secret "$SMB_PASSWORD_FILE")"
SFTP_PASSWORD="$(read_or_create_secret "$SFTP_PASSWORD_FILE")"
WEBDAV_PASSWORD="$(read_or_create_secret "$WEBDAV_PASSWORD_FILE")"
RESTIC_PASSWORD="$(read_or_create_secret "$RESTIC_PASSWORD_FILE")"
if [[ ! -f "$KEY_PATH" || ! -f "$KEY_PATH.pub" ]]; then
ssh-keygen -q -t ed25519 -N "" -C "zerobyte-backend-integration@$TARGET_HOST" -f "$KEY_PATH"
chmod 600 "$KEY_PATH"
fi
PUBLIC_KEY_BASE64="$(base64 <"$KEY_PATH.pub" | tr -d '\n')"
ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" "$RESTIC_PASSWORD" "$PUBLIC_KEY_BASE64" <<'REMOTE'
ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" <<'REMOTE'
set -euo pipefail
fixture_uid="$1"
@@ -52,11 +42,6 @@ fixture_gid="$2"
smb_password="$3"
sftp_password="$4"
webdav_password="$5"
restic_password="$6"
public_key="$(printf '%s' "$7" | base64 -d)"
repo_path="/srv/zerobyte-backend-integration/restic-repo"
repo_password_fingerprint_path="$repo_path/.zerobyte-password-sha256"
repo_password_fingerprint="$(printf '%s' "$restic_password" | sha256sum | cut -d' ' -f1)"
legacy_sshd_dir="/etc/ssh/zerobyte-backend-integration-legacy"
export DEBIAN_FRONTEND=noninteractive
@@ -66,25 +51,8 @@ write_file() {
cat >"$file_path"
}
initialize_restic_repo() {
local password_file
rm -rf "$repo_path"
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 "$repo_path"
password_file="$(mktemp)"
printf '%s\n' "$restic_password" >"$password_file"
chown zerobyte-sftp:zerobyte-sftp "$password_file"
chmod 0600 "$password_file"
su -s /bin/sh -c "restic init --repo '$repo_path' --password-file '$password_file'" zerobyte-sftp
rm -f "$password_file"
printf '%s\n' "$repo_password_fingerprint" >"$repo_password_fingerprint_path"
chmod 0600 "$repo_password_fingerprint_path"
}
apt-get update
apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server restic rpcbind samba
apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server rpcbind samba
id -u zerobyte-sftp >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-sftp --shell /bin/bash zerobyte-sftp
id -u zerobyte-smb >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-smb --shell /bin/bash zerobyte-smb
@@ -96,24 +64,12 @@ chown -R "$fixture_uid:$fixture_gid" /srv/zerobyte-backend-integration/fixtures
find /srv/zerobyte-backend-integration/fixtures -type d -exec chmod 0755 {} +
find /srv/zerobyte-backend-integration/fixtures -type f -exec chmod 0644 {} +
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp/.ssh
printf '%s\n' "$public_key" >/home/zerobyte-sftp/.ssh/authorized_keys
chown zerobyte-sftp:zerobyte-sftp /home/zerobyte-sftp/.ssh/authorized_keys
chmod 0600 /home/zerobyte-sftp/.ssh/authorized_keys
printf '%s\n%s\n' "$smb_password" "$smb_password" | smbpasswd -a -s zerobyte-smb >/dev/null
smbpasswd -e zerobyte-smb >/dev/null
printf 'zerobyte-sftp:%s\n' "$sftp_password" | chpasswd
passwd -u zerobyte-sftp >/dev/null 2>&1 || true
htpasswd -bc /etc/apache2/zerobyte-backend-integration.htpasswd zerobyte-webdav "$webdav_password" >/dev/null
if [[ ! -f "$repo_path/config" ]]; then
initialize_restic_repo
elif [[ ! -f "$repo_password_fingerprint_path" ]] || [[ "$(cat "$repo_password_fingerprint_path")" != "$repo_password_fingerprint" ]]; then
initialize_restic_repo
fi
write_file /etc/exports <<'EOF'
/srv/zerobyte-backend-integration/fixtures *(ro,sync,no_subtree_check,insecure)
EOF
@@ -239,8 +195,6 @@ INTEGRATION_HOST="$TARGET_HOST" \
SMB_PASSWORD="$SMB_PASSWORD" \
SFTP_PASSWORD="$SFTP_PASSWORD" \
WEBDAV_PASSWORD="$WEBDAV_PASSWORD" \
RESTIC_PASSWORD="$RESTIC_PASSWORD" \
SFTP_KEY_PATH="$KEY_PATH" \
KNOWN_HOSTS_PATH="$KNOWN_HOSTS_PATH" \
CONFIG_PATH="$CONFIG_PATH" \
bun run "$SCRIPT_DIR/write-generated-config.ts"

View File

@@ -14,8 +14,6 @@ const fixtureGid = getRequiredNumberEnv("FIXTURE_GID");
const smbPassword = getRequiredEnv("SMB_PASSWORD");
const sftpPassword = getRequiredEnv("SFTP_PASSWORD");
const webdavPassword = getRequiredEnv("WEBDAV_PASSWORD");
const resticPassword = getRequiredEnv("RESTIC_PASSWORD");
const privateKey = fs.readFileSync(getRequiredEnv("SFTP_KEY_PATH"), "utf8");
const knownHosts = fs.readFileSync(getRequiredEnv("KNOWN_HOSTS_PATH"), "utf8");
const configPath = getRequiredEnv("CONFIG_PATH");
@@ -74,40 +72,6 @@ const config = {
fixtureRoot: "case-a",
expectedEntries: smbEntries,
},
{
id: "sftp-local-repo",
volume: {
backend: "sftp",
host,
port: 22,
username: "zerobyte-sftp",
privateKey,
path: "/srv/zerobyte-backend-integration/fixtures",
readOnly: true,
skipHostKeyCheck: false,
knownHosts,
},
repository: { backend: "local", path: "repo-sftp-volume" },
fixtureRoot: "case-a",
expectedEntries: contentOnlyEntries,
},
{
id: "sftp-password-local-repo",
volume: {
backend: "sftp",
host,
port: 22,
username: "zerobyte-sftp",
password: sftpPassword,
path: "/srv/zerobyte-backend-integration/fixtures",
readOnly: true,
skipHostKeyCheck: false,
knownHosts,
},
repository: { backend: "local", path: "repo-sftp-password-volume" },
fixtureRoot: "case-a",
expectedEntries: contentOnlyEntries,
},
{
id: "sftp-legacy-rsa-hostkey-local-repo",
volume: {
@@ -142,31 +106,6 @@ const config = {
fixtureRoot: "case-a",
expectedEntries: contentOnlyEntries,
},
{
id: "nfs-sftp-repository",
volume: {
backend: "nfs",
server: host,
exportPath: "/srv/zerobyte-backend-integration/fixtures",
port: 2049,
version: "4.1",
readOnly: true,
},
repository: {
backend: "sftp",
host,
port: 22,
user: "zerobyte-sftp",
path: "/srv/zerobyte-backend-integration/restic-repo",
privateKey,
skipHostKeyCheck: false,
knownHosts,
isExistingRepository: true,
customPassword: resticPassword,
},
fixtureRoot: "case-a",
expectedEntries: nfsEntries,
},
],
};

View File

@@ -23,6 +23,22 @@ services:
done
mc mb --ignore-existing --region us-east-1 rustfs/zerobyte-integration
sftp:
build:
context: ./sftp
environment:
SFTP_USER: "zerobyte-sftp"
SFTP_PASSWORD: "zerobyte-sftp-password"
SFTP_PUBLIC_KEY_PATH: "/run/zerobyte/sftp/id_ed25519.pub"
volumes:
- ../artifacts/sftp/id_ed25519.pub:/run/zerobyte/sftp/id_ed25519.pub:ro
- sftp-data:/srv/zerobyte-integration
healthcheck:
test: ["CMD", "ssh-keyscan", "-T", "2", "localhost"]
interval: 1s
timeout: 5s
retries: 30
integration:
build:
context: ../../../..
@@ -32,10 +48,19 @@ services:
depends_on:
rustfs-setup:
condition: service_completed_successfully
sftp:
condition: service_healthy
cap_add:
- SYS_ADMIN
devices:
- /dev/fuse:/dev/fuse
volumes:
- ../artifacts:/app/app/test/integration/artifacts
- ../artifacts/sftp/id_ed25519:/run/zerobyte/sftp/id_ed25519:ro
environment:
LOG_LEVEL: "info"
LOG_LEVEL: "error"
SFTP_PRIVATE_KEY_PATH: "/run/zerobyte/sftp/id_ed25519"
volumes:
rustfs-data:
sftp-data:

View File

@@ -1,5 +1,9 @@
#!/usr/bin/env sh
set -eu
if [ -w /etc/fuse.conf ] && ! grep -q '^user_allow_other$' /etc/fuse.conf; then
printf '\nuser_allow_other\n' >>/etc/fuse.conf
fi
bun app/test/integration/src/write-rclone-config.ts
exec bunx --bun vitest run --config app/test/integration/vitest.config.ts

View File

@@ -0,0 +1,10 @@
FROM alpine:3.21
RUN apk add --no-cache openssh-client openssh-server
COPY entrypoint.sh /usr/local/bin/zerobyte-sftp-entrypoint
RUN chmod +x /usr/local/bin/zerobyte-sftp-entrypoint
EXPOSE 22
ENTRYPOINT ["/usr/local/bin/zerobyte-sftp-entrypoint"]

View File

@@ -0,0 +1,46 @@
#!/bin/sh
set -eu
SFTP_USER="${SFTP_USER:-zerobyte-sftp}"
SFTP_PASSWORD="${SFTP_PASSWORD:-zerobyte-sftp-password}"
SFTP_PUBLIC_KEY_PATH="${SFTP_PUBLIC_KEY_PATH:-/run/zerobyte/sftp/id_ed25519.pub}"
SERVICE_ROOT="/srv/zerobyte-integration"
ssh-keygen -A
install -d -m 0755 /run/sshd
if ! id "$SFTP_USER" >/dev/null 2>&1; then
addgroup -S "$SFTP_USER"
adduser -S -D -h "/home/$SFTP_USER" -s /bin/sh -G "$SFTP_USER" "$SFTP_USER"
fi
printf '%s:%s\n' "$SFTP_USER" "$SFTP_PASSWORD" | chpasswd
install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0700 "/home/$SFTP_USER/.ssh"
install -o "$SFTP_USER" -g "$SFTP_USER" -m 0600 "$SFTP_PUBLIC_KEY_PATH" "/home/$SFTP_USER/.ssh/authorized_keys"
install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0755 "$SERVICE_ROOT/fixtures/case-a/docs"
install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0755 "$SERVICE_ROOT/repos/sftp"
printf 'hello from zerobyte integration\n' >"$SERVICE_ROOT/fixtures/case-a/hello.txt"
printf 'fixture documentation\n' >"$SERVICE_ROOT/fixtures/case-a/docs/readme.md"
chown -R "$SFTP_USER:$SFTP_USER" "$SERVICE_ROOT"
cat >/etc/ssh/sshd_config <<EOF
Port 22
HostKey /etc/ssh/ssh_host_ed25519_key
HostKey /etc/ssh/ssh_host_rsa_key
PermitRootLogin no
PasswordAuthentication yes
PubkeyAuthentication yes
KbdInteractiveAuthentication no
Subsystem sftp internal-sftp
Match User $SFTP_USER
ForceCommand internal-sftp
AllowTcpForwarding no
X11Forwarding no
PasswordAuthentication yes
PubkeyAuthentication yes
EOF
exec /usr/sbin/sshd -D -e

View File

@@ -6,14 +6,21 @@ repo_root="$(cd "$script_dir/../../.." && pwd)"
base_image="zerobyte-integration-runtime-base:latest"
compose_project="zerobyte-integration-$(basename "$repo_root" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9_-')"
artifacts_dir="$script_dir/artifacts"
sftp_artifacts_dir="$artifacts_dir/sftp"
compose_file="$script_dir/infra/docker-compose.yml"
mkdir -p "$artifacts_dir"
rm -rf "$artifacts_dir/runs"
rm -rf "$sftp_artifacts_dir"
rm -f "$artifacts_dir/compose.log"
mkdir -p "$artifacts_dir/runs"
mkdir -p "$sftp_artifacts_dir"
docker build --target production -t "$base_image" "$repo_root"
ssh-keygen -q -t ed25519 -N "" -f "$sftp_artifacts_dir/id_ed25519"
chmod 600 "$sftp_artifacts_dir/id_ed25519"
chmod 644 "$sftp_artifacts_dir/id_ed25519.pub"
docker build --target runtime-tools -t "$base_image" "$repo_root"
exit_code=0
docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup || exit_code=$?

View File

@@ -13,32 +13,30 @@ const nodeByPath = (nodes: ResticLsNode[]) => new Map(nodes.map((node) => [path.
export const assertSnapshotContainsFixture = (sourceRoot: string, nodes: ResticLsNode[], fixture: ScenarioFixture) => {
const nodesByPath = nodeByPath(nodes);
const regularFile = nodesByPath.get(path.join(sourceRoot, fixture.regularFile.relativePath));
expect(regularFile?.type).toBe("file");
const nestedDirectory = nodesByPath.get(path.join(sourceRoot, fixture.nestedDirectory.relativePath));
expect(nestedDirectory?.type).toBe("dir");
const nestedFile = nodesByPath.get(path.join(sourceRoot, fixture.nestedFile.relativePath));
expect(nestedFile?.type).toBe("file");
const symlink = nodesByPath.get(path.join(sourceRoot, fixture.symlink.relativePath));
expect(symlink?.type).toBe("symlink");
for (const entry of fixture.entries) {
const node = nodesByPath.get(path.join(sourceRoot, entry.relativePath));
expect(node?.type).toBe(entry.type === "directory" ? "dir" : entry.type);
}
};
export const assertRestoredFixture = async (restoreRoot: string, fixture: ScenarioFixture) => {
await expect(fs.readFile(path.join(restoreRoot, fixture.regularFile.relativePath), "utf8")).resolves.toBe(
fixture.regularFile.content,
);
const nestedDirectoryStat = await fs.stat(path.join(restoreRoot, fixture.nestedDirectory.relativePath));
expect(nestedDirectoryStat.isDirectory()).toBe(true);
await expect(fs.readFile(path.join(restoreRoot, fixture.nestedFile.relativePath), "utf8")).resolves.toBe(
fixture.nestedFile.content,
);
await expect(fs.readlink(path.join(restoreRoot, fixture.symlink.relativePath))).resolves.toBe(
fixture.symlink.target,
);
for (const entry of fixture.entries) {
const entryPath = path.join(restoreRoot, entry.relativePath);
if (entry.type === "file") {
await expect(fs.readFile(entryPath, "utf8")).resolves.toBe(entry.content);
}
if (entry.type === "directory") {
const stats = await fs.stat(entryPath);
expect(stats.isDirectory()).toBe(true);
}
if (entry.type === "symlink") {
await expect(fs.readlink(entryPath)).resolves.toBe(entry.target);
}
}
};
export const assertFixtureSourceExists = async (fixture: ScenarioFixture) => {
const stats = await fs.stat(fixture.sourceRoot);
expect(stats.isDirectory()).toBe(true);
await assertRestoredFixture(fixture.sourceRoot, fixture);
};

View File

@@ -2,53 +2,90 @@ import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
export type FixtureEntry =
| {
type: "file";
relativePath: string;
content: string;
}
| {
type: "directory";
relativePath: string;
}
| {
type: "symlink";
relativePath: string;
target: string;
};
export type ScenarioFixture = {
sourceRoot: string;
regularFile: {
relativePath: string;
content: string;
};
nestedDirectory: {
relativePath: string;
};
nestedFile: {
relativePath: string;
content: string;
};
symlink: {
relativePath: string;
target: string;
};
entries: FixtureEntry[];
};
export const createScenarioFixture = async (workspace: string, scenarioId: string): Promise<ScenarioFixture> => {
const sourceRoot = path.join(workspace, "source");
const nestedDirectory = path.join(sourceRoot, "nested");
const randomSuffix = crypto.randomBytes(8).toString("hex");
const fixture: ScenarioFixture = {
sourceRoot,
regularFile: {
relativePath: "regular.txt",
content: `regular fixture for ${scenarioId} (${randomSuffix})\n`,
},
nestedDirectory: {
relativePath: "nested",
},
nestedFile: {
relativePath: "nested/deep.txt",
content: `nested fixture for ${scenarioId} (${randomSuffix})\n`,
},
symlink: {
relativePath: "regular-link",
target: "regular.txt",
},
entries: [
{
type: "file",
relativePath: "regular.txt",
content: `regular fixture for ${scenarioId} (${randomSuffix})\n`,
},
{
type: "directory",
relativePath: "nested",
},
{
type: "file",
relativePath: "nested/deep.txt",
content: `nested fixture for ${scenarioId} (${randomSuffix})\n`,
},
{
type: "symlink",
relativePath: "regular-link",
target: "regular.txt",
},
],
};
await fs.mkdir(nestedDirectory, { recursive: true });
await fs.writeFile(path.join(sourceRoot, fixture.regularFile.relativePath), fixture.regularFile.content);
await fs.writeFile(path.join(sourceRoot, fixture.nestedFile.relativePath), fixture.nestedFile.content);
await fs.symlink(fixture.symlink.target, path.join(sourceRoot, fixture.symlink.relativePath));
for (const entry of fixture.entries) {
const entryPath = path.join(sourceRoot, entry.relativePath);
if (entry.type === "file") {
await fs.mkdir(path.dirname(entryPath), { recursive: true });
await fs.writeFile(entryPath, entry.content);
}
if (entry.type === "directory") {
await fs.mkdir(entryPath, { recursive: true });
}
if (entry.type === "symlink") {
await fs.mkdir(path.dirname(entryPath), { recursive: true });
await fs.symlink(entry.target, entryPath);
}
}
return fixture;
};
export const createStaticSftpFixture = (sourceRoot: string): ScenarioFixture => ({
sourceRoot,
entries: [
{
type: "file",
relativePath: "hello.txt",
content: "hello from zerobyte integration\n",
},
{
type: "directory",
relativePath: "docs",
},
{
type: "file",
relativePath: "docs/readme.md",
content: "fixture documentation\n",
},
],
});

View File

@@ -0,0 +1,86 @@
import fs from "node:fs/promises";
import type { BackendConfig } from "@zerobyte/contracts/volumes";
import { safeExec } from "@zerobyte/core/node";
import type { RepositoryConfig } from "@zerobyte/core/restic";
export const SFTP_HOST = "sftp";
export const SFTP_PORT = 22;
export const SFTP_USERNAME = "zerobyte-sftp";
export const SFTP_PASSWORD = "zerobyte-sftp-password";
export const SFTP_FIXTURE_ROOT = "/srv/zerobyte-integration/fixtures";
export const SFTP_REPOSITORY_ROOT = "/srv/zerobyte-integration/repos";
export const readSftpPrivateKey = async () => {
const privateKeyPath = process.env.SFTP_PRIVATE_KEY_PATH;
if (!privateKeyPath) {
throw new Error("SFTP_PRIVATE_KEY_PATH is required for SFTP integration tests");
}
return fs.readFile(privateKeyPath, "utf8");
};
export const scanSftpKnownHosts = async () => {
const result = await safeExec({
command: "ssh-keyscan",
args: ["-T", "5", SFTP_HOST],
timeout: 10_000,
});
if (result.exitCode !== 0 || result.stdout.trim().length === 0) {
throw new Error(`Failed to scan SFTP known hosts: ${result.stderr || result.stdout}`);
}
return result.stdout;
};
export const buildSftpPrivateKeyVolumeConfig = ({
privateKey,
knownHosts,
}: {
privateKey: string;
knownHosts: string;
}): BackendConfig => ({
backend: "sftp",
host: SFTP_HOST,
port: SFTP_PORT,
username: SFTP_USERNAME,
privateKey,
path: SFTP_FIXTURE_ROOT,
readOnly: true,
skipHostKeyCheck: false,
knownHosts,
allowLegacySshRsa: false,
});
export const buildSftpPasswordVolumeConfig = ({ knownHosts }: { knownHosts: string }): BackendConfig => ({
backend: "sftp",
host: SFTP_HOST,
port: SFTP_PORT,
username: SFTP_USERNAME,
password: SFTP_PASSWORD,
path: SFTP_FIXTURE_ROOT,
readOnly: true,
skipHostKeyCheck: false,
knownHosts,
allowLegacySshRsa: false,
});
export const buildSftpRepositoryConfig = ({
privateKey,
knownHosts,
path,
}: {
privateKey: string;
knownHosts: string;
path: string;
}): RepositoryConfig => ({
backend: "sftp",
host: SFTP_HOST,
port: SFTP_PORT,
user: SFTP_USERNAME,
path,
privateKey,
skipHostKeyCheck: false,
knownHosts,
allowLegacySshRsa: false,
});

View File

@@ -13,14 +13,20 @@ import {
RUSTFS_ENDPOINT,
RUSTFS_SECRET_ACCESS_KEY,
} from "./constants";
import { assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
import { createScenarioFixture } from "./helpers/fixture";
import { createIntegrationRestic } from "./helpers/restic";
import {
buildSftpRepositoryConfig,
readSftpPrivateKey,
scanSftpKnownHosts,
SFTP_REPOSITORY_ROOT,
} from "./helpers/sftp";
type RepositoryScenario = {
id: string;
name: string;
createRepositoryConfig: (prefix: string) => RepositoryConfig;
createRepositoryConfig: (prefix: string) => RepositoryConfig | Promise<RepositoryConfig>;
};
const scenarios: RepositoryScenario[] = [
@@ -44,6 +50,16 @@ const scenarios: RepositoryScenario[] = [
path: `${RUSTFS_BUCKET}/${prefix}`,
}),
},
{
id: "sftp",
name: "SFTP repository",
createRepositoryConfig: async (prefix) =>
buildSftpRepositoryConfig({
privateKey: await readSftpPrivateKey(),
knownHosts: await scanSftpKnownHosts(),
path: `${SFTP_REPOSITORY_ROOT}/${prefix}`,
}),
},
];
test.concurrent.each(scenarios)("$name can backup, list, and restore fixture data", async (scenario) => {
@@ -53,7 +69,7 @@ test.concurrent.each(scenarios)("$name can backup, list, and restore fixture dat
const restoreTarget = path.join(workspace, "restore");
const backupTag = `zerobyte-integration-${scenario.id}-${runId}`;
const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`;
const repositoryConfig = scenario.createRepositoryConfig(repositoryPrefix);
const repositoryConfig = await scenario.createRepositoryConfig(repositoryPrefix);
const restic = createIntegrationRestic(workspace, resticPassword);
let passed = false;
@@ -61,6 +77,7 @@ test.concurrent.each(scenarios)("$name can backup, list, and restore fixture dat
try {
await fs.mkdir(workspace, { recursive: true });
const fixture = await createScenarioFixture(workspace, scenario.id);
await assertFixtureSourceExists(fixture);
const initResult = await Effect.runPromise(
restic.init(repositoryConfig, {
@@ -107,6 +124,8 @@ test.concurrent.each(scenarios)("$name can backup, list, and restore fixture dat
);
assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture);
await fs.rm(fixture.sourceRoot, { recursive: true, force: true });
await Effect.runPromise(
restic.restore(repositoryConfig, snapshotId, restoreTarget, {
organizationId: INTEGRATION_ORGANIZATION_ID,

View File

@@ -0,0 +1,148 @@
import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import type { BackendConfig } from "@zerobyte/contracts/volumes";
import type { RepositoryConfig } from "@zerobyte/core/restic";
import { Effect } from "effect";
import { expect, test } from "vitest";
import { makeSftpBackend } from "../../../../apps/agent/src/volume-host/backends/sftp";
import { INTEGRATION_ORGANIZATION_ID, INTEGRATION_RUNS_DIR } from "./constants";
import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
import { createStaticSftpFixture } from "./helpers/fixture";
import { createIntegrationRestic } from "./helpers/restic";
import {
buildSftpPasswordVolumeConfig,
buildSftpPrivateKeyVolumeConfig,
readSftpPrivateKey,
scanSftpKnownHosts,
} from "./helpers/sftp";
type SftpVolumeScenario = {
id: string;
name: string;
createVolumeConfig: (runtime: { privateKey: string; knownHosts: string }) => BackendConfig;
};
const scenarios: SftpVolumeScenario[] = [
{
id: "sftp-local-repo",
name: "SFTP volume with private key auth and local repository",
createVolumeConfig: ({ privateKey, knownHosts }) => buildSftpPrivateKeyVolumeConfig({ privateKey, knownHosts }),
},
{
id: "sftp-password-local-repo",
name: "SFTP volume with password auth and local repository",
createVolumeConfig: ({ knownHosts }) => buildSftpPasswordVolumeConfig({ knownHosts }),
},
];
test.concurrent.each(scenarios)("$name can backup and restore static fixture data", async (scenario) => {
const runId = crypto.randomUUID();
const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`);
const mountPath = path.join(workspace, "mount");
const repositoryPath = path.join(workspace, "repo");
const restoreTarget = path.join(workspace, "restore");
const backupTag = `zerobyte-integration-${scenario.id}-${runId}`;
const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`;
const repositoryConfig: RepositoryConfig = { backend: "local", path: repositoryPath };
const restic = createIntegrationRestic(workspace, resticPassword);
let backend: ReturnType<typeof makeSftpBackend> | undefined;
let passed = false;
let unmountFailed = false;
try {
await fs.mkdir(workspace, { recursive: true });
const knownHosts = await scanSftpKnownHosts();
const privateKey = await readSftpPrivateKey();
const volumeConfig = scenario.createVolumeConfig({ privateKey, knownHosts });
backend = makeSftpBackend(volumeConfig, mountPath);
const mountResult = await backend.mount();
expect(mountResult.status).toBe("mounted");
const healthResult = await backend.checkHealth();
expect(healthResult.status).toBe("mounted");
const fixture = createStaticSftpFixture(path.join(mountPath, "case-a"));
await assertFixtureSourceExists(fixture);
const initResult = await Effect.runPromise(
restic.init(repositoryConfig, {
organizationId: INTEGRATION_ORGANIZATION_ID,
timeoutMs: 120_000,
}),
);
expect(initResult.success).toBe(true);
expect(initResult.error).toBeNull();
const backupResult = await Effect.runPromise(
restic.backup(repositoryConfig, fixture.sourceRoot, {
organizationId: INTEGRATION_ORGANIZATION_ID,
tags: [backupTag],
}),
);
expect(backupResult.exitCode).toBe(0);
expect(backupResult.warningDetails).toBeNull();
expect(backupResult.result?.snapshot_id).toEqual(expect.any(String));
const snapshotId = backupResult.result?.snapshot_id;
if (!snapshotId) {
throw new Error("Restic backup completed without a snapshot id");
}
const snapshots = await Effect.runPromise(
restic.snapshots(repositoryConfig, {
organizationId: INTEGRATION_ORGANIZATION_ID,
tags: [backupTag],
}),
);
const snapshot = snapshots.find(
(candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId,
);
expect(snapshot).toBeDefined();
expect(snapshot?.paths).toContain(fixture.sourceRoot);
const lsResult = await Effect.runPromise(
restic.ls(repositoryConfig, snapshotId, undefined, {
organizationId: INTEGRATION_ORGANIZATION_ID,
limit: 100,
}),
);
assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture);
const unmountResult = await backend.unmount();
expect(unmountResult.status).toBe("unmounted");
backend = undefined;
await Effect.runPromise(
restic.restore(repositoryConfig, snapshotId, restoreTarget, {
organizationId: INTEGRATION_ORGANIZATION_ID,
basePath: fixture.sourceRoot,
}),
);
await assertRestoredFixture(restoreTarget, fixture);
passed = true;
} finally {
if (backend) {
const unmountResult = await backend.unmount();
if (unmountResult.status === "error") {
unmountFailed = true;
console.error(`Failed to unmount SFTP volume at ${mountPath}: ${unmountResult.error}`);
}
}
if (passed && !unmountFailed) {
await fs.rm(workspace, { recursive: true, force: true });
} else {
console.error(`Integration scenario artifacts retained in ${workspace}`);
}
if (passed && unmountFailed) {
throw new Error(`Failed to unmount SFTP volume at ${mountPath}`);
}
}
});