mirror of
https://github.com/nicotsx/zerobyte.git
synced 2026-06-02 13:13:43 -04:00
test: automate SFTP integration coverage
This commit is contained in:
@@ -24,3 +24,4 @@
|
||||
node_modules/**
|
||||
dist/**
|
||||
.output/**
|
||||
app/test/integration/artifacts/**
|
||||
|
||||
@@ -44,6 +44,15 @@ RUN bzip2 -d restic.bz2 && chmod +x restic
|
||||
RUN mv rclone-v*-linux-*/rclone /deps/rclone && chmod +x /deps/rclone
|
||||
RUN tar -xzf shoutrrr.tar.gz && chmod +x shoutrrr
|
||||
|
||||
# ------------------------------
|
||||
# RUNTIME TOOLS
|
||||
# ------------------------------
|
||||
FROM base AS runtime-tools
|
||||
|
||||
COPY --from=deps /deps/restic /usr/local/bin/restic
|
||||
COPY --from=deps /deps/rclone /usr/local/bin/rclone
|
||||
COPY --from=deps /deps/shoutrrr /usr/local/bin/shoutrrr
|
||||
|
||||
# ------------------------------
|
||||
# DEVELOPMENT
|
||||
# ------------------------------
|
||||
|
||||
@@ -8,14 +8,12 @@ FIXTURE_UID="1000"
|
||||
FIXTURE_GID="1000"
|
||||
|
||||
ARTIFACTS_DIR="$SCRIPT_DIR/artifacts/$TARGET_HOST"
|
||||
KEY_PATH="$ARTIFACTS_DIR/zerobyte-sftp-ed25519"
|
||||
KNOWN_HOSTS_PATH="$ARTIFACTS_DIR/known_hosts"
|
||||
CONFIG_PATH="$ARTIFACTS_DIR/config.generated.json"
|
||||
|
||||
SMB_PASSWORD_FILE="$ARTIFACTS_DIR/smb-password.txt"
|
||||
SFTP_PASSWORD_FILE="$ARTIFACTS_DIR/sftp-password.txt"
|
||||
WEBDAV_PASSWORD_FILE="$ARTIFACTS_DIR/webdav-password.txt"
|
||||
RESTIC_PASSWORD_FILE="$ARTIFACTS_DIR/restic-password.txt"
|
||||
|
||||
read_or_create_secret() {
|
||||
local file_path="$1"
|
||||
@@ -35,16 +33,8 @@ chmod 700 "$ARTIFACTS_DIR"
|
||||
SMB_PASSWORD="$(read_or_create_secret "$SMB_PASSWORD_FILE")"
|
||||
SFTP_PASSWORD="$(read_or_create_secret "$SFTP_PASSWORD_FILE")"
|
||||
WEBDAV_PASSWORD="$(read_or_create_secret "$WEBDAV_PASSWORD_FILE")"
|
||||
RESTIC_PASSWORD="$(read_or_create_secret "$RESTIC_PASSWORD_FILE")"
|
||||
|
||||
if [[ ! -f "$KEY_PATH" || ! -f "$KEY_PATH.pub" ]]; then
|
||||
ssh-keygen -q -t ed25519 -N "" -C "zerobyte-backend-integration@$TARGET_HOST" -f "$KEY_PATH"
|
||||
chmod 600 "$KEY_PATH"
|
||||
fi
|
||||
|
||||
PUBLIC_KEY_BASE64="$(base64 <"$KEY_PATH.pub" | tr -d '\n')"
|
||||
|
||||
ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" "$RESTIC_PASSWORD" "$PUBLIC_KEY_BASE64" <<'REMOTE'
|
||||
ssh "$TARGET" bash -s -- "$FIXTURE_UID" "$FIXTURE_GID" "$SMB_PASSWORD" "$SFTP_PASSWORD" "$WEBDAV_PASSWORD" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
fixture_uid="$1"
|
||||
@@ -52,11 +42,6 @@ fixture_gid="$2"
|
||||
smb_password="$3"
|
||||
sftp_password="$4"
|
||||
webdav_password="$5"
|
||||
restic_password="$6"
|
||||
public_key="$(printf '%s' "$7" | base64 -d)"
|
||||
repo_path="/srv/zerobyte-backend-integration/restic-repo"
|
||||
repo_password_fingerprint_path="$repo_path/.zerobyte-password-sha256"
|
||||
repo_password_fingerprint="$(printf '%s' "$restic_password" | sha256sum | cut -d' ' -f1)"
|
||||
legacy_sshd_dir="/etc/ssh/zerobyte-backend-integration-legacy"
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
@@ -66,25 +51,8 @@ write_file() {
|
||||
cat >"$file_path"
|
||||
}
|
||||
|
||||
initialize_restic_repo() {
|
||||
local password_file
|
||||
|
||||
rm -rf "$repo_path"
|
||||
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 "$repo_path"
|
||||
|
||||
password_file="$(mktemp)"
|
||||
printf '%s\n' "$restic_password" >"$password_file"
|
||||
chown zerobyte-sftp:zerobyte-sftp "$password_file"
|
||||
chmod 0600 "$password_file"
|
||||
su -s /bin/sh -c "restic init --repo '$repo_path' --password-file '$password_file'" zerobyte-sftp
|
||||
rm -f "$password_file"
|
||||
|
||||
printf '%s\n' "$repo_password_fingerprint" >"$repo_password_fingerprint_path"
|
||||
chmod 0600 "$repo_password_fingerprint_path"
|
||||
}
|
||||
|
||||
apt-get update
|
||||
apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server restic rpcbind samba
|
||||
apt-get install -y apache2 apache2-utils nfs-kernel-server openssh-server rpcbind samba
|
||||
|
||||
id -u zerobyte-sftp >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-sftp --shell /bin/bash zerobyte-sftp
|
||||
id -u zerobyte-smb >/dev/null 2>&1 || useradd --create-home --home-dir /home/zerobyte-smb --shell /bin/bash zerobyte-smb
|
||||
@@ -96,24 +64,12 @@ chown -R "$fixture_uid:$fixture_gid" /srv/zerobyte-backend-integration/fixtures
|
||||
find /srv/zerobyte-backend-integration/fixtures -type d -exec chmod 0755 {} +
|
||||
find /srv/zerobyte-backend-integration/fixtures -type f -exec chmod 0644 {} +
|
||||
|
||||
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp
|
||||
install -d -o zerobyte-sftp -g zerobyte-sftp -m 0700 /home/zerobyte-sftp/.ssh
|
||||
printf '%s\n' "$public_key" >/home/zerobyte-sftp/.ssh/authorized_keys
|
||||
chown zerobyte-sftp:zerobyte-sftp /home/zerobyte-sftp/.ssh/authorized_keys
|
||||
chmod 0600 /home/zerobyte-sftp/.ssh/authorized_keys
|
||||
|
||||
printf '%s\n%s\n' "$smb_password" "$smb_password" | smbpasswd -a -s zerobyte-smb >/dev/null
|
||||
smbpasswd -e zerobyte-smb >/dev/null
|
||||
printf 'zerobyte-sftp:%s\n' "$sftp_password" | chpasswd
|
||||
passwd -u zerobyte-sftp >/dev/null 2>&1 || true
|
||||
htpasswd -bc /etc/apache2/zerobyte-backend-integration.htpasswd zerobyte-webdav "$webdav_password" >/dev/null
|
||||
|
||||
if [[ ! -f "$repo_path/config" ]]; then
|
||||
initialize_restic_repo
|
||||
elif [[ ! -f "$repo_password_fingerprint_path" ]] || [[ "$(cat "$repo_password_fingerprint_path")" != "$repo_password_fingerprint" ]]; then
|
||||
initialize_restic_repo
|
||||
fi
|
||||
|
||||
write_file /etc/exports <<'EOF'
|
||||
/srv/zerobyte-backend-integration/fixtures *(ro,sync,no_subtree_check,insecure)
|
||||
EOF
|
||||
@@ -239,8 +195,6 @@ INTEGRATION_HOST="$TARGET_HOST" \
|
||||
SMB_PASSWORD="$SMB_PASSWORD" \
|
||||
SFTP_PASSWORD="$SFTP_PASSWORD" \
|
||||
WEBDAV_PASSWORD="$WEBDAV_PASSWORD" \
|
||||
RESTIC_PASSWORD="$RESTIC_PASSWORD" \
|
||||
SFTP_KEY_PATH="$KEY_PATH" \
|
||||
KNOWN_HOSTS_PATH="$KNOWN_HOSTS_PATH" \
|
||||
CONFIG_PATH="$CONFIG_PATH" \
|
||||
bun run "$SCRIPT_DIR/write-generated-config.ts"
|
||||
|
||||
@@ -14,8 +14,6 @@ const fixtureGid = getRequiredNumberEnv("FIXTURE_GID");
|
||||
const smbPassword = getRequiredEnv("SMB_PASSWORD");
|
||||
const sftpPassword = getRequiredEnv("SFTP_PASSWORD");
|
||||
const webdavPassword = getRequiredEnv("WEBDAV_PASSWORD");
|
||||
const resticPassword = getRequiredEnv("RESTIC_PASSWORD");
|
||||
const privateKey = fs.readFileSync(getRequiredEnv("SFTP_KEY_PATH"), "utf8");
|
||||
const knownHosts = fs.readFileSync(getRequiredEnv("KNOWN_HOSTS_PATH"), "utf8");
|
||||
const configPath = getRequiredEnv("CONFIG_PATH");
|
||||
|
||||
@@ -74,40 +72,6 @@ const config = {
|
||||
fixtureRoot: "case-a",
|
||||
expectedEntries: smbEntries,
|
||||
},
|
||||
{
|
||||
id: "sftp-local-repo",
|
||||
volume: {
|
||||
backend: "sftp",
|
||||
host,
|
||||
port: 22,
|
||||
username: "zerobyte-sftp",
|
||||
privateKey,
|
||||
path: "/srv/zerobyte-backend-integration/fixtures",
|
||||
readOnly: true,
|
||||
skipHostKeyCheck: false,
|
||||
knownHosts,
|
||||
},
|
||||
repository: { backend: "local", path: "repo-sftp-volume" },
|
||||
fixtureRoot: "case-a",
|
||||
expectedEntries: contentOnlyEntries,
|
||||
},
|
||||
{
|
||||
id: "sftp-password-local-repo",
|
||||
volume: {
|
||||
backend: "sftp",
|
||||
host,
|
||||
port: 22,
|
||||
username: "zerobyte-sftp",
|
||||
password: sftpPassword,
|
||||
path: "/srv/zerobyte-backend-integration/fixtures",
|
||||
readOnly: true,
|
||||
skipHostKeyCheck: false,
|
||||
knownHosts,
|
||||
},
|
||||
repository: { backend: "local", path: "repo-sftp-password-volume" },
|
||||
fixtureRoot: "case-a",
|
||||
expectedEntries: contentOnlyEntries,
|
||||
},
|
||||
{
|
||||
id: "sftp-legacy-rsa-hostkey-local-repo",
|
||||
volume: {
|
||||
@@ -142,31 +106,6 @@ const config = {
|
||||
fixtureRoot: "case-a",
|
||||
expectedEntries: contentOnlyEntries,
|
||||
},
|
||||
{
|
||||
id: "nfs-sftp-repository",
|
||||
volume: {
|
||||
backend: "nfs",
|
||||
server: host,
|
||||
exportPath: "/srv/zerobyte-backend-integration/fixtures",
|
||||
port: 2049,
|
||||
version: "4.1",
|
||||
readOnly: true,
|
||||
},
|
||||
repository: {
|
||||
backend: "sftp",
|
||||
host,
|
||||
port: 22,
|
||||
user: "zerobyte-sftp",
|
||||
path: "/srv/zerobyte-backend-integration/restic-repo",
|
||||
privateKey,
|
||||
skipHostKeyCheck: false,
|
||||
knownHosts,
|
||||
isExistingRepository: true,
|
||||
customPassword: resticPassword,
|
||||
},
|
||||
fixtureRoot: "case-a",
|
||||
expectedEntries: nfsEntries,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
|
||||
@@ -23,6 +23,22 @@ services:
|
||||
done
|
||||
mc mb --ignore-existing --region us-east-1 rustfs/zerobyte-integration
|
||||
|
||||
sftp:
|
||||
build:
|
||||
context: ./sftp
|
||||
environment:
|
||||
SFTP_USER: "zerobyte-sftp"
|
||||
SFTP_PASSWORD: "zerobyte-sftp-password"
|
||||
SFTP_PUBLIC_KEY_PATH: "/run/zerobyte/sftp/id_ed25519.pub"
|
||||
volumes:
|
||||
- ../artifacts/sftp/id_ed25519.pub:/run/zerobyte/sftp/id_ed25519.pub:ro
|
||||
- sftp-data:/srv/zerobyte-integration
|
||||
healthcheck:
|
||||
test: ["CMD", "ssh-keyscan", "-T", "2", "localhost"]
|
||||
interval: 1s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
integration:
|
||||
build:
|
||||
context: ../../../..
|
||||
@@ -32,10 +48,19 @@ services:
|
||||
depends_on:
|
||||
rustfs-setup:
|
||||
condition: service_completed_successfully
|
||||
sftp:
|
||||
condition: service_healthy
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- /dev/fuse:/dev/fuse
|
||||
volumes:
|
||||
- ../artifacts:/app/app/test/integration/artifacts
|
||||
- ../artifacts/sftp/id_ed25519:/run/zerobyte/sftp/id_ed25519:ro
|
||||
environment:
|
||||
LOG_LEVEL: "info"
|
||||
LOG_LEVEL: "error"
|
||||
SFTP_PRIVATE_KEY_PATH: "/run/zerobyte/sftp/id_ed25519"
|
||||
|
||||
volumes:
|
||||
rustfs-data:
|
||||
sftp-data:
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
if [ -w /etc/fuse.conf ] && ! grep -q '^user_allow_other$' /etc/fuse.conf; then
|
||||
printf '\nuser_allow_other\n' >>/etc/fuse.conf
|
||||
fi
|
||||
|
||||
bun app/test/integration/src/write-rclone-config.ts
|
||||
exec bunx --bun vitest run --config app/test/integration/vitest.config.ts
|
||||
|
||||
10
app/test/integration/infra/sftp/Dockerfile
Normal file
10
app/test/integration/infra/sftp/Dockerfile
Normal file
@@ -0,0 +1,10 @@
|
||||
FROM alpine:3.21
|
||||
|
||||
RUN apk add --no-cache openssh-client openssh-server
|
||||
|
||||
COPY entrypoint.sh /usr/local/bin/zerobyte-sftp-entrypoint
|
||||
RUN chmod +x /usr/local/bin/zerobyte-sftp-entrypoint
|
||||
|
||||
EXPOSE 22
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/zerobyte-sftp-entrypoint"]
|
||||
46
app/test/integration/infra/sftp/entrypoint.sh
Normal file
46
app/test/integration/infra/sftp/entrypoint.sh
Normal file
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SFTP_USER="${SFTP_USER:-zerobyte-sftp}"
|
||||
SFTP_PASSWORD="${SFTP_PASSWORD:-zerobyte-sftp-password}"
|
||||
SFTP_PUBLIC_KEY_PATH="${SFTP_PUBLIC_KEY_PATH:-/run/zerobyte/sftp/id_ed25519.pub}"
|
||||
SERVICE_ROOT="/srv/zerobyte-integration"
|
||||
|
||||
ssh-keygen -A
|
||||
install -d -m 0755 /run/sshd
|
||||
|
||||
if ! id "$SFTP_USER" >/dev/null 2>&1; then
|
||||
addgroup -S "$SFTP_USER"
|
||||
adduser -S -D -h "/home/$SFTP_USER" -s /bin/sh -G "$SFTP_USER" "$SFTP_USER"
|
||||
fi
|
||||
|
||||
printf '%s:%s\n' "$SFTP_USER" "$SFTP_PASSWORD" | chpasswd
|
||||
|
||||
install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0700 "/home/$SFTP_USER/.ssh"
|
||||
install -o "$SFTP_USER" -g "$SFTP_USER" -m 0600 "$SFTP_PUBLIC_KEY_PATH" "/home/$SFTP_USER/.ssh/authorized_keys"
|
||||
|
||||
install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0755 "$SERVICE_ROOT/fixtures/case-a/docs"
|
||||
install -d -o "$SFTP_USER" -g "$SFTP_USER" -m 0755 "$SERVICE_ROOT/repos/sftp"
|
||||
printf 'hello from zerobyte integration\n' >"$SERVICE_ROOT/fixtures/case-a/hello.txt"
|
||||
printf 'fixture documentation\n' >"$SERVICE_ROOT/fixtures/case-a/docs/readme.md"
|
||||
chown -R "$SFTP_USER:$SFTP_USER" "$SERVICE_ROOT"
|
||||
|
||||
cat >/etc/ssh/sshd_config <<EOF
|
||||
Port 22
|
||||
HostKey /etc/ssh/ssh_host_ed25519_key
|
||||
HostKey /etc/ssh/ssh_host_rsa_key
|
||||
PermitRootLogin no
|
||||
PasswordAuthentication yes
|
||||
PubkeyAuthentication yes
|
||||
KbdInteractiveAuthentication no
|
||||
Subsystem sftp internal-sftp
|
||||
|
||||
Match User $SFTP_USER
|
||||
ForceCommand internal-sftp
|
||||
AllowTcpForwarding no
|
||||
X11Forwarding no
|
||||
PasswordAuthentication yes
|
||||
PubkeyAuthentication yes
|
||||
EOF
|
||||
|
||||
exec /usr/sbin/sshd -D -e
|
||||
@@ -6,14 +6,21 @@ repo_root="$(cd "$script_dir/../../.." && pwd)"
|
||||
base_image="zerobyte-integration-runtime-base:latest"
|
||||
compose_project="zerobyte-integration-$(basename "$repo_root" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9_-')"
|
||||
artifacts_dir="$script_dir/artifacts"
|
||||
sftp_artifacts_dir="$artifacts_dir/sftp"
|
||||
compose_file="$script_dir/infra/docker-compose.yml"
|
||||
|
||||
mkdir -p "$artifacts_dir"
|
||||
rm -rf "$artifacts_dir/runs"
|
||||
rm -rf "$sftp_artifacts_dir"
|
||||
rm -f "$artifacts_dir/compose.log"
|
||||
mkdir -p "$artifacts_dir/runs"
|
||||
mkdir -p "$sftp_artifacts_dir"
|
||||
|
||||
docker build --target production -t "$base_image" "$repo_root"
|
||||
ssh-keygen -q -t ed25519 -N "" -f "$sftp_artifacts_dir/id_ed25519"
|
||||
chmod 600 "$sftp_artifacts_dir/id_ed25519"
|
||||
chmod 644 "$sftp_artifacts_dir/id_ed25519.pub"
|
||||
|
||||
docker build --target runtime-tools -t "$base_image" "$repo_root"
|
||||
|
||||
exit_code=0
|
||||
docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup || exit_code=$?
|
||||
|
||||
@@ -13,32 +13,30 @@ const nodeByPath = (nodes: ResticLsNode[]) => new Map(nodes.map((node) => [path.
|
||||
export const assertSnapshotContainsFixture = (sourceRoot: string, nodes: ResticLsNode[], fixture: ScenarioFixture) => {
|
||||
const nodesByPath = nodeByPath(nodes);
|
||||
|
||||
const regularFile = nodesByPath.get(path.join(sourceRoot, fixture.regularFile.relativePath));
|
||||
expect(regularFile?.type).toBe("file");
|
||||
|
||||
const nestedDirectory = nodesByPath.get(path.join(sourceRoot, fixture.nestedDirectory.relativePath));
|
||||
expect(nestedDirectory?.type).toBe("dir");
|
||||
|
||||
const nestedFile = nodesByPath.get(path.join(sourceRoot, fixture.nestedFile.relativePath));
|
||||
expect(nestedFile?.type).toBe("file");
|
||||
|
||||
const symlink = nodesByPath.get(path.join(sourceRoot, fixture.symlink.relativePath));
|
||||
expect(symlink?.type).toBe("symlink");
|
||||
for (const entry of fixture.entries) {
|
||||
const node = nodesByPath.get(path.join(sourceRoot, entry.relativePath));
|
||||
expect(node?.type).toBe(entry.type === "directory" ? "dir" : entry.type);
|
||||
}
|
||||
};
|
||||
|
||||
export const assertRestoredFixture = async (restoreRoot: string, fixture: ScenarioFixture) => {
|
||||
await expect(fs.readFile(path.join(restoreRoot, fixture.regularFile.relativePath), "utf8")).resolves.toBe(
|
||||
fixture.regularFile.content,
|
||||
);
|
||||
|
||||
const nestedDirectoryStat = await fs.stat(path.join(restoreRoot, fixture.nestedDirectory.relativePath));
|
||||
expect(nestedDirectoryStat.isDirectory()).toBe(true);
|
||||
|
||||
await expect(fs.readFile(path.join(restoreRoot, fixture.nestedFile.relativePath), "utf8")).resolves.toBe(
|
||||
fixture.nestedFile.content,
|
||||
);
|
||||
|
||||
await expect(fs.readlink(path.join(restoreRoot, fixture.symlink.relativePath))).resolves.toBe(
|
||||
fixture.symlink.target,
|
||||
);
|
||||
for (const entry of fixture.entries) {
|
||||
const entryPath = path.join(restoreRoot, entry.relativePath);
|
||||
if (entry.type === "file") {
|
||||
await expect(fs.readFile(entryPath, "utf8")).resolves.toBe(entry.content);
|
||||
}
|
||||
if (entry.type === "directory") {
|
||||
const stats = await fs.stat(entryPath);
|
||||
expect(stats.isDirectory()).toBe(true);
|
||||
}
|
||||
if (entry.type === "symlink") {
|
||||
await expect(fs.readlink(entryPath)).resolves.toBe(entry.target);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const assertFixtureSourceExists = async (fixture: ScenarioFixture) => {
|
||||
const stats = await fs.stat(fixture.sourceRoot);
|
||||
expect(stats.isDirectory()).toBe(true);
|
||||
await assertRestoredFixture(fixture.sourceRoot, fixture);
|
||||
};
|
||||
|
||||
@@ -2,53 +2,90 @@ import crypto from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export type FixtureEntry =
|
||||
| {
|
||||
type: "file";
|
||||
relativePath: string;
|
||||
content: string;
|
||||
}
|
||||
| {
|
||||
type: "directory";
|
||||
relativePath: string;
|
||||
}
|
||||
| {
|
||||
type: "symlink";
|
||||
relativePath: string;
|
||||
target: string;
|
||||
};
|
||||
|
||||
export type ScenarioFixture = {
|
||||
sourceRoot: string;
|
||||
regularFile: {
|
||||
relativePath: string;
|
||||
content: string;
|
||||
};
|
||||
nestedDirectory: {
|
||||
relativePath: string;
|
||||
};
|
||||
nestedFile: {
|
||||
relativePath: string;
|
||||
content: string;
|
||||
};
|
||||
symlink: {
|
||||
relativePath: string;
|
||||
target: string;
|
||||
};
|
||||
entries: FixtureEntry[];
|
||||
};
|
||||
|
||||
export const createScenarioFixture = async (workspace: string, scenarioId: string): Promise<ScenarioFixture> => {
|
||||
const sourceRoot = path.join(workspace, "source");
|
||||
const nestedDirectory = path.join(sourceRoot, "nested");
|
||||
const randomSuffix = crypto.randomBytes(8).toString("hex");
|
||||
|
||||
const fixture: ScenarioFixture = {
|
||||
sourceRoot,
|
||||
regularFile: {
|
||||
relativePath: "regular.txt",
|
||||
content: `regular fixture for ${scenarioId} (${randomSuffix})\n`,
|
||||
},
|
||||
nestedDirectory: {
|
||||
relativePath: "nested",
|
||||
},
|
||||
nestedFile: {
|
||||
relativePath: "nested/deep.txt",
|
||||
content: `nested fixture for ${scenarioId} (${randomSuffix})\n`,
|
||||
},
|
||||
symlink: {
|
||||
relativePath: "regular-link",
|
||||
target: "regular.txt",
|
||||
},
|
||||
entries: [
|
||||
{
|
||||
type: "file",
|
||||
relativePath: "regular.txt",
|
||||
content: `regular fixture for ${scenarioId} (${randomSuffix})\n`,
|
||||
},
|
||||
{
|
||||
type: "directory",
|
||||
relativePath: "nested",
|
||||
},
|
||||
{
|
||||
type: "file",
|
||||
relativePath: "nested/deep.txt",
|
||||
content: `nested fixture for ${scenarioId} (${randomSuffix})\n`,
|
||||
},
|
||||
{
|
||||
type: "symlink",
|
||||
relativePath: "regular-link",
|
||||
target: "regular.txt",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
await fs.mkdir(nestedDirectory, { recursive: true });
|
||||
await fs.writeFile(path.join(sourceRoot, fixture.regularFile.relativePath), fixture.regularFile.content);
|
||||
await fs.writeFile(path.join(sourceRoot, fixture.nestedFile.relativePath), fixture.nestedFile.content);
|
||||
await fs.symlink(fixture.symlink.target, path.join(sourceRoot, fixture.symlink.relativePath));
|
||||
for (const entry of fixture.entries) {
|
||||
const entryPath = path.join(sourceRoot, entry.relativePath);
|
||||
if (entry.type === "file") {
|
||||
await fs.mkdir(path.dirname(entryPath), { recursive: true });
|
||||
await fs.writeFile(entryPath, entry.content);
|
||||
}
|
||||
if (entry.type === "directory") {
|
||||
await fs.mkdir(entryPath, { recursive: true });
|
||||
}
|
||||
if (entry.type === "symlink") {
|
||||
await fs.mkdir(path.dirname(entryPath), { recursive: true });
|
||||
await fs.symlink(entry.target, entryPath);
|
||||
}
|
||||
}
|
||||
|
||||
return fixture;
|
||||
};
|
||||
|
||||
export const createStaticSftpFixture = (sourceRoot: string): ScenarioFixture => ({
|
||||
sourceRoot,
|
||||
entries: [
|
||||
{
|
||||
type: "file",
|
||||
relativePath: "hello.txt",
|
||||
content: "hello from zerobyte integration\n",
|
||||
},
|
||||
{
|
||||
type: "directory",
|
||||
relativePath: "docs",
|
||||
},
|
||||
{
|
||||
type: "file",
|
||||
relativePath: "docs/readme.md",
|
||||
content: "fixture documentation\n",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
86
app/test/integration/src/helpers/sftp.ts
Normal file
86
app/test/integration/src/helpers/sftp.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
import fs from "node:fs/promises";
|
||||
import type { BackendConfig } from "@zerobyte/contracts/volumes";
|
||||
import { safeExec } from "@zerobyte/core/node";
|
||||
import type { RepositoryConfig } from "@zerobyte/core/restic";
|
||||
|
||||
export const SFTP_HOST = "sftp";
|
||||
export const SFTP_PORT = 22;
|
||||
export const SFTP_USERNAME = "zerobyte-sftp";
|
||||
export const SFTP_PASSWORD = "zerobyte-sftp-password";
|
||||
export const SFTP_FIXTURE_ROOT = "/srv/zerobyte-integration/fixtures";
|
||||
export const SFTP_REPOSITORY_ROOT = "/srv/zerobyte-integration/repos";
|
||||
|
||||
export const readSftpPrivateKey = async () => {
|
||||
const privateKeyPath = process.env.SFTP_PRIVATE_KEY_PATH;
|
||||
if (!privateKeyPath) {
|
||||
throw new Error("SFTP_PRIVATE_KEY_PATH is required for SFTP integration tests");
|
||||
}
|
||||
|
||||
return fs.readFile(privateKeyPath, "utf8");
|
||||
};
|
||||
|
||||
export const scanSftpKnownHosts = async () => {
|
||||
const result = await safeExec({
|
||||
command: "ssh-keyscan",
|
||||
args: ["-T", "5", SFTP_HOST],
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
if (result.exitCode !== 0 || result.stdout.trim().length === 0) {
|
||||
throw new Error(`Failed to scan SFTP known hosts: ${result.stderr || result.stdout}`);
|
||||
}
|
||||
|
||||
return result.stdout;
|
||||
};
|
||||
|
||||
export const buildSftpPrivateKeyVolumeConfig = ({
|
||||
privateKey,
|
||||
knownHosts,
|
||||
}: {
|
||||
privateKey: string;
|
||||
knownHosts: string;
|
||||
}): BackendConfig => ({
|
||||
backend: "sftp",
|
||||
host: SFTP_HOST,
|
||||
port: SFTP_PORT,
|
||||
username: SFTP_USERNAME,
|
||||
privateKey,
|
||||
path: SFTP_FIXTURE_ROOT,
|
||||
readOnly: true,
|
||||
skipHostKeyCheck: false,
|
||||
knownHosts,
|
||||
allowLegacySshRsa: false,
|
||||
});
|
||||
|
||||
export const buildSftpPasswordVolumeConfig = ({ knownHosts }: { knownHosts: string }): BackendConfig => ({
|
||||
backend: "sftp",
|
||||
host: SFTP_HOST,
|
||||
port: SFTP_PORT,
|
||||
username: SFTP_USERNAME,
|
||||
password: SFTP_PASSWORD,
|
||||
path: SFTP_FIXTURE_ROOT,
|
||||
readOnly: true,
|
||||
skipHostKeyCheck: false,
|
||||
knownHosts,
|
||||
allowLegacySshRsa: false,
|
||||
});
|
||||
|
||||
export const buildSftpRepositoryConfig = ({
|
||||
privateKey,
|
||||
knownHosts,
|
||||
path,
|
||||
}: {
|
||||
privateKey: string;
|
||||
knownHosts: string;
|
||||
path: string;
|
||||
}): RepositoryConfig => ({
|
||||
backend: "sftp",
|
||||
host: SFTP_HOST,
|
||||
port: SFTP_PORT,
|
||||
user: SFTP_USERNAME,
|
||||
path,
|
||||
privateKey,
|
||||
skipHostKeyCheck: false,
|
||||
knownHosts,
|
||||
allowLegacySshRsa: false,
|
||||
});
|
||||
@@ -13,14 +13,20 @@ import {
|
||||
RUSTFS_ENDPOINT,
|
||||
RUSTFS_SECRET_ACCESS_KEY,
|
||||
} from "./constants";
|
||||
import { assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
|
||||
import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
|
||||
import { createScenarioFixture } from "./helpers/fixture";
|
||||
import { createIntegrationRestic } from "./helpers/restic";
|
||||
import {
|
||||
buildSftpRepositoryConfig,
|
||||
readSftpPrivateKey,
|
||||
scanSftpKnownHosts,
|
||||
SFTP_REPOSITORY_ROOT,
|
||||
} from "./helpers/sftp";
|
||||
|
||||
type RepositoryScenario = {
|
||||
id: string;
|
||||
name: string;
|
||||
createRepositoryConfig: (prefix: string) => RepositoryConfig;
|
||||
createRepositoryConfig: (prefix: string) => RepositoryConfig | Promise<RepositoryConfig>;
|
||||
};
|
||||
|
||||
const scenarios: RepositoryScenario[] = [
|
||||
@@ -44,6 +50,16 @@ const scenarios: RepositoryScenario[] = [
|
||||
path: `${RUSTFS_BUCKET}/${prefix}`,
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "sftp",
|
||||
name: "SFTP repository",
|
||||
createRepositoryConfig: async (prefix) =>
|
||||
buildSftpRepositoryConfig({
|
||||
privateKey: await readSftpPrivateKey(),
|
||||
knownHosts: await scanSftpKnownHosts(),
|
||||
path: `${SFTP_REPOSITORY_ROOT}/${prefix}`,
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
test.concurrent.each(scenarios)("$name can backup, list, and restore fixture data", async (scenario) => {
|
||||
@@ -53,7 +69,7 @@ test.concurrent.each(scenarios)("$name can backup, list, and restore fixture dat
|
||||
const restoreTarget = path.join(workspace, "restore");
|
||||
const backupTag = `zerobyte-integration-${scenario.id}-${runId}`;
|
||||
const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`;
|
||||
const repositoryConfig = scenario.createRepositoryConfig(repositoryPrefix);
|
||||
const repositoryConfig = await scenario.createRepositoryConfig(repositoryPrefix);
|
||||
const restic = createIntegrationRestic(workspace, resticPassword);
|
||||
|
||||
let passed = false;
|
||||
@@ -61,6 +77,7 @@ test.concurrent.each(scenarios)("$name can backup, list, and restore fixture dat
|
||||
try {
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
const fixture = await createScenarioFixture(workspace, scenario.id);
|
||||
await assertFixtureSourceExists(fixture);
|
||||
|
||||
const initResult = await Effect.runPromise(
|
||||
restic.init(repositoryConfig, {
|
||||
@@ -107,6 +124,8 @@ test.concurrent.each(scenarios)("$name can backup, list, and restore fixture dat
|
||||
);
|
||||
assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture);
|
||||
|
||||
await fs.rm(fixture.sourceRoot, { recursive: true, force: true });
|
||||
|
||||
await Effect.runPromise(
|
||||
restic.restore(repositoryConfig, snapshotId, restoreTarget, {
|
||||
organizationId: INTEGRATION_ORGANIZATION_ID,
|
||||
|
||||
148
app/test/integration/src/volume-backends.test.ts
Normal file
148
app/test/integration/src/volume-backends.test.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { BackendConfig } from "@zerobyte/contracts/volumes";
|
||||
import type { RepositoryConfig } from "@zerobyte/core/restic";
|
||||
import { Effect } from "effect";
|
||||
import { expect, test } from "vitest";
|
||||
import { makeSftpBackend } from "../../../../apps/agent/src/volume-host/backends/sftp";
|
||||
import { INTEGRATION_ORGANIZATION_ID, INTEGRATION_RUNS_DIR } from "./constants";
|
||||
import { assertFixtureSourceExists, assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
|
||||
import { createStaticSftpFixture } from "./helpers/fixture";
|
||||
import { createIntegrationRestic } from "./helpers/restic";
|
||||
import {
|
||||
buildSftpPasswordVolumeConfig,
|
||||
buildSftpPrivateKeyVolumeConfig,
|
||||
readSftpPrivateKey,
|
||||
scanSftpKnownHosts,
|
||||
} from "./helpers/sftp";
|
||||
|
||||
type SftpVolumeScenario = {
|
||||
id: string;
|
||||
name: string;
|
||||
createVolumeConfig: (runtime: { privateKey: string; knownHosts: string }) => BackendConfig;
|
||||
};
|
||||
|
||||
const scenarios: SftpVolumeScenario[] = [
|
||||
{
|
||||
id: "sftp-local-repo",
|
||||
name: "SFTP volume with private key auth and local repository",
|
||||
createVolumeConfig: ({ privateKey, knownHosts }) => buildSftpPrivateKeyVolumeConfig({ privateKey, knownHosts }),
|
||||
},
|
||||
{
|
||||
id: "sftp-password-local-repo",
|
||||
name: "SFTP volume with password auth and local repository",
|
||||
createVolumeConfig: ({ knownHosts }) => buildSftpPasswordVolumeConfig({ knownHosts }),
|
||||
},
|
||||
];
|
||||
|
||||
test.concurrent.each(scenarios)("$name can backup and restore static fixture data", async (scenario) => {
|
||||
const runId = crypto.randomUUID();
|
||||
const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`);
|
||||
const mountPath = path.join(workspace, "mount");
|
||||
const repositoryPath = path.join(workspace, "repo");
|
||||
const restoreTarget = path.join(workspace, "restore");
|
||||
const backupTag = `zerobyte-integration-${scenario.id}-${runId}`;
|
||||
const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`;
|
||||
const repositoryConfig: RepositoryConfig = { backend: "local", path: repositoryPath };
|
||||
const restic = createIntegrationRestic(workspace, resticPassword);
|
||||
|
||||
let backend: ReturnType<typeof makeSftpBackend> | undefined;
|
||||
let passed = false;
|
||||
let unmountFailed = false;
|
||||
|
||||
try {
|
||||
await fs.mkdir(workspace, { recursive: true });
|
||||
|
||||
const knownHosts = await scanSftpKnownHosts();
|
||||
const privateKey = await readSftpPrivateKey();
|
||||
const volumeConfig = scenario.createVolumeConfig({ privateKey, knownHosts });
|
||||
backend = makeSftpBackend(volumeConfig, mountPath);
|
||||
|
||||
const mountResult = await backend.mount();
|
||||
expect(mountResult.status).toBe("mounted");
|
||||
|
||||
const healthResult = await backend.checkHealth();
|
||||
expect(healthResult.status).toBe("mounted");
|
||||
|
||||
const fixture = createStaticSftpFixture(path.join(mountPath, "case-a"));
|
||||
await assertFixtureSourceExists(fixture);
|
||||
|
||||
const initResult = await Effect.runPromise(
|
||||
restic.init(repositoryConfig, {
|
||||
organizationId: INTEGRATION_ORGANIZATION_ID,
|
||||
timeoutMs: 120_000,
|
||||
}),
|
||||
);
|
||||
expect(initResult.success).toBe(true);
|
||||
expect(initResult.error).toBeNull();
|
||||
|
||||
const backupResult = await Effect.runPromise(
|
||||
restic.backup(repositoryConfig, fixture.sourceRoot, {
|
||||
organizationId: INTEGRATION_ORGANIZATION_ID,
|
||||
tags: [backupTag],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(backupResult.exitCode).toBe(0);
|
||||
expect(backupResult.warningDetails).toBeNull();
|
||||
expect(backupResult.result?.snapshot_id).toEqual(expect.any(String));
|
||||
|
||||
const snapshotId = backupResult.result?.snapshot_id;
|
||||
if (!snapshotId) {
|
||||
throw new Error("Restic backup completed without a snapshot id");
|
||||
}
|
||||
|
||||
const snapshots = await Effect.runPromise(
|
||||
restic.snapshots(repositoryConfig, {
|
||||
organizationId: INTEGRATION_ORGANIZATION_ID,
|
||||
tags: [backupTag],
|
||||
}),
|
||||
);
|
||||
const snapshot = snapshots.find(
|
||||
(candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId,
|
||||
);
|
||||
expect(snapshot).toBeDefined();
|
||||
expect(snapshot?.paths).toContain(fixture.sourceRoot);
|
||||
|
||||
const lsResult = await Effect.runPromise(
|
||||
restic.ls(repositoryConfig, snapshotId, undefined, {
|
||||
organizationId: INTEGRATION_ORGANIZATION_ID,
|
||||
limit: 100,
|
||||
}),
|
||||
);
|
||||
assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture);
|
||||
|
||||
const unmountResult = await backend.unmount();
|
||||
expect(unmountResult.status).toBe("unmounted");
|
||||
backend = undefined;
|
||||
|
||||
await Effect.runPromise(
|
||||
restic.restore(repositoryConfig, snapshotId, restoreTarget, {
|
||||
organizationId: INTEGRATION_ORGANIZATION_ID,
|
||||
basePath: fixture.sourceRoot,
|
||||
}),
|
||||
);
|
||||
await assertRestoredFixture(restoreTarget, fixture);
|
||||
|
||||
passed = true;
|
||||
} finally {
|
||||
if (backend) {
|
||||
const unmountResult = await backend.unmount();
|
||||
if (unmountResult.status === "error") {
|
||||
unmountFailed = true;
|
||||
console.error(`Failed to unmount SFTP volume at ${mountPath}: ${unmountResult.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (passed && !unmountFailed) {
|
||||
await fs.rm(workspace, { recursive: true, force: true });
|
||||
} else {
|
||||
console.error(`Integration scenario artifacts retained in ${workspace}`);
|
||||
}
|
||||
|
||||
if (passed && unmountFailed) {
|
||||
throw new Error(`Failed to unmount SFTP volume at ${mountPath}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user