test(integration): s3 repository with rustfs & rclone (#933)

* test(integration): s3 repository with rustfs

* ci: run integration tests before release

* chore: fix linting issue

* ci: persist-creds -> false

.dockerignore

@@ -3,6 +3,7 @@
 !bun.lock
 !package.json
 !.gitignore
+!bunfig.toml
 
 !**/package.json
 !**/bun.lock

.github/workflows/integration.yml

@@ -0,0 +1,30 @@
+name: Integration Tests
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  integration:
+    name: Integration
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+
+      - name: Install dependencies
+        uses: "./.github/actions/install-dependencies"
+
+      - name: Run integration tests
+        run: bun run test:integration
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: failure()
+        with:
+          name: integration-artifacts
+          path: app/test/integration/artifacts/
+          retention-days: 5

.github/workflows/release.yml

@@ -43,10 +43,13 @@ jobs:
   e2e-tests:
     uses: ./.github/workflows/e2e.yml
 
+  integration-tests:
+    uses: ./.github/workflows/integration.yml
+
   build-images:
     environment: release
     timeout-minutes: 15
-    needs: [determine-release-type, checks, e2e-tests]
+    needs: [determine-release-type, checks, e2e-tests, integration-tests]
     runs-on: ubuntu-latest
     permissions:
       contents: read

app/test/integration/.gitignore

@@ -0,0 +1,2 @@
+artifacts/*
+!artifacts/.gitignore

app/test/integration/artifacts/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

app/test/integration/infra/Dockerfile

@@ -0,0 +1,18 @@
+ARG BASE_IMAGE=zerobyte-integration-runtime-base:latest
+FROM ${BASE_IMAGE}
+
+WORKDIR /app
+
+ENV NODE_ENV=test
+
+COPY ./package.json ./bun.lock ./
+COPY ./packages/core/package.json ./packages/core/package.json
+COPY ./packages/contracts/package.json ./packages/contracts/package.json
+COPY ./apps/agent/package.json ./apps/agent/package.json
+COPY ./apps/docs/package.json ./apps/docs/package.json
+
+RUN VITE_GIT_HOOKS=0 bun install --frozen-lockfile
+
+COPY . .
+
+CMD ["sh", "app/test/integration/infra/entrypoint.sh"]

app/test/integration/infra/docker-compose.yml

@@ -0,0 +1,41 @@
+services:
+  rustfs:
+    image: rustfs/rustfs:latest
+    command: ["/data"]
+    environment:
+      RUSTFS_ADDRESS: ":9000"
+      RUSTFS_ACCESS_KEY: "rustfsadmin"
+      RUSTFS_SECRET_KEY: "rustfsadmin"
+      RUSTFS_CONSOLE_ENABLE: "false"
+    volumes:
+      - rustfs-data:/data
+
+  rustfs-setup:
+    image: minio/mc:latest
+    depends_on:
+      - rustfs
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        until mc alias set rustfs http://rustfs:9000 rustfsadmin rustfsadmin --api S3v4 --path on; do
+          echo "Waiting for RustFS..."
+          sleep 1
+        done
+        mc mb --ignore-existing --region us-east-1 rustfs/zerobyte-integration
+
+  integration:
+    build:
+      context: ../../../..
+      dockerfile: app/test/integration/infra/Dockerfile
+      args:
+        BASE_IMAGE: zerobyte-integration-runtime-base:latest
+    depends_on:
+      rustfs-setup:
+        condition: service_completed_successfully
+    volumes:
+      - ../artifacts:/app/app/test/integration/artifacts
+    environment:
+      LOG_LEVEL: "info"
+
+volumes:
+  rustfs-data:

app/test/integration/infra/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+set -eu
+
+bun app/test/integration/src/write-rclone-config.ts
+exec bunx --bun vitest run --config app/test/integration/vitest.config.ts

app/test/integration/run.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$script_dir/../../.." && pwd)"
+base_image="zerobyte-integration-runtime-base:latest"
+compose_project="zerobyte-integration-$(basename "$repo_root" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9_-')"
+artifacts_dir="$script_dir/artifacts"
+compose_file="$script_dir/infra/docker-compose.yml"
+
+mkdir -p "$artifacts_dir"
+rm -rf "$artifacts_dir/runs"
+rm -f "$artifacts_dir/compose.log"
+mkdir -p "$artifacts_dir/runs"
+
+docker build --target production -t "$base_image" "$repo_root"
+
+exit_code=0
+docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup || exit_code=$?
+
+if [[ "$exit_code" -eq 0 ]]; then
+	docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from integration integration || exit_code=$?
+fi
+
+if [[ "$exit_code" -ne 0 ]]; then
+	docker compose -f "$compose_file" -p "$compose_project" logs --no-color >"$artifacts_dir/compose.log" || true
+fi
+
+docker compose -f "$compose_file" -p "$compose_project" down --volumes --remove-orphans || true
+
+exit "$exit_code"

app/test/integration/src/constants.ts

@@ -0,0 +1,14 @@
+import path from "node:path";
+
+export const RUSTFS_ENDPOINT = "http://rustfs:9000";
+export const RUSTFS_ACCESS_KEY_ID = "rustfsadmin";
+export const RUSTFS_SECRET_ACCESS_KEY = "rustfsadmin";
+export const RUSTFS_BUCKET = "zerobyte-integration";
+
+export const RCLONE_REMOTE = "e2e-rustfs";
+export const RCLONE_CONFIG_DIR = "/root/.config/rclone";
+export const RCLONE_CONFIG_FILE = path.join(RCLONE_CONFIG_DIR, "rclone.conf");
+
+export const INTEGRATION_ORGANIZATION_ID = "integration-suite";
+export const INTEGRATION_ARTIFACTS_DIR = path.resolve(import.meta.dirname, "../artifacts");
+export const INTEGRATION_RUNS_DIR = path.join(INTEGRATION_ARTIFACTS_DIR, "runs");

app/test/integration/src/helpers/assertions.ts

@@ -0,0 +1,44 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { expect } from "vitest";
+import type { ScenarioFixture } from "./fixture";
+
+type ResticLsNode = {
+	path: string;
+	type: string;
+};
+
+const nodeByPath = (nodes: ResticLsNode[]) => new Map(nodes.map((node) => [path.normalize(node.path), node]));
+
+export const assertSnapshotContainsFixture = (sourceRoot: string, nodes: ResticLsNode[], fixture: ScenarioFixture) => {
+	const nodesByPath = nodeByPath(nodes);
+
+	const regularFile = nodesByPath.get(path.join(sourceRoot, fixture.regularFile.relativePath));
+	expect(regularFile?.type).toBe("file");
+
+	const nestedDirectory = nodesByPath.get(path.join(sourceRoot, fixture.nestedDirectory.relativePath));
+	expect(nestedDirectory?.type).toBe("dir");
+
+	const nestedFile = nodesByPath.get(path.join(sourceRoot, fixture.nestedFile.relativePath));
+	expect(nestedFile?.type).toBe("file");
+
+	const symlink = nodesByPath.get(path.join(sourceRoot, fixture.symlink.relativePath));
+	expect(symlink?.type).toBe("symlink");
+};
+
+export const assertRestoredFixture = async (restoreRoot: string, fixture: ScenarioFixture) => {
+	await expect(fs.readFile(path.join(restoreRoot, fixture.regularFile.relativePath), "utf8")).resolves.toBe(
+		fixture.regularFile.content,
+	);
+
+	const nestedDirectoryStat = await fs.stat(path.join(restoreRoot, fixture.nestedDirectory.relativePath));
+	expect(nestedDirectoryStat.isDirectory()).toBe(true);
+
+	await expect(fs.readFile(path.join(restoreRoot, fixture.nestedFile.relativePath), "utf8")).resolves.toBe(
+		fixture.nestedFile.content,
+	);
+
+	await expect(fs.readlink(path.join(restoreRoot, fixture.symlink.relativePath))).resolves.toBe(
+		fixture.symlink.target,
+	);
+};

app/test/integration/src/helpers/fixture.ts

@@ -0,0 +1,54 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export type ScenarioFixture = {
+	sourceRoot: string;
+	regularFile: {
+		relativePath: string;
+		content: string;
+	};
+	nestedDirectory: {
+		relativePath: string;
+	};
+	nestedFile: {
+		relativePath: string;
+		content: string;
+	};
+	symlink: {
+		relativePath: string;
+		target: string;
+	};
+};
+
+export const createScenarioFixture = async (workspace: string, scenarioId: string): Promise<ScenarioFixture> => {
+	const sourceRoot = path.join(workspace, "source");
+	const nestedDirectory = path.join(sourceRoot, "nested");
+	const randomSuffix = crypto.randomBytes(8).toString("hex");
+
+	const fixture: ScenarioFixture = {
+		sourceRoot,
+		regularFile: {
+			relativePath: "regular.txt",
+			content: `regular fixture for ${scenarioId} (${randomSuffix})\n`,
+		},
+		nestedDirectory: {
+			relativePath: "nested",
+		},
+		nestedFile: {
+			relativePath: "nested/deep.txt",
+			content: `nested fixture for ${scenarioId} (${randomSuffix})\n`,
+		},
+		symlink: {
+			relativePath: "regular-link",
+			target: "regular.txt",
+		},
+	};
+
+	await fs.mkdir(nestedDirectory, { recursive: true });
+	await fs.writeFile(path.join(sourceRoot, fixture.regularFile.relativePath), fixture.regularFile.content);
+	await fs.writeFile(path.join(sourceRoot, fixture.nestedFile.relativePath), fixture.nestedFile.content);
+	await fs.symlink(fixture.symlink.target, path.join(sourceRoot, fixture.symlink.relativePath));
+
+	return fixture;
+};

app/test/integration/src/helpers/restic.ts

@@ -0,0 +1,14 @@
+import path from "node:path";
+import { createRestic } from "@zerobyte/core/restic/server";
+import { RCLONE_CONFIG_FILE } from "../constants";
+
+export const createIntegrationRestic = (workspace: string, resticPassword: string) => {
+	return createRestic({
+		resolveSecret: async (value: string) => value,
+		getOrganizationResticPassword: async () => resticPassword,
+		resticCacheDir: path.join(workspace, "restic-cache"),
+		resticPassFile: path.join(workspace, "restic.pass"),
+		defaultExcludes: [],
+		rcloneConfigFile: RCLONE_CONFIG_FILE,
+	});
+};

app/test/integration/src/repositories.test.ts

@@ -0,0 +1,126 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { Effect } from "effect";
+import type { RepositoryConfig } from "@zerobyte/core/restic";
+import { expect, test } from "vitest";
+import {
+	INTEGRATION_ORGANIZATION_ID,
+	INTEGRATION_RUNS_DIR,
+	RCLONE_REMOTE,
+	RUSTFS_ACCESS_KEY_ID,
+	RUSTFS_BUCKET,
+	RUSTFS_ENDPOINT,
+	RUSTFS_SECRET_ACCESS_KEY,
+} from "./constants";
+import { assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions";
+import { createScenarioFixture } from "./helpers/fixture";
+import { createIntegrationRestic } from "./helpers/restic";
+
+type RepositoryScenario = {
+	id: string;
+	name: string;
+	createRepositoryConfig: (prefix: string) => RepositoryConfig;
+};
+
+const scenarios: RepositoryScenario[] = [
+	{
+		id: "direct-s3",
+		name: "direct S3 repository",
+		createRepositoryConfig: (prefix) => ({
+			backend: "s3",
+			endpoint: RUSTFS_ENDPOINT,
+			bucket: `${RUSTFS_BUCKET}/${prefix}`,
+			accessKeyId: RUSTFS_ACCESS_KEY_ID,
+			secretAccessKey: RUSTFS_SECRET_ACCESS_KEY,
+		}),
+	},
+	{
+		id: "rclone-s3",
+		name: "rclone repository over RustFS S3",
+		createRepositoryConfig: (prefix) => ({
+			backend: "rclone",
+			remote: RCLONE_REMOTE,
+			path: `${RUSTFS_BUCKET}/${prefix}`,
+		}),
+	},
+];
+
+test.concurrent.each(scenarios)("$name can backup, list, and restore fixture data", async (scenario) => {
+	const runId = crypto.randomUUID();
+	const repositoryPrefix = `${scenario.id}/${runId}`;
+	const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`);
+	const restoreTarget = path.join(workspace, "restore");
+	const backupTag = `zerobyte-integration-${scenario.id}-${runId}`;
+	const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`;
+	const repositoryConfig = scenario.createRepositoryConfig(repositoryPrefix);
+	const restic = createIntegrationRestic(workspace, resticPassword);
+
+	let passed = false;
+
+	try {
+		await fs.mkdir(workspace, { recursive: true });
+		const fixture = await createScenarioFixture(workspace, scenario.id);
+
+		const initResult = await Effect.runPromise(
+			restic.init(repositoryConfig, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				timeoutMs: 120_000,
+			}),
+		);
+		expect(initResult.success).toBe(true);
+		expect(initResult.error).toBeNull();
+
+		const backupResult = await Effect.runPromise(
+			restic.backup(repositoryConfig, fixture.sourceRoot, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				tags: [backupTag],
+			}),
+		);
+
+		expect(backupResult.exitCode).toBe(0);
+		expect(backupResult.warningDetails).toBeNull();
+		expect(backupResult.result?.snapshot_id).toEqual(expect.any(String));
+
+		const snapshotId = backupResult.result?.snapshot_id;
+		if (!snapshotId) {
+			throw new Error("Restic backup completed without a snapshot id");
+		}
+
+		const snapshots = await Effect.runPromise(
+			restic.snapshots(repositoryConfig, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				tags: [backupTag],
+			}),
+		);
+		const snapshot = snapshots.find(
+			(candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId,
+		);
+		expect(snapshot).toBeDefined();
+		expect(snapshot?.paths).toContain(fixture.sourceRoot);
+
+		const lsResult = await Effect.runPromise(
+			restic.ls(repositoryConfig, snapshotId, undefined, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				limit: 100,
+			}),
+		);
+		assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture);
+
+		await Effect.runPromise(
+			restic.restore(repositoryConfig, snapshotId, restoreTarget, {
+				organizationId: INTEGRATION_ORGANIZATION_ID,
+				basePath: fixture.sourceRoot,
+			}),
+		);
+		await assertRestoredFixture(restoreTarget, fixture);
+
+		passed = true;
+	} finally {
+		if (passed) {
+			await fs.rm(workspace, { recursive: true, force: true });
+		} else {
+			console.error(`Integration scenario artifacts retained in ${workspace}`);
+		}
+	}
+});

app/test/integration/src/write-rclone-config.ts

@@ -0,0 +1,24 @@
+import fs from "node:fs/promises";
+import {
+	RCLONE_CONFIG_DIR,
+	RCLONE_CONFIG_FILE,
+	RCLONE_REMOTE,
+	RUSTFS_ACCESS_KEY_ID,
+	RUSTFS_ENDPOINT,
+	RUSTFS_SECRET_ACCESS_KEY,
+} from "./constants";
+
+const rcloneConfig = `[${RCLONE_REMOTE}]
+type = s3
+provider = Minio
+env_auth = false
+access_key_id = ${RUSTFS_ACCESS_KEY_ID}
+secret_access_key = ${RUSTFS_SECRET_ACCESS_KEY}
+endpoint = ${RUSTFS_ENDPOINT}
+force_path_style = true
+acl = private
+`;
+
+await fs.mkdir(RCLONE_CONFIG_DIR, { recursive: true });
+await fs.writeFile(RCLONE_CONFIG_FILE, rcloneConfig, { mode: 0o600 });
+await fs.chmod(RCLONE_CONFIG_FILE, 0o600);

app/test/integration/vitest.config.ts

@@ -0,0 +1,23 @@
+import path from "node:path";
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+	resolve: {
+		alias: {
+			"~": path.resolve(import.meta.dirname, "../.."),
+		},
+	},
+	test: {
+		server: {
+			deps: {
+				inline: ["zod"],
+			},
+		},
+		name: "integration",
+		environment: "node",
+		include: ["app/test/integration/src/**/*.test.ts"],
+		testTimeout: 300_000,
+		hookTimeout: 300_000,
+		maxConcurrency: 4,
+	},
+});

package.json

@@ -15,6 +15,7 @@
 		"build": "bunx --bun vite build",
 		"start": "bun run .output/server/index.mjs",
 		"build:backend-integration": "bun build app/test/backend-integration/index.ts --outdir .output/backend-integration --target bun",
+		"test:integration": "bash app/test/integration/run.sh",
 		"test:integration:backends": "bash app/test/backend-integration/run.sh",
 		"preview": "bunx --bun vite preview",
 		"cli:dev": "bun run app/server/cli/main.ts",

vitest.config.ts

@@ -25,6 +25,8 @@ export default defineConfig({
 						"app/client/**/*.test.tsx",
 						"app/client/**/*.spec.ts",
 						"app/client/**/*.spec.tsx",
+						"app/test/integration/**/*.test.ts",
+						"app/test/integration/**/*.spec.ts",
 					],
 					setupFiles: ["./app/test/setup.ts"],
 				},