From 648ccae5fcfaf84df20fb4b49465768224a6f2d4 Mon Sep 17 00:00:00 2001 From: Nico <47644445+nicotsx@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:37:12 +0200 Subject: [PATCH] test(integration): s3 repository with rustfs & rclone (#933) * test(integration): s3 repository with rustfs * ci: run integration tests before release * chore: fix linting issue * ci: persist-creds -> false --- .dockerignore | 1 + .github/workflows/integration.yml | 30 +++++ .github/workflows/release.yml | 5 +- app/test/integration/.gitignore | 2 + app/test/integration/artifacts/.gitignore | 2 + app/test/integration/infra/Dockerfile | 18 +++ app/test/integration/infra/docker-compose.yml | 41 ++++++ app/test/integration/infra/entrypoint.sh | 5 + app/test/integration/run.sh | 31 +++++ app/test/integration/src/constants.ts | 14 ++ .../integration/src/helpers/assertions.ts | 44 ++++++ app/test/integration/src/helpers/fixture.ts | 54 ++++++++ app/test/integration/src/helpers/restic.ts | 14 ++ app/test/integration/src/repositories.test.ts | 126 ++++++++++++++++++ .../integration/src/write-rclone-config.ts | 24 ++++ app/test/integration/vitest.config.ts | 23 ++++ package.json | 1 + vitest.config.ts | 2 + 18 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/integration.yml create mode 100644 app/test/integration/.gitignore create mode 100644 app/test/integration/artifacts/.gitignore create mode 100644 app/test/integration/infra/Dockerfile create mode 100644 app/test/integration/infra/docker-compose.yml create mode 100644 app/test/integration/infra/entrypoint.sh create mode 100644 app/test/integration/run.sh create mode 100644 app/test/integration/src/constants.ts create mode 100644 app/test/integration/src/helpers/assertions.ts create mode 100644 app/test/integration/src/helpers/fixture.ts create mode 100644 app/test/integration/src/helpers/restic.ts create mode 100644 app/test/integration/src/repositories.test.ts create mode 100644 app/test/integration/src/write-rclone-config.ts create mode 100644 app/test/integration/vitest.config.ts diff --git a/.dockerignore b/.dockerignore index ed763eb8..07a672b3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ !bun.lock !package.json !.gitignore +!bunfig.toml !**/package.json !**/bun.lock diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 00000000..ac7a0338 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,30 @@ +name: Integration Tests +permissions: + contents: read + +on: + workflow_dispatch: + workflow_call: + +jobs: + integration: + name: Integration + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Install dependencies + uses: "./.github/actions/install-dependencies" + + - name: Run integration tests + run: bun run test:integration + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: failure() + with: + name: integration-artifacts + path: app/test/integration/artifacts/ + retention-days: 5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 75c69b9c..1d4ab31e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,10 +43,13 @@ jobs: e2e-tests: uses: ./.github/workflows/e2e.yml + integration-tests: + uses: ./.github/workflows/integration.yml + build-images: environment: release timeout-minutes: 15 - needs: [determine-release-type, checks, e2e-tests] + needs: [determine-release-type, checks, e2e-tests, integration-tests] runs-on: ubuntu-latest permissions: contents: read diff --git a/app/test/integration/.gitignore b/app/test/integration/.gitignore new file mode 100644 index 00000000..20754bf0 --- /dev/null +++ b/app/test/integration/.gitignore @@ -0,0 +1,2 @@ +artifacts/* +!artifacts/.gitignore diff --git a/app/test/integration/artifacts/.gitignore b/app/test/integration/artifacts/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/app/test/integration/artifacts/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/app/test/integration/infra/Dockerfile b/app/test/integration/infra/Dockerfile new file mode 100644 index 00000000..4099b0c6 --- /dev/null +++ b/app/test/integration/infra/Dockerfile @@ -0,0 +1,18 @@ +ARG BASE_IMAGE=zerobyte-integration-runtime-base:latest +FROM ${BASE_IMAGE} + +WORKDIR /app + +ENV NODE_ENV=test + +COPY ./package.json ./bun.lock ./ +COPY ./packages/core/package.json ./packages/core/package.json +COPY ./packages/contracts/package.json ./packages/contracts/package.json +COPY ./apps/agent/package.json ./apps/agent/package.json +COPY ./apps/docs/package.json ./apps/docs/package.json + +RUN VITE_GIT_HOOKS=0 bun install --frozen-lockfile + +COPY . . + +CMD ["sh", "app/test/integration/infra/entrypoint.sh"] diff --git a/app/test/integration/infra/docker-compose.yml b/app/test/integration/infra/docker-compose.yml new file mode 100644 index 00000000..dee7aa52 --- /dev/null +++ b/app/test/integration/infra/docker-compose.yml @@ -0,0 +1,41 @@ +services: + rustfs: + image: rustfs/rustfs:latest + command: ["/data"] + environment: + RUSTFS_ADDRESS: ":9000" + RUSTFS_ACCESS_KEY: "rustfsadmin" + RUSTFS_SECRET_KEY: "rustfsadmin" + RUSTFS_CONSOLE_ENABLE: "false" + volumes: + - rustfs-data:/data + + rustfs-setup: + image: minio/mc:latest + depends_on: + - rustfs + entrypoint: ["/bin/sh", "-c"] + command: + - | + until mc alias set rustfs http://rustfs:9000 rustfsadmin rustfsadmin --api S3v4 --path on; do + echo "Waiting for RustFS..." + sleep 1 + done + mc mb --ignore-existing --region us-east-1 rustfs/zerobyte-integration + + integration: + build: + context: ../../../.. + dockerfile: app/test/integration/infra/Dockerfile + args: + BASE_IMAGE: zerobyte-integration-runtime-base:latest + depends_on: + rustfs-setup: + condition: service_completed_successfully + volumes: + - ../artifacts:/app/app/test/integration/artifacts + environment: + LOG_LEVEL: "info" + +volumes: + rustfs-data: diff --git a/app/test/integration/infra/entrypoint.sh b/app/test/integration/infra/entrypoint.sh new file mode 100644 index 00000000..95d30793 --- /dev/null +++ b/app/test/integration/infra/entrypoint.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh +set -eu + +bun app/test/integration/src/write-rclone-config.ts +exec bunx --bun vitest run --config app/test/integration/vitest.config.ts diff --git a/app/test/integration/run.sh b/app/test/integration/run.sh new file mode 100644 index 00000000..7a362898 --- /dev/null +++ b/app/test/integration/run.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +base_image="zerobyte-integration-runtime-base:latest" +compose_project="zerobyte-integration-$(basename "$repo_root" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9_-')" +artifacts_dir="$script_dir/artifacts" +compose_file="$script_dir/infra/docker-compose.yml" + +mkdir -p "$artifacts_dir" +rm -rf "$artifacts_dir/runs" +rm -f "$artifacts_dir/compose.log" +mkdir -p "$artifacts_dir/runs" + +docker build --target production -t "$base_image" "$repo_root" + +exit_code=0 +docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from rustfs-setup rustfs-setup || exit_code=$? + +if [[ "$exit_code" -eq 0 ]]; then + docker compose -f "$compose_file" -p "$compose_project" up --build --abort-on-container-exit --exit-code-from integration integration || exit_code=$? +fi + +if [[ "$exit_code" -ne 0 ]]; then + docker compose -f "$compose_file" -p "$compose_project" logs --no-color >"$artifacts_dir/compose.log" || true +fi + +docker compose -f "$compose_file" -p "$compose_project" down --volumes --remove-orphans || true + +exit "$exit_code" diff --git a/app/test/integration/src/constants.ts b/app/test/integration/src/constants.ts new file mode 100644 index 00000000..8387b016 --- /dev/null +++ b/app/test/integration/src/constants.ts @@ -0,0 +1,14 @@ +import path from "node:path"; + +export const RUSTFS_ENDPOINT = "http://rustfs:9000"; +export const RUSTFS_ACCESS_KEY_ID = "rustfsadmin"; +export const RUSTFS_SECRET_ACCESS_KEY = "rustfsadmin"; +export const RUSTFS_BUCKET = "zerobyte-integration"; + +export const RCLONE_REMOTE = "e2e-rustfs"; +export const RCLONE_CONFIG_DIR = "/root/.config/rclone"; +export const RCLONE_CONFIG_FILE = path.join(RCLONE_CONFIG_DIR, "rclone.conf"); + +export const INTEGRATION_ORGANIZATION_ID = "integration-suite"; +export const INTEGRATION_ARTIFACTS_DIR = path.resolve(import.meta.dirname, "../artifacts"); +export const INTEGRATION_RUNS_DIR = path.join(INTEGRATION_ARTIFACTS_DIR, "runs"); diff --git a/app/test/integration/src/helpers/assertions.ts b/app/test/integration/src/helpers/assertions.ts new file mode 100644 index 00000000..6ee6997d --- /dev/null +++ b/app/test/integration/src/helpers/assertions.ts @@ -0,0 +1,44 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { expect } from "vitest"; +import type { ScenarioFixture } from "./fixture"; + +type ResticLsNode = { + path: string; + type: string; +}; + +const nodeByPath = (nodes: ResticLsNode[]) => new Map(nodes.map((node) => [path.normalize(node.path), node])); + +export const assertSnapshotContainsFixture = (sourceRoot: string, nodes: ResticLsNode[], fixture: ScenarioFixture) => { + const nodesByPath = nodeByPath(nodes); + + const regularFile = nodesByPath.get(path.join(sourceRoot, fixture.regularFile.relativePath)); + expect(regularFile?.type).toBe("file"); + + const nestedDirectory = nodesByPath.get(path.join(sourceRoot, fixture.nestedDirectory.relativePath)); + expect(nestedDirectory?.type).toBe("dir"); + + const nestedFile = nodesByPath.get(path.join(sourceRoot, fixture.nestedFile.relativePath)); + expect(nestedFile?.type).toBe("file"); + + const symlink = nodesByPath.get(path.join(sourceRoot, fixture.symlink.relativePath)); + expect(symlink?.type).toBe("symlink"); +}; + +export const assertRestoredFixture = async (restoreRoot: string, fixture: ScenarioFixture) => { + await expect(fs.readFile(path.join(restoreRoot, fixture.regularFile.relativePath), "utf8")).resolves.toBe( + fixture.regularFile.content, + ); + + const nestedDirectoryStat = await fs.stat(path.join(restoreRoot, fixture.nestedDirectory.relativePath)); + expect(nestedDirectoryStat.isDirectory()).toBe(true); + + await expect(fs.readFile(path.join(restoreRoot, fixture.nestedFile.relativePath), "utf8")).resolves.toBe( + fixture.nestedFile.content, + ); + + await expect(fs.readlink(path.join(restoreRoot, fixture.symlink.relativePath))).resolves.toBe( + fixture.symlink.target, + ); +}; diff --git a/app/test/integration/src/helpers/fixture.ts b/app/test/integration/src/helpers/fixture.ts new file mode 100644 index 00000000..e10d9e17 --- /dev/null +++ b/app/test/integration/src/helpers/fixture.ts @@ -0,0 +1,54 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; + +export type ScenarioFixture = { + sourceRoot: string; + regularFile: { + relativePath: string; + content: string; + }; + nestedDirectory: { + relativePath: string; + }; + nestedFile: { + relativePath: string; + content: string; + }; + symlink: { + relativePath: string; + target: string; + }; +}; + +export const createScenarioFixture = async (workspace: string, scenarioId: string): Promise => { + const sourceRoot = path.join(workspace, "source"); + const nestedDirectory = path.join(sourceRoot, "nested"); + const randomSuffix = crypto.randomBytes(8).toString("hex"); + + const fixture: ScenarioFixture = { + sourceRoot, + regularFile: { + relativePath: "regular.txt", + content: `regular fixture for ${scenarioId} (${randomSuffix})\n`, + }, + nestedDirectory: { + relativePath: "nested", + }, + nestedFile: { + relativePath: "nested/deep.txt", + content: `nested fixture for ${scenarioId} (${randomSuffix})\n`, + }, + symlink: { + relativePath: "regular-link", + target: "regular.txt", + }, + }; + + await fs.mkdir(nestedDirectory, { recursive: true }); + await fs.writeFile(path.join(sourceRoot, fixture.regularFile.relativePath), fixture.regularFile.content); + await fs.writeFile(path.join(sourceRoot, fixture.nestedFile.relativePath), fixture.nestedFile.content); + await fs.symlink(fixture.symlink.target, path.join(sourceRoot, fixture.symlink.relativePath)); + + return fixture; +}; diff --git a/app/test/integration/src/helpers/restic.ts b/app/test/integration/src/helpers/restic.ts new file mode 100644 index 00000000..9420e3f9 --- /dev/null +++ b/app/test/integration/src/helpers/restic.ts @@ -0,0 +1,14 @@ +import path from "node:path"; +import { createRestic } from "@zerobyte/core/restic/server"; +import { RCLONE_CONFIG_FILE } from "../constants"; + +export const createIntegrationRestic = (workspace: string, resticPassword: string) => { + return createRestic({ + resolveSecret: async (value: string) => value, + getOrganizationResticPassword: async () => resticPassword, + resticCacheDir: path.join(workspace, "restic-cache"), + resticPassFile: path.join(workspace, "restic.pass"), + defaultExcludes: [], + rcloneConfigFile: RCLONE_CONFIG_FILE, + }); +}; diff --git a/app/test/integration/src/repositories.test.ts b/app/test/integration/src/repositories.test.ts new file mode 100644 index 00000000..6a244bfa --- /dev/null +++ b/app/test/integration/src/repositories.test.ts @@ -0,0 +1,126 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { Effect } from "effect"; +import type { RepositoryConfig } from "@zerobyte/core/restic"; +import { expect, test } from "vitest"; +import { + INTEGRATION_ORGANIZATION_ID, + INTEGRATION_RUNS_DIR, + RCLONE_REMOTE, + RUSTFS_ACCESS_KEY_ID, + RUSTFS_BUCKET, + RUSTFS_ENDPOINT, + RUSTFS_SECRET_ACCESS_KEY, +} from "./constants"; +import { assertRestoredFixture, assertSnapshotContainsFixture } from "./helpers/assertions"; +import { createScenarioFixture } from "./helpers/fixture"; +import { createIntegrationRestic } from "./helpers/restic"; + +type RepositoryScenario = { + id: string; + name: string; + createRepositoryConfig: (prefix: string) => RepositoryConfig; +}; + +const scenarios: RepositoryScenario[] = [ + { + id: "direct-s3", + name: "direct S3 repository", + createRepositoryConfig: (prefix) => ({ + backend: "s3", + endpoint: RUSTFS_ENDPOINT, + bucket: `${RUSTFS_BUCKET}/${prefix}`, + accessKeyId: RUSTFS_ACCESS_KEY_ID, + secretAccessKey: RUSTFS_SECRET_ACCESS_KEY, + }), + }, + { + id: "rclone-s3", + name: "rclone repository over RustFS S3", + createRepositoryConfig: (prefix) => ({ + backend: "rclone", + remote: RCLONE_REMOTE, + path: `${RUSTFS_BUCKET}/${prefix}`, + }), + }, +]; + +test.concurrent.each(scenarios)("$name can backup, list, and restore fixture data", async (scenario) => { + const runId = crypto.randomUUID(); + const repositoryPrefix = `${scenario.id}/${runId}`; + const workspace = path.join(INTEGRATION_RUNS_DIR, `${scenario.id}-${runId}`); + const restoreTarget = path.join(workspace, "restore"); + const backupTag = `zerobyte-integration-${scenario.id}-${runId}`; + const resticPassword = `zerobyte-integration-${scenario.id}-${runId}-${crypto.randomBytes(16).toString("hex")}`; + const repositoryConfig = scenario.createRepositoryConfig(repositoryPrefix); + const restic = createIntegrationRestic(workspace, resticPassword); + + let passed = false; + + try { + await fs.mkdir(workspace, { recursive: true }); + const fixture = await createScenarioFixture(workspace, scenario.id); + + const initResult = await Effect.runPromise( + restic.init(repositoryConfig, { + organizationId: INTEGRATION_ORGANIZATION_ID, + timeoutMs: 120_000, + }), + ); + expect(initResult.success).toBe(true); + expect(initResult.error).toBeNull(); + + const backupResult = await Effect.runPromise( + restic.backup(repositoryConfig, fixture.sourceRoot, { + organizationId: INTEGRATION_ORGANIZATION_ID, + tags: [backupTag], + }), + ); + + expect(backupResult.exitCode).toBe(0); + expect(backupResult.warningDetails).toBeNull(); + expect(backupResult.result?.snapshot_id).toEqual(expect.any(String)); + + const snapshotId = backupResult.result?.snapshot_id; + if (!snapshotId) { + throw new Error("Restic backup completed without a snapshot id"); + } + + const snapshots = await Effect.runPromise( + restic.snapshots(repositoryConfig, { + organizationId: INTEGRATION_ORGANIZATION_ID, + tags: [backupTag], + }), + ); + const snapshot = snapshots.find( + (candidate) => candidate.id === snapshotId || candidate.short_id === snapshotId, + ); + expect(snapshot).toBeDefined(); + expect(snapshot?.paths).toContain(fixture.sourceRoot); + + const lsResult = await Effect.runPromise( + restic.ls(repositoryConfig, snapshotId, undefined, { + organizationId: INTEGRATION_ORGANIZATION_ID, + limit: 100, + }), + ); + assertSnapshotContainsFixture(fixture.sourceRoot, lsResult.nodes, fixture); + + await Effect.runPromise( + restic.restore(repositoryConfig, snapshotId, restoreTarget, { + organizationId: INTEGRATION_ORGANIZATION_ID, + basePath: fixture.sourceRoot, + }), + ); + await assertRestoredFixture(restoreTarget, fixture); + + passed = true; + } finally { + if (passed) { + await fs.rm(workspace, { recursive: true, force: true }); + } else { + console.error(`Integration scenario artifacts retained in ${workspace}`); + } + } +}); diff --git a/app/test/integration/src/write-rclone-config.ts b/app/test/integration/src/write-rclone-config.ts new file mode 100644 index 00000000..0ff8eb2d --- /dev/null +++ b/app/test/integration/src/write-rclone-config.ts @@ -0,0 +1,24 @@ +import fs from "node:fs/promises"; +import { + RCLONE_CONFIG_DIR, + RCLONE_CONFIG_FILE, + RCLONE_REMOTE, + RUSTFS_ACCESS_KEY_ID, + RUSTFS_ENDPOINT, + RUSTFS_SECRET_ACCESS_KEY, +} from "./constants"; + +const rcloneConfig = `[${RCLONE_REMOTE}] +type = s3 +provider = Minio +env_auth = false +access_key_id = ${RUSTFS_ACCESS_KEY_ID} +secret_access_key = ${RUSTFS_SECRET_ACCESS_KEY} +endpoint = ${RUSTFS_ENDPOINT} +force_path_style = true +acl = private +`; + +await fs.mkdir(RCLONE_CONFIG_DIR, { recursive: true }); +await fs.writeFile(RCLONE_CONFIG_FILE, rcloneConfig, { mode: 0o600 }); +await fs.chmod(RCLONE_CONFIG_FILE, 0o600); diff --git a/app/test/integration/vitest.config.ts b/app/test/integration/vitest.config.ts new file mode 100644 index 00000000..2fda2d1d --- /dev/null +++ b/app/test/integration/vitest.config.ts @@ -0,0 +1,23 @@ +import path from "node:path"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + resolve: { + alias: { + "~": path.resolve(import.meta.dirname, "../.."), + }, + }, + test: { + server: { + deps: { + inline: ["zod"], + }, + }, + name: "integration", + environment: "node", + include: ["app/test/integration/src/**/*.test.ts"], + testTimeout: 300_000, + hookTimeout: 300_000, + maxConcurrency: 4, + }, +}); diff --git a/package.json b/package.json index 5a20d47b..8175db3e 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build": "bunx --bun vite build", "start": "bun run .output/server/index.mjs", "build:backend-integration": "bun build app/test/backend-integration/index.ts --outdir .output/backend-integration --target bun", + "test:integration": "bash app/test/integration/run.sh", "test:integration:backends": "bash app/test/backend-integration/run.sh", "preview": "bunx --bun vite preview", "cli:dev": "bun run app/server/cli/main.ts", diff --git a/vitest.config.ts b/vitest.config.ts index 3445aa13..13665528 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -25,6 +25,8 @@ export default defineConfig({ "app/client/**/*.test.tsx", "app/client/**/*.spec.ts", "app/client/**/*.spec.tsx", + "app/test/integration/**/*.test.ts", + "app/test/integration/**/*.spec.ts", ], setupFiles: ["./app/test/setup.ts"], },