diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml index 575647ac9..5817aff76 100644 --- a/.github/workflows/core_tests.yml +++ b/.github/workflows/core_tests.yml @@ -53,12 +53,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + # Skip Rust toolchain setup on self-hosted runners (macOS) + # Assumes Rust is pre-installed and maintained on self-hosted machines - name: Setup Rust + if: ${{ matrix.settings.os != 'macos' }} uses: dtolnay/rust-toolchain@stable with: targets: ${{ matrix.settings.target }} - name: Setup System and Rust + if: ${{ matrix.settings.os != 'macos' }} uses: ./.github/actions/setup-system with: token: ${{ secrets.GITHUB_TOKEN }} @@ -82,6 +86,8 @@ jobs: path: target key: ${{ matrix.settings.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }} + # Native deps setup is fast on self-hosted if deps already cached in apps/.deps/ + # Keep it to ensure correct versions even on self-hosted runners - name: Setup native dependencies run: cargo xtask setup diff --git a/core/examples/library_demo.rs b/core/examples/library_demo.rs index 634d9b5ef..3be9554bc 100644 --- a/core/examples/library_demo.rs +++ b/core/examples/library_demo.rs @@ -126,6 +126,7 @@ async fn main() -> Result<(), Box> { accessed_at: Set(None), indexed_at: Set(None), permissions: Set(None), + device_id: Set(Some(inserted_device.id)), inode: Set(None), }; let entry_record = entry.insert(db.conn()).await?; diff --git a/core/tests/job_resumption_integration_test.rs b/core/tests/job_resumption_integration_test.rs index 7008257ef..e113d7213 100644 --- a/core/tests/job_resumption_integration_test.rs +++ b/core/tests/job_resumption_integration_test.rs @@ -81,8 +81,8 @@ async fn test_job_resumption_at_various_points() { .await .expect("Failed to prepare test data"); - // Define interruption points to test with realistic event counts for smaller datasets - // For Downloads folder, use lower event counts since there are fewer files + // Define interruption points to test with realistic event counts + // Use lower event counts for faster test execution let interruption_points = vec![ InterruptionPoint::DiscoveryAfterEvents(2), // Interrupt early in discovery InterruptionPoint::ProcessingAfterEvents(2), // Interrupt early in processing @@ -128,25 +128,24 @@ async fn test_job_resumption_at_various_points() { info!("Test logs and data available in: test_data/"); } -/// Generate test data using benchmark data generation +/// Generate test data using Spacedrive source code for deterministic testing async fn generate_test_data() -> Result> { - // Use Downloads folder instead of benchmark data - let home_dir = std::env::var("HOME") - .map(PathBuf::from) - .or_else(|_| std::env::current_dir())?; - - let indexing_data_path = home_dir.join("Downloads"); + // Use Spacedrive core/src directory for deterministic cross-platform testing + let indexing_data_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("Failed to get project root")? + .join("core/src"); if !indexing_data_path.exists() { return Err(format!( - "Downloads folder does not exist at: {}", + "Spacedrive core/src folder does not exist at: {}", indexing_data_path.display() ) .into()); } info!( - "Using Downloads folder at: {}", + "Using Spacedrive core/src folder at: {}", indexing_data_path.display() ); Ok(indexing_data_path) diff --git a/core/tests/sync_backfill_race_test.rs b/core/tests/sync_backfill_race_test.rs index 8a074449f..201a9b1ff 100644 --- a/core/tests/sync_backfill_race_test.rs +++ b/core/tests/sync_backfill_race_test.rs @@ -340,10 +340,20 @@ async fn test_backfill_with_concurrent_indexing() -> anyhow::Result<()> { let harness = BackfillRaceHarness::new("backfill_race").await?; // Step 1: Alice indexes first location - let downloads_path = std::env::var("HOME").unwrap() + "/Downloads"; - tracing::info!("Step 1: Alice indexes Downloads"); + // Use Spacedrive crates directory for deterministic testing + let project_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + let crates_path = project_root.join("crates"); + tracing::info!("Step 1: Alice indexes crates"); - add_and_index_location(&harness.library_alice, &downloads_path, "Downloads").await?; + add_and_index_location( + &harness.library_alice, + crates_path.to_str().unwrap(), + "crates", + ) + .await?; let alice_entries_after_loc1 = entities::entry::Entity::find() .count(harness.library_alice.db().conn()) @@ -373,10 +383,18 @@ async fn test_backfill_with_concurrent_indexing() -> anyhow::Result<()> { // Step 2: Start backfill on Bob while Alice continues indexing tracing::info!("Step 2: Starting Bob's backfill AND Alice's second indexing concurrently"); - let desktop_path = std::env::var("HOME").unwrap() + "/Desktop"; + // Use Spacedrive source code for deterministic testing across all environments + let test_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); let backfill_future = harness.trigger_bob_backfill(); - let indexing_future = add_and_index_location(&harness.library_alice, &desktop_path, "Desktop"); + let indexing_future = add_and_index_location( + &harness.library_alice, + test_path.to_str().unwrap(), + "spacedrive", + ); // Run concurrently - this is the key to triggering the race let (backfill_result, indexing_result) = tokio::join!(backfill_future, indexing_future); @@ -448,13 +466,18 @@ async fn test_sequential_backfill_control() -> anyhow::Result<()> { let harness = BackfillRaceHarness::new("sequential_control").await?; // Alice indexes both locations first - let downloads_path = std::env::var("HOME").unwrap() + "/Downloads"; - let desktop_path = std::env::var("HOME").unwrap() + "/Desktop"; + // Use Spacedrive source code for deterministic testing + let project_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + let core_path = project_root.join("core"); + let apps_path = project_root.join("apps"); tracing::info!("Indexing both locations on Alice first"); - add_and_index_location(&harness.library_alice, &downloads_path, "Downloads").await?; - add_and_index_location(&harness.library_alice, &desktop_path, "Desktop").await?; + add_and_index_location(&harness.library_alice, core_path.to_str().unwrap(), "core").await?; + add_and_index_location(&harness.library_alice, apps_path.to_str().unwrap(), "apps").await?; let alice_entries = entities::entry::Entity::find() .count(harness.library_alice.db().conn()) diff --git a/core/tests/sync_backfill_test.rs b/core/tests/sync_backfill_test.rs index 6ab6e4488..5b1abd1fc 100644 --- a/core/tests/sync_backfill_test.rs +++ b/core/tests/sync_backfill_test.rs @@ -16,9 +16,8 @@ use sd_core::{ Core, }; use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter}; -use std::{path::PathBuf, sync::Arc}; +use std::sync::Arc; use tokio::{fs, time::Duration}; -use uuid::Uuid; #[tokio::test] async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> { @@ -61,10 +60,14 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> { .await? .ok_or_else(|| anyhow::anyhow!("Device not found"))?; - let desktop_path = std::env::var("HOME").unwrap() + "/Desktop"; + // Use Spacedrive source code for deterministic testing across all environments + let test_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); let location_args = LocationCreateArgs { - path: std::path::PathBuf::from(&desktop_path), - name: Some("Desktop".to_string()), + path: test_path.clone(), + name: Some("spacedrive".to_string()), index_mode: IndexMode::Content, }; diff --git a/core/tests/sync_realtime_test.rs b/core/tests/sync_realtime_test.rs index 95e78a08e..2f5f317df 100644 --- a/core/tests/sync_realtime_test.rs +++ b/core/tests/sync_realtime_test.rs @@ -5,7 +5,7 @@ //! //! ## Features //! - Pre-paired devices (Alice & Bob) -//! - Indexes real folders +//! - Indexes Spacedrive source code for deterministic testing //! - Event-driven architecture //! - Captures sync logs, databases, and event bus events //! - Timestamped snapshot folders for each run @@ -39,9 +39,13 @@ async fn test_realtime_sync_alice_to_bob() -> anyhow::Result<()> { // Phase 1: Add location on Alice tracing::info!("=== Phase 1: Adding location on Alice ==="); - let desktop_path = std::env::var("HOME").unwrap() + "/Desktop"; + // Use Spacedrive source code for deterministic testing across all environments + let test_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); let location_uuid = harness - .add_and_index_location_alice(&desktop_path, "Desktop") + .add_and_index_location_alice(test_path.to_str().unwrap(), "spacedrive") .await?; tracing::info!( @@ -144,9 +148,14 @@ async fn test_realtime_sync_bob_to_alice() -> anyhow::Result<()> { .await?; // Add location on Bob (reverse direction) - let downloads_path = std::env::var("HOME").unwrap() + "/Downloads"; + // Use Spacedrive crates directory for deterministic testing + let project_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + let crates_path = project_root.join("crates"); harness - .add_and_index_location_bob(&downloads_path, "Downloads") + .add_and_index_location_bob(crates_path.to_str().unwrap(), "crates") .await?; // Wait for sync @@ -184,12 +193,17 @@ async fn test_concurrent_indexing() -> anyhow::Result<()> { .await?; // Add different locations on both devices simultaneously - let downloads_path = std::env::var("HOME").unwrap() + "/Downloads"; - let desktop_path = std::env::var("HOME").unwrap() + "/Desktop"; + // Use Spacedrive source code for deterministic testing + let project_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + let core_path = project_root.join("core"); + let apps_path = project_root.join("apps"); // Start indexing on both - let alice_task = harness.add_and_index_location_alice(&downloads_path, "Downloads"); - let bob_task = harness.add_and_index_location_bob(&desktop_path, "Desktop"); + let alice_task = harness.add_and_index_location_alice(core_path.to_str().unwrap(), "core"); + let bob_task = harness.add_and_index_location_bob(apps_path.to_str().unwrap(), "apps"); // Wait for both tokio::try_join!(alice_task, bob_task)?; @@ -223,9 +237,14 @@ async fn test_content_identity_linkage() -> anyhow::Result<()> { .await?; // Index on Alice - let downloads_path = std::env::var("HOME").unwrap() + "/Downloads"; + // Use Spacedrive docs directory for deterministic testing + let project_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + let docs_path = project_root.join("docs"); harness - .add_and_index_location_alice(&downloads_path, "Downloads") + .add_and_index_location_alice(docs_path.to_str().unwrap(), "docs") .await?; // Wait for content identification to complete diff --git a/docs/core/testing.mdx b/docs/core/testing.mdx index 6132f88b4..dc7ff1f16 100644 --- a/docs/core/testing.mdx +++ b/docs/core/testing.mdx @@ -15,6 +15,7 @@ Spacedrive Core provides two primary testing approaches: ### Test Organization Tests live in two locations: + - `core/tests/` - Integration tests that verify complete workflows - `core/src/testing/` - Test framework utilities and helpers @@ -27,12 +28,12 @@ For single-device tests, use Tokio's async test framework: async fn test_library_creation() { let setup = IntegrationTestSetup::new("library_test").await.unwrap(); let core = setup.create_core().await.unwrap(); - + let library = core.libraries .create_library("Test Library", None) .await .unwrap(); - + assert!(!library.id.is_empty()); } ``` @@ -55,6 +56,7 @@ let setup = IntegrationTestSetup::with_config("test_name", |builder| { ``` Key features: + - Isolated temporary directories per test - Structured logging to `test_data/{test_name}/library/logs/` - Automatic cleanup on drop @@ -67,6 +69,7 @@ Spacedrive provides two approaches for testing multi-device scenarios: ### When to Use Subprocess Framework **Use `CargoTestRunner` subprocess framework when:** + - Testing **real networking** with actual network discovery, NAT traversal, and connections - Testing **device pairing** workflows that require independent network stacks - Scenarios need **true process isolation** (separate memory spaces, different ports) @@ -85,6 +88,7 @@ let mut runner = CargoTestRunner::new() ### When to Use Custom Transport/Harness **Use custom harness with mock transport when:** + - Testing **sync logic** without network overhead - Fast iteration on **data synchronization** algorithms - Testing **deterministic scenarios** without network timing issues @@ -103,14 +107,14 @@ let harness = TwoDeviceHarnessBuilder::new("sync_test") ### Comparison -| Aspect | Subprocess Framework | Custom Harness | -|--------|---------------------|----------------| -| **Speed** | Slower (real networking) | Fast (in-memory) | -| **Networking** | Real (discovery, NAT) | Mock transport | -| **Isolation** | True process isolation | Shared process | -| **Debugging** | Harder (multiple processes) | Easier (single process) | -| **Determinism** | Network timing varies | Fully deterministic | -| **Use Case** | Network features | Sync/data logic | +| Aspect | Subprocess Framework | Custom Harness | +| --------------- | --------------------------- | ----------------------- | +| **Speed** | Slower (real networking) | Fast (in-memory) | +| **Networking** | Real (discovery, NAT) | Mock transport | +| **Isolation** | True process isolation | Shared process | +| **Debugging** | Harder (multiple processes) | Easier (single process) | +| **Determinism** | Network timing varies | Fully deterministic | +| **Use Case** | Network features | Sync/data logic | ## Subprocess Testing Framework @@ -137,7 +141,7 @@ async fn test_device_pairing() { let mut runner = CargoTestRunner::new() .add_subprocess("alice", "alice_pairing") .add_subprocess("bob", "bob_pairing"); - + runner.run_until_success(|outputs| { outputs.values().all(|o| o.contains("PAIRING_SUCCESS")) }).await.unwrap(); @@ -149,14 +153,14 @@ async fn alice_pairing() { if env::var("TEST_ROLE").unwrap_or_default() != "alice" { return; } - + let data_dir = PathBuf::from(env::var("TEST_DATA_DIR").unwrap()); let core = create_test_core(data_dir).await.unwrap(); - + // Alice initiates pairing let (code, _) = core.start_pairing_as_initiator().await.unwrap(); fs::write("/tmp/pairing_code.txt", &code).unwrap(); - + // Wait for connection wait_for_connection(&core).await; println!("PAIRING_SUCCESS"); @@ -164,12 +168,14 @@ async fn alice_pairing() { ``` -Device scenario functions must be marked with `#[ignore]` to prevent direct execution. They only run when called by the subprocess framework. + Device scenario functions must be marked with `#[ignore]` to prevent direct + execution. They only run when called by the subprocess framework. ### Process Coordination Processes coordinate through: + - **Environment variables**: `TEST_ROLE` and `TEST_DATA_DIR` - **Temporary files**: Share data like pairing codes - **Output patterns**: Success markers for the runner to detect @@ -240,7 +246,8 @@ watcher.watch_ephemeral(dest_dir.clone()).await?; ``` -The `IndexerJob` automatically calls `watch_ephemeral()` after successful indexing, so manual registration is only needed when bypassing the indexer. + The `IndexerJob` automatically calls `watch_ephemeral()` after successful + indexing, so manual registration is only needed when bypassing the indexer. #### Persistent Location Watching @@ -290,6 +297,7 @@ assert!(stats.resource_changed.get("file").copied().unwrap_or(0) >= 2); ``` The `EventCollector` automatically filters out: + - Library statistics updates (`LibraryStatisticsUpdated`) - Library resource events (non-file/entry events) @@ -365,12 +373,14 @@ let indexing_events = collector.get_events_by_type("IndexingCompleted").await; ``` The `EventCollector` tracks: + - **ResourceChanged/ResourceChangedBatch** events by resource type - **Indexing** start/completion events - **Job** lifecycle events (started/completed) - **Entry** events (created/modified/deleted/moved) **Statistics Output:** + ``` Event Statistics: ================== @@ -394,6 +404,7 @@ Job events: ``` **Detailed Event Output (with `with_capture()`):** + ``` === Collected Events (8) === @@ -418,6 +429,7 @@ Job events: ``` **Use Cases:** + - Verifying watcher events during file operations - Testing normalized cache updates - Debugging event emission patterns @@ -449,7 +461,7 @@ let job_id = core.jobs.dispatch(IndexingJob::new(...)).await?; // Monitor progress wait_for_event(&mut events, |e| matches!( - e, + e, Event::JobProgress { id, .. } if *id == job_id ), timeout).await?; @@ -483,27 +495,32 @@ wait_for_sync(&core_b).await?; The framework provides comprehensive test helpers in `core/tests/helpers/`: **Event Collection:** + - `EventCollector` - Collect and analyze all events from the event bus - `EventStats` - Statistics about collected events with formatted output **Indexing Tests:** + - `IndexingHarnessBuilder` - Create isolated test environments with indexing support - `TestLocation` - Builder for test locations with files - `LocationHandle` - Handle to indexed locations with verification methods **Sync Tests:** + - `TwoDeviceHarnessBuilder` - Pre-configured two-device sync test environments - `MockTransport` - Mock network transport for deterministic sync testing - `wait_for_sync()` - Sophisticated sync completion detection - `TestConfigBuilder` - Custom test configurations **Database & Jobs:** + - `wait_for_event()` - Wait for specific events with timeout - `wait_for_indexing()` - Wait for indexing job completion - `register_device()` - Register a device in a library -See `core/tests/helpers/README.md` for detailed documentation on all available helpers including usage examples and migration guides. + See `core/tests/helpers/README.md` for detailed documentation on all available + helpers including usage examples and migration guides. ### Test Volumes @@ -518,25 +535,201 @@ let volume = test_volumes::create_test_volume().await?; test_volumes::cleanup_test_volume(volume).await?; ``` +## Core Integration Test Suite + +Spacedrive maintains a curated suite of core integration tests that run in CI and during local development. These tests are defined in a single source of truth using the `xtask` pattern. + +### Running the Core Test Suite + +The `cargo xtask test-core` command runs all core integration tests with progress tracking: + +```bash +# Run all core tests (minimal output) +cargo xtask test-core + +# Run with full test output +cargo xtask test-core --verbose +``` + +**Example output:** + +``` +════════════════════════════════════════════════════════════════ + Spacedrive Core Tests Runner + Running 13 test suite(s) +════════════════════════════════════════════════════════════════ + +[1/13] Running: Library tests +──────────────────────────────────────────────────────────────── +✓ PASSED (2s) + +[2/13] Running: Indexing test +──────────────────────────────────────────────────────────────── +✓ PASSED (15s) + +... + +════════════════════════════════════════════════════════════════ + Test Results Summary +════════════════════════════════════════════════════════════════ + +Total time: 7m 24s + +✓ Passed (11/13): + ✓ Library tests + ✓ Indexing test + ... + +✗ Failed (2/13): + ✗ Sync realtime test + ✗ File sync test +``` + +### Single Source of Truth + +All core integration tests are defined in `xtask/src/test_core.rs` in the `CORE_TESTS` constant: + +```rust +pub const CORE_TESTS: &[TestSuite] = &[ + TestSuite { + name: "Library tests", + args: &["test", "-p", "sd-core", "--lib", "--", "--test-threads=1"], + }, + TestSuite { + name: "Indexing test", + args: &["test", "-p", "sd-core", "--test", "indexing_test", "--", "--test-threads=1"], + }, + // ... more tests +]; +``` + +**Benefits:** + +- CI and local development use identical test definitions +- Add or remove tests in one place +- Automatic progress tracking and result summary +- Continues running even if some tests fail + +### CI Integration + +The GitHub Actions workflow runs the core test suite on all platforms: + +```yaml +# .github/workflows/core_tests.yml +- name: Run all tests + run: cargo xtask test-core --verbose +``` + +Tests run in parallel on: + +- **macOS** (ARM64 self-hosted) +- **Linux** (Ubuntu 22.04) +- **Windows** (latest) + +With `fail-fast: false`, all platforms complete even if one fails. + +### Deterministic Test Data + +Core integration tests use the Spacedrive source code itself as test data instead of user directories. This ensures: + +- **Consistent results** across all machines and CI +- **No user data access** required +- **Cross-platform compatibility** without setup +- **Predictable file structure** for test assertions + +```rust +// Tests index the Spacedrive project root +let test_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + +let location = harness + .add_and_index_location(test_path.to_str().unwrap(), "spacedrive") + .await?; +``` + +Tests that need multiple locations use different subdirectories: + +```rust +let project_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); +let core_path = project_root.join("core"); +let apps_path = project_root.join("apps"); +``` + +### Adding Tests to the Suite + +To add a new test to the core suite: + +1. Create your test in `core/tests/your_test.rs` +2. Add it to `CORE_TESTS` in `xtask/src/test_core.rs`: + +```rust +pub const CORE_TESTS: &[TestSuite] = &[ + // ... existing tests + TestSuite { + name: "Your new test", + args: &[ + "test", + "-p", + "sd-core", + "--test", + "your_test", + "--", + "--test-threads=1", + "--nocapture", + ], + }, +]; +``` + +The test will automatically: + +- Run in CI on all platforms +- Appear in `cargo xtask test-core` output +- Show in progress tracking and summary + + + Core integration tests use `--test-threads=1` to avoid conflicts when + accessing the same locations or performing filesystem operations. + + ## Running Tests ### All Tests + ```bash cargo test --workspace ``` +### Core Integration Tests + +```bash +# Run curated core test suite +cargo xtask test-core + +# With full output +cargo xtask test-core --verbose +``` + ### Specific Test + ```bash cargo test test_device_pairing --nocapture ``` ### Debug Subprocess Tests + ```bash # Run individual scenario TEST_ROLE=alice TEST_DATA_DIR=/tmp/test cargo test alice_scenario -- --ignored --nocapture ``` ### With Logging + ```bash RUST_LOG=debug cargo test test_name --nocapture ``` @@ -548,6 +741,25 @@ RUST_LOG=debug cargo test test_name --nocapture 1. **Use descriptive names**: `test_cross_device_file_transfer` over `test_transfer` 2. **One concern per test**: Focus on a single feature or workflow 3. **Clean up resources**: Use RAII patterns or explicit cleanup +4. **Use deterministic test data**: Index Spacedrive source code instead of user directories + +### Test Data + +1. **Prefer project source code**: Use `env!("CARGO_MANIFEST_DIR")` to locate the Spacedrive repo +2. **Avoid user directories**: Don't hardcode paths like `$HOME/Desktop` or `$HOME/Downloads` +3. **Use subdirectories for multiple locations**: `core/`, `apps/`, etc. when testing multi-location scenarios +4. **Cross-platform paths**: Ensure test paths work on Linux, macOS, and Windows + +```rust +// ✅ Good: Uses project source code (deterministic) +let test_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf(); + +// ❌ Bad: Uses user directory (non-deterministic) +let desktop_path = std::env::var("HOME").unwrap() + "/Desktop"; +``` ### Subprocess Tests @@ -559,10 +771,12 @@ RUST_LOG=debug cargo test test_name --nocapture ### Debugging -When tests fail, check the logs in `test_data/{test_name}/library/logs/` for detailed information about what went wrong. + When tests fail, check the logs in `test_data/{test_name}/library/logs/` for + detailed information about what went wrong. Common debugging approaches: + - Run with `--nocapture` to see all output - Check job logs in `test_data/{test_name}/library/job_logs/` - Run scenarios individually with manual environment variables @@ -583,6 +797,7 @@ Common debugging approaches: - [ ] Wait for events instead of sleeping - [ ] Verify both positive and negative cases - [ ] Clean up temporary files +- [ ] Use deterministic test data (project source code, not user directories) ### Multi-Device Test Checklist @@ -592,6 +807,17 @@ Common debugging approaches: - [ ] Define clear success patterns - [ ] Handle process coordination properly - [ ] Set reasonable timeouts +- [ ] Use deterministic test data for cross-platform compatibility + +### Core Integration Test Checklist + +When adding a test to the core suite (`cargo xtask test-core`): + +- [ ] Test uses deterministic data (Spacedrive source code) +- [ ] Test runs reliably on Linux, macOS, and Windows +- [ ] Test includes `--test-threads=1` if accessing shared resources +- [ ] Add test definition to `xtask/src/test_core.rs` +- [ ] Verify test runs successfully in CI workflow ## TypeScript Integration Testing @@ -676,7 +902,8 @@ async fn test_typescript_cache_updates() -> anyhow::Result<()> { ``` -Use `.enable_daemon()` on `IndexingHarnessBuilder` to start the RPC server. The daemon listens on a random TCP port returned by `.daemon_socket_addr()`. + Use `.enable_daemon()` on `IndexingHarnessBuilder` to start the RPC server. + The daemon listens on a random TCP port returned by `.daemon_socket_addr()`. #### TypeScript Side @@ -692,60 +919,61 @@ import { SpacedriveProvider } from "../../src/hooks/useClient"; import { useNormalizedQuery } from "../../src/hooks/useNormalizedQuery"; interface BridgeConfig { - socket_addr: string; - library_id: string; - location_db_id: number; - location_path: string; - test_data_path: string; + socket_addr: string; + library_id: string; + location_db_id: number; + location_path: string; + test_data_path: string; } let bridgeConfig: BridgeConfig; let client: SpacedriveClient; beforeAll(async () => { - // Read bridge config from Rust test - const configPath = process.env.BRIDGE_CONFIG_PATH; - const configJson = await readFile(configPath, "utf-8"); - bridgeConfig = JSON.parse(configJson); + // Read bridge config from Rust test + const configPath = process.env.BRIDGE_CONFIG_PATH; + const configJson = await readFile(configPath, "utf-8"); + bridgeConfig = JSON.parse(configJson); - // Connect to daemon via TCP socket - client = SpacedriveClient.fromTcpSocket(bridgeConfig.socket_addr); - client.setCurrentLibrary(bridgeConfig.library_id); + // Connect to daemon via TCP socket + client = SpacedriveClient.fromTcpSocket(bridgeConfig.socket_addr); + client.setCurrentLibrary(bridgeConfig.library_id); }); describe("Cache Update Tests", () => { - test("should update cache when files move", async () => { - const wrapper = ({ children }) => - React.createElement(SpacedriveProvider, { client }, children); + test("should update cache when files move", async () => { + const wrapper = ({ children }) => + React.createElement(SpacedriveProvider, { client }, children); - // Query directory listing with useNormalizedQuery - const { result } = renderHook( - () => useNormalizedQuery({ - wireMethod: "query:files.directory_listing", - input: { path: { Physical: { path: folderPath } } }, - resourceType: "file", - pathScope: { Physical: { path: folderPath } }, - debug: true, // Enable debug logging - }), - { wrapper } - ); + // Query directory listing with useNormalizedQuery + const { result } = renderHook( + () => + useNormalizedQuery({ + wireMethod: "query:files.directory_listing", + input: { path: { Physical: { path: folderPath } } }, + resourceType: "file", + pathScope: { Physical: { path: folderPath } }, + debug: true, // Enable debug logging + }), + { wrapper }, + ); - // Wait for initial data - await waitFor(() => { - expect(result.current.data).toBeDefined(); - }); + // Wait for initial data + await waitFor(() => { + expect(result.current.data).toBeDefined(); + }); - // Perform file operation - await rename(oldPath, newPath); + // Perform file operation + await rename(oldPath, newPath); - // Wait for watcher to detect change (500ms buffer + processing) - await new Promise(resolve => setTimeout(resolve, 2000)); + // Wait for watcher to detect change (500ms buffer + processing) + await new Promise((resolve) => setTimeout(resolve, 2000)); - // Verify cache updated - expect(result.current.data.files).toContainEqual( - expect.objectContaining({ name: "newfile" }) - ); - }); + // Verify cache updated + expect(result.current.data.files).toContainEqual( + expect.objectContaining({ name: "newfile" }), + ); + }); }); ``` @@ -764,6 +992,7 @@ const client = new SpacedriveClient(transport); ``` The TCP transport: + - Uses JSON-RPC 2.0 over TCP - Supports WebSocket-style subscriptions for events - Automatically reconnects on connection loss @@ -783,26 +1012,35 @@ The primary use case for bridge tests is verifying that `useNormalizedQuery` cac ```typescript // Enable debug logging const { result } = renderHook( - () => useNormalizedQuery({ - wireMethod: "query:files.directory_listing", - input: { /* ... */ }, - resourceType: "file", - pathScope: { /* ... */ }, - debug: true, // Logs event processing - }), - { wrapper } + () => + useNormalizedQuery({ + wireMethod: "query:files.directory_listing", + input: { + /* ... */ + }, + resourceType: "file", + pathScope: { + /* ... */ + }, + debug: true, // Logs event processing + }), + { wrapper }, ); // Collect all events for debugging const allEvents: any[] = []; -const originalCreateSubscription = (client as any).subscriptionManager.createSubscription; -(client as any).subscriptionManager.createSubscription = function(filter: any, callback: any) { - const wrappedCallback = (event: any) => { - allEvents.push({ timestamp: new Date().toISOString(), event }); - console.log(`Event received:`, JSON.stringify(event, null, 2)); - callback(event); - }; - return originalCreateSubscription.call(this, filter, wrappedCallback); +const originalCreateSubscription = (client as any).subscriptionManager + .createSubscription; +(client as any).subscriptionManager.createSubscription = function ( + filter: any, + callback: any, +) { + const wrappedCallback = (event: any) => { + allEvents.push({ timestamp: new Date().toISOString(), event }); + console.log(`Event received:`, JSON.stringify(event, null, 2)); + callback(event); + }; + return originalCreateSubscription.call(this, filter, wrappedCallback); }; ``` @@ -821,24 +1059,29 @@ BRIDGE_CONFIG_PATH=/path/to/config.json bun test tests/integration/mytest.test.t ``` -Use `--nocapture` to see TypeScript test output. The Rust test prints all stdout/stderr from the TypeScript test process. + Use `--nocapture` to see TypeScript test output. The Rust test prints all + stdout/stderr from the TypeScript test process. ### Common Scenarios **File moves between folders:** + - Tests that files removed from one directory appear in another - Verifies UUID preservation (move detection vs delete+create) **Folder renames:** + - Tests that nested files update their paths correctly - Verifies parent path updates propagate to descendants **Bulk operations:** + - Tests 20+ file moves with mixed Physical/Content paths - Verifies cache updates don't miss files during batched events **Content-addressed files:** + - Uses `IndexMode::Content` to enable content identification - Tests that files with `alternate_paths` update correctly - Verifies metadata-only updates don't add duplicate cache entries @@ -846,17 +1089,20 @@ Use `--nocapture` to see TypeScript test output. The Rust test prints all stdout ### Debugging Bridge Tests **Check Rust logs:** + ```bash RUST_LOG=debug cargo test typescript_bridge -- --nocapture ``` **Check TypeScript output:** The Rust test prints all TypeScript stdout/stderr. Look for: + - `[TS]` prefixed log messages - Event payloads with `🔔` emoji - Final event summary at test end **Verify daemon is running:** + ```bash # In Rust test output, look for: Socket address: 127.0.0.1:XXXXX @@ -864,12 +1110,14 @@ Library ID: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx ``` **Check bridge config:** + ```bash # The config file is written to test_data directory cat /tmp/test_data/typescript_bridge_test/bridge_config.json ``` **Common issues:** + - **TypeScript test times out**: Increase watcher wait time (filesystem events can be slow) - **Cache not updating**: Enable `debug: true` to see if events are received - **Connection refused**: Verify daemon started with `.enable_daemon()` @@ -879,20 +1127,30 @@ cat /tmp/test_data/typescript_bridge_test/bridge_config.json For complete examples, refer to: +**Core Test Infrastructure:** + +- `xtask/src/test_core.rs` - Single source of truth for all core integration tests +- `.github/workflows/core_tests.yml` - CI workflow using xtask test runner + **Single Device Tests:** + - `tests/copy_action_test.rs` - Event collection during file operations (persistent + ephemeral) - `tests/job_resumption_integration_test.rs` - Job interruption handling **Subprocess Framework (Real Networking):** + - `tests/device_pairing_test.rs` - Device pairing with real network discovery **Custom Harness (Mock Transport):** -- `tests/sync_realtime_test.rs` - Real-time sync testing with deterministic transport -- `tests/sync_integration_test.rs` - Complex sync scenarios with mock networking + +- `tests/sync_realtime_test.rs` - Real-time sync testing with deterministic transport using Spacedrive source code +- `tests/sync_backfill_test.rs` - Backfill sync with deterministic test data +- `tests/sync_backfill_race_test.rs` - Race condition testing with concurrent operations - `tests/file_transfer_test.rs` - Cross-device file operations **TypeScript Bridge Tests:** + - `tests/typescript_bridge_test.rs` - Rust harness that spawns TypeScript tests - `packages/ts-client/tests/integration/useNormalizedQuery.test.ts` - File move cache updates - `packages/ts-client/tests/integration/useNormalizedQuery.folder-rename.test.ts` - Folder rename propagation -- `packages/ts-client/tests/integration/useNormalizedQuery.bulk-moves.test.ts` - Bulk operations with content-addressed files \ No newline at end of file +- `packages/ts-client/tests/integration/useNormalizedQuery.bulk-moves.test.ts` - Bulk operations with content-addressed files