From 03cb29868301ba6ec267efb5cf069f754e7df752 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Tue, 7 Oct 2025 20:31:12 -0700 Subject: [PATCH] a truck load of docs --- .tasks/CORE-011-unified-event-system.md | 57 + .../CORE-012-resource-type-registry-swift.md | 72 + ...E-013-resource-type-registry-typescript.md | 78 + .tasks/CORE-014-specta-codegen-events.md | 75 + .tasks/CORE-015-normalized-cache-swift.md | 68 + .../CORE-016-normalized-cache-typescript.md | 77 + .tasks/CORE-017-optimistic-updates.md | 67 + .tasks/LSYNC-000-library-sync.md | 59 +- .../LSYNC-001-design-library-sync-protocol.md | 36 +- .tasks/LSYNC-002-metadata-sync.md | 42 +- .tasks/LSYNC-003-file-op-sync.md | 45 +- .tasks/LSYNC-005-library-sync-setup.md | 31 + .tasks/LSYNC-006-transaction-manager-core.md | 46 + .tasks/LSYNC-007-syncable-trait.md | 61 + .tasks/LSYNC-008-sync-log-schema.md | 107 + .tasks/LSYNC-009-leader-election.md | 58 + .tasks/LSYNC-010-sync-service.md | 162 + .tasks/LSYNC-011-conflict-resolution.md | 68 + .../LSYNC-012-entry-sync-bulk-optimization.md | 73 + .tasks/LSYNC-013-sync-protocol-handler.md | 154 + docs/core/design/RELAY_FLOW_DIAGRAM.md | 1 + docs/core/design/RELAY_INTEGRATION_SUMMARY.md | 1 + docs/core/design/frontend_graphql_usage.tsx | 132 - .../design/sync/NORMALIZED_CACHE_DESIGN.md | 2673 +++++++++++++++++ docs/core/design/sync/README.md | 180 ++ .../design/{ => sync}/SYNC_CONDUIT_DESIGN.md | 0 docs/core/design/{ => sync}/SYNC_DESIGN.md | 0 .../{ => sync}/SYNC_DESIGN_2025_08_19.md | 0 .../{ => sync}/SYNC_FIRST_DRAFT_DESIGN.md | 0 .../{ => sync}/SYNC_INTEGRATION_NOTES.md | 0 .../design/sync/SYNC_TX_CACHE_MINI_SPEC.md | 271 ++ .../sync/TRANSACTION_MANAGER_COMPATIBILITY.md | 604 ++++ .../design/sync/UNIFIED_RESOURCE_EVENTS.md | 740 +++++ .../UNIFIED_TRANSACTIONAL_SYNC_AND_CACHE.md | 2294 ++++++++++++++ docs/core/devices.md | 1290 ++++++++ docs/core/events.md | 688 +++++ docs/core/normalized_cache.md | 933 ++++++ docs/core/sync.md | 561 ++++ 38 files changed, 11635 insertions(+), 169 deletions(-) create mode 100644 .tasks/CORE-011-unified-event-system.md create mode 100644 .tasks/CORE-012-resource-type-registry-swift.md create mode 100644 .tasks/CORE-013-resource-type-registry-typescript.md create mode 100644 .tasks/CORE-014-specta-codegen-events.md create mode 100644 .tasks/CORE-015-normalized-cache-swift.md create mode 100644 .tasks/CORE-016-normalized-cache-typescript.md create mode 100644 .tasks/CORE-017-optimistic-updates.md create mode 100644 .tasks/LSYNC-005-library-sync-setup.md create mode 100644 .tasks/LSYNC-006-transaction-manager-core.md create mode 100644 .tasks/LSYNC-007-syncable-trait.md create mode 100644 .tasks/LSYNC-008-sync-log-schema.md create mode 100644 .tasks/LSYNC-009-leader-election.md create mode 100644 .tasks/LSYNC-010-sync-service.md create mode 100644 .tasks/LSYNC-011-conflict-resolution.md create mode 100644 .tasks/LSYNC-012-entry-sync-bulk-optimization.md create mode 100644 .tasks/LSYNC-013-sync-protocol-handler.md delete mode 100644 docs/core/design/frontend_graphql_usage.tsx create mode 100644 docs/core/design/sync/NORMALIZED_CACHE_DESIGN.md create mode 100644 docs/core/design/sync/README.md rename docs/core/design/{ => sync}/SYNC_CONDUIT_DESIGN.md (100%) rename docs/core/design/{ => sync}/SYNC_DESIGN.md (100%) rename docs/core/design/{ => sync}/SYNC_DESIGN_2025_08_19.md (100%) rename docs/core/design/{ => sync}/SYNC_FIRST_DRAFT_DESIGN.md (100%) rename docs/core/design/{ => sync}/SYNC_INTEGRATION_NOTES.md (100%) create mode 100644 docs/core/design/sync/SYNC_TX_CACHE_MINI_SPEC.md create mode 100644 docs/core/design/sync/TRANSACTION_MANAGER_COMPATIBILITY.md create mode 100644 docs/core/design/sync/UNIFIED_RESOURCE_EVENTS.md create mode 100644 docs/core/design/sync/UNIFIED_TRANSACTIONAL_SYNC_AND_CACHE.md create mode 100644 docs/core/devices.md create mode 100644 docs/core/events.md create mode 100644 docs/core/normalized_cache.md create mode 100644 docs/core/sync.md diff --git a/.tasks/CORE-011-unified-event-system.md b/.tasks/CORE-011-unified-event-system.md new file mode 100644 index 000000000..24e84bfdb --- /dev/null +++ b/.tasks/CORE-011-unified-event-system.md @@ -0,0 +1,57 @@ +--- +id: CORE-011 +title: Unified Resource Event System +status: To Do +assignee: unassigned +priority: High +tags: [core, events, architecture, refactor] +--- + +## Description + +Refactor the event system from 40+ specialized event variants to a unified generic resource event architecture. This eliminates boilerplate and enables horizontal scaling - adding new resources requires zero event handling code changes. + +## Current Problem + +- 40+ event variants in `core/src/infra/event/mod.rs` +- Manual event emission scattered across codebase (easy to forget) +- Adding new resource = new event variant + client code changes +- No type safety between events and resources + +## Solution + +- Generic `ResourceChanged`, `ResourceBatchChanged`, `ResourceDeleted` events +- TransactionManager emits automatically (no manual emission) +- Client type registries handle deserialization generically +- Infrastructure events remain specific (CoreStarted, Job, etc.) + +## Implementation Steps + +1. Define new `Event` struct with `EventEnvelope` and `EventKind` +2. Add `ResourceChanged` and related variants to `EventKind` +3. Update TransactionManager to emit resource events automatically +4. Keep infrastructure events as specific variants +5. Mark old event variants as `#[deprecated]` +6. Migrate Albums/Tags/Locations to new events (parallel systems) +7. Remove old variants after full migration + +## Acceptance Criteria + +- [ ] New event structure defined +- [ ] EventEnvelope includes id, timestamp, library_id, sequence +- [ ] ResourceChanged auto-emitted by TransactionManager +- [ ] Infrastructure events (Job, CoreStarted) preserved +- [ ] No breaking changes (parallel systems initially) +- [ ] Documentation updated + +## Migration Strategy + +- Phase 1: Additive (both old and new events) +- Phase 2: Parallel (new resources use unified events only) +- Phase 3: Deprecation (mark old events deprecated) +- Phase 4: Cleanup (remove old events) + +## References + +- `docs/core/events.md` - Complete specification +- Current: `core/src/infra/event/mod.rs` diff --git a/.tasks/CORE-012-resource-type-registry-swift.md b/.tasks/CORE-012-resource-type-registry-swift.md new file mode 100644 index 000000000..2d35ad6bf --- /dev/null +++ b/.tasks/CORE-012-resource-type-registry-swift.md @@ -0,0 +1,72 @@ +--- +id: CORE-012 +title: Resource Type Registry (Swift) +status: To Do +assignee: unassigned +parent: CORE-011 +priority: High +tags: [client, swift, codegen, cache] +depends_on: [CORE-011] +--- + +## Description + +Create the Swift ResourceTypeRegistry that enables generic deserialization of resource events. This is the key component that makes unified events zero-friction on the client side. + +## Implementation Steps + +1. Define `CacheableResource` protocol +2. Create `ResourceTypeRegistry` class with decoder map +3. Implement `register()` method +4. Implement `decode(resourceType:from:)` method +5. Generate registry entries from specta codegen +6. Add auto-registration on app startup +7. Integrate with event handler + +## Technical Details + +- Location: `packages/client-swift/Sources/SpacedriveCore/Cache/ResourceTypeRegistry.swift` +- Protocol: `CacheableResource: Identifiable, Codable` +- Registry: `[String: (Data) throws -> any CacheableResource]` +- Auto-generated via specta codegen + +## Example + +```swift +// Protocol +protocol CacheableResource: Identifiable, Codable { + static var resourceType: String { get } +} + +// Registry +class ResourceTypeRegistry { + private static var decoders: [String: (Data) throws -> any CacheableResource] = [:] + + static func register(_ type: T.Type) { + decoders[T.resourceType] = { data in + try JSONDecoder().decode(T.self, from: data) + } + } + + static func decode(resourceType: String, from data: Data) throws -> any CacheableResource { + guard let decoder = decoders[resourceType] else { + throw CacheError.unknownResourceType(resourceType) + } + return try decoder(data) + } +} +``` + +## Acceptance Criteria + +- [ ] ResourceTypeRegistry implemented +- [ ] All domain resources conform to CacheableResource +- [ ] Auto-registration on app init +- [ ] Error handling for unknown types +- [ ] Unit tests for registration and decoding +- [ ] Integration with EventCacheUpdater + +## References + +- `docs/core/events.md` lines 213-298 +- `docs/core/normalized_cache.md` - Cache integration diff --git a/.tasks/CORE-013-resource-type-registry-typescript.md b/.tasks/CORE-013-resource-type-registry-typescript.md new file mode 100644 index 000000000..ab336d562 --- /dev/null +++ b/.tasks/CORE-013-resource-type-registry-typescript.md @@ -0,0 +1,78 @@ +--- +id: CORE-013 +title: Resource Type Registry (TypeScript) +status: To Do +assignee: unassigned +parent: CORE-011 +priority: High +tags: [client, typescript, codegen, cache] +depends_on: [CORE-011] +--- + +## Description + +Create the TypeScript ResourceTypeRegistry for web/desktop clients. Enables generic deserialization of resource events with type safety maintained through generated bindings. + +## Implementation Steps + +1. Create `ResourceTypeRegistry` class +2. Implement `register()` method with validators +3. Implement `decode()` method +4. Auto-generate `resourceTypeMap` via specta +5. Add auto-registration from generated map +6. Integrate with EventCacheUpdater +7. Add TypeScript type safety + +## Technical Details + +- Location: `packages/client/src/core/ResourceTypeRegistry.ts` +- Auto-generated: `packages/client/src/bindings/resourceRegistry.ts` +- Type-safe: TypeScript types generated from Rust +- Validation: Runtime type checking (optional) + +## Example + +```typescript +class ResourceTypeRegistry { + private static validators = new Map any>(); + + static register(resourceType: string, validator: (data: unknown) => T) { + this.validators.set(resourceType, validator); + } + + static decode(resourceType: string, data: unknown): any { + const validator = this.validators.get(resourceType); + if (!validator) { + throw new Error(`Unknown resource type: ${resourceType}`); + } + return validator(data); + } +} + +// Auto-generated from specta +export const resourceTypeMap = { + 'file': File, + 'album': Album, + 'tag': Tag, + 'location': Location, +} as const; + +// Auto-registration +Object.entries(resourceTypeMap).forEach(([type, TypeClass]) => { + ResourceTypeRegistry.register(type, (data) => data as InstanceType); +}); +``` + +## Acceptance Criteria + +- [ ] ResourceTypeRegistry implemented +- [ ] Auto-generated resourceTypeMap from specta +- [ ] Type safety preserved through generics +- [ ] Error handling for unknown types +- [ ] Unit tests for registration and decoding +- [ ] Integration with EventCacheUpdater + +## References + +- `docs/core/events.md` lines 302-389 +- Specta codegen: `xtask/src/specta_gen.rs` diff --git a/.tasks/CORE-014-specta-codegen-events.md b/.tasks/CORE-014-specta-codegen-events.md new file mode 100644 index 000000000..f13f16d85 --- /dev/null +++ b/.tasks/CORE-014-specta-codegen-events.md @@ -0,0 +1,75 @@ +--- +id: CORE-014 +title: Specta Codegen for Resource Events +status: To Do +assignee: unassigned +parent: CORE-011 +priority: High +tags: [codegen, specta, typescript, swift] +depends_on: [CORE-011] +--- + +## Description + +Extend the existing specta codegen system to auto-generate resource type registries for TypeScript and Swift. This ensures client-side type registries stay in sync with Rust domain models. + +## Implementation Steps + +1. Update `xtask/src/specta_gen.rs` to collect all `Identifiable` types +2. Generate TypeScript `resourceTypeMap` with all resource types +3. Generate Swift `ResourceTypeRegistry+Generated.swift` with registrations +4. Add build verification that all Identifiable types are registered +5. Update CI to regenerate on every commit +6. Document regeneration process for developers + +## Generated Output + +### TypeScript +```typescript +// packages/client/src/bindings/resourceRegistry.ts +export const resourceTypeMap = { + 'file': File, + 'album': Album, + 'tag': Tag, + 'location': Location, + 'device': Device, + 'volume': Volume, + 'content_identity': ContentIdentity, + // ... all Identifiable types +} as const; +``` + +### Swift (Future) +```swift +// SpacedriveCore/Generated/ResourceTypeRegistry+Generated.swift +extension ResourceTypeRegistry { + static func registerAllTypes() { + register(File.self) + register(Album.self) + register(Tag.self) + // ... all Identifiable types + } +} +``` + +## Technical Details + +- Location: `xtask/src/specta_gen.rs` +- Trait marker: Check for `impl Identifiable` +- Output: `packages/client/src/bindings/resourceRegistry.ts` +- Build step: `cargo xtask specta-gen` +- CI: Auto-run on pre-commit or CI build + +## Acceptance Criteria + +- [ ] Specta codegen extended for resource types +- [ ] TypeScript resourceTypeMap auto-generated +- [ ] Build verification ensures all types registered +- [ ] CI/CD regenerates on every commit +- [ ] Developer documentation updated +- [ ] Diff checking prevents manual edits + +## References + +- `docs/core/events.md` lines 391-434 +- Existing: `xtask/src/specta_gen.rs` diff --git a/.tasks/CORE-015-normalized-cache-swift.md b/.tasks/CORE-015-normalized-cache-swift.md new file mode 100644 index 000000000..e034965ea --- /dev/null +++ b/.tasks/CORE-015-normalized-cache-swift.md @@ -0,0 +1,68 @@ +--- +id: CORE-015 +title: Normalized Client Cache (Swift) +status: To Do +assignee: unassigned +priority: High +tags: [client, swift, cache, performance] +depends_on: [CORE-012] +--- + +## Description + +Implement the normalized client cache for iOS/macOS apps. Provides instant UI updates, offline support, and massive bandwidth savings by normalizing all resources by ID and updating atomically when events arrive. + +## Implementation Steps + +1. Create `NormalizedCache` actor with two-level structure: + - Level 1: Entity store (normalized by ID) + - Level 2: Query index (maps queries to entity IDs) +2. Implement `updateEntity()` - updates entity and notifies observers +3. Implement `query()` - caches queries and results +4. Implement `deleteEntity()` - removes entity and updates indices +5. Implement `invalidateQueriesForResource()` - bulk operation handling +6. Add LRU eviction (max 10K entities) +7. Add SQLite persistence for offline support +8. Create `EventCacheUpdater` for event integration + +## Cache Architecture + +``` +┌─────────────────────────────────────────┐ +│ Entity Store (Level 1) │ +│ "file:uuid-1" → File { ... } │ +│ "album:uuid-2" → Album { ... } │ +└─────────────────────────────────────────┘ + ↑ + │ Atomic updates + │ +┌─────────────────────────────────────────┐ +│ Query Index (Level 2) │ +│ "search:photos" → ["file:uuid-1", ...] │ +│ "albums.list" → ["album:uuid-2"] │ +└─────────────────────────────────────────┘ +``` + +## Technical Details + +- Location: `packages/client-swift/Sources/SpacedriveCore/Cache/NormalizedCache.swift` +- Actor for thread-safety +- Max entities: 10,000 (configurable) +- TTL: 5 minutes default (query-specific) +- Persistence: SQLite in app cache directory + +## Acceptance Criteria + +- [ ] NormalizedCache actor implemented +- [ ] Entity store with LRU eviction +- [ ] Query index with TTL +- [ ] SQLite persistence +- [ ] EventCacheUpdater integration +- [ ] ObservableObject wrapper for SwiftUI +- [ ] Memory stays under 15MB with 10K entities +- [ ] Unit tests for cache operations +- [ ] Integration tests with events + +## References + +- `docs/core/normalized_cache.md` - Complete specification diff --git a/.tasks/CORE-016-normalized-cache-typescript.md b/.tasks/CORE-016-normalized-cache-typescript.md new file mode 100644 index 000000000..0fb9dd2fd --- /dev/null +++ b/.tasks/CORE-016-normalized-cache-typescript.md @@ -0,0 +1,77 @@ +--- +id: CORE-016 +title: Normalized Client Cache (TypeScript) +status: To Do +assignee: unassigned +priority: High +tags: [client, typescript, react, cache, performance] +depends_on: [CORE-013] +--- + +## Description + +Implement the normalized client cache for web/desktop (Electron) apps. Same architecture as Swift version but with React integration via hooks. + +## Implementation Steps + +1. Create `NormalizedCache` class with entity store + query index +2. Implement `updateEntity()` with subscription notifications +3. Implement `query()` with caching +4. Implement `deleteEntity()` and query invalidation +5. Add LRU eviction +6. Add IndexedDB persistence for offline support +7. Create `useCachedQuery` React hook +8. Create `EventCacheUpdater` for event integration + +## React Integration + +```typescript +function useCachedQuery( + method: string, + input: any, +): { data: T[] | null; loading: boolean; error: Error | null } { + const cache = useContext(CacheContext); + const [data, setData] = useState(null); + + useEffect(() => { + const queryKey = cache.generateQueryKey(method, input); + + // Subscribe to cache changes + const unsubscribe = cache.subscribe(queryKey, () => { + const result = cache.getQueryResult(queryKey); + setData(result); + }); + + // Initial fetch + cache.query(method, input).then(setData); + + return unsubscribe; + }, [method, JSON.stringify(input)]); + + return { data, loading: data === null, error: null }; +} +``` + +## Technical Details + +- Location: `packages/client/src/core/NormalizedCache.ts` +- React hook: `packages/client/src/hooks/useCachedQuery.ts` +- Max entities: 10,000 +- TTL: 5 minutes default +- Persistence: IndexedDB + +## Acceptance Criteria + +- [ ] NormalizedCache class implemented +- [ ] Entity store with LRU eviction +- [ ] Query index with TTL +- [ ] IndexedDB persistence +- [ ] useCachedQuery hook +- [ ] EventCacheUpdater integration +- [ ] Memory stays under 15MB +- [ ] Unit tests for cache operations +- [ ] Integration tests with React components + +## References + +- `docs/core/normalized_cache.md` lines 188-279 diff --git a/.tasks/CORE-017-optimistic-updates.md b/.tasks/CORE-017-optimistic-updates.md new file mode 100644 index 000000000..c8cee2cac --- /dev/null +++ b/.tasks/CORE-017-optimistic-updates.md @@ -0,0 +1,67 @@ +--- +id: CORE-017 +title: Optimistic Updates for Client Cache +status: To Do +assignee: unassigned +parent: CORE-015 +priority: Medium +tags: [client, cache, ux, optimistic] +depends_on: [CORE-015, CORE-016] +--- + +## Description + +Implement optimistic updates in the normalized cache, allowing instant UI feedback before server confirmation. If the action fails, the update is rolled back automatically. + +## Implementation Steps + +1. Add `optimisticUpdates` map to cache (pending_id → resource) +2. Implement `updateOptimistically()` - applies change immediately +3. Implement `commitOptimisticUpdate()` - replaces with confirmed data +4. Implement `rollbackOptimisticUpdate()` - reverts on error +5. Integrate with action execution flow +6. Add visual indicators for pending changes (optional) + +## Flow Example + +```typescript +// 1. Optimistic update (instant UI) +const pendingId = uuid(); +await cache.updateOptimistically(pendingId, { + id: albumId, + name: newName, + ...optimisticAlbum +}); + +try { + // 2. Send action to server + const confirmed = await client.action('albums.rename.v1', { id: albumId, name: newName }); + + // 3. Commit (replace optimistic with confirmed) + await cache.commitOptimisticUpdate(pendingId, confirmed); +} catch (error) { + // 4. Rollback on error + await cache.rollbackOptimisticUpdate(pendingId); + throw error; +} +``` + +## Technical Details + +- Optimistic updates stored separately from confirmed entities +- UI sees merged view (optimistic + confirmed) +- Pending changes visually indicated (future) +- Automatic rollback on action failure + +## Acceptance Criteria + +- [ ] Optimistic update API implemented +- [ ] UI updates instantly before server response +- [ ] Rollback works on errors +- [ ] No flickering during commit +- [ ] Unit tests for optimistic flow +- [ ] Integration tests validate error scenarios + +## References + +- `docs/core/normalized_cache.md` lines 685-741 diff --git a/.tasks/LSYNC-000-library-sync.md b/.tasks/LSYNC-000-library-sync.md index 3227ba347..721a3c04c 100644 --- a/.tasks/LSYNC-000-library-sync.md +++ b/.tasks/LSYNC-000-library-sync.md @@ -1,8 +1,8 @@ --- id: LSYNC-000 title: "Epic: Library-based Synchronization" -status: To Do -assignee: unassigned +status: In Progress +assignee: james priority: High tags: [epic, sync, networking, library-sync] whitepaper: Section 4.5.1 @@ -10,4 +10,57 @@ whitepaper: Section 4.5.1 ## Description -This epic covers the implementation of the "Library Sync" model, a novel approach to synchronization that leverages the VDFS index to avoid the complexities of CRDTs. It treats metadata (index, audit log) and file operations as separate, coordinated streams, enabling efficient and robust syncing between peers. +This epic covers the implementation of the "Library Sync" system, enabling real-time, multi-device synchronization of library metadata. The architecture consists of three pillars: TransactionManager (write gatekeeper), Sync Log (append-only change log), and Sync Service (pull-based replication). + +## Current Status + +**Completed (Phase 1)**: +- NET-001: Iroh P2P stack +- NET-002: Device pairing protocol +- LSYNC-004: SyncRelationship schema +- LSYNC-005: Library sync setup (device discovery & registration) +- LSYNC-001: Protocol design (documented in `docs/core/`) + +**In Progress (Phase 2)**: +- LSYNC-006: TransactionManager core +- LSYNC-007: Syncable trait & derives +- LSYNC-008: Sync log schema +- LSYNC-009: Leader election + +**Upcoming (Phase 3)**: +- LSYNC-013: Sync protocol handler (message-based) +- LSYNC-010: Sync service (leader & follower) +- LSYNC-011: Conflict resolution +- LSYNC-002: Metadata sync (albums/tags) +- LSYNC-012: Entry sync (bulk optimization) + +## Architecture + +**Message-based Sync**: Push notifications via dedicated sync protocol instead of polling for better performance, lower latency, and battery efficiency. + +See `docs/core/sync.md` for complete specification. + +## Subtasks + +### Phase 1: Foundation (Completed) +- LSYNC-001: Protocol design ✅ +- LSYNC-004: Database schema ✅ +- LSYNC-005: Sync setup ✅ + +### Phase 2: Core Infrastructure (In Progress) +- LSYNC-006: TransactionManager +- LSYNC-007: Syncable trait +- LSYNC-008: Sync log schema (separate DB) +- LSYNC-009: Leader election + +### Phase 3: Sync Services (Next) +- LSYNC-013: Sync protocol handler (push-based) +- LSYNC-010: Sync service (leader & follower) +- LSYNC-011: Conflict resolution + +### Phase 4: Application (After Phase 3) +- LSYNC-002: Metadata sync (albums/tags) +- LSYNC-012: Entry sync (bulk optimization) + +### Future +- LSYNC-003: File operations (sync conduits) diff --git a/.tasks/LSYNC-001-design-library-sync-protocol.md b/.tasks/LSYNC-001-design-library-sync-protocol.md index a0099247a..eb05a9a5a 100644 --- a/.tasks/LSYNC-001-design-library-sync-protocol.md +++ b/.tasks/LSYNC-001-design-library-sync-protocol.md @@ -1,8 +1,8 @@ --- id: LSYNC-001 title: Design Library Sync Protocol -status: To Do -assignee: unassigned +status: Done +assignee: james parent: LSYNC-000 priority: High tags: [sync, networking, protocol, design] @@ -13,14 +13,30 @@ whitepaper: Section 4.5.1 Design the detailed protocol for Library Sync. This includes defining the communication flow between peers, the format of the messages, and the logic for initiating and managing a sync session. -## Implementation Steps +**Update**: The sync protocol has been fully designed and documented as the "Three Pillars" architecture: +1. TransactionManager (sole gatekeeper for writes) +2. Sync Log (append-only, sequentially-ordered) +3. Sync Service (pull-based replication) -1. Define the state machine for a sync session (e.g., negotiation, metadata exchange, file operations, completion). -2. Specify the protocol messages for each stage of the sync process. -3. Design the error handling and recovery mechanisms. -4. Create a sequence diagram to illustrate the protocol flow. +## Design Documents + +- `docs/core/sync.md` - Complete sync system specification +- `docs/core/sync-setup.md` - Library sync setup flow +- `docs/core/events.md` - Unified event system +- `docs/core/normalized_cache.md` - Client-side cache +- `docs/core/devices.md` - Device system and leadership + +## Architecture Decisions + +- **Pull-based sync**: Followers poll leader every 5 seconds +- **Bulk optimization**: 1K+ items create metadata-only sync logs +- **Leader election**: Single leader per library assigns sequences +- **Conflict resolution**: Last-Write-Wins via version field +- **No CRDTs**: Simpler approach, sufficient for metadata ## Acceptance Criteria -- [ ] A detailed protocol design document is created. -- [ ] The protocol addresses all aspects of the Library Sync model. -- [ ] The design is approved and ready for implementation. + +- [x] Detailed protocol design document created +- [x] Protocol addresses all aspects of Library Sync +- [x] Design reviewed and approved +- [x] Implementation tasks created (LSYNC-006 through LSYNC-011) diff --git a/.tasks/LSYNC-002-metadata-sync.md b/.tasks/LSYNC-002-metadata-sync.md index c474ff260..77ae5d068 100644 --- a/.tasks/LSYNC-002-metadata-sync.md +++ b/.tasks/LSYNC-002-metadata-sync.md @@ -1,26 +1,46 @@ --- id: LSYNC-002 -title: Metadata Sync (Index & Audit Log) +title: Metadata Sync (Albums, Tags, Locations) status: To Do assignee: unassigned parent: LSYNC-000 priority: High -tags: [sync, networking, database, metadata] -whitepaper: Section 4.5.1 +tags: [sync, metadata, albums, tags] +depends_on: [LSYNC-006, LSYNC-010] --- ## Description -Implement the metadata synchronization part of the Library Sync protocol. This involves efficiently syncing the VDFS index and the audit log between two peers. +Implement metadata synchronization for user-created resources (Albums, Tags, Locations) using the sync system. This is the first practical application of the TransactionManager and sync follower service. + +**Note**: This task focuses on rich metadata (albums, tags), NOT file entries. File entry sync is handled separately with bulk optimization (see LSYNC-012-entry-sync.md). ## Implementation Steps -1. Develop a mechanism to efficiently diff the SQLite databases of the two peers. -2. Implement the logic to transfer the missing index and audit log entries. -3. Ensure that the metadata sync is atomic and consistent. -4. Optimize the data transfer to minimize network usage. +1. Implement `Syncable` trait for `albums::Model` +2. Implement `Syncable` trait for `tags::Model` +3. Implement `Syncable` trait for `locations::Model` +4. Update album/tag/location actions to use TransactionManager +5. Verify sync logs created on leader device +6. Verify follower service applies changes correctly +7. Test cross-device album/tag creation and updates + +## Technical Details + +- Albums: Sync name, cover, description +- Tags: Sync name, color +- Locations: Sync metadata only (path is device-specific) +- Entry relationships: Sync via junction tables ## Acceptance Criteria -- [ ] Two peers can successfully sync their VDFS index and audit log. -- [ ] The metadata sync is efficient and scalable. -- [ ] The synced data is consistent and correct on both peers. + +- [ ] Albums sync between devices +- [ ] Tags sync between devices +- [ ] Location metadata syncs +- [ ] Changes appear instantly in client cache +- [ ] Conflict resolution works for concurrent edits +- [ ] Integration tests validate cross-device sync + +## References + +- See `docs/core/sync.md` for domain sync strategy diff --git a/.tasks/LSYNC-003-file-op-sync.md b/.tasks/LSYNC-003-file-op-sync.md index 6753c91fa..e456da6e0 100644 --- a/.tasks/LSYNC-003-file-op-sync.md +++ b/.tasks/LSYNC-003-file-op-sync.md @@ -1,26 +1,45 @@ --- id: LSYNC-003 -title: File Operation Sync (via Action System) +title: Cross-Device File Operations (Future Phase) status: To Do assignee: unassigned parent: LSYNC-000 -priority: High -tags: [sync, networking, actions, jobs] -whitepaper: Section 4.5.1 +priority: Low +tags: [sync, file-ops, future] --- ## Description -Implement the file operation synchronization part of the Library Sync protocol. This involves using the Action System to replicate file operations (copy, move, delete) between peers based on the synced metadata. +Enable file operations (copy, move, delete) to be executed across devices. This is a **future phase** feature - not part of the initial sync implementation. -## Implementation Steps +**Current Architecture**: File operations are device-local. If you delete a file on Device A, only the metadata syncs (the Entry is marked deleted). Device B sees the metadata change but does NOT delete its local file copy. -1. Develop a mechanism to translate the diff of the audit logs into a series of `Action`s. -2. Implement the logic to dispatch these `Action`s to the `ActionManager` on the target peer. -3. Ensure that the file operations are executed in the correct order and with the correct context. -4. Integrate this with the overall Library Sync protocol. +**Future Goal**: User can optionally enable "sync conduits" where file operations replicate across devices. Example: Delete on Device A → Device B also deletes local file. + +## Implementation Steps (Future) + +1. Design "sync conduit" configuration (which locations participate) +2. File operation actions emit special sync log entries +3. Follower service recognizes file-op entries +4. Follower executes corresponding local file operation +5. Handle conflicts (file already deleted, moved, etc.) +6. Add user controls for sync conduit policies + +## Why Not Phase 1? + +- Metadata sync is complex enough initially +- File operations need robust conflict resolution +- Users may not want all devices to mirror operations +- Bandwidth/storage considerations (mobile devices) ## Acceptance Criteria -- [ ] File operations are correctly replicated on the target peer. -- [ ] The system can handle conflicts and errors during file operation sync. -- [ ] The file operation sync is integrated seamlessly into the Library Sync protocol. + +- [ ] Sync conduit configuration schema +- [ ] File operations create special sync log type +- [ ] Follower can execute file operations +- [ ] Conflict resolution for file ops +- [ ] User can enable/disable per location pair + +## References + +- `docs/core/sync.md` - Sync domains (Content future phase) diff --git a/.tasks/LSYNC-005-library-sync-setup.md b/.tasks/LSYNC-005-library-sync-setup.md new file mode 100644 index 000000000..12cf77dfa --- /dev/null +++ b/.tasks/LSYNC-005-library-sync-setup.md @@ -0,0 +1,31 @@ +--- +id: LSYNC-005 +title: Library Sync Setup (Device Registration & Discovery) +status: Done +assignee: james +parent: LSYNC-000 +priority: High +tags: [sync, networking, library-setup, device-pairing] +--- + +## Description + +Implement the library sync setup flow that enables paired devices to discover each other's libraries and register for synchronization. This is Phase 1 of the sync system (RegisterOnly mode). + +## Implementation Notes + +- Complete implementation detailed in `docs/core/sync-setup.md` +- Devices must be paired (NET-002) before library sync setup +- Two-phase process: Discovery → Registration +- Bidirectional device registration in each library's database +- Leader election during setup +- No actual sync replication in Phase 1 (just registration) + +## Acceptance Criteria + +- [x] Paired devices can discover each other's libraries +- [x] Devices can be registered in remote library databases +- [x] Sync leadership assigned during setup +- [x] `sync_setup.discover.v1` query implemented +- [x] `sync_setup.input.v1` action implemented +- [x] Integration tests validate cross-device setup diff --git a/.tasks/LSYNC-006-transaction-manager-core.md b/.tasks/LSYNC-006-transaction-manager-core.md new file mode 100644 index 000000000..4cae306f3 --- /dev/null +++ b/.tasks/LSYNC-006-transaction-manager-core.md @@ -0,0 +1,46 @@ +--- +id: LSYNC-006 +title: TransactionManager Core Implementation +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: Critical +tags: [sync, database, transaction, architecture] +--- + +## Description + +Implement the TransactionManager, the sole gatekeeper for all syncable database writes. It guarantees atomic DB commits + sync log creation, ensuring that state changes are always logged for synchronization. + +## Implementation Steps + +1. Create `TransactionManager` struct with event bus and sync sequence tracking +2. Implement `commit()` for single resource changes +3. Implement `commit_batch()` for 10-1K item batches +4. Implement `commit_bulk()` for 1K+ items (metadata-only sync logs) +5. Add sequence number generation (only on leader devices) +6. Integrate with event emission (automatic ResourceChanged events) +7. Add `with_tx()` for raw SQL compatibility + +## Technical Details + +- Location: `core/src/infra/transaction/manager.rs` +- Must check leader status before assigning sequence numbers +- Atomic transaction: DB write + sync log entry creation +- Auto-emit events via EventBus after commit +- Bulk operations create single metadata sync log (not per-item) + +## Acceptance Criteria + +- [ ] TransactionManager can commit single resources with sync logs +- [ ] Batch operations create per-item sync logs +- [ ] Bulk operations create metadata-only sync logs +- [ ] Events emitted automatically after commits +- [ ] Leader check prevents non-leaders from creating sync logs +- [ ] Unit tests verify atomicity +- [ ] Integration tests validate sync log creation + +## References + +- `docs/core/sync.md` - Complete specification +- Phase 1 dependency for sync system diff --git a/.tasks/LSYNC-007-syncable-trait.md b/.tasks/LSYNC-007-syncable-trait.md new file mode 100644 index 000000000..84b41ef54 --- /dev/null +++ b/.tasks/LSYNC-007-syncable-trait.md @@ -0,0 +1,61 @@ +--- +id: LSYNC-007 +title: Syncable Trait & Derive Macros +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: High +tags: [sync, trait, codegen, macro] +--- + +## Description + +Create the `Syncable` trait that database models implement to enable automatic sync log creation. Includes derive macro for ergonomic implementation. + +## Implementation Steps + +1. Define `Syncable` trait with required methods: + - `SYNC_MODEL: &'static str` - Model identifier + - `sync_id() -> Uuid` - Global resource ID + - `version() -> i64` - For conflict resolution + - `exclude_fields()` - Optional field exclusion + - `to_sync_json()` - Optional custom serialization +2. Create `#[derive(Syncable)]` macro +3. Implement for initial models: Album, Tag, Location +4. Add validation that `sync_id` is unique across model +5. Document field exclusion patterns (db IDs, timestamps) + +## Technical Details + +- Location: `core/src/infra/sync/syncable.rs` +- Macro location: `crates/sync-derive/src/lib.rs` +- Must integrate with SeaORM models +- `exclude_fields()` prevents platform-specific data from syncing + +## Example Usage + +```rust +impl Syncable for albums::Model { + const SYNC_MODEL: &'static str = "album"; + + fn sync_id(&self) -> Uuid { self.uuid } + fn version(&self) -> i64 { self.version } + + fn exclude_fields() -> Option<&'static [&'static str]> { + Some(&["id", "created_at", "updated_at"]) + } +} +``` + +## Acceptance Criteria + +- [ ] `Syncable` trait defined +- [ ] Derive macro implemented +- [ ] Works with SeaORM models +- [ ] Field exclusion functional +- [ ] Documentation with examples +- [ ] Unit tests for derive macro + +## References + +- `docs/core/sync.md` lines 60-118 diff --git a/.tasks/LSYNC-008-sync-log-schema.md b/.tasks/LSYNC-008-sync-log-schema.md new file mode 100644 index 000000000..efd24377a --- /dev/null +++ b/.tasks/LSYNC-008-sync-log-schema.md @@ -0,0 +1,107 @@ +--- +id: LSYNC-008 +title: Sync Log Database Schema & Entity (Separate DB) +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: High +tags: [sync, database, schema, migration] +--- + +## Description + +Create the sync log database schema and SeaORM entity. The sync log is an append-only, sequentially-ordered log of all state changes per library, maintained by the leader device. + +**Architecture Decision**: The sync log lives in its own separate database (`sync_log.db`) in the Library data folder rather than in the main library database. This provides: +- Better performance (no query contention) +- Easier maintenance (vacuum, archive old entries) +- Cleaner separation (infrastructure vs domain data) +- Simpler backup/restore (library can be backed up without sync log) + +## Implementation Steps + +1. Create sync log database connection per library +2. Create migration for `sync_log` table (in sync DB) +3. Create SeaORM entity `core/src/infra/db/entities/sync_log.rs` +4. Add indexes for efficient querying: + - `(sequence)` - Primary lookup for sync (library_id not needed since it's per-library DB) + - `(device_id)` - Filter by originating device + - `(model_type, record_id)` - Find changes to specific records +5. Add unique constraint on `(sequence)` +6. Create `SyncLogDb` wrapper for database lifecycle +7. Create helper methods for querying sync entries + +## Schema + +```sql +CREATE TABLE sync_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sequence INTEGER NOT NULL UNIQUE, -- Monotonic per library (unique since this DB is per-library) + device_id TEXT NOT NULL, -- Device that created this entry + timestamp TEXT NOT NULL, + + -- Change details + model_type TEXT NOT NULL, -- "album", "tag", "entry", "bulk_operation" + record_id TEXT NOT NULL, -- UUID of changed record + change_type TEXT NOT NULL, -- "insert", "update", "delete", "bulk_insert" + version INTEGER NOT NULL DEFAULT 1, -- Optimistic concurrency + + -- Data payload (JSON) + data TEXT NOT NULL +); + +CREATE INDEX idx_sync_log_sequence ON sync_log(sequence); +CREATE INDEX idx_sync_log_device ON sync_log(device_id); +CREATE INDEX idx_sync_log_model_record ON sync_log(model_type, record_id); +CREATE INDEX idx_sync_log_timestamp ON sync_log(timestamp); +``` + +**Note**: `library_id` field removed since each library has its own sync log database. + +## Database Location + +- Path: `~/.spacedrive/libraries/{library_uuid}/sync.db` +- One sync log DB per library +- Created automatically when library is opened +- Managed by `SyncLogDb` wrapper + +## SyncLogDb Wrapper + +```rust +pub struct SyncLogDb { + library_id: Uuid, + conn: DatabaseConnection, +} + +impl SyncLogDb { + /// Open or create sync log DB for library + pub async fn open(library_id: Uuid, data_dir: &Path) -> Result; + + /// Append entry to sync log (leader only) + pub async fn append(&self, entry: SyncLogEntry) -> Result; + + /// Fetch entries since sequence + pub async fn fetch_since(&self, sequence: u64, limit: usize) -> Result, DbError>; + + /// Get latest sequence number + pub async fn latest_sequence(&self) -> Result; + + /// Vacuum old entries (> 30 days) + pub async fn vacuum_old_entries(&self, before: DateTime) -> Result; +} +``` + +## Acceptance Criteria + +- [ ] Per-library sync log database created +- [ ] Migration created and tested +- [ ] SeaORM entity implemented +- [ ] Indexes created for performance +- [ ] SyncLogDb wrapper implemented +- [ ] Helper methods for common queries +- [ ] Database lifecycle managed correctly +- [ ] Documentation of schema design + +## References + +- `docs/core/sync.md` lines 211-236 diff --git a/.tasks/LSYNC-009-leader-election.md b/.tasks/LSYNC-009-leader-election.md new file mode 100644 index 000000000..069f2b2dc --- /dev/null +++ b/.tasks/LSYNC-009-leader-election.md @@ -0,0 +1,58 @@ +--- +id: LSYNC-009 +title: Sync Leader Election & Lease Management +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: High +tags: [sync, leadership, distributed-systems] +--- + +## Description + +Implement the leader election mechanism that ensures each library has a single leader device responsible for assigning sync log sequence numbers. This prevents sequence collisions and ensures consistent ordering. + +## Implementation Steps + +1. Create `SyncLeader` struct with lease tracking +2. Implement `request_leadership()` method +3. Implement `is_leader()` check +4. Add heartbeat mechanism (leader sends every 30s) +5. Implement re-election on leader timeout (>60s) +6. Use highest device_id as tiebreaker +7. Integrate with TransactionManager + +## Election Strategy + +- **Initial leader**: Device that creates the library +- **Heartbeat**: Leader sends heartbeat every 30 seconds +- **Re-election**: If leader offline >60s, devices elect new leader +- **Tiebreaker**: Highest device_id wins +- **Lease**: Leader holds exclusive write lease + +## Technical Details + +- Location: `core/src/infra/sync/leader.rs` +- Leadership state stored in device's `sync_leadership` field +- Lease expires_at tracked per library +- TransactionManager checks leadership before assigning sequences + +## Acceptance Criteria + +- [ ] Leader election on library creation +- [ ] Heartbeat mechanism prevents false timeouts +- [ ] Re-election works when leader goes offline +- [ ] Only leader can create sync log entries +- [ ] Follower devices reject write attempts with clear error +- [ ] Integration tests validate failover scenarios + +## Future Enhancements + +- Multi-leader support with sequence partitioning +- Manual leader reassignment via admin action +- Leader election metrics and monitoring + +## References + +- `docs/core/sync.md` lines 238-280 +- `docs/core/devices.md` - Sync leadership model diff --git a/.tasks/LSYNC-010-sync-service.md b/.tasks/LSYNC-010-sync-service.md new file mode 100644 index 000000000..3bfd3608f --- /dev/null +++ b/.tasks/LSYNC-010-sync-service.md @@ -0,0 +1,162 @@ +--- +id: LSYNC-010 +title: Sync Service (Leader & Follower) +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: High +tags: [sync, replication, service, leader, follower] +depends_on: [LSYNC-006, LSYNC-008, LSYNC-009, LSYNC-013] +--- + +## Description + +Implement the complete sync service with both leader and follower functionality. The leader pushes notifications to followers when new sync log entries are created, and followers receive these notifications and apply changes locally. + +**Architecture**: Message-based (push) instead of polling for better performance, lower latency, and battery efficiency. + +## Implementation Steps + +### Core Service +1. Create `SyncService` struct with role-specific behavior +2. Initialize service when library opens (determines leader/follower role) +3. Integrate with SyncProtocolHandler for messaging +4. Handle role transitions (leader election changes) + +### Leader Functionality +5. Subscribe to TransactionManager commit events +6. Implement `on_commit()` - called after sync log entry created +7. Implement `notify_followers()` - sends NewEntries to all followers +8. Implement batch notification logic (debounce within 100ms) +9. Implement message handler for `SyncMessage::FetchEntries` +10. Implement message handler for `SyncMessage::Heartbeat` +11. Track connected followers per library + +### Follower Functionality +12. Implement message handler for `SyncMessage::NewEntries` +13. Implement `request_entries()` - uses SyncProtocolHandler to fetch +14. Implement `apply_sync_entry()` - deserializes and applies changes +15. Track `last_synced_sequence` per library +16. Handle bulk operation metadata (trigger local indexing) +17. Handle connection loss with reconnect logic +18. Send heartbeats to leader + +## Technical Details + +- Location: `core/src/service/sync/` + - `mod.rs` - SyncService orchestrator + - `leader.rs` - Leader-specific logic + - `follower.rs` - Follower-specific logic +- Push-based: Leader notifies when changes happen +- Batch size: Max 100 entries per request +- Error handling: Retry with exponential backoff +- Gap detection: Detect missed entries and reconcile + +## Complete Flow + +### Leader Side +``` +1. TransactionManager commits change + creates sync log entry +2. Leader receives commit event +3. Leader groups entries (if multiple commits in <100ms) +4. Leader sends SyncMessage::NewEntries to all followers: + - library_id + - from_sequence (start of batch) + - to_sequence (end of batch) + - entry_count +5. Follower requests entries with FetchEntries +6. Leader responds with EntriesResponse (up to 100 entries) +7. Follower sends Acknowledge +``` + +### Follower Side +``` +1. Listen for SyncMessage::NewEntries from leader +2. On notification: + - Send FetchEntries request (since last_synced_sequence) + - Receive EntriesResponse + - For each entry: + - Deserialize model + - Apply change to local DB + - Update last_synced_sequence + - Send Acknowledge +3. On connection loss: + - Reconnect + - Send Heartbeat with current sequence + - Leader sends SyncRequired if needed +``` + +## Service Structure + +```rust +pub struct SyncService { + library_id: Uuid, + role: SyncRole, + sync_log_db: Arc, + protocol_handler: Arc, + event_bus: Arc, + + // Leader-specific + pending_batches: Arc>>, + followers: Arc>>, + + // Follower-specific + last_synced_sequence: Arc>, +} + +impl SyncService { + /// Create and start sync service for library + pub async fn start( + library_id: Uuid, + role: SyncRole, + sync_log_db: Arc, + protocol_handler: Arc, + ) -> Result; + + /// Leader: Notify followers of new entries + async fn notify_followers(&self, from_seq: u64, to_seq: u64); + + /// Follower: Apply sync entry locally + async fn apply_sync_entry(&self, entry: SyncLogEntry); + + /// Handle role transition (election change) + pub async fn transition_role(&mut self, new_role: SyncRole); +} + +## Acceptance Criteria + +### Leader +- [ ] Leader service receives commit events +- [ ] Notifications sent to all followers instantly +- [ ] Batch notifications for rapid commits (100ms window) +- [ ] Handles FetchEntries requests +- [ ] Responds with EntriesResponse +- [ ] Tracks connected followers +- [ ] Handles disconnections gracefully + +### Follower +- [ ] Follower receives NewEntries push notifications +- [ ] Entries fetched and applied correctly +- [ ] Sequence tracking prevents duplicate application +- [ ] Bulk operations trigger local jobs (not replication) +- [ ] Connection loss handled with reconnect +- [ ] Gap detection triggers full reconciliation + +### Integration +- [ ] Service starts correctly based on device role +- [ ] Role transitions handled (leader election) +- [ ] Integration tests validate device-to-device sync +- [ ] Multi-follower scenario tested + +## Performance Benefits vs Polling + +- **Latency**: Instant (push) vs 5s average (polling) +- **Bandwidth**: Only when changes occur vs constant polls +- **Battery**: Idle until notification vs wake every 5s + +## References + +- `docs/core/sync.md` - Complete sync specification +- Protocol: LSYNC-013 (Sync protocol handler) +- Leader election: LSYNC-009 +- TransactionManager: LSYNC-006 diff --git a/.tasks/LSYNC-011-conflict-resolution.md b/.tasks/LSYNC-011-conflict-resolution.md new file mode 100644 index 000000000..772213590 --- /dev/null +++ b/.tasks/LSYNC-011-conflict-resolution.md @@ -0,0 +1,68 @@ +--- +id: LSYNC-011 +title: Sync Conflict Resolution (Optimistic Concurrency) +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: Medium +tags: [sync, conflict-resolution, versioning] +depends_on: [LSYNC-010] +--- + +## Description + +Implement conflict resolution for sync entries using optimistic concurrency control. All Syncable models have a version field; when applying updates, the system compares versions to determine which change wins. + +## Implementation Steps + +1. Implement `apply_model_change()` with version checking +2. Add Last-Write-Wins (LWW) strategy +3. Handle Insert/Update/Delete operations +4. Skip updates when local version is newer +5. Log conflicts for debugging/monitoring +6. Add optional conflict resolution UI hooks (future) + +## Conflict Strategy + +- **Last-Write-Wins (LWW)**: Use version field to determine winner +- **Insert**: Always apply (no conflict possible) +- **Update**: Compare versions, skip if local >= remote +- **Delete**: Always apply (tombstone) +- **User Metadata** (tags, albums): Union merge (future) + +## Technical Details + +- Location: `core/src/service/sync/conflict.rs` +- Version field: Monotonically increasing integer +- Timestamp-based versioning for some models +- No CRDTs in Phase 1 (simpler, sufficient for metadata) + +## Example Logic + +```rust +if remote_model.version > local_model.version { + // Remote is newer - apply update + remote_model.update(db).await?; +} else { + // Local is newer or same - skip + tracing::debug!("Skipping sync entry: local version is newer"); +} +``` + +## Acceptance Criteria + +- [ ] Version comparison logic implemented +- [ ] Conflicts resolved automatically +- [ ] Conflicts logged for monitoring +- [ ] Unit tests cover all conflict scenarios +- [ ] Integration tests validate cross-device conflicts + +## Future Enhancements + +- CRDT-based merge for rich text fields +- User-facing conflict resolution UI +- Conflict metrics and alerting + +## References + +- `docs/core/sync.md` lines 403-443 diff --git a/.tasks/LSYNC-012-entry-sync-bulk-optimization.md b/.tasks/LSYNC-012-entry-sync-bulk-optimization.md new file mode 100644 index 000000000..4fb4649dc --- /dev/null +++ b/.tasks/LSYNC-012-entry-sync-bulk-optimization.md @@ -0,0 +1,73 @@ +--- +id: LSYNC-012 +title: Entry Sync with Bulk Optimization +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: High +tags: [sync, indexing, bulk, performance] +depends_on: [LSYNC-006, LSYNC-010] +--- + +## Description + +Implement entry (file/folder) synchronization with bulk optimization. When a device indexes 1M files, it creates ONE metadata sync log instead of 1M individual entries. Other devices trigger their own local indexing when they see this notification. + +## The Problem + +Naive approach: Index 1M files → Create 1M sync log entries → 500MB sync log → 10 minutes to replicate + +**This doesn't scale.** + +## The Solution + +Bulk operations create metadata-only sync logs: +```json +{ + "sequence": 1234, + "model_type": "bulk_operation", + "operation": "InitialIndex", + "location_id": "uuid-...", + "affected_count": 1000000, + "hints": { "location_path": "/Users/alice/Photos" } +} +``` + +Other devices see this, check if they have a matching location, and trigger their own indexing job. + +## Implementation Steps + +1. Create `BulkOperation` enum (InitialIndex, WatcherBatch) +2. Update `commit_bulk()` in TransactionManager +3. Create bulk operation sync log entries +4. Implement `handle_bulk_operation()` in sync follower +5. Match location by path/fingerprint on remote device +6. Queue local IndexerJob when match found +7. Handle watcher batches (10-1K items) with per-item logs + +## Performance Impact + +- **Before**: 1M entries, 500MB, 10 minutes, 3M operations +- **After**: 1 entry, 500 bytes, <1 second, 1M operations +- **Result**: 10x faster, 1 million times smaller! + +## Technical Details + +- Initial indexing: Always bulk (1K+ items) +- Watcher events: Batch if 10-1K, per-item if <10 +- User operations: Always per-item (instant sync) +- Location matching: By path or volume fingerprint + +## Acceptance Criteria + +- [ ] Bulk operations create metadata-only sync logs +- [ ] Follower triggers local indexing on bulk notification +- [ ] 1M file indexing creates <10KB sync log +- [ ] Watcher batches use appropriate strategy +- [ ] Location matching across devices works +- [ ] Performance tests validate 10x improvement + +## References + +- `docs/core/sync.md` lines 172-196 (bulk operations) +- `docs/core/sync.md` lines 495-502 (performance metrics) diff --git a/.tasks/LSYNC-013-sync-protocol-handler.md b/.tasks/LSYNC-013-sync-protocol-handler.md new file mode 100644 index 000000000..67f7e0112 --- /dev/null +++ b/.tasks/LSYNC-013-sync-protocol-handler.md @@ -0,0 +1,154 @@ +--- +id: LSYNC-013 +title: Sync Protocol Handler (Message-based) +status: To Do +assignee: unassigned +parent: LSYNC-000 +priority: High +tags: [sync, networking, protocol, push] +depends_on: [LSYNC-008, LSYNC-009] +--- + +## Description + +Create a dedicated sync protocol handler for the networking layer that enables push-based sync via `SyncMessage` enum. This replaces polling with efficient message-passing between leader and follower devices. + +## Architecture Decision + +**Before**: Follower polls leader every 5 seconds (`sync_iteration()`) +- High latency (up to 5s) +- Wasted bandwidth (empty polls) +- Battery drain on mobile + +**After**: Push-based messaging via dedicated protocol +- Instant updates (pushed when changes happen) +- No empty polls +- Bi-directional: Leader pushes, follower can request + +## SyncMessage Enum + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum SyncMessage { + // Leader → Follower: New entries available + NewEntries { + library_id: Uuid, + from_sequence: u64, + to_sequence: u64, + entry_count: usize, + }, + + // Follower → Leader: Request entries + FetchEntries { + library_id: Uuid, + since_sequence: u64, + limit: usize, + }, + + // Leader → Follower: Response with entries + EntriesResponse { + library_id: Uuid, + entries: Vec, + has_more: bool, + }, + + // Follower → Leader: Acknowledge received + Acknowledge { + library_id: Uuid, + up_to_sequence: u64, + }, + + // Bi-directional: Heartbeat + Heartbeat { + library_id: Uuid, + current_sequence: u64, + role: SyncRole, + }, + + // Leader → Follower: You're behind, full sync needed + SyncRequired { + library_id: Uuid, + reason: String, + }, +} +``` + +## Implementation Steps + +1. Create `core/src/service/network/protocol/sync/` directory +2. Create `sync/mod.rs` - Main protocol handler +3. Create `sync/messages.rs` - SyncMessage enum +4. Create `sync/leader.rs` - Leader-side message handling +5. Create `sync/follower.rs` - Follower-side message handling +6. Register protocol with ALPN: `/spacedrive/sync/1.0.0` +7. Integrate with DeviceRegistry for connection lookup +8. Add connection lifecycle management + +## Protocol Handler Structure + +```rust +// core/src/service/network/protocol/sync/mod.rs +pub struct SyncProtocolHandler { + library_id: Uuid, + sync_log_db: Arc, + device_registry: Arc>, + event_bus: Arc, + role: SyncRole, +} + +impl ProtocolHandler for SyncProtocolHandler { + const ALPN: &'static [u8] = b"/spacedrive/sync/1.0.0"; + + async fn handle_connection( + &self, + stream: BiStream, + peer_device_id: Uuid, + ) -> Result<(), NetworkingError>; +} + +impl SyncProtocolHandler { + // Leader: Push notification when new entries created + pub async fn notify_new_entries( + &self, + from_seq: u64, + to_seq: u64, + ) -> Result<(), SyncError>; + + // Follower: Request entries from leader + pub async fn request_entries( + &self, + since_seq: u64, + ) -> Result, SyncError>; + + // Handle incoming message + async fn handle_message( + &self, + msg: SyncMessage, + stream: &mut BiStream, + ) -> Result<(), SyncError>; +} +``` + +## Connection Management + +- Protocol uses Iroh BiStreams for bi-directional communication +- Leader maintains open connections to all follower devices +- Follower connects to leader on library open +- Heartbeat every 30 seconds to detect disconnections +- Auto-reconnect on connection loss + +## Acceptance Criteria + +- [ ] SyncProtocolHandler implemented +- [ ] SyncMessage enum defined +- [ ] Leader can push NewEntries notifications +- [ ] Follower can request entries +- [ ] BiStream communication working +- [ ] Protocol registered with correct ALPN +- [ ] Connection lifecycle managed +- [ ] Integration tests validate message flow + +## References + +- Existing protocols: `core/src/service/network/protocol/pairing/` +- Protocol registry: `core/src/service/network/protocol/registry.rs` diff --git a/docs/core/design/RELAY_FLOW_DIAGRAM.md b/docs/core/design/RELAY_FLOW_DIAGRAM.md index b42987675..bb2eb56ca 100644 --- a/docs/core/design/RELAY_FLOW_DIAGRAM.md +++ b/docs/core/design/RELAY_FLOW_DIAGRAM.md @@ -329,3 +329,4 @@ Week 4: Observability + metrics + documentation Week 5-6: Beta testing with various network configs Week 7: Production rollout ``` + diff --git a/docs/core/design/RELAY_INTEGRATION_SUMMARY.md b/docs/core/design/RELAY_INTEGRATION_SUMMARY.md index 65287aa65..96e6374b5 100644 --- a/docs/core/design/RELAY_INTEGRATION_SUMMARY.md +++ b/docs/core/design/RELAY_INTEGRATION_SUMMARY.md @@ -188,3 +188,4 @@ These are production-grade, handling 200k+ concurrent connections. --- **See detailed plan**: [IROH_RELAY_INTEGRATION.md](./IROH_RELAY_INTEGRATION.md) + diff --git a/docs/core/design/frontend_graphql_usage.tsx b/docs/core/design/frontend_graphql_usage.tsx deleted file mode 100644 index 62d7ba190..000000000 --- a/docs/core/design/frontend_graphql_usage.tsx +++ /dev/null @@ -1,132 +0,0 @@ -// Example: How the frontend would use the GraphQL API with full type safety - -import { useQuery, useMutation, gql } from '@apollo/client'; -import type { - Library, - CreateLibraryInput, - UpdateLibrarySettingsInput -} from './generated/graphql'; - -// GraphQL queries and mutations -const GET_LIBRARIES = gql` - query GetLibraries { - libraries { - id - name - path - description - totalFiles - totalSize - createdAt - updatedAt - } - } -`; - -const CREATE_LIBRARY = gql` - mutation CreateLibrary($input: CreateLibraryInput!) { - createLibrary(input: $input) { - id - name - path - description - } - } -`; - -const UPDATE_LIBRARY_SETTINGS = gql` - mutation UpdateLibrarySettings($input: UpdateLibrarySettingsInput!) { - updateLibrarySettings(input: $input) { - id - name - } - } -`; - -// React component with full type safety -export function LibraryManager() { - // Fully typed query - TypeScript knows the shape of data - const { data, loading, error } = useQuery<{ libraries: Library[] }>(GET_LIBRARIES); - - // Fully typed mutation - const [createLibrary] = useMutation< - { createLibrary: Library }, - { input: CreateLibraryInput } - >(CREATE_LIBRARY); - - const [updateSettings] = useMutation< - { updateLibrarySettings: Library }, - { input: UpdateLibrarySettingsInput } - >(UPDATE_LIBRARY_SETTINGS); - - const handleCreateLibrary = async () => { - // TypeScript enforces correct input shape - const result = await createLibrary({ - variables: { - input: { - name: "My Photos", - description: "Personal photo collection", - location: "/Users/me/Pictures" - } - } - }); - - // result.data.createLibrary is fully typed as Library - console.log("Created library:", result.data?.createLibrary.name); - }; - - const handleUpdateSettings = async (libraryId: string) => { - // TypeScript ensures we pass valid settings - await updateSettings({ - variables: { - input: { - id: libraryId, - generateThumbnails: true, - thumbnailQuality: 90, - enableAiTagging: false - } - } - }); - }; - - if (loading) return
Loading...
; - if (error) return
Error: {error.message}
; - - return ( -
-

Libraries

- {data?.libraries.map(library => ( -
-

{library.name}

-

Files: {library.totalFiles}

-

Size: {library.totalSize}

- {/* TypeScript knows all available fields */} - -
- ))} - -
- ); -} - -// Example: Using with React hooks for even better DX -import { useGetLibrariesQuery, useCreateLibraryMutation } from './generated/graphql'; - -export function LibraryManagerWithHooks() { - // Even simpler with generated hooks! - const { data, loading } = useGetLibrariesQuery(); - const [createLibrary] = useCreateLibraryMutation(); - - // Full intellisense and type checking - const libraries = data?.libraries ?? []; - - return ( -
- {libraries.map(lib => ( -
{lib.name}
- ))} -
- ); -} \ No newline at end of file diff --git a/docs/core/design/sync/NORMALIZED_CACHE_DESIGN.md b/docs/core/design/sync/NORMALIZED_CACHE_DESIGN.md new file mode 100644 index 000000000..5013ac1af --- /dev/null +++ b/docs/core/design/sync/NORMALIZED_CACHE_DESIGN.md @@ -0,0 +1,2673 @@ +# Normalized Resource Cache Design + +**Status**: RFC / Design Document +**Author**: AI Assistant with James Pine +**Date**: 2025-01-07 +**Version**: 1.0 +**Related**: INFRA_LAYER_SEPARATION.md + +## Executive Summary + +This document proposes a **normalized client-side cache** with **event-driven atomic updates** for Spacedrive. Instead of invalidating entire query results when a single resource changes, we: + +1. **Normalize resources** by identity (UUID) in a client-side entity store +2. **Map queries to resources** they contain (query → [resource IDs]) +3. **Listen to events** and perform atomic updates to cached resources +4. **Automatically update UI** when resources change + +This enables: +- ✅ **Efficient search** - 1000 files returned, 1 file updated → update 1 entity, not re-fetch 1000 +- ✅ **Real-time UI** - File renamed? Update visible immediately across all views +- ✅ **Bandwidth savings** - Only send deltas, not full result sets +- ✅ **Optimistic updates** - Update cache immediately, sync in background + +## Core Concept: Resource Normalization + +### The Problem + +**Current approach** (query-based caching): + +```swift +// Query returns full result +let searchResults = try await client.query("search:files.v1", input: searchInput) +// Cache: { "search:xyz": [File1, File2, File3, ...] } + +// File2 gets renamed via event +event: .EntryModified { entry_id: file2_uuid } + +// Problem: Have to invalidate entire search cache and re-fetch! +cache.invalidate("search:xyz") +let newResults = try await client.query("search:files.v1", input: searchInput) // 😢 +``` + +**Normalized approach** (resource-based caching): + +```swift +// Query returns full result +let searchResults = try await client.query("search:files.v1", input: searchInput) + +// Cache structure: +// entities: { +// "File:uuid1": File1, +// "File:uuid2": File2, +// "File:uuid3": File3 +// } +// queries: { +// "search:xyz": ["File:uuid1", "File:uuid2", "File:uuid3"] +// } + +// File2 gets renamed via event +event: .EntryModified { entry_id: file2_uuid, updated_data: {...} } + +// Atomic update: Update single entity, UI updates automatically! +cache.update(entity: "File:file2_uuid", delta: {...}) // ✅ +``` + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Client Application (Swift UI, React) │ +│ ────────────────────────────────────────────────────────────│ +│ CLIENT-SIDE ONLY │ +│ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ UI Components │ │ +│ │ • Observe normalized cache via SwiftUI/React hooks │ │ +│ │ • Automatically re-render on cache updates │ │ +│ └───────────────────────┬──────────────────────────────┘ │ +│ │ │ +│ ┌───────────────────────▼──────────────────────────────┐ │ +│ │ Normalized Resource Cache (CLIENT ONLY) │ │ +│ │ │ │ +│ │ Entity Store: │ │ +│ │ ┌───────────────────────────────────────────────┐ │ │ +│ │ │ "File:uuid1" → File { id, name, tags, ... } │ │ │ +│ │ │ "File:uuid2" → File { id, name, tags, ... } │ │ │ +│ │ │ "Tag:tag1" → Tag { id, name, color, ... } │ │ │ +│ │ │ "Location:loc1" → Location { id, ... } │ │ │ +│ │ └───────────────────────────────────────────────┘ │ │ +│ │ │ │ +│ │ Query Index: │ │ +│ │ ┌───────────────────────────────────────────────┐ │ │ +│ │ │ "search:abc" → ["File:uuid1", "File:uuid2"] │ │ │ +│ │ │ "directory:/photos" → ["File:uuid3", ...] │ │ │ +│ │ │ "tags:list" → ["Tag:tag1", "Tag:tag2"] │ │ │ +│ │ └───────────────────────────────────────────────┘ │ │ +│ └──────────────────────┬────────┬──────────────────────┘ │ +│ │ │ │ +│ ┌──────────────────────▼────┐ │ │ +│ │ Query Client │ │ │ +│ │ • Execute queries │ │ │ +│ │ • Normalize responses │ │ │ +│ └──────────────────────┬────┘ │ │ +│ │ │ │ +│ ┌──────────────────────▼────────▼──────────────────────┐ │ +│ │ Event Stream Handler │ │ +│ │ • Subscribe to core events │ │ +│ │ • Map events → cache updates │ │ +│ │ • Apply atomic updates to entity store │ │ +│ └──────────────────────┬──────────────────────────────┘ │ +│ │ │ +└─────────────────────────┼───────────────────────────────────┘ + │ Unix Socket / JSON-RPC + │ (Events stream down) + │ +┌─────────────────────────▼───────────────────────────────────┐ +│ Spacedrive Core (Rust) - NO CACHE LAYER │ +│ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ Database (Source of Truth) │ │ +│ │ • SeaORM entities │ │ +│ │ • Single source of truth │ │ +│ │ • Already optimized with indexes │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ Event Bus (Broadcast Only) │ │ +│ │ • FileUpdated { file: File {...} } │ │ +│ │ • TagApplied { entry_ids, tag_id } │ │ +│ │ • LocationUpdated { location: Location {...} } │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ QueryManager (Stateless) │ │ +│ │ • Returns data from database │ │ +│ │ • Includes cache metadata for client │ │ +│ │ • No caching layer - clients handle that │ │ +│ └──────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────┘ +``` + +## Layer 1: Rust Core Infrastructure + +### 1.1 Identifiable Trait for Domain Models + +```rust +// core/src/domain/identifiable.rs + +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::hash::Hash; +use uuid::Uuid; + +/// Marker trait for domain models that can be cached by identity +pub trait Identifiable: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static { + /// The type of ID used (usually Uuid, sometimes i32) + type Id: Clone + Hash + Eq + Serialize + for<'de> Deserialize<'de> + std::fmt::Display; + + /// Get the primary key for this resource + fn resource_id(&self) -> Self::Id; + + /// Get the resource type name for cache keys + /// Returns something like "File", "Tag", "Location" + fn resource_type() -> &'static str + where + Self: Sized; + + /// Get the full cache key: "ResourceType:id" + fn cache_key(&self) -> String { + format!("{}:{}", Self::resource_type(), self.resource_id()) + } + + /// Get cache key from just the ID + fn cache_key_from_id(id: &Self::Id) -> String + where + Self: Sized, + { + format!("{}:{}", Self::resource_type(), id) + } + + /// Extract relationships to other resources + /// Returns map of: relationship_name → [resource_cache_keys] + fn extract_relationships(&self) -> ResourceRelationships { + ResourceRelationships::default() + } +} + +/// Relationships this resource has to other cached resources +#[derive(Debug, Clone, Default, Serialize, Deserialize, Type)] +pub struct ResourceRelationships { + /// One-to-one relationships (e.g., File → Location) + pub singular: HashMap, + + /// One-to-many relationships (e.g., File → [Tags]) + pub plural: HashMap>, +} + +impl ResourceRelationships { + pub fn new() -> Self { + Self::default() + } + + pub fn add_singular(&mut self, name: impl Into, cache_key: impl Into) { + self.singular.insert(name.into(), cache_key.into()); + } + + pub fn add_plural(&mut self, name: impl Into, cache_keys: Vec) { + self.plural.insert(name.into(), cache_keys); + } +} +``` + +### 1.2 Implement Identifiable for Domain Models + +```rust +// core/src/domain/file.rs + +use super::identifiable::{Identifiable, ResourceRelationships}; + +impl Identifiable for File { + type Id = Uuid; + + fn resource_id(&self) -> Self::Id { + self.id + } + + fn resource_type() -> &'static str { + "File" + } + + fn extract_relationships(&self) -> ResourceRelationships { + let mut rels = ResourceRelationships::new(); + + // Tags relationship + if !self.tags.is_empty() { + let tag_keys: Vec = self + .tags + .iter() + .map(|t| Tag::cache_key_from_id(&t.id)) + .collect(); + rels.add_plural("tags", tag_keys); + } + + // Content identity relationship + if let Some(content) = &self.content_identity { + rels.add_singular("content_identity", ContentIdentity::cache_key_from_id(&content.uuid)); + } + + // Location relationship (from sd_path) + // Note: This requires parsing the location from sd_path context + // For now, we'll handle this in query-specific logic + + rels + } +} + +impl Identifiable for Tag { + type Id = Uuid; + + fn resource_id(&self) -> Self::Id { + self.id + } + + fn resource_type() -> &'static str { + "Tag" + } +} + +impl Identifiable for Location { + type Id = Uuid; + + fn resource_id(&self) -> Self::Id { + self.id + } + + fn resource_type() -> &'static str { + "Location" + } +} + +// Note: Entry is a low-level database entity. For client-side caching, +// we use higher-level File domain objects instead. Entry → File conversion +// happens on the Rust side before sending to clients. + +impl Identifiable for crate::infra::job::types::JobInfo { + type Id = Uuid; + + fn resource_id(&self) -> Self::Id { + self.id + } + + fn resource_type() -> &'static str { + "Job" + } + + fn extract_relationships(&self) -> ResourceRelationships { + let mut rels = ResourceRelationships::new(); + + // Parent job relationship + if let Some(parent_id) = self.parent_job_id { + rels.add_singular("parent_job", Self::cache_key_from_id(&parent_id)); + } + + rels + } +} + +impl Identifiable for crate::library::Library { + type Id = Uuid; + + fn resource_id(&self) -> Self::Id { + self.id() + } + + fn resource_type() -> &'static str { + "Library" + } +} + +// Similarly for Volume, Device, etc. +``` + +### 1.3 Cache Metadata in Query Results + +```rust +// core/src/infra/query/cache_metadata.rs + +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::collections::HashMap; + +/// Metadata about what resources are included in a query result +/// This enables the client to normalize and cache properly +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct CacheMetadata { + /// Map of resource type → list of IDs in this response + /// e.g., { "File": ["uuid1", "uuid2"], "Tag": ["tag1", "tag2"] } + pub resources: HashMap>, + + /// Whether this query result should be cached + pub cacheable: bool, + + /// Cache duration (in seconds, None = indefinite) + pub cache_duration: Option, + + /// Cache invalidation strategy + pub invalidation: InvalidationStrategy, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum InvalidationStrategy { + /// Invalidate when any listed resource types change + OnResourceChange { resource_types: Vec }, + + /// Invalidate when specific events occur + OnEvents { event_types: Vec }, + + /// Manual invalidation only + Manual, + + /// Never invalidate (static data) + Never, +} + +impl CacheMetadata { + pub fn new() -> Self { + Self { + resources: HashMap::new(), + cacheable: true, + cache_duration: None, + invalidation: InvalidationStrategy::OnResourceChange { + resource_types: Vec::new(), + }, + } + } + + /// Add a batch of identifiable resources + pub fn add_resources(&mut self, resources: &[T]) { + let resource_type = T::resource_type(); + let ids: Vec = resources + .iter() + .map(|r| r.resource_id().to_string()) + .collect(); + + self.resources + .entry(resource_type.to_string()) + .or_insert_with(Vec::new) + .extend(ids); + } + + /// Add a single resource + pub fn add_resource(&mut self, resource: &T) { + let resource_type = T::resource_type(); + let id = resource.resource_id().to_string(); + + self.resources + .entry(resource_type.to_string()) + .or_insert_with(Vec::new) + .push(id); + } +} + +/// Trait for queries to declare their cache behavior +pub trait CacheableQuery: LibraryQuery + Sized { + /// Generate cache metadata for this query's result + /// + /// This is an instance method (not static) to allow the query to inspect + /// its input parameters and customize metadata generation accordingly. + fn generate_cache_metadata(&self, result: &Self::Output) -> CacheMetadata { + let mut metadata = CacheMetadata::new(); + + // Default implementation: try to extract identifiable resources + // Queries should override this to handle complex result structures + metadata.cacheable = Self::is_cacheable(); + metadata.cache_duration = Self::cache_duration(); + metadata.invalidation = Self::invalidation_strategy(); + + metadata + } + + /// Whether this query type should be cached + fn is_cacheable() -> bool { + true + } + + /// Cache duration in seconds + fn cache_duration() -> Option { + None // Indefinite by default + } + + /// Invalidation strategy + fn invalidation_strategy() -> InvalidationStrategy { + InvalidationStrategy::OnResourceChange { + resource_types: Vec::new(), + } + } +} +``` + +### 1.4 Enhanced Query Response Wrapper + +```rust +// core/src/infra/query/response.rs + +use super::cache_metadata::CacheMetadata; +use serde::{Deserialize, Serialize}; +use specta::Type; + +/// Wrapper for query responses that includes cache metadata +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct QueryResponse { + /// The actual query result data + pub data: T, + + /// Cache metadata for normalization + pub cache: CacheMetadata, + + /// Query execution metadata + pub meta: QueryMeta, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct QueryMeta { + /// Query execution time in milliseconds + pub execution_time_ms: u64, + + /// Query ID for debugging + pub query_id: String, + + /// Timestamp of when this query was executed + pub executed_at: chrono::DateTime, +} + +impl QueryResponse { + pub fn new(data: T, cache: CacheMetadata, execution_time_ms: u64) -> Self { + Self { + data, + cache, + meta: QueryMeta { + execution_time_ms, + query_id: uuid::Uuid::new_v4().to_string(), + executed_at: chrono::Utc::now(), + }, + } + } +} +``` + +### 1.5 Update QueryManager to Generate Cache Metadata + +```rust +// core/src/infra/query/manager.rs (additions) + +impl QueryManager { + pub async fn dispatch_library_with_cache( + &self, + query: Q, + library_id: Uuid, + session: SessionContext, + ) -> QueryResult> + where + Q::Output: Serialize, + { + let start = std::time::Instant::now(); + + // Execute query normally + let result = self.dispatch_library(query, library_id, session).await?; + + // Generate cache metadata based on query configuration + let cache_metadata = Q::cache_metadata(&result); // Assumes result is &[Identifiable] + + let execution_time_ms = start.elapsed().as_millis() as u64; + + Ok(QueryResponse::new(result, cache_metadata, execution_time_ms)) + } +} +``` + +### 1.6 Enhanced Events with Resource Deltas + +```rust +// core/src/infra/event/mod.rs (additions) + +/// Enhanced entry event with full resource data for cache updates +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum Event { + // ... existing events ... + + /// Entry was modified - includes delta for cache update + EntryUpdated { + library_id: Uuid, + entry: Entry, // Full Entry data for cache update + }, + + /// File was updated - includes full File for cache + FileUpdated { + library_id: Uuid, + file: File, // Full File domain object + }, + + /// Tag was modified + TagUpdated { + library_id: Uuid, + tag: Tag, + }, + + /// Tag was applied to entries + TagApplied { + library_id: Uuid, + tag_id: Uuid, + entry_ids: Vec, + }, + + /// Tag was removed from entries + TagRemoved { + library_id: Uuid, + tag_id: Uuid, + entry_ids: Vec, + }, + + /// Location was updated + LocationUpdated { + library_id: Uuid, + location: Location, + }, + + /// Job was updated + JobUpdated { + library_id: Uuid, + job_id: Uuid, + status: JobStatus, + progress: Option, + }, + + // ... other events ... +} + +/// Trait for events that contain resource updates +pub trait ResourceEvent { + /// Extract the resource type and ID from this event + fn resource_identity(&self) -> Option<(String, String)>; + + /// Extract the full resource data if available + fn resource_data(&self) -> Option; + + /// Get the resource type this event affects + fn resource_types(&self) -> Vec; +} + +impl ResourceEvent for Event { + fn resource_identity(&self) -> Option<(String, String)> { + match self { + Event::FileUpdated { file, .. } => { + Some(("File".to_string(), file.id.to_string())) + } + Event::TagUpdated { tag, .. } => { + Some(("Tag".to_string(), tag.id.to_string())) + } + Event::LocationUpdated { location, .. } => { + Some(("Location".to_string(), location.id.to_string())) + } + Event::EntryUpdated { entry, .. } => { + Some(("Entry".to_string(), entry.id.to_string())) + } + _ => None, + } + } + + fn resource_data(&self) -> Option { + match self { + Event::FileUpdated { file, .. } => serde_json::to_value(file).ok(), + Event::TagUpdated { tag, .. } => serde_json::to_value(tag).ok(), + Event::LocationUpdated { location, .. } => serde_json::to_value(location).ok(), + Event::EntryUpdated { entry, .. } => serde_json::to_value(entry).ok(), + _ => None, + } + } + + fn resource_types(&self) -> Vec { + match self { + Event::FileUpdated { .. } => vec!["File".to_string()], + Event::TagApplied { .. } | Event::TagRemoved { .. } => { + vec!["File".to_string(), "Tag".to_string()] + } + Event::LocationUpdated { .. } => vec!["Location".to_string()], + _ => Vec::new(), + } + } +} +``` + +## Layer 2: Client-Side Implementation (Generic) + +### 2.1 Normalized Cache Store (Swift) + +```swift +// packages/swift-client/Sources/SpacedriveCache/NormalizedCache.swift + +import Foundation +import Combine + +/// Normalized entity cache with automatic UI updates +@MainActor +public class NormalizedCache: ObservableObject { + // MARK: - Entity Store + + /// Normalized entity storage: "ResourceType:id" → JSON data + private var entities: [String: Any] = [:] + + /// Query result index: queryKey → [resource cache keys] + private var queryIndex: [String: [String]] = [:] + + /// Reverse index: resource cache key → [query keys that contain it] + private var resourceQueries: [String: Set] = [:] + + /// Published to trigger UI updates + @Published private var updateTrigger: Int = 0 + + // MARK: - Public API + + /// Store query result with normalization + public func storeQueryResult( + queryKey: String, + data: [T], + metadata: CacheMetadata + ) { + // 1. Store entities in normalized form + for item in data { + let cacheKey = item.cacheKey() + entities[cacheKey] = item + + // Update reverse index + resourceQueries[cacheKey, default: []].insert(queryKey) + } + + // 2. Store query index + let resourceKeys = data.map { $0.cacheKey() } + queryIndex[queryKey] = resourceKeys + + triggerUpdate() + } + + /// Get query result from cache (reconstructed from entities) + public func getQueryResult(queryKey: String) -> [T]? { + guard let resourceKeys = queryIndex[queryKey] else { + return nil + } + + // Reconstruct result from entities + return resourceKeys.compactMap { key in + entities[key] as? T + } + } + + /// Update a single entity atomically + public func updateEntity(_ entity: T) { + let cacheKey = entity.cacheKey() + entities[cacheKey] = entity + + // Trigger updates for all queries containing this entity + if let affectedQueries = resourceQueries[cacheKey] { + print("📦 Updated \(cacheKey) → \(affectedQueries.count) queries affected") + } + + triggerUpdate() + } + + /// Update entity by ID with partial data (merge) + public func patchEntity( + resourceType: String, + id: String, + patch: [String: Any] + ) { + let cacheKey = "\(resourceType):\(id)" + + guard var entity = entities[cacheKey] as? [String: Any] else { + print("⚠️ Entity \(cacheKey) not in cache, skipping patch") + return + } + + // Merge patch into entity + for (key, value) in patch { + entity[key] = value + } + + entities[cacheKey] = entity + triggerUpdate() + } + + /// Remove entity from cache + public func removeEntity(resourceType: String, id: String) { + let cacheKey = "\(resourceType):\(id)" + entities.removeValue(forKey: cacheKey) + + // Update affected queries + if let affectedQueries = resourceQueries[cacheKey] { + for queryKey in affectedQueries { + // Remove from query index + queryIndex[queryKey]?.removeAll { $0 == cacheKey } + } + resourceQueries.removeValue(forKey: cacheKey) + } + + triggerUpdate() + } + + /// Invalidate entire query (remove from index, keep entities) + public func invalidateQuery(queryKey: String) { + // Remove query index, but keep entities (might be used by other queries) + if let resourceKeys = queryIndex[queryKey] { + for resourceKey in resourceKeys { + resourceQueries[resourceKey]?.remove(queryKey) + } + } + queryIndex.removeValue(forKey: queryKey) + } + + // MARK: - Observation Helpers + + /// Get observable query result for SwiftUI + public func observeQuery(queryKey: String) -> some Publisher { + $updateTrigger + .compactMap { [weak self] _ in + self?.getQueryResult(queryKey) as [T]? + } + .eraseToAnyPublisher() + } + + /// Get observable single entity + public func observeEntity(id: T.Id) -> some Publisher { + let cacheKey = T.cacheKeyFromId(id) + + return $updateTrigger + .compactMap { [weak self] _ in + self?.entities[cacheKey] as? T + } + .eraseToAnyPublisher() + } + + private func triggerUpdate() { + updateTrigger += 1 + } +} +``` + +### 2.2 Event-Driven Cache Updater + +```swift +// packages/swift-client/Sources/SpacedriveCache/EventCacheUpdater.swift + +import Foundation + +/// Handles event stream and applies atomic cache updates +public class EventCacheUpdater { + private let cache: NormalizedCache + private let eventStream: AsyncThrowingStream + private var task: Task? + + public init(cache: NormalizedCache, eventStream: AsyncThrowingStream) { + self.cache = cache + self.eventStream = eventStream + } + + /// Start listening to events and updating cache + public func start() { + task = Task { [weak self] in + guard let self = self else { return } + + do { + for try await event in self.eventStream { + await self.handleEvent(event) + } + } catch { + print("❌ Event stream error: \(error)") + } + } + } + + /// Stop listening to events + public func stop() { + task?.cancel() + task = nil + } + + @MainActor + private func handleEvent(_ event: Event) { + switch event { + case .FileUpdated(let libraryId, let file): + cache.updateEntity(file) + + case .TagUpdated(let libraryId, let tag): + cache.updateEntity(tag) + + case .LocationUpdated(let libraryId, let location): + cache.updateEntity(location) + + case .TagApplied(let libraryId, let tagId, let entryIds): + // Update multiple File entities to include this tag + for entryId in entryIds { + // Fetch tag from cache + guard let tag = cache.getEntity(Tag.self, id: tagId) else { continue } + + // Update file's tags array + if let file = cache.getEntity(File.self, id: entryId) { + var updatedFile = file + updatedFile.tags.append(tag) + cache.updateEntity(updatedFile) + } + } + + case .TagRemoved(let libraryId, let tagId, let entryIds): + // Remove tag from multiple files + for entryId in entryIds { + if let file = cache.getEntity(File.self, id: entryId) { + var updatedFile = file + updatedFile.tags.removeAll { $0.id == tagId } + cache.updateEntity(updatedFile) + } + } + + case .EntryModified(let libraryId, let entryId): + // For lightweight events without full data, invalidate specific queries + // that contain this entry + cache.invalidateQueriesContaining(resourceType: "File", id: entryId) + + case .JobUpdated(let libraryId, let jobId, let status, let progress): + // Patch job entity + cache.patchEntity( + resourceType: "Job", + id: jobId.uuidString, + patch: [ + "status": status, + "progress": progress as Any + ] + ) + + default: + break + } + } +} +``` + +### 2.3 Query Client with Cache Integration + +```swift +// packages/swift-client/Sources/SpacedriveCache/CachedQueryClient.swift + +import Foundation + +/// Query client with automatic normalization and caching +public class CachedQueryClient { + private let client: SpacedriveClient + private let cache: NormalizedCache + + public init(client: SpacedriveClient, cache: NormalizedCache = NormalizedCache()) { + self.client = client + self.cache = cache + } + + /// Execute query with automatic caching + public func query( + _ method: String, + input: Input, + cachePolicy: CachePolicy = .cacheFirst + ) async throws -> Output { + let queryKey = generateQueryKey(method: method, input: input) + + switch cachePolicy { + case .cacheFirst: + // Check cache first + if let cached: Output = cache.getQueryResult(queryKey: queryKey) { + print("📦 Cache HIT: \(queryKey)") + return cached + } + fallthrough + + case .networkOnly: + print("🌐 Fetching from network: \(queryKey)") + let response: QueryResponse = try await client.query(method, input: input) + + // Normalize and cache response + if let identifiableArray = response.data as? [any Identifiable] { + await cache.storeQueryResult( + queryKey: queryKey, + data: identifiableArray, + metadata: response.cache + ) + } + + return response.data + + case .cacheOnly: + guard let cached: Output = cache.getQueryResult(queryKey: queryKey) else { + throw CacheError.cacheMiss(queryKey: queryKey) + } + return cached + } + } + + /// Observe query result with automatic updates from cache + public func observeQuery( + _ method: String, + input: some Encodable + ) -> AsyncThrowingStream<[Output], Error> { + let queryKey = generateQueryKey(method: method, input: input) + + return AsyncThrowingStream { continuation in + Task { + // Initial fetch + do { + let result: [Output] = try await self.query(method, input: input) + continuation.yield(result) + } catch { + continuation.finish(throwing: error) + return + } + + // Subscribe to cache updates + let cancellable = cache.observeQuery(queryKey: queryKey) + .sink { (result: [Output]) in + continuation.yield(result) + } + + continuation.onTermination = { _ in + cancellable.cancel() + } + } + } + } + + private func generateQueryKey(method: String, input: some Encodable) -> String { + // Hash input to create stable query key + let inputData = try! JSONEncoder().encode(input) + let inputHash = inputData.hashValue + return "\(method):\(inputHash)" + } +} + +public enum CachePolicy { + case cacheFirst // Check cache, fallback to network + case networkOnly // Always fetch from network, update cache + case cacheOnly // Only use cache, error if miss +} + +public enum CacheError: Error { + case cacheMiss(queryKey: String) +} +``` + +## Layer 3: Query Implementation Examples + +### Example 1: File Search Query with Cache Metadata + +```rust +// core/src/ops/search/query.rs (additions) + +use crate::infra::query::{CacheableQuery, CacheMetadata, QueryResponse}; + +impl CacheableQuery for FileSearchQuery { + fn cache_metadata(files: &[T]) -> CacheMetadata { + let mut metadata = CacheMetadata::new(); + + // Add all file resources + metadata.add_resources(files); + + // Configure invalidation + metadata.invalidation = InvalidationStrategy::OnEvents { + event_types: vec![ + "FileUpdated".to_string(), + "TagApplied".to_string(), + "TagRemoved".to_string(), + ], + }; + + metadata.cacheable = true; + metadata.cache_duration = Some(300); // 5 minutes + + metadata + } + + fn is_cacheable() -> bool { + true + } + + fn invalidation_strategy() -> InvalidationStrategy { + InvalidationStrategy::OnEvents { + event_types: vec!["FileUpdated", "TagApplied", "TagRemoved"] + .into_iter() + .map(String::from) + .collect(), + } + } +} + +// Enhanced query execution +impl FileSearchQuery { + pub async fn execute_with_cache_metadata( + self, + context: Arc, + session: SessionContext, + ) -> QueryResult>> { + let start = std::time::Instant::now(); + + // Execute search normally + let files = self.execute(context, session).await?; + + // Generate cache metadata + let mut cache_metadata = Self::cache_metadata(&files); + + // Add tag entities too (extracted from files) + let mut all_tags = Vec::new(); + for file in &files { + all_tags.extend(file.tags.iter().cloned()); + } + all_tags.dedup_by_key(|t| t.id); + cache_metadata.add_resources(&all_tags); + + let execution_time = start.elapsed().as_millis() as u64; + + Ok(QueryResponse::new(files, cache_metadata, execution_time)) + } +} +``` + +### Example 2: Directory Listing with Cache + +```rust +// core/src/ops/files/query/directory_listing.rs (additions) + +impl CacheableQuery for DirectoryListingQuery { + fn cache_metadata(files: &[T]) -> CacheMetadata { + let mut metadata = CacheMetadata::new(); + metadata.add_resources(files); + + // Directory listings should invalidate when entries are added/removed/moved + metadata.invalidation = InvalidationStrategy::OnEvents { + event_types: vec![ + "EntryCreated".to_string(), + "EntryDeleted".to_string(), + "EntryMoved".to_string(), + "FileUpdated".to_string(), + ], + }; + + // Cache for 60 seconds (directories change less frequently) + metadata.cache_duration = Some(60); + + metadata + } +} +``` + +### Example 3: Tag List Query + +```rust +// core/src/ops/tags/list/query.rs (new) + +pub struct ListTagsQuery; + +impl LibraryQuery for ListTagsQuery { + type Input = (); + type Output = Vec; + + fn from_input(_input: Self::Input) -> QueryResult { + Ok(Self) + } + + async fn execute( + self, + context: Arc, + session: SessionContext, + ) -> QueryResult { + // Fetch all tags from database + // ... + } +} + +impl CacheableQuery for ListTagsQuery { + fn cache_metadata(tags: &[T]) -> CacheMetadata { + let mut metadata = CacheMetadata::new(); + metadata.add_resources(tags); + + // Tags change rarely, cache indefinitely + metadata.invalidation = InvalidationStrategy::OnEvents { + event_types: vec![ + "TagCreated".to_string(), + "TagUpdated".to_string(), + "TagDeleted".to_string(), + ], + }; + + metadata.cache_duration = None; // Indefinite + + metadata + } +} +``` + +## Layer 4: SwiftUI Integration + +### Example: Self-Updating Search View + +```swift +// apps/ios/Spacedrive/Views/Search/SearchView.swift + +import SwiftUI +import SpacedriveCache + +struct SearchView: View { + @StateObject private var cache = NormalizedCache.shared + @State private var searchQuery: String = "" + @State private var files: [File] = [] + + var body: some View { + VStack { + SearchBar(text: $searchQuery, onSubmit: executeSearch) + + List(files, id: \.id) { file in + FileRow(file: file) + // Each file automatically updates when EntryModified event arrives! + .id(file.id) // SwiftUI tracks by ID + } + } + .onAppear { + // Subscribe to cache updates for this query + subscribeToCache() + } + } + + private func executeSearch() { + Task { + do { + // Query with cache + files = try await cache.client.query( + "query:files.search.v1", + input: FileSearchInput(query: searchQuery), + cachePolicy: .cacheFirst + ) + } catch { + print("Search error: \(error)") + } + } + } + + private func subscribeToCache() { + let queryKey = "search:\(searchQuery.hashValue)" + + Task { + // Observe cache updates + for await updatedFiles in cache.observeQuery(queryKey) as AsyncThrowingStream<[File], Error> { + await MainActor.run { + self.files = updatedFiles + } + } + } + } +} +``` + +## Layer 5: Event Emission from Core + +### Update Actions to Emit Resource Events + +```rust +// core/src/ops/files/rename/action.rs (example) + +impl LibraryAction for FileRenameAction { + async fn execute( + self, + library: Arc, + context: Arc, + ) -> ActionResult { + // 1. Perform the rename + let entry_id = self.entry_id; + let new_name = self.new_name.clone(); + + // Database update... + let updated_entry = update_entry_name(library, entry_id, new_name).await?; + + // 2. ✨ Emit event with full resource data for cache update + if let Some(file) = construct_full_file_from_entry(library, &updated_entry).await? { + context.events.emit(Event::FileUpdated { + library_id: library.id(), + file, // Full File object for cache replacement + }); + } + + Ok(RenameOutput { success: true }) + } +} +``` + +### Helper: Construct File from Entry + +```rust +// core/src/domain/file.rs (additions) + +impl File { + /// Construct a complete File from an entry ID by fetching all related data + /// This is used when emitting events that need full resource data + pub async fn from_entry_id( + library: Arc, + entry_id: Uuid, + ) -> QueryResult { + let db = library.db().conn(); + + // Fetch entry + let entry_model = entry::Entity::find() + .filter(entry::Column::Uuid.eq(entry_id)) + .one(db) + .await? + .ok_or(QueryError::Internal(format!("Entry {} not found", entry_id)))?; + + // Fetch content identity + let content_identity = if let Some(content_id) = entry_model.content_id { + ContentIdentity::from_id(db, content_id).await? + } else { + None + }; + + // Fetch tags + let tags = Tag::for_entry(db, entry_model.id).await?; + + // Fetch sidecars + let sidecars = Sidecar::for_entry(db, entry_model.id).await?; + + // Fetch alternate paths (duplicates) + let alternate_paths = Entry::find_by_content_id(db, entry_model.content_id).await?; + + // Construct Entry domain object + let entry = Entry::from_model(entry_model)?; + + Ok(File::from_data(FileConstructionData { + entry, + content_identity, + tags, + sidecars, + alternate_paths, + })) + } +} +``` + +## Event Design Patterns + +### Pattern 1: Full Resource Events (Recommended) + +**Use when**: Resource is small enough to send in full + +```rust +Event::TagUpdated { + library_id: Uuid, + tag: Tag { /* full data */ }, +} +``` + +**Benefit**: Client can atomically replace cached entity without re-fetching + +### Pattern 2: Lightweight Events with Delta + +**Use when**: Resource is large, only specific fields changed + +```rust +Event::FileMetadataUpdated { + library_id: Uuid, + entry_id: Uuid, + delta: FileMetadataDelta { + name: Some("new_name.txt"), + modified_at: Some(timestamp), + // Other fields: None (unchanged) + }, +} +``` + +**Benefit**: Lower bandwidth, client merges delta into cached entity + +### Pattern 3: Relationship Events + +**Use when**: Relationship changed but resources unchanged + +```rust +Event::TagApplied { + library_id: Uuid, + tag_id: Uuid, + entry_ids: Vec, +} +``` + +**Benefit**: Client can update relationships without re-fetching full resources + +## Cache Consistency Guarantees + +### Optimistic Updates + +```swift +// Example: Tag a file optimistically +func tagFile(fileId: UUID, tagId: UUID) async throws { + // 1. Update cache immediately (optimistic) + var file = cache.getEntity(File.self, id: fileId)! + let tag = cache.getEntity(Tag.self, id: tagId)! + file.tags.append(tag) + cache.updateEntity(file) + // UI updates immediately! ✨ + + // 2. Send action to server + do { + try await client.action("action:tags.apply.v1", input: ApplyTagInput( + fileId: fileId, + tagId: tagId + )) + // Server confirms, event arrives, cache updated again (same state) + } catch { + // 3. Rollback on error + file.tags.removeLast() + cache.updateEntity(file) + throw error + } +} +``` + +### Eventual Consistency + +- Client cache is **eventually consistent** with server +- Events provide the synchronization mechanism +- Optimistic updates improve perceived performance +- Conflicts handled by "last write wins" or custom merge logic + +## Query Annotation API + +### Declarative Cache Configuration + +```rust +// core/src/ops/search/query.rs + +impl FileSearchQuery { + /// Declare what resources this query returns + pub fn declares_resources() -> Vec { + vec![ + ResourceTypeDeclaration { + resource_type: "File", + extraction: ResourceExtraction::Direct, // Files are top-level result + includes_relationships: vec!["tags", "content_identity"], + }, + ResourceTypeDeclaration { + resource_type: "Tag", + extraction: ResourceExtraction::Nested { path: "tags" }, // Tags nested in files + includes_relationships: vec![], + }, + ] + } + + /// Declare what events should invalidate this query + pub fn invalidation_events() -> Vec { + vec![ + InvalidationRule::OnResourceChange { + resource_type: "File", + // Only invalidate if changed file is in our result set + condition: InvalidationCondition::InResultSet, + }, + InvalidationRule::OnEvent { + event_type: "TagApplied", + // Re-fetch if tag applied to file in our results + condition: InvalidationCondition::InResultSet, + }, + ] + } +} + +pub struct ResourceTypeDeclaration { + pub resource_type: &'static str, + pub extraction: ResourceExtraction, + pub includes_relationships: Vec<&'static str>, +} + +pub enum ResourceExtraction { + /// Resources are top-level in result (e.g., Vec) + Direct, + + /// Resources are nested (e.g., file.tags) + Nested { path: &'static str }, + + /// Resources are in a map (e.g., HashMap) + InMap { key_path: &'static str }, +} + +pub enum InvalidationCondition { + /// Invalidate only if changed resource is in this query's result + InResultSet, + + /// Always invalidate when this event occurs + Always, + + /// Custom condition (library_id matches, etc.) + Custom(fn(&Event) -> bool), +} +``` + +## Advanced: Relationship Updates + +### Nested Resource Updates + +When a File's Tag changes, we need to update: +1. The Tag entity itself +2. All File entities that reference this Tag + +```rust +// Event with cascade information +Event::TagUpdated { + library_id: Uuid, + tag: Tag { /* updated tag */ }, + affects_entities: EntityAffectMap { + "File": vec![uuid1, uuid2, uuid3], // Files that have this tag + }, +} +``` + +```swift +// Client handles cascading updates +case .TagUpdated(let libraryId, let tag, let affects): + // 1. Update tag entity + cache.updateEntity(tag) + + // 2. Update all files that reference this tag + if let fileIds = affects["File"] { + for fileId in fileIds { + // Re-fetch file to get updated tag data + // OR merge tag into file's tags array + if var file = cache.getEntity(File.self, id: fileId) { + if let tagIndex = file.tags.firstIndex(where: { $0.id == tag.id }) { + file.tags[tagIndex] = tag + cache.updateEntity(file) + } + } + } + } +``` + +## Performance Considerations + +### Memory Management + +**Problem**: Unbounded cache growth + +**Solution**: LRU eviction with size limits + +```swift +class NormalizedCache { + private var lruOrder: [String] = [] // Cache keys in LRU order + private let maxEntities: Int = 10_000 + private let maxMemoryMB: Int = 100 + + private func evictIfNeeded() { + while entities.count > maxEntities { + // Remove oldest entity + guard let oldestKey = lruOrder.first else { break } + entities.removeValue(forKey: oldestKey) + lruOrder.removeFirst() + + // Clean up query indexes + cleanupQueryIndexes(for: oldestKey) + } + } + + private func touchEntity(_ cacheKey: String) { + // Move to end of LRU + lruOrder.removeAll { $0 == cacheKey } + lruOrder.append(cacheKey) + } +} +``` + +### Network Efficiency + +**Batch Event Updates**: Instead of sending individual events, batch related updates: + +```rust +Event::BatchUpdate { + library_id: Uuid, + updates: Vec, + transaction_id: Uuid, +} + +pub struct ResourceUpdate { + pub resource_type: String, + pub resource_id: String, + pub update_type: UpdateType, + pub data: serde_json::Value, +} + +pub enum UpdateType { + Create, + Update, + Delete, + Patch { fields: Vec }, +} +``` + +## Implementation Roadmap + +### Phase 1: Core Infrastructure (Week 1) +- [ ] Create `Identifiable` trait in `core/src/domain/identifiable.rs` +- [ ] Implement `Identifiable` for File, Tag, Location, Entry, Job, Library +- [ ] Create `CacheMetadata` and `QueryResponse` wrapper types +- [ ] Add `CacheableQuery` trait to query infrastructure + +### Phase 2: Event Enhancement (Week 1-2) +- [ ] Add `*Updated` events with full resource data to Event enum +- [ ] Add `ResourceEvent` trait for extracting resource identities +- [ ] Update key actions to emit resource events (rename, tag, move) +- [ ] Add relationship events (TagApplied, TagRemoved) + +### Phase 3: Swift Cache Implementation (Week 2-3) +- [ ] Create `NormalizedCache.swift` with entity store +- [ ] Create `EventCacheUpdater.swift` for event handling +- [ ] Create `CachedQueryClient.swift` wrapper +- [ ] Implement LRU eviction and memory management + +### Phase 4: SwiftUI Integration (Week 3-4) +- [ ] Create `@CachedQuery` property wrapper for views +- [ ] Create `ObservedEntity` for individual resource observation +- [ ] Update existing views to use cached queries +- [ ] Add loading states and error handling + +### Phase 5: TypeScript Implementation (Week 4-5) +- [ ] Port NormalizedCache to TypeScript +- [ ] Create React hooks: `useCachedQuery`, `useEntity` +- [ ] Update web app to use normalized cache + +### Phase 6: Optimization (Ongoing) +- [ ] Add query deduplication (merge concurrent queries) +- [ ] Add prefetching strategies +- [ ] Add cache persistence (SQLite for offline) +- [ ] Add cache statistics and monitoring + +## Benefits + +### For Users +- ⚡ **Instant updates** - UI updates immediately when data changes +- 📶 **Works offline** - Cached data available when disconnected +- 🎯 **Lower battery usage** - Fewer network requests + +### For Developers +- 🎨 **Simple API** - Just use `@CachedQuery`, updates happen automatically +- 🔧 **Type-safe** - Identifiable trait ensures consistency +- 🧪 **Testable** - Mock cache for UI tests + +### For System +- 📉 **Lower bandwidth** - Atomic updates instead of full re-fetches +- 🚀 **Better performance** - Client-side joins eliminate network roundtrips +- 🔄 **Real-time sync** - Event bus provides immediate updates + +## Example: Complete Flow + +```swift +// 1. USER SEARCHES FOR FILES +let files = try await cache.client.query( + "query:files.search.v1", + input: FileSearchInput(query: "photos") +) +// Returns 1000 files, all normalized in cache + +// 2. USER RENAMES ONE FILE (on another device or in another view) +// Action executes → Core emits event +Event::FileUpdated { + library_id: lib_uuid, + file: File { id: file_123, name: "new_name.jpg", ... } +} + +// 3. EVENT ARRIVES AT CLIENT +// EventCacheUpdater handles it: +cache.updateEntity(file) // Atomic update of 1 entity + +// 4. UI AUTOMATICALLY UPDATES +// All views displaying this file re-render with new name +// Search results update +// Directory listings update +// Inspector panel updates +// All without re-fetching! ✨ +``` + +## Comparison to Other Systems + +| Feature | Apollo Client | React Query | Spacedrive Cache | +|---------|---------------|-------------|------------------| +| Normalization | ✅ GraphQL IDs | ❌ Query-based | ✅ UUID-based | +| Event-driven | ❌ Subscriptions | ❌ Manual invalidation | ✅ Event bus | +| Optimistic updates | ✅ Yes | ✅ Yes | ✅ Yes | +| Offline support | ⚠️ Apollo Persist | ⚠️ Manual | ✅ Planned | +| Cross-platform | ❌ JS only | ❌ JS only | ✅ Swift + TS + Rust | +| Type safety | ⚠️ Codegen | ⚠️ Generics | ✅ Derive-based | + +## Critical Implementation Concerns + +### 1. Concurrency Safety in Client Cache + +**Problem**: Multiple threads updating cache simultaneously can cause race conditions + +**Solution**: Thread-safe client-side cache implementation + +**Swift Implementation**: + +```swift +// For SwiftUI apps: Use @MainActor for UI thread safety +@MainActor +public class NormalizedCache: ObservableObject { + // All mutations happen on main thread - simple and safe + private var entities: [String: Any] = [:] + private var queryIndex: [String: [String]] = [:] + + func updateEntity(_ entity: T) { + entities[entity.cacheKey()] = entity + objectWillChange.send() // Trigger SwiftUI updates + } +} + +// For background processing (network, event handling): +// Use actor isolation for concurrent access +actor BackgroundCacheUpdater { + private let mainCache: NormalizedCache + + func processEvent(_ event: Event) async { + // Parse event + // ... + + // Apply to main cache on main thread + await MainActor.run { + mainCache.updateEntity(updatedFile) + } + } +} +``` + +**TypeScript Implementation**: + +```typescript +// For React/web: Use immutable updates with locks +export class NormalizedCache { + private entities: Map = new Map(); + private queryIndex: Map = new Map(); + private updateLock: Promise = Promise.resolve(); + + async updateEntity(entity: T): Promise { + // Serialize updates to prevent race conditions + this.updateLock = this.updateLock.then(async () => { + const key = entity.cacheKey(); + this.entities.set(key, entity); + this.notifySubscribers(key); + }); + + await this.updateLock; + } +} +``` + +**Note**: The Rust core does **not** need a cache - it already has the database as the source of truth. The cache is purely client-side. + +### 2. Event Ordering and Consistency + +**Problem**: Events can arrive out of order, especially during network issues + +**Solution**: Event versioning with reconciliation + +```rust +// Add version numbers to all events +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct EventEnvelope { + /// Sequential event number per library + pub sequence: u64, + + /// Library this event belongs to + pub library_id: Uuid, + + /// Timestamp when event was created + pub timestamp: DateTime, + + /// The actual event + pub event: Event, +} + +// Track sequence numbers +pub struct EventSequenceTracker { + /// Last seen sequence per library + last_sequence: HashMap, +} + +impl EventSequenceTracker { + pub fn check_for_gaps(&mut self, envelope: &EventEnvelope) -> EventGapStatus { + let last_seen = self.last_sequence + .get(&envelope.library_id) + .copied() + .unwrap_or(0); + + if envelope.sequence == last_seen + 1 { + // Expected sequence, no gap + self.last_sequence.insert(envelope.library_id, envelope.sequence); + EventGapStatus::Ok + } else if envelope.sequence > last_seen + 1 { + // Gap detected! Missed events + EventGapStatus::Gap { + expected: last_seen + 1, + received: envelope.sequence, + missing_count: (envelope.sequence - last_seen - 1) as usize, + } + } else { + // Duplicate or old event + EventGapStatus::Duplicate + } + } +} + +pub enum EventGapStatus { + Ok, + Gap { expected: u64, received: u64, missing_count: usize }, + Duplicate, +} +``` + +**Client-side gap handling**: +```swift +class EventCacheUpdater { + private var sequenceTracker = EventSequenceTracker() + + private func handleEvent(_ envelope: EventEnvelope) async { + let gapStatus = sequenceTracker.checkForGaps(envelope) + + switch gapStatus { + case .ok: + // Process event normally + await applyEventToCache(envelope.event) + + case .gap(let expected, let received, let missing): + print("⚠️ Event gap detected: expected \(expected), got \(received)") + + // Invalidate affected queries to force refetch + await invalidateAffectedQueries(envelope.event) + + // Background reconciliation: fetch missing state + Task.detached { + await self.reconcileState(libraryId: envelope.libraryId) + } + + case .duplicate: + // Ignore duplicate events + break + } + } + + private func reconcileState(libraryId: UUID) async { + // Re-fetch critical queries to ensure consistency + // This is a "catch-up" mechanism after missed events + print("🔄 Reconciling state for library \(libraryId)") + + // Invalidate all queries for this library + cache.invalidateLibrary(libraryId) + } +} +``` + +### 3. Centralized Event Emission + +**Problem**: Events emitted from multiple places can be inconsistent + +**Solution**: EventEmitter service with transactional guarantees + +```rust +// core/src/infra/event/emitter.rs + +use super::EventBus; +use crate::domain::{File, Location, Tag}; +use uuid::Uuid; + +/// Centralized service for emitting cache-update events +/// Ensures events are created consistently and include proper resource data +pub struct CacheEventEmitter { + event_bus: Arc, + sequence_generator: Arc>>, // library_id → sequence +} + +impl CacheEventEmitter { + pub fn new(event_bus: Arc) -> Self { + Self { + event_bus, + sequence_generator: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Emit a file update event with full resource data + pub fn emit_file_updated(&self, library_id: Uuid, file: File) { + let sequence = self.next_sequence(library_id); + + let envelope = EventEnvelope { + sequence, + library_id, + timestamp: Utc::now(), + event: Event::FileUpdated { library_id, file }, + }; + + self.event_bus.emit(Event::Envelope(Box::new(envelope))); + + tracing::debug!( + library_id = %library_id, + sequence = sequence, + resource = "File", + "Emitted cache update event" + ); + } + + /// Emit a tag update event + pub fn emit_tag_updated(&self, library_id: Uuid, tag: Tag) { + let sequence = self.next_sequence(library_id); + + let envelope = EventEnvelope { + sequence, + library_id, + timestamp: Utc::now(), + event: Event::TagUpdated { library_id, tag }, + }; + + self.event_bus.emit(Event::Envelope(Box::new(envelope))); + } + + /// Emit a relationship change event + pub fn emit_tag_applied(&self, library_id: Uuid, tag_id: Uuid, entry_ids: Vec) { + let sequence = self.next_sequence(library_id); + + let envelope = EventEnvelope { + sequence, + library_id, + timestamp: Utc::now(), + event: Event::TagApplied { library_id, tag_id, entry_ids }, + }; + + self.event_bus.emit(Event::Envelope(Box::new(envelope))); + } + + /// Emit multiple events in a transaction (atomic batch) + pub fn emit_transaction(&self, library_id: Uuid, events: Vec) { + let sequence = self.next_sequence(library_id); + + let envelope = EventEnvelope { + sequence, + library_id, + timestamp: Utc::now(), + event: Event::BatchUpdate { + library_id, + updates: events, + transaction_id: Uuid::new_v4(), + }, + }; + + self.event_bus.emit(Event::Envelope(Box::new(envelope))); + } + + fn next_sequence(&self, library_id: Uuid) -> u64 { + let mut sequences = self.sequence_generator.lock().unwrap(); + let sequence = sequences.entry(library_id).or_insert(0); + *sequence += 1; + *sequence + } +} + +// Add to CoreContext +impl CoreContext { + pub fn cache_events(&self) -> &CacheEventEmitter { + &self.cache_event_emitter + } +} +``` + +**Usage in Actions**: +```rust +// core/src/ops/files/rename/action.rs + +impl LibraryAction for FileRenameAction { + async fn execute( + self, + library: Arc, + context: Arc, + ) -> ActionResult { + let entry_id = self.entry_id; + + // Perform rename in database + let updated_entry = rename_entry(&library, entry_id, &self.new_name).await?; + + // Construct full File domain object + let file = File::from_entry_id(library.clone(), entry_id).await?; + + // ✅ Emit through centralized emitter + context.cache_events().emit_file_updated(library.id(), file); + + Ok(RenameOutput { success: true }) + } +} +``` + +### 4. Resource Versioning for Conflict Resolution + +**Problem**: Optimistic updates need conflict detection + +**Solution**: Add version field to domain models + +```rust +// core/src/domain/file.rs (additions) + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct File { + pub id: Uuid, + + /// Resource version - incremented on each update + /// Used for optimistic concurrency control + pub version: u64, + + // ... rest of fields +} + +// Update strategy +pub enum MergeStrategy { + /// Always use server version (default) + ServerWins, + + /// Keep client version, reject server update + ClientWins, + + /// Merge fields if both changed different things + FieldLevelMerge, + + /// Use version with higher timestamp + LastWriteWins, +} +``` + +**Client-side conflict handling**: +```swift +func handleFileUpdate(_ event: Event.FileUpdated) { + let incomingFile = event.file + + guard let cachedFile = cache.getEntity(File.self, id: incomingFile.id) else { + // Not in cache, just add it + cache.updateEntity(incomingFile) + return + } + + // Check for conflicts + if cachedFile.version > incomingFile.version { + // Client has newer version - possible if optimistic update happened + print("⚠️ Version conflict: client=\(cachedFile.version) server=\(incomingFile.version)") + + // Strategy: Server wins (safest), but log the conflict + cache.updateEntity(incomingFile) + + // Could implement more sophisticated merging here + } else { + // Normal case: server has newer or same version + cache.updateEntity(incomingFile) + } +} +``` + +### 5. Memory Management and GC + +**Problem**: Unbounded cache growth consumes memory + +**Solution**: Multi-tiered eviction strategy + +```swift +class NormalizedCache { + // Configuration + private let maxEntities: Int = 10_000 + private let maxMemoryMB: Int = 100 + private let entityTTL: TimeInterval = 3600 // 1 hour + + // Tracking + private var lruOrder: [String] = [] + private var accessTimestamps: [String: Date] = [:] + private var referenceCount: [String: Int] = [:] // How many queries reference this + + /// Update entity with automatic GC + func updateEntity(_ entity: T) { + let cacheKey = entity.cacheKey() + + // Store entity + entities[cacheKey] = entity + accessTimestamps[cacheKey] = Date() + + // Update LRU + touchEntity(cacheKey) + + // Check if eviction needed + if entities.count > maxEntities { + evictLRU() + } + + triggerUpdate() + } + + private func evictLRU() { + // Sort by: refCount (0 first) → lastAccess (oldest first) + let candidates = entities.keys.sorted { key1, key2 in + let ref1 = referenceCount[key1] ?? 0 + let ref2 = referenceCount[key2] ?? 0 + + if ref1 != ref2 { + return ref1 < ref2 // Unreferenced first + } + + let time1 = accessTimestamps[key1] ?? Date.distantPast + let time2 = accessTimestamps[key2] ?? Date.distantPast + return time1 < time2 // Older first + } + + // Evict until under limit + let toEvict = entities.count - (maxEntities * 90 / 100) // Evict to 90% + + for i in 0.. 0 { + continue + } + + entities.removeValue(forKey: key) + accessTimestamps.removeValue(forKey: key) + referenceCount.removeValue(forKey: key) + + print("🗑️ Evicted: \(key)") + } + } + + /// Increment reference count when query adds entity + func incrementRefCount(_ cacheKey: String) { + referenceCount[cacheKey, default: 0] += 1 + } + + /// Decrement reference count when query is invalidated + func decrementRefCount(_ cacheKey: String) { + if let count = referenceCount[cacheKey], count > 0 { + referenceCount[cacheKey] = count - 1 + } + } +} +``` + +### 6. Background Reconciliation for Missed Events + +**Problem**: Client disconnects, misses events, cache becomes stale + +**Solution**: State reconciliation on reconnect + +```rust +// core/src/infra/sync/reconciliation.rs + +pub struct StateReconciliationService; + +impl StateReconciliationService { + /// Get all changes since a specific event sequence + pub async fn get_changes_since( + &self, + library_id: Uuid, + since_sequence: u64, + ) -> QueryResult> { + // Query audit log / event log for changes + // Return list of resources that changed + todo!() + } + + /// Full state snapshot for complete cache rebuild + pub async fn get_full_state_snapshot( + &self, + library_id: Uuid, + resource_types: Vec, + ) -> QueryResult { + // Return all entities of requested types + todo!() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResourceChange { + pub resource_type: String, + pub resource_id: Uuid, + pub change_type: ChangeType, + pub data: Option, + pub sequence: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ChangeType { + Created, + Updated, + Deleted, +} +``` + +**Client usage**: +```swift +class CacheReconciliationService { + func reconcileOnReconnect(libraryId: UUID) async throws { + let lastSequence = cache.getLastSequence(libraryId: libraryId) + + print("🔄 Reconciling from sequence \(lastSequence)") + + // Fetch all changes since last known sequence + let changes = try await client.query( + "query:sync.changes_since.v1", + input: ChangesSinceInput( + libraryId: libraryId, + sinceSequence: lastSequence + ) + ) + + // Apply changes in order + for change in changes.sorted(by: { $0.sequence < $1.sequence }) { + switch change.changeType { + case .created, .updated: + if let data = change.data { + await cache.updateFromJSON( + resourceType: change.resourceType, + id: change.resourceId, + json: data + ) + } + case .deleted: + await cache.removeEntity( + resourceType: change.resourceType, + id: change.resourceId + ) + } + } + + print("✅ Reconciliation complete: applied \(changes.count) changes") + } +} +``` + +## Implementation Strategy Refinements + +### Refinement 1: Instance Method for cache_metadata + +**Original**: `fn cache_metadata(result: &[T]) -> CacheMetadata` +**Improved**: `fn generate_cache_metadata(&self, result: &Self::Output) -> CacheMetadata` + +```rust +impl CacheableQuery for FileSearchQuery { + fn generate_cache_metadata(&self, result: &Self::Output) -> CacheMetadata { + let mut metadata = CacheMetadata::new(); + + // Access query input to customize caching + if self.input.query.len() < 3 { + // Don't cache very short searches (too dynamic) + metadata.cacheable = false; + return metadata; + } + + // Extract files from search output + for search_result in &result.results { + // Handle the actual result structure + metadata.add_resource(&search_result.file); + + // Add nested resources (tags) + for tag in &search_result.file.tags { + metadata.add_resource(tag); + } + } + + // Configure based on search mode + metadata.cache_duration = match self.input.mode { + SearchMode::Fast => Some(300), // 5 minutes + SearchMode::Normal => Some(60), // 1 minute (less stable) + SearchMode::Full => Some(600), // 10 minutes (expensive to recompute) + }; + + metadata + } +} +``` + +### Refinement 2: Centralized Event Creation in Actions + +**Pattern**: All events emitted at end of action execution + +```rust +// core/src/infra/action/manager.rs (additions) + +impl ActionManager { + pub async fn dispatch_library( + &self, + library_id: Option, + action: A, + ) -> Result { + let library_id = library_id.ok_or(/*...*/)?; + let library = self.context.get_library(library_id).await?; + + // Execute action + let result = action.execute(library.clone(), self.context.clone()).await; + + // ✅ Emit cache events AFTER successful execution + if let Ok(ref output) = result { + // Actions can optionally implement CacheEventEmitter trait + if let Some(events) = action.generate_cache_events(library_id, output) { + for event in events { + self.context.cache_events().emit(library_id, event); + } + } + } + + result + } +} + +/// Optional trait for actions to declare what cache events they generate +pub trait CacheEventEmitter { + type Output; + + /// Generate cache events after successful execution + fn generate_cache_events( + &self, + library_id: Uuid, + output: &Self::Output, + ) -> Option> { + None // Default: no special cache events + } +} + +pub enum CacheableEvent { + FileUpdated(File), + TagUpdated(Tag), + LocationUpdated(Location), + RelationshipChanged { + resource_type: String, + resource_id: Uuid, + relationship: String, + added: Vec, + removed: Vec, + }, +} +``` + +### Refinement 3: Use File Instead of Entry for Clients + +**Rationale**: File is richer, Entry is database-level + +```rust +// Don't implement Identifiable for Entry (keep it internal) +// Only expose File to clients + +impl Event { + // ❌ Don't emit Entry events to clients + // EntryModified { entry_id: Uuid } + + // ✅ Emit File events with full data + FileUpdated { + library_id: Uuid, + file: File, // Complete File domain object + }, + + // For lightweight updates, use delta pattern + FileMetadataChanged { + library_id: Uuid, + file_id: Uuid, + changes: FileMetadataDelta, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct FileMetadataDelta { + pub name: Option, + pub size: Option, + pub modified_at: Option>, + // Only include fields that changed +} +``` + +**Entry → File conversion happens server-side**: +```rust +// When indexer updates an entry, emit File event +impl IndexingJob { + async fn process_entry(&mut self, entry: entry::Model) { + // Update database... + + // Construct File domain object + let file = File::from_entry_id(self.library.clone(), entry.uuid?).await?; + + // Emit to clients (not Entry, but File!) + self.context.cache_events().emit_file_updated(self.library.id(), file); + } +} +``` + +## Open Questions (Revised) + +1. **Partial events**: Should we always send full resources, or support delta updates? + - ✅ **Decision**: Start with full resources for File/Tag/Location (< 10KB typically) + - ✅ Add `FileMetadataDelta` for large objects with many relationships + - ✅ Client merges deltas into cached entities + +2. **Cache persistence**: Should cache survive app restarts? + - ✅ **Decision**: Phase 2 feature - persist to SQLite for offline access + - ✅ Use sequence numbers to validate cache on startup + - ✅ Implement "stale while revalidate" pattern + +3. **Cache invalidation**: What if event is missed (network drop)? + - ✅ **Solved**: Event versioning with sequence numbers + - ✅ Gap detection triggers background reconciliation + - ✅ Fallback: invalidate affected queries, force refetch + +4. **Resource versions**: Should resources have version numbers for conflict resolution? + - ✅ **Solved**: Add `version: u64` field to all Identifiable resources + - ✅ Increment on each update + - ✅ Client checks version before applying optimistic updates + +5. **Garbage collection**: When to remove entities no longer in any query? + - ✅ **Solved**: Reference counting + LRU eviction + - ✅ Evict entities with refCount = 0 and not accessed recently + - ✅ Configurable limits: maxEntities, maxMemoryMB, entityTTL + +## Handling Complex Relationships + +### The Challenge + +The `extract_relationships()` method can become complex for deeply nested domain models. Consider `File`: + +```rust +pub struct File { + pub id: Uuid, + pub sd_path: SdPath, // Contains device_id (relationship!) + pub tags: Vec, // Many-to-many relationship + pub sidecars: Vec, // One-to-many relationship + pub content_identity: Option, // One-to-one relationship + pub alternate_paths: Vec, // Implicit relationship to other Files + // ... +} +``` + +### Solution: Layered Relationship Extraction + +```rust +impl Identifiable for File { + fn extract_relationships(&self) -> ResourceRelationships { + let mut rels = ResourceRelationships::new(); + + // Layer 1: Direct relationships (IDs are explicit) + for tag in &self.tags { + rels.add_to_collection("tags", Tag::cache_key_from_id(&tag.id)); + } + + if let Some(content) = &self.content_identity { + rels.add_singular("content_identity", ContentIdentity::cache_key_from_id(&content.uuid)); + } + + // Layer 2: Derived relationships (require parsing) + // Extract location from sd_path + if let Some(location_id) = self.infer_location_id() { + rels.add_singular("location", Location::cache_key_from_id(&location_id)); + } + + // Extract device from sd_path + if let SdPath::Physical { device_id, .. } = &self.sd_path { + rels.add_singular("device", Device::cache_key_from_id(device_id)); + } + + // Layer 3: Implicit relationships (duplicates) + // Note: alternate_paths represent other Files with same content + // We don't extract these as explicit relationships to avoid circular deps + // The client can query for duplicates when needed + + rels + } + + /// Helper: Infer location ID from sd_path + /// This requires looking up which location contains this path + fn infer_location_id(&self) -> Option { + // Implementation would query location registry + // For now, we can include location_id explicitly in File struct + // See improvement below + None + } +} + +// IMPROVEMENT: Add explicit location_id to File +// This avoids complex inference logic +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct File { + pub id: Uuid, + pub location_id: Option, // ✅ Explicit relationship + pub sd_path: SdPath, + pub tags: Vec, + pub content_identity: Option, + // ... +} + +impl Identifiable for File { + fn extract_relationships(&self) -> ResourceRelationships { + let mut rels = ResourceRelationships::new(); + + // Much simpler now! + if let Some(loc_id) = self.location_id { + rels.add_singular("location", Location::cache_key_from_id(&loc_id)); + } + + for tag in &self.tags { + rels.add_to_collection("tags", Tag::cache_key_from_id(&tag.id)); + } + + if let Some(content) = &self.content_identity { + rels.add_singular("content_identity", ContentIdentity::cache_key_from_id(&content.uuid)); + } + + rels + } +} +``` + +### Circular Relationship Handling + +**Problem**: File references Tag, Tag might reference Files (via search) + +**Solution**: One-directional relationships in cache graph + +```rust +// File → Tag (stored) +// Tag → Files (not stored, computed via reverse lookup) + +impl NormalizedCache { + /// Get all files that have a specific tag (reverse lookup) + fn files_with_tag(&self, tag_id: Uuid) -> Vec { + let tag_cache_key = Tag::cache_key_from_id(&tag_id); + + self.entities + .values() + .filter_map(|entity| entity as? File) + .filter(|file| { + file.tags.iter().any(|t| t.id == tag_id) + }) + .collect() + } +} +``` + +### Relationship Update Patterns + +**Pattern 1**: Many-to-many (Tag ↔ File) + +```rust +// When tag is applied to file +Event::TagApplied { + library_id: Uuid, + tag_id: Uuid, + entry_ids: Vec, // Files affected +} + +// Client handler: +// 1. Fetch tag entity from cache +// 2. For each entry_id, update that File's tags array +// 3. Don't update Tag entity (it doesn't store reverse refs) +``` + +**Pattern 2**: One-to-many (Location → Files) + +```rust +// When location is updated +Event::LocationUpdated { + library_id: Uuid, + location: Location, // Full location data +} + +// Client handler: +// 1. Update Location entity +// 2. Don't need to update Files (they reference location_id, not vice versa) +// 3. UI will see new location data automatically via relationships +``` + +**Pattern 3**: Cascading updates (rename Location → all Files in it) + +```rust +// When location is renamed +Event::LocationRenamed { + library_id: Uuid, + location: Location, // Updated location + affected_file_count: usize, // For UI feedback +} + +// Client handler: +// 1. Update Location entity +// 2. All Files with this location_id will show new location name +// automatically via join (no need to update each File!) +``` + +## Phased Rollout Strategy + +### Phase 1A: Core Infrastructure (Week 1) +- ✅ Create `Identifiable` trait +- ✅ Implement for File, Tag, Location, Job +- ✅ Add `version` field to domain models +- ✅ Create `CacheMetadata` and `QueryResponse` +- ✅ Add `CacheableQuery` trait with instance method + +### Phase 1B: Event Infrastructure (Week 1-2) +- ✅ Create `EventEnvelope` with sequence numbers +- ✅ Create `CacheEventEmitter` service +- ✅ Add to `CoreContext` +- ✅ Create new event types: `FileUpdated`, `TagUpdated`, etc. + +### Phase 2A: Swift Prototype (Week 2-3) +- ✅ Implement `NormalizedCache` for File only (narrow scope) +- ✅ Test with file search query +- ✅ Implement `EventCacheUpdater` for File events +- ✅ Measure performance vs query-based approach + +### Phase 2B: Expand to More Resources (Week 3-4) +- ✅ Add Tag, Location, Job to cache +- ✅ Test relationship updates +- ✅ Implement reference counting and GC + +### Phase 3: Production Hardening (Week 4-6) +- ✅ Add event versioning and gap detection +- ✅ Implement reconciliation service +- ✅ Add conflict resolution for optimistic updates +- ✅ Performance testing and optimization +- ✅ Memory profiling and tuning + +### Phase 4: TypeScript Port (Week 6-8) +- ✅ Port NormalizedCache to TypeScript +- ✅ Create React hooks +- ✅ Update web app + +### Phase 5: Advanced Features (Ongoing) +- ✅ Cache persistence (SQLite) +- ✅ Prefetching strategies +- ✅ Query deduplication +- ✅ Analytics and monitoring + +## Risk Mitigation + +| Risk | Likelihood | Impact | Mitigation | +|------|------------|--------|------------| +| Complexity overwhelms team | Medium | High | Start with File only, iterate | +| Cache becomes stale | Medium | High | Event versioning + reconciliation | +| Memory issues on mobile | High | Medium | Aggressive LRU eviction, configurable limits | +| Relationship logic bugs | High | Medium | Comprehensive tests, start simple | +| Event order issues | Medium | High | Sequence numbers + gap detection | +| Performance regression | Low | High | Benchmark before/after, A/B test | + +## Success Metrics + +### Performance Targets +- ⚡ **UI responsiveness**: < 16ms for cache hits (60fps) +- 📉 **Network reduction**: 80% fewer queries after initial load +- 💾 **Memory usage**: < 100MB for 10k cached entities +- 🔄 **Event latency**: < 100ms from action → cache update → UI + +### User Experience Goals +- ✨ Instant UI updates when data changes +- 📶 App works offline with cached data +- 🔋 50% reduction in battery usage from fewer network calls +- 🎯 Real-time sync across devices + +## Why Client-Side Only? + +### Server-Side Cache is Redundant + +The Rust core **should not** have a cache layer because: + +1. **Database IS the cache** - SeaORM with PostgreSQL/SQLite is already highly optimized + - Indexes provide fast lookups + - Query planner optimizes joins + - Connection pooling handles concurrency + - Adding another cache layer would just duplicate data + +2. **Different problems being solved**: + - **Database**: Persistent storage, ACID guarantees, query optimization + - **Client cache**: Network latency, offline access, instant UI updates + - These are orthogonal concerns! + +3. **Complexity without benefit**: + - Server cache needs invalidation logic (when DB updates) + - Cache coherency between cache and DB + - More memory usage on server + - More code to maintain + - Minimal performance gain (DB queries are already fast locally) + +4. **Queries should be fast enough**: + - Core is local (same machine or local network) + - Database queries are microseconds to milliseconds + - The bottleneck is network latency (client → core), not DB queries + +### The Client-Side Cache Solves Real Problems + +The normalized cache on **clients** makes sense because: + +- ✅ **Network latency**: 100ms+ round trip vs 0ms cache hit +- ✅ **Bandwidth**: Don't re-fetch unchanged data +- ✅ **Offline**: App works when disconnected +- ✅ **Real-time UI**: Atomic updates instead of full refreshes +- ✅ **Battery life**: Fewer network operations on mobile + +### Architecture Clarity + +``` +┌──────────────┐ ┌──────────────┐ ┌──────────────┐ +│ Swift Client │ │ Web Client │ │ CLI Client │ +│ │ │ │ │ │ +│ ✅ Cache │ │ ✅ Cache │ │ ❌ No Cache │ +│ (Memory) │ │ (Memory) │ │ (Stateless) │ +└──────┬───────┘ └──────┬───────┘ └──────┬───────┘ + │ │ │ + └────────────────────┼────────────────────┘ + │ Network (bottleneck!) + │ + ┌──────▼───────┐ + │ Rust Core │ + │ │ + │ ❌ No Cache │ + │ ✅ Database │ ← Single source of truth + └──────────────┘ +``` + +**Takeaway**: Cache at the network boundary (clients), not at the data source (core). + +## Next Steps + +1. **✅ Design approved** - Incorporate review feedback (DONE) +2. **Start Phase 1A** - Implement `Identifiable` trait in Rust +3. **Prototype Phase 2A** - Build Swift NormalizedCache for File +4. **Measure and iterate** - Compare performance metrics +5. **Expand gradually** - Add more resource types based on learnings + +--- + +This design provides a **foundation for instant, real-time UI updates** across all Spacedrive clients while minimizing network overhead and enabling offline functionality. The phased approach mitigates risk while delivering value incrementally. + +**Critical Design Principle**: Cache where the latency is (client ↔ core), not where the data is (core ↔ database). diff --git a/docs/core/design/sync/README.md b/docs/core/design/sync/README.md new file mode 100644 index 000000000..2f13414e7 --- /dev/null +++ b/docs/core/design/sync/README.md @@ -0,0 +1,180 @@ +# Sync System Design Documentation + +This directory contains **detailed design documents** for Spacedrive's multi-device synchronization and client-side caching architecture. + +## 🎯 Implementation Guides (Start Here!) + +For implementation, read these **root-level guides**: + +1. **[../../sync.md](../../sync.md)** ⭐ **Sync System Implementation Guide** + - TransactionManager API and usage + - Syncable trait specification + - Leader election protocol + - Sync service implementation + - Production-ready reference + +2. **[../../events.md](../../events.md)** ⭐ **Unified Event System** + - Generic resource events + - Type registry pattern (zero switch statements!) + - Client integration (Swift + TypeScript) + - Migration strategy + +3. **[../../normalized_cache.md](../../normalized_cache.md)** ⭐ **Client-Side Normalized Cache** + - Cache architecture and implementation + - Memory management (LRU, TTL, ref counting) + - React and SwiftUI integration + - Optimistic updates and offline support + +--- + +## 📚 Design Documents (Deep Dives) + +The documents in this directory provide comprehensive design rationale and detailed exploration. Read these for context and decision history: + +### 1. Foundation & Context +- **[SYNC_DESIGN.md](./SYNC_DESIGN.md)** - The original comprehensive sync architecture + - Covers: Sync domains (Index, Metadata, Content, State), conflict resolution, leader election + - Start here for foundational understanding + +### 2. Core Implementation Specs +- **[SYNC_TX_CACHE_MINI_SPEC.md](./SYNC_TX_CACHE_MINI_SPEC.md)** ⭐ **START HERE FOR IMPLEMENTATION** + - Concise, actionable spec for `Syncable`/`Identifiable` traits + - TransactionManager API and semantics + - BulkChangeSet mechanism for efficient bulk operations + - Albums example with minimal boilerplate + - Raw SQL compatibility notes + +- **[UNIFIED_RESOURCE_EVENTS.md](./UNIFIED_RESOURCE_EVENTS.md)** ⭐ **CRITICAL FOR EVENT SYSTEM** + - Generic resource event design (eliminates ~40 specialized event variants) + - Type registry pattern for zero-friction horizontal scaling + - Swift and TypeScript examples with auto-generation via specta + - **Key insight**: Zero switch statements when adding new resources + +### 3. Unified Architecture +- **[UNIFIED_TRANSACTIONAL_SYNC_AND_CACHE.md](./UNIFIED_TRANSACTIONAL_SYNC_AND_CACHE.md)** + - Complete end-to-end architecture integrating sync + cache + - Context-aware commits: `transactional` vs `bulk` vs `silent` + - **Critical**: Bulk operations create ONE metadata sync entry (not millions) + - Performance analysis and decision rationale + - 2295 lines of comprehensive design (reference doc, not reading material) + +### 4. Client-Side Caching +- **[NORMALIZED_CACHE_DESIGN.md](./NORMALIZED_CACHE_DESIGN.md)** + - Client-side normalized entity cache (similar to Apollo Client) + - Event-driven invalidation and atomic updates + - Memory management (LRU, TTL, reference counting) + - Swift and TypeScript implementation patterns + - 2674 lines covering edge cases and advanced scenarios + +### 5. Implementation Analysis +- **[TRANSACTION_MANAGER_COMPATIBILITY.md](./TRANSACTION_MANAGER_COMPATIBILITY.md)** + - Compatibility analysis with existing codebase + - Current write patterns (SeaORM, transactions, raw SQL) + - Migration strategy with code examples + - Risk analysis and mitigation + - **Verdict**: Fully compatible, ready to implement ✅ + +### 6. Historical & Supplementary +- **[SYNC_DESIGN_2025_08_19.md](./SYNC_DESIGN_2025_08_19.md)** - Updated sync design iteration +- **[SYNC_FIRST_DRAFT_DESIGN.md](./SYNC_FIRST_DRAFT_DESIGN.md)** - Early draft (historical context) +- **[SYNC_INTEGRATION_NOTES.md](./SYNC_INTEGRATION_NOTES.md)** - Integration notes and considerations +- **[SYNC_CONDUIT_DESIGN.md](./SYNC_CONDUIT_DESIGN.md)** - Sync conduit specific design + +--- + +## 🎯 Quick Reference + +### Key Concepts + +**Syncable** (Rust persistence models) +```rust +pub trait Syncable { + const SYNC_MODEL: &'static str; + fn sync_id(&self) -> Uuid; + fn version(&self) -> i64; +} +``` + +**Identifiable** (Client-facing resources) +```rust +pub trait Identifiable { + type Id; + fn resource_id(&self) -> Self::Id; + fn resource_type() -> &'static str; +} +``` + +**TransactionManager** (Sole write gateway) +- `commit()` - Single resource, per-entry sync log +- `commit_batch()` - Micro-batch (10-1K), per-entry sync logs +- `commit_bulk()` - Bulk (1K+), ONE metadata sync entry + +**Event System** (Generic, horizontally scalable) +- `ResourceChanged { resource_type, resource }` +- `ResourceBatchChanged { resource_type, resources }` +- `BulkOperationCompleted { resource_type, affected_count, hints }` + +### Critical Design Decisions + +1. **Indexing ≠ Sync**: Each device indexes its own filesystem. Bulk operations create metadata notifications, not individual entry replications. + +2. **Leader Election**: One device per library assigns sync log sequence numbers. Prevents collisions. + +3. **Zero Manual Sync Logging**: TransactionManager automatically creates sync logs. Application code never touches sync infrastructure. + +4. **Type Registry Pattern**: Clients use type registries (auto-generated via specta) to handle all resource events generically. No switch statements per resource type. + +5. **Client-Side Cache**: Normalized entity store + query index. Events trigger atomic updates. Cache persistence for offline mode. + +--- + +## 📋 Implementation Status + +- [x] Design documentation complete +- [ ] Phase 1: Core infrastructure (TM, traits, events) +- [ ] Phase 2: Client prototype (Swift cache + event handler) +- [ ] Phase 3: Expansion (migrate all ops to TM) +- [ ] Phase 4: TypeScript port + advanced features + +--- + +## 🔗 Related Documentation + +**Implementation Guides** (Root Level): +- `../../sync.md` - Sync system implementation +- `../../events.md` - Unified event system +- `../../normalized_cache.md` - Client cache implementation +- `../../sync-setup.md` - Library sync setup (Phase 1) + +**Infrastructure**: +- `../INFRA_LAYER_SEPARATION.md` - Infrastructure layer architecture +- `../JOB_SYSTEM_DESIGN.md` - Job system (indexing jobs integrate with TM) +- `../DEVICE_PAIRING_PROTOCOL.md` - Device pairing (prerequisite for sync) + +--- + +## 📖 Documentation Philosophy + +**Root-level docs** (`docs/core/*.md`): +- Implementation-ready guides +- Concise, actionable specifications +- Code examples and usage patterns +- Reference during development + +**Design docs** (`docs/core/design/sync/*.md`): +- Comprehensive exploration +- Decision rationale and alternatives +- Edge cases and advanced scenarios +- Historical context + +--- + +## 💡 Contributing + +**Adding implementation guidance**: Update root-level docs (`sync.md`, `events.md`, `normalized_cache.md`) + +**Adding design exploration**: Create new document in this directory: +1. Follow naming: `SYNC__DESIGN.md` +2. Update this README +3. Reference related documents +4. Include comprehensive examples diff --git a/docs/core/design/SYNC_CONDUIT_DESIGN.md b/docs/core/design/sync/SYNC_CONDUIT_DESIGN.md similarity index 100% rename from docs/core/design/SYNC_CONDUIT_DESIGN.md rename to docs/core/design/sync/SYNC_CONDUIT_DESIGN.md diff --git a/docs/core/design/SYNC_DESIGN.md b/docs/core/design/sync/SYNC_DESIGN.md similarity index 100% rename from docs/core/design/SYNC_DESIGN.md rename to docs/core/design/sync/SYNC_DESIGN.md diff --git a/docs/core/design/SYNC_DESIGN_2025_08_19.md b/docs/core/design/sync/SYNC_DESIGN_2025_08_19.md similarity index 100% rename from docs/core/design/SYNC_DESIGN_2025_08_19.md rename to docs/core/design/sync/SYNC_DESIGN_2025_08_19.md diff --git a/docs/core/design/SYNC_FIRST_DRAFT_DESIGN.md b/docs/core/design/sync/SYNC_FIRST_DRAFT_DESIGN.md similarity index 100% rename from docs/core/design/SYNC_FIRST_DRAFT_DESIGN.md rename to docs/core/design/sync/SYNC_FIRST_DRAFT_DESIGN.md diff --git a/docs/core/design/SYNC_INTEGRATION_NOTES.md b/docs/core/design/sync/SYNC_INTEGRATION_NOTES.md similarity index 100% rename from docs/core/design/SYNC_INTEGRATION_NOTES.md rename to docs/core/design/sync/SYNC_INTEGRATION_NOTES.md diff --git a/docs/core/design/sync/SYNC_TX_CACHE_MINI_SPEC.md b/docs/core/design/sync/SYNC_TX_CACHE_MINI_SPEC.md new file mode 100644 index 000000000..6e4abc882 --- /dev/null +++ b/docs/core/design/sync/SYNC_TX_CACHE_MINI_SPEC.md @@ -0,0 +1,271 @@ +# Sync + Transaction Manager + Normalized Cache: Mini Spec + +## Scope +A concise specification aligning TransactionManager, Syncable/Identifiable traits, bulk change handling, raw query compatibility, and leader election. Includes a concrete Albums example with minimal boilerplate. + +## Goals +- Zero manual sync-log creation in application code +- Keep raw SQL for complex reads; writes go through TransactionManager +- Bulk = mechanism (generic changeset), not hard-coded enum cases +- Clear trait-based configuration, minimal boilerplate +- Compatible with existing SeaORM patterns + +## Core Traits + +```rust +// client-facing identity for cache normalization +pub trait Identifiable { + type Id: Into + Copy + Eq + std::hash::Hash + Serialize + for<'de> Deserialize<'de>; + fn id(&self) -> Self::Id; + fn resource_type() -> &'static str; +} + +// persistence-facing for sync logging +pub trait Syncable { + // stable model type name used in sync log + const SYNC_MODEL: &'static str; + + // globally unique logical id for sync (Uuid recommended) + fn sync_id(&self) -> Uuid; + + // optimistic concurrency + fn version(&self) -> i64; + + // minimal payload for replication (defaults to full serde) + fn to_sync_json(&self) -> serde_json::Value where Self: Serialize { + serde_json::to_value(self).unwrap_or(serde_json::json!({})) + } + + // optional field allow/deny (minimize boilerplate: both optional) + fn include_fields() -> Option<&'static [&'static str]> { None } + fn exclude_fields() -> Option<&'static [&'static str]> { None } +} +``` + +Notes: +- App code should not construct sync logs; TransactionManager derives them from `Syncable`. +- `include_fields`/`exclude_fields` are optional knobs. If both None, default to `to_sync_json()`. + +## TransactionManager Responsibilities + +- Enforce atomic DB write + sync log creation +- Emit rich events post-commit for client cache +- Support single, batch, and bulk change sets +- Provide a transaction-bound context for raw SQL when needed + +### API (sketch) +```rust +pub struct TransactionManager { /* event bus, seq allocator, leader state */ } + +pub struct ChangeSet { pub items: Vec } // generic mechanism for bulk + +impl TransactionManager { + // single model + pub async fn commit( + &self, + library: Arc, + model: M, + ) -> Result; + + // micro-batch (10–1k), produces per-item sync entries + pub async fn commit_batch( + &self, + library: Arc, + models: Vec, + ) -> Result, TxError>; + + // bulk (1k+), produces ONE metadata sync entry with ChangeSet descriptor + pub async fn commit_bulk( + &self, + library: Arc, + changes: ChangeSet, + ) -> Result; +} + +pub struct BulkAck { pub affected: usize, pub token: Uuid } +``` + +### Sync Log Semantics +- commit: one sync entry per item +- commit_batch: one per item (same txn), event may be batched +- commit_bulk: ONE metadata sync entry: +```json +{ + "sequence": 1234, + "model_type": "bulk_changeset", + "token": "uuid-token", + "affected": 1000000, + "model": "entry", // derived from Syncable::SYNC_MODEL + "mode": "insert|update|delete", + "hints": { "location_id": "..." } +} +``` +Followers treat this as a notification; they DO NOT pull all items. They trigger local indexing where applicable. + +## Raw Query Compatibility +- Reads: unrestricted (SeaORM query builder or raw SQL) +- Writes: perform inside TM-provided transaction handle + - TM exposes `with_tx(|txn| async { /* raw SQL writes */ })` that auto sync-logs via `Syncable` wrappers or explicit `commit_*` calls. + +## Leader Election (Minimum) +- Single leader per library for assigning sync sequences +- Election strategy per SYNC_DESIGN.md (initial leader = creator; re-elect via heartbeat timeout) +- TM refuses sync-log creation if not leader (or buffers and requests lease) + +## Albums Example (Concrete) + +Schema (SeaORM model): +```rust +#[derive(Clone, Debug, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "albums")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub uuid: Uuid, + pub name: String, + pub cover_entry_uuid: Option, + pub version: i64, + pub created_at: DateTime, + pub updated_at: DateTime, +} +``` + +Implement traits: +```rust +impl Syncable for albums::Model { + const SYNC_MODEL: &'static str = "album"; + fn sync_id(&self) -> Uuid { self.uuid } + fn version(&self) -> i64 { self.version } + fn exclude_fields() -> Option<&'static [&'static str]> { + // example: exclude timestamps from replication + Some(&["created_at", "updated_at", "id"]) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Album { pub id: Uuid, pub name: String, pub cover: Option } + +impl Identifiable for Album { + type Id = Uuid; + fn id(&self) -> Self::Id { self.id } + fn resource_type() -> &'static str { "album" } +} +``` + +Create action (no manual sync logging): +```rust +pub async fn create_album( + tm: &TransactionManager, + library: Arc, + name: String, +) -> Result { + let model = albums::Model { + id: 0, + uuid: Uuid::new_v4(), + name, + cover_entry_uuid: None, + version: 1, + created_at: Utc::now(), + updated_at: Utc::now(), + }; + + // TM writes + sync logs atomically + let saved = tm.commit(library.clone(), model).await?; + + // Build client model (query layer) + let album = Album { id: saved.uuid, name: saved.name, cover: saved.cover_entry_uuid }; + + // TM (post-commit) emits Event::AlbumUpdated { album } automatically + Ok(album) +} +``` + +Bulk import albums: +```rust +pub async fn import_albums( + tm: &TransactionManager, + library: Arc, + names: Vec, +) -> Result { + let models: Vec = names.into_iter().map(|n| albums::Model { + id: 0, + uuid: Uuid::new_v4(), + name: n, + cover_entry_uuid: None, + version: 1, + created_at: Utc::now(), + updated_at: Utc::now(), + }).collect(); + + let ack = tm.commit_bulk(library, ChangeSet { items: models }).await?; + Ok(ack.affected) +} +``` + +## Boilerplate Minimization +- Derive macros can implement `Syncable` and `Identifiable` from annotations: +```rust +#[derive(Syncable)] +#[syncable(model="album", id="uuid", version="version", exclude=["created_at","updated_at","id"])] +struct albums::Model { /* ... */ } + +#[derive(Identifiable)] +#[identifiable(resource="album", id="id")] +struct Album { /* ... */ } +``` + +## Event Emission (Unified System) + +See `UNIFIED_RESOURCE_EVENTS.md` for complete design. + +**Key Points**: +- TM emits generic `ResourceChanged` events automatically +- No manual `event_bus.emit()` in application code +- Clients handle resources generically via `resource_type` field +- Event structure: + ```rust + Event { + envelope: { id, timestamp, library_id, sequence }, + kind: ResourceChanged { resource_type, resource } + | ResourceBatchChanged { resource_type, resources, operation } + | BulkOperationCompleted { resource_type, affected_count, token, hints } + | ResourceDeleted { resource_type, resource_id } + } + ``` + +**Example**: +```rust +// Rust: Automatic emission +let album = tm.commit::(library, model).await?; +// → Emits: ResourceChanged { resource_type: "album", resource: album } + +// Swift: Generic handling +case .ResourceChanged(let type, let json): + switch type { + case "album": cache.updateEntity(try decode(Album.self, json)) + case "file": cache.updateEntity(try decode(File.self, json)) + // Add new resources without changing event code! + } +``` + +Benefits: +- Zero boilerplate for new resources +- Type-safe on both ends +- Cache integration automatic +- ~35 specialized event variants eliminated + +## Consistency Rules +- All sync-worthy writes go through TM +- Reads, including raw SQL, remain unrestricted +- Followers treat bulk metadata as notification; they re-index locally if applicable + +## Appendix: Raw SQL inside TM +```rust +tm.with_tx(library, |txn| async move { + // raw SQL writes + txn.execute(Statement::from_sql_and_values(DbBackend::Sqlite, "UPDATE albums SET name=? WHERE uuid=?", vec![name.into(), uuid.into()])).await?; + // tell TM to record sync for this model change + tm.sync_log_for::(txn, uuid).await?; + Ok(()) +}).await?; +``` diff --git a/docs/core/design/sync/TRANSACTION_MANAGER_COMPATIBILITY.md b/docs/core/design/sync/TRANSACTION_MANAGER_COMPATIBILITY.md new file mode 100644 index 000000000..1cc69e810 --- /dev/null +++ b/docs/core/design/sync/TRANSACTION_MANAGER_COMPATIBILITY.md @@ -0,0 +1,604 @@ +# TransactionManager Compatibility Analysis + +## Executive Summary + +**Status**: ✅ **FULLY COMPATIBLE** with existing codebase patterns + +The `TransactionManager` design is **fully compatible** with the current database write patterns. The codebase uses **SeaORM exclusively** with well-structured transaction patterns that the TransactionManager can enhance without requiring major refactoring. + +**Key Finding**: No sync log infrastructure exists yet - the TransactionManager will be the **first implementation** of transactional sync. + +--- + +## Current Database Write Patterns + +### 1. SeaORM-Only Architecture ✅ + +**Good news**: The codebase uses **SeaORM exclusively** for database operations. No raw SQL for writes (except for optimized bulk operations). + +```rust +// Pattern 1: Single insert with ActiveModel +let new_entry = entry::ActiveModel { + uuid: Set(Uuid::new_v4()), + name: Set(entry_name), + size: Set(file_size), + // ... +}; +let result = new_entry.insert(db).await?; +``` + +```rust +// Pattern 2: Batch insert +let entries: Vec = /* ... */; +entry::Entity::insert_many(entries) + .exec(db) + .await?; +``` + +```rust +// Pattern 3: Transaction-wrapped operations +let txn = db.begin().await?; + +// Multiple operations +let result1 = model1.insert(&txn).await?; +let result2 = model2.insert(&txn).await?; + +txn.commit().await?; +``` + +**TransactionManager Compatibility**: ✅ **Perfect fit** +- Can wrap existing ActiveModel operations +- Can use SeaORM's transaction support +- No need to change ORM layer + +--- + +## Where Writes Currently Happen + +### 1. **Indexer** (Bulk Operations) + +**Location**: `core/src/ops/indexing/` + +**Pattern**: Batch transactions with bulk inserts + +```rust +// Current indexer pattern +let txn = ctx.library_db().begin().await?; + +// Accumulate entries in memory +let mut bulk_self_closures: Vec = Vec::new(); +let mut bulk_dir_paths: Vec = Vec::new(); + +// Process batch +for entry in batch { + EntryProcessor::create_entry_in_conn( + state, ctx, &entry, device_id, location_root_path, + &txn, // ← Single transaction for whole batch + &mut bulk_self_closures, + &mut bulk_dir_paths, + ).await?; +} + +// Bulk insert related tables +entry_closure::Entity::insert_many(bulk_self_closures) + .exec(&txn).await?; +directory_paths::Entity::insert_many(bulk_dir_paths) + .exec(&txn).await?; + +txn.commit().await?; +``` + +**TransactionManager Integration**: +```rust +// New pattern with TransactionManager +let entries: Vec = /* collect in memory */; + +tx_manager.commit_bulk( + library, + entries, + BulkOperation::InitialIndex { location_id } +).await?; +// ✅ ONE sync log entry created automatically +// ✅ Event emitted automatically +``` + +**Refactoring Required**: ⚠️ **Moderate** +- Replace batch transaction with `commit_bulk` call +- Remove manual transaction management +- Add BulkOperation context +- **Benefit**: 10x performance improvement + sync log integration + +--- + +### 2. **User Actions** (Single Operations) + +**Location**: `core/src/ops/tags/apply/action.rs`, `core/src/ops/locations/add/action.rs` + +**Pattern**: Direct inserts via managers/services + +```rust +// Current action pattern +impl LibraryAction for ApplyTagsAction { + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + let db = library.db(); + let metadata_manager = UserMetadataManager::new(db.conn().clone()); + + // Apply tags (internally does inserts) + metadata_manager.apply_semantic_tags( + entry_uuid, + tag_applications, + device_id + ).await?; + + Ok(output) + } +} +``` + +**TransactionManager Integration**: +```rust +// New pattern with TransactionManager +impl LibraryAction for ApplyTagsAction { + async fn execute( + self, + library: Arc, + context: Arc, + ) -> Result { + let tx_manager = context.transaction_manager(); + + // Prepare models + let entry_model = /* ... */; + let tag_link_model = /* ... */; + + // Commit transactionally (creates sync log + event) + let file = tx_manager.commit_tag_addition( + library, + entry_model, + tag_link_model, + ).await?; + + Ok(output) + } +} +``` + +**Refactoring Required**: ⚠️ **Moderate** +- Inject TransactionManager from CoreContext +- Replace direct DB writes with tx_manager calls +- **Benefit**: Automatic sync log + event emission + audit trail + +--- + +### 3. **TagManager** (Service Layer) + +**Location**: `core/src/ops/tags/manager.rs` + +**Pattern**: Direct ActiveModel inserts + +```rust +// Current tag manager pattern +pub async fn create_tag(&self, canonical_name: String, ...) -> Result { + let db = &*self.db; + + let active_model = tag::ActiveModel { + uuid: Set(tag.id), + canonical_name: Set(canonical_name), + // ... + }; + + let result = active_model.insert(db).await?; + + Ok(tag) +} +``` + +**TransactionManager Integration**: +```rust +// New pattern with TransactionManager +pub async fn create_tag(&self, canonical_name: String, ...) -> Result { + let tx_manager = self.tx_manager.clone(); + + let active_model = tag::ActiveModel { + uuid: Set(tag.id), + canonical_name: Set(canonical_name), + // ... + }; + + // If sync-worthy: + let tag = tx_manager.commit_transactional( + self.library, + active_model, + ).await?; + + // If not sync-worthy (internal operation): + let tag = tx_manager.commit_silent( + self.library, + active_model, + ).await?; + + Ok(tag) +} +``` + +**Refactoring Required**: ⚠️ **Minor** +- Inject TransactionManager into service constructors +- Replace .insert(db) with appropriate commit method +- **Benefit**: Sync-aware services + +--- + +## Raw SQL Usage Analysis + +### Current Raw SQL Patterns + +**Pattern 1**: Optimized bulk operations (closure table population) + +```rust +// core/src/ops/indexing/persistence.rs +txn.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + "INSERT INTO entry_closure (ancestor_id, descendant_id, depth) \ + SELECT ancestor_id, ?, depth + 1 \ + FROM entry_closure \ + WHERE descendant_id = ?", + vec![result.id.into(), parent_id.into()], +)).await?; +``` + +**TransactionManager Compatibility**: ✅ **Fully compatible** +- Raw SQL operations happen **inside the transaction** +- TransactionManager provides the transaction context +- No changes needed to these optimizations + +**Pattern 2**: FTS5 search queries (read-only) + +```rust +// core/src/ops/search/query.rs +db.query_all( + Statement::from_string( + DatabaseBackend::Sqlite, + format!("SELECT rowid FROM search_index WHERE search_index MATCH '{}'", query) + ) +).await?; +``` + +**TransactionManager Compatibility**: ✅ **No conflict** +- Read-only operations don't need TransactionManager +- Queries remain unchanged + +--- + +## Sync Log Infrastructure + +### Current State: ❌ **Does Not Exist** + +**Finding**: No `sync_log` table or entity exists in the current database schema. + +**Files Checked**: +- `core/src/infra/db/entities/`: No sync_log.rs +- No SyncLog ActiveModel +- No sync log creation in any write operations + +**Existing Related Infrastructure**: +1. ✅ **Audit Log** (`core/src/infra/db/entities/audit_log.rs`): Tracks user actions + - Used by ActionManager + - Tracks action status, errors, results + - NOT used for sync (library-local only) + +2. ✅ **Job Database** (`core/src/infra/job/database.rs`): Tracks job execution + - Separate database from library DB + - NOT synced between devices + - Used for resumable jobs + +3. ❌ **Sync Log**: Not implemented yet + +--- + +## TransactionManager Implementation Strategy + +### Phase 1: Create Sync Infrastructure + +**Step 1**: Create sync_log entity + +```rust +// core/src/infra/db/entities/sync_log.rs + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "sync_log")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + + // Core fields + pub sequence: i64, // Monotonically increasing per library + pub library_id: Uuid, + pub device_id: Uuid, + pub timestamp: chrono::DateTime, + + // Change tracking + pub model_type: String, // "entry", "tag", "bulk_operation" + pub record_id: String, // UUID of changed record + pub change_type: String, // "insert", "update", "delete", "bulk_insert" + pub version: i32, // Optimistic concurrency version + + // Data payload + pub data: serde_json::Value, // Full model data or metadata +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} + +// Add to core/src/infra/db/entities/mod.rs +pub mod sync_log; +pub use sync_log::Entity as SyncLog; +``` + +**Step 2**: Create migration + +```rust +// Add to database migrations +CREATE TABLE sync_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sequence INTEGER NOT NULL, + library_id TEXT NOT NULL, + device_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + model_type TEXT NOT NULL, + record_id TEXT NOT NULL, + change_type TEXT NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + data TEXT NOT NULL, -- JSON + + UNIQUE(library_id, sequence) +); + +CREATE INDEX idx_sync_log_library_sequence ON sync_log(library_id, sequence); +CREATE INDEX idx_sync_log_device ON sync_log(device_id); +CREATE INDEX idx_sync_log_model_record ON sync_log(model_type, record_id); +``` + +--- + +### Phase 2: Implement TransactionManager + +**File**: `core/src/infra/transaction/manager.rs` + +```rust +pub struct TransactionManager { + event_bus: Arc, + sync_sequence: Arc>>, +} + +impl TransactionManager { + /// Transactional commit: DB + Sync Log + Event + pub async fn commit_transactional( + &self, + library: Arc, + model: M::ActiveModel, + ) -> Result { + let library_id = library.id(); + let db = library.db().conn(); + + // Atomic transaction + let saved_model = db.transaction(|txn| async move { + // 1. Save main model + let saved = model.save(txn).await?; + + // 2. Create sync log entry + let sync_entry = self.create_sync_log_entry( + library_id, + &saved, + ChangeType::Upsert, + )?; + sync_entry.insert(txn).await?; + + Ok::<_, TransactionError>(saved) + }).await?; + + // 3. Emit event (outside transaction) + let event = self.build_event(&library_id, &saved_model); + self.event_bus.emit(event); + + Ok(saved_model) + } + + /// Bulk commit: DB + ONE metadata sync log + pub async fn commit_bulk( + &self, + library: Arc, + models: Vec, + operation: BulkOperation, + ) -> Result { + let library_id = library.id(); + let db = library.db().conn(); + + db.transaction(|txn| async move { + // 1. Bulk insert models + M::Entity::insert_many(models) + .exec(txn) + .await?; + + // 2. ONE sync log with metadata + let bulk_sync = self.create_bulk_sync_entry( + library_id, + &operation, + models.len(), + )?; + bulk_sync.insert(txn).await?; + + Ok::<_, TransactionError>(()) + }).await?; + + // 3. Summary event + self.event_bus.emit(Event::BulkOperationCompleted { + library_id, + operation, + affected_count: models.len(), + }); + + Ok(BulkResult { count: models.len() }) + } +} +``` + +--- + +### Phase 3: Refactor Existing Code + +**Priority 1: Indexer** (Highest impact) + +```rust +// Before +let txn = db.begin().await?; +for entry in entries { + entry.insert(&txn).await?; +} +txn.commit().await?; + +// After +tx_manager.commit_bulk( + library, + entries, + BulkOperation::InitialIndex { location_id } +).await?; +``` + +**Priority 2: User Actions** (Highest value) + +```rust +// Before +let model = entry::ActiveModel { /* ... */ }; +model.insert(db).await?; + +// After +tx_manager.commit_transactional(library, model).await?; +``` + +**Priority 3: Services** (TagManager, etc.) + +```rust +// Inject tx_manager into constructors +impl TagManager { + pub fn new( + db: Arc, + tx_manager: Arc, // ← NEW + ) -> Self { + // ... + } +} +``` + +--- + +## Compatibility Matrix + +| Component | Current Pattern | TransactionManager Method | Refactor Effort | Benefit | +|-----------|----------------|---------------------------|-----------------|---------| +| **Indexer** | Batch txn + bulk insert | `commit_bulk` | Moderate | 10x faster, sync aware | +| **Actions** | Direct insert via services | `commit_transactional` | Moderate | Auto sync + event | +| **TagManager** | Direct ActiveModel insert | `commit_transactional` or `commit_silent` | Minor | Sync aware | +| **LocationManager** | Spawns indexer job | Use indexer's commit_bulk | None | Inherits benefits | +| **Watcher** | Individual inserts | `commit_transactional_batch` | Minor | Batch optimization | +| **Raw SQL optimizations** | Inside transactions | Unchanged (use txn from manager) | None | Fully compatible | +| **Queries** | Read-only | Unchanged | None | No conflict | + +--- + +## Migration Path + +### Step 1: Foundation (Week 1) +- [ ] Create `sync_log` entity and migration +- [ ] Implement `TransactionManager` core +- [ ] Add to `CoreContext` +- [ ] Write unit tests + +### Step 2: Indexer (Week 2) +- [ ] Refactor indexer to use `commit_bulk` +- [ ] Benchmark before/after +- [ ] Integration tests +- [ ] Deploy to test library + +### Step 3: User Actions (Week 3) +- [ ] Refactor file operations (rename, tag, move) +- [ ] Refactor location operations +- [ ] Test sync log creation +- [ ] Test event emission + +### Step 4: Services (Week 4) +- [ ] Inject TransactionManager into TagManager +- [ ] Inject into other services +- [ ] Update all write operations +- [ ] Comprehensive integration tests + +### Step 5: Client Integration (Week 5+) +- [ ] Implement sync follower service +- [ ] Implement client cache +- [ ] Test end-to-end sync +- [ ] Performance testing + +--- + +## Risk Analysis + +### Low Risk ✅ + +1. **SeaORM Compatibility**: ✅ Perfect fit + - TransactionManager uses SeaORM's native transaction support + - No ORM layer changes needed + +2. **Raw SQL Compatibility**: ✅ No issues + - Raw SQL stays inside transactions + - TransactionManager provides transaction context + +3. **Backward Compatibility**: ✅ Non-breaking + - Existing code continues to work + - Gradual migration possible + - No API changes for external callers + +### Medium Risk ⚠️ + +1. **Refactoring Effort**: ⚠️ Moderate work required + - ~50 write locations across codebase + - Need to inject TransactionManager into services + - Testing effort substantial but manageable + +2. **Performance Impact**: ⚠️ Need validation + - Sync log writes add overhead + - Mitigated by bulk operations + - Need benchmarks before/after + +### Mitigation Strategies + +1. **Gradual Migration**: Start with indexer, then actions, then services +2. **Feature Flag**: Gate sync log creation behind config flag during rollout +3. **Performance Testing**: Benchmark each phase before moving to next +4. **Rollback Plan**: Keep old code paths until validated + +--- + +## Conclusion + +**Verdict**: ✅ **FULLY COMPATIBLE AND READY TO IMPLEMENT** + +The TransactionManager design is **architecturally sound** and **fully compatible** with the existing codebase: + +1. ✅ **No conflicts** with existing patterns +2. ✅ **Enhances** rather than replaces current code +3. ✅ **Gradual migration** path available +4. ✅ **Significant benefits**: Sync support, event emission, audit trail +5. ✅ **Performance improvements** for bulk operations + +**Recommendation**: **Proceed with implementation using the phased approach outlined above.** + +The TransactionManager will be the **foundation** for Spacedrive's sync architecture, and the current codebase is **well-structured** to integrate it cleanly. + diff --git a/docs/core/design/sync/UNIFIED_RESOURCE_EVENTS.md b/docs/core/design/sync/UNIFIED_RESOURCE_EVENTS.md new file mode 100644 index 000000000..42b91e8a8 --- /dev/null +++ b/docs/core/design/sync/UNIFIED_RESOURCE_EVENTS.md @@ -0,0 +1,740 @@ +# Unified Resource Event System + +## Problem Statement + +Current event system has ~40 specialized variants (`EntryCreated`, `VolumeAdded`, `JobStarted`, etc.), leading to: +- Manual event emission scattered across codebase +- No type safety between events and resources +- Clients must handle each variant specifically +- Adding new resources requires new event variants +- TransactionManager cannot automatically emit events + +**Observation from code**: Line 353 has a TODO: "events should have an envelope that contains the library_id instead of this" + +## Solution: Generic Resource Events + +All resources implementing `Identifiable` can use a unified event structure. TransactionManager emits these automatically. + +### Design + +```rust +// core/src/infra/event/mod.rs + +/// Unified event envelope wrapping all resource events +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct Event { + /// Event metadata + pub envelope: EventEnvelope, + + /// The actual event payload + pub kind: EventKind, +} + +/// Standard envelope for all events +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct EventEnvelope { + /// Event ID for deduplication/tracking + pub id: Uuid, + + /// When this event was created + pub timestamp: DateTime, + + /// Library context (if applicable) + pub library_id: Option, + + /// Sequence number for ordering (optional) + pub sequence: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +#[serde(tag = "type", content = "data")] +pub enum EventKind { + // ======================================== + // GENERIC RESOURCE EVENTS + // ======================================== + + /// A resource was created/updated (single) + ResourceChanged { + /// Resource type identifier (from Identifiable::resource_type) + resource_type: String, + + /// The full resource data (must implement Identifiable) + #[specta(skip)] // Clients reconstruct from JSON + resource: serde_json::Value, + }, + + /// Multiple resources changed in a batch + ResourceBatchChanged { + resource_type: String, + resources: Vec, + operation: BatchOperation, + }, + + /// A resource was deleted + ResourceDeleted { + resource_type: String, + resource_id: Uuid, + }, + + /// Bulk operation completed (notification only, no data transfer) + BulkOperationCompleted { + /// Type of resource affected + resource_type: String, + + /// Summary info + affected_count: usize, + operation_token: Uuid, + hints: serde_json::Value, // location_id, etc. + }, + + // ======================================== + // LIFECYCLE EVENTS (no resources) + // ======================================== + + CoreStarted, + CoreShutdown, + + LibraryOpened { id: Uuid, name: String }, + LibraryClosed { id: Uuid }, + + // ======================================== + // INFRASTRUCTURE EVENTS + // ======================================== + + /// Job lifecycle (not a domain resource) + Job { + job_id: String, + status: JobStatus, + progress: Option, + message: Option, + }, + + /// Raw filesystem changes (before DB resolution) + FsRawChange { + kind: FsRawEventKind, + }, + + /// Log streaming + LogMessage { + timestamp: DateTime, + level: String, + target: String, + message: String, + job_id: Option, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum BatchOperation { + Index, + Search, + Update, + WatcherBatch, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum JobStatus { + Queued, + Started, + Progress, + Completed { output: JobOutput }, + Failed { error: String }, + Cancelled, + Paused, + Resumed, +} +``` + +### TransactionManager Integration + +```rust +impl TransactionManager { + /// Emit a resource changed event (automatic) + fn emit_resource_changed( + &self, + library_id: Uuid, + resource: &R, + ) { + let event = Event { + envelope: EventEnvelope { + id: Uuid::new_v4(), + timestamp: Utc::now(), + library_id: Some(library_id), + sequence: None, + }, + kind: EventKind::ResourceChanged { + resource_type: R::resource_type().to_string(), + resource: serde_json::to_value(resource).unwrap(), + }, + }; + + self.event_bus.emit(event); + } + + /// Commit single resource (emits ResourceChanged) + pub async fn commit>( + &self, + library: Arc, + model: M, + ) -> Result { + let library_id = library.id(); + + // Atomic: DB + sync log + let saved = /* transaction logic */; + + // Build client resource + let resource = R::from(saved); + + // Auto-emit + self.emit_resource_changed(library_id, &resource); + + Ok(resource) + } + + /// Commit batch (emits ResourceBatchChanged) + pub async fn commit_batch( + &self, + library: Arc, + models: Vec, + ) -> Result, TxError> + where + M: Syncable + IntoActiveModel, + R: Identifiable + From, + { + let library_id = library.id(); + + // Atomic batch transaction + let saved_models = /* batch transaction */; + + // Build resources + let resources: Vec = saved_models.into_iter().map(R::from).collect(); + + // Emit batch event + let event = Event { + envelope: EventEnvelope { + id: Uuid::new_v4(), + timestamp: Utc::now(), + library_id: Some(library_id), + sequence: None, + }, + kind: EventKind::ResourceBatchChanged { + resource_type: R::resource_type().to_string(), + resources: resources.iter() + .map(|r| serde_json::to_value(r).unwrap()) + .collect(), + operation: BatchOperation::Update, + }, + }; + + self.event_bus.emit(event); + + Ok(resources) + } + + /// Bulk operation (emits BulkOperationCompleted) + pub async fn commit_bulk( + &self, + library: Arc, + changes: ChangeSet, + ) -> Result { + let library_id = library.id(); + + // Atomic bulk insert + metadata sync log + let token = /* bulk transaction */; + + // Emit summary event (no resource data!) + let event = Event { + envelope: EventEnvelope { + id: Uuid::new_v4(), + timestamp: Utc::now(), + library_id: Some(library_id), + sequence: None, + }, + kind: EventKind::BulkOperationCompleted { + resource_type: M::SYNC_MODEL.to_string(), + affected_count: changes.items.len(), + operation_token: token, + hints: changes.hints, + }, + }; + + self.event_bus.emit(event); + + Ok(BulkAck { affected: changes.items.len(), token }) + } +} +``` + +### Client Handling (Swift Example) + +```swift +// ZERO-FRICTION: Type registry (auto-generated from Rust via specta) +protocol CacheableResource: Identifiable, Codable { + static var resourceType: String { get } +} + +// Auto-generated registry (no manual maintenance!) +class ResourceTypeRegistry { + private static var decoders: [String: (Data) throws -> any CacheableResource] = [:] + + // Called automatically when types are loaded + static func register(_ type: T.Type) { + decoders[T.resourceType] = { data in + try JSONDecoder().decode(T.self, from: data) + } + } + + static func decode(resourceType: String, from data: Data) throws -> any CacheableResource { + guard let decoder = decoders[resourceType] else { + throw CacheError.unknownResourceType(resourceType) + } + return try decoder(data) + } +} + +// Types auto-register via property wrapper or extension +extension File: CacheableResource { + static let resourceType = "file" +} + +extension Album: CacheableResource { + static let resourceType = "album" +} + +extension Tag: CacheableResource { + static let resourceType = "tag" +} + +// Add new resources without touching ANY event handling code! +extension Location: CacheableResource { + static let resourceType = "location" +} + +// GENERIC event handler (ZERO switch statements!) +actor ResourceCache { + func handleEvent(_ event: Event) async { + switch event.kind { + case .ResourceChanged(let resourceType, let resourceJSON): + do { + // ✅ Generic decode - works for ALL resources! + let resource = try ResourceTypeRegistry.decode( + resourceType: resourceType, + from: resourceJSON + ) + updateEntity(resource) + } catch { + print("Failed to decode \(resourceType): \(error)") + } + + case .ResourceBatchChanged(let resourceType, let resourcesJSON, let operation): + // ✅ Generic batch decode + let resources = resourcesJSON.compactMap { json in + try? ResourceTypeRegistry.decode(resourceType: resourceType, from: json) + } + resources.forEach { updateEntity($0) } + + case .BulkOperationCompleted(let resourceType, let count, let token, let hints): + // Invalidate queries + print("Bulk op on \(resourceType): \(count) items") + invalidateQueriesForResource(resourceType, hints: hints) + + case .ResourceDeleted(let resourceType, let resourceId): + // ✅ Generic deletion + deleteEntity(resourceType: resourceType, id: resourceId) + + // Infrastructure events + case .Job(let jobId, let status, _, _): + updateJobStatus(jobId: jobId, status: status) + + default: + break + } + } + + // Generic entity update (works for all Identifiable resources) + func updateEntity(_ resource: any CacheableResource) { + let cacheKey = type(of: resource).resourceType + ":" + resource.id.uuidString + entityStore[cacheKey] = resource + + // Update all queries that reference this resource + invalidateQueriesContaining(cacheKey) + } + + // Generic deletion + func deleteEntity(resourceType: String, id: UUID) { + let cacheKey = resourceType + ":" + id.uuidString + entityStore.removeValue(forKey: cacheKey) + invalidateQueriesContaining(cacheKey) + } +} +``` + +**Key Innovation**: Type registry eliminates all switch statements! + +**Adding a new resource**: +```swift +// 1. Define type (auto-generated from Rust via specta) +struct Photo: CacheableResource { + let id: UUID + let albumId: UUID + let path: String + static let resourceType = "photo" +} + +// 2. That's it! Event handling automatically works. +// No changes to ResourceCache, no switch cases, nothing! +``` +``` + +### TypeScript Client Example + +```typescript +// ZERO-FRICTION: Type registry (auto-generated from Rust via specta) +interface CacheableResource { + id: string; +} + +// Auto-generated type map (from Rust types via specta) +type ResourceTypeMap = { + file: File; + album: Album; + tag: Tag; + location: Location; + // New types added automatically by codegen! +}; + +// Generic decoder with type safety +class ResourceTypeRegistry { + private static validators: Map CacheableResource> = new Map(); + + // Auto-register types (called during module init) + static register( + resourceType: string, + validator: (data: unknown) => T + ) { + this.validators.set(resourceType, validator); + } + + static decode(resourceType: string, data: unknown): CacheableResource { + const validator = this.validators.get(resourceType); + if (!validator) { + throw new Error(`Unknown resource type: ${resourceType}`); + } + return validator(data); + } +} + +// Types auto-register (could use decorators or explicit calls) +ResourceTypeRegistry.register('file', (data) => data as File); +ResourceTypeRegistry.register('album', (data) => data as Album); +ResourceTypeRegistry.register('tag', (data) => data as Tag); +// Add new types without touching event handler! + +// GENERIC event handler (ZERO switch statements!) +export class NormalizedCache { + handleEvent(event: Event) { + switch (event.kind.type) { + case 'ResourceChanged': { + const { resource_type, resource } = event.kind.data; + // ✅ Generic decode - works for ALL resources! + const decoded = ResourceTypeRegistry.decode(resource_type, resource); + this.updateEntity(resource_type, decoded); + break; + } + + case 'ResourceBatchChanged': { + const { resource_type, resources } = event.kind.data; + // ✅ Generic batch + resources.forEach(r => { + const decoded = ResourceTypeRegistry.decode(resource_type, r); + this.updateEntity(resource_type, decoded); + }); + break; + } + + case 'BulkOperationCompleted': { + const { resource_type, hints } = event.kind.data; + this.invalidateQueries(resource_type, hints); + break; + } + + case 'ResourceDeleted': { + const { resource_type, resource_id } = event.kind.data; + this.deleteEntity(resource_type, resource_id); + break; + } + } + } + + // ✅ Automatic cache update for ANY resource + private updateEntity(resourceType: string, resource: CacheableResource) { + const cacheKey = `${resourceType}:${resource.id}`; + this.entities.set(cacheKey, resource); + + // Trigger UI updates for queries using this resource + this.notifyQueries(cacheKey); + } + + // ✅ Generic deletion + private deleteEntity(resourceType: string, resourceId: string) { + const cacheKey = `${resourceType}:${resourceId}`; + this.entities.delete(cacheKey); + this.notifyQueries(cacheKey); + } +} + +// Adding a new resource (Photo): +// 1. Rust: impl Identifiable for Photo { resource_type() = "photo" } +// 2. Run: cargo run --bin specta-gen (regenerates TypeScript types) +// 3. TypeScript: import { Photo } from './bindings/Photo.ts' +// 4. ResourceTypeRegistry.register('photo', (data) => data as Photo); +// 5. Done! No changes to event handling, cache logic, nothing! +``` + +**With Build Script Automation** (fully automatic): +```typescript +// Auto-generated file: src/bindings/resourceRegistry.ts +// This file is generated by: cargo run --bin specta-gen +// DO NOT EDIT MANUALLY + +import { File } from './File'; +import { Album } from './Album'; +import { Tag } from './Tag'; +import { Location } from './Location'; +// ... all other Identifiable types + +// Registry is populated at module load time +export const resourceTypeMap = { + 'file': File, + 'album': Album, + 'tag': Tag, + 'location': Location, + // ... all other types +} as const; + +// Zero-config setup +Object.entries(resourceTypeMap).forEach(([type, validator]) => { + ResourceTypeRegistry.register(type, validator as any); +}); +``` + +**Result**: Adding a new Identifiable resource in Rust automatically: +1. Generates TypeScript type +2. Registers in type map +3. Works with event handling +4. **Zero manual client changes!** + +## Migration Strategy + +### Phase 1: Add Unified Events (Additive) +- Keep existing Event variants +- Add new `ResourceChanged`, `ResourceBatchChanged`, etc. +- TransactionManager emits new events +- Clients can start consuming new events + +### Phase 2: Migrate Resources One-by-One +For each resource (File, Album, Tag, Location, etc.): +1. Implement `Identifiable` trait +2. Switch from manual `event_bus.emit(Event::EntryCreated)` to TM +3. Update client to consume `ResourceChanged` for that type +4. Mark old event variant as deprecated + +### Phase 3: Remove Old Events +Once all resources migrated: +- Remove `EntryCreated`, `VolumeAdded`, etc. +- Keep infrastructure events (Job, Log, FsRawChange) +- Remove manual event emission from ops code + +## Benefits + +### For Rust Core +✅ **Zero boilerplate**: No manual event emission +✅ **Type safety**: TM ensures events match resources +✅ **Automatic**: Emit on every commit +✅ **Uniform**: All resources handled same way + +### For Clients +✅ **ZERO switch statements**: Type registry handles all resources +✅ **Type-safe deserialization**: JSON → typed resource +✅ **Zero-friction scaling**: Add 100 resources, no client changes +✅ **Auto-generated**: specta codegen creates registry automatically +✅ **Cache-friendly**: Direct integration with normalized cache + +### Horizontal Scaling +✅ **Rust**: Add `impl Identifiable` → automatic events +✅ **TypeScript**: Run codegen → automatic type + registry +✅ **Swift**: Add `CacheableResource` conformance → automatic handling +✅ **New platforms**: Implement type registry once, scales infinitely + +### For Maintenance +✅ **Less code**: ~40 variants → ~5 generic variants +✅ **No manual updates**: Adding File → Album → Tag reuses same code +✅ **Clear semantics**: Resource events vs infrastructure events +✅ **Centralized**: All emission in TransactionManager + +## Examples by Resource Type + +### Files (Entry → File) +```rust +// Rust +let file = tm.commit::(library, entry_model).await?; +// → Emits: ResourceChanged { resource_type: "file", resource: file } + +// Swift +case .ResourceChanged("file", let json): + let file = try decode(File.self, json) + cache.updateEntity(file) +``` + +### Albums +```rust +// Rust +let album = tm.commit::(library, album_model).await?; +// → Emits: ResourceChanged { resource_type: "album", resource: album } + +// Swift +case .ResourceChanged("album", let json): + let album = try decode(Album.self, json) + cache.updateEntity(album) +``` + +### Tags +```rust +// Rust +let tag = tm.commit::(library, tag_model).await?; +// → Emits: ResourceChanged { resource_type: "tag", resource: tag } + +// Swift +case .ResourceChanged("tag", let json): + let tag = try decode(Tag.self, json) + cache.updateEntity(tag) +``` + +### Locations +```rust +// Rust +let location = tm.commit::(library, location_model).await?; +// → Emits: ResourceChanged { resource_type: "location", resource: location } + +// Swift +case .ResourceChanged("location", let json): + let location = try decode(Location.self, json) + cache.updateEntity(location) +``` + +## Infrastructure Events (Not Resources) + +Some events are not domain resources: +- **Jobs**: Ephemeral, not cached, different lifecycle +- **Logs**: Streaming, not state +- **FsRawChange**: Pre-database, becomes Entry later +- **Core lifecycle**: System-level + +These keep specialized variants under `EventKind`. + +## Comparison: Before vs After + +### Before (Current) +```rust +// Scattered manual emission +pub async fn create_album(library: Arc, name: String) -> Result { + let model = albums::ActiveModel { /* ... */ }; + let saved = model.insert(db).await?; + + // Manual event emission + event_bus.emit(Event::AlbumCreated { + library_id: library.id(), + album_id: saved.uuid, + }); + + Ok(album) +} + +// Client must handle specific variant + switch case +case .AlbumCreated(let libraryId, let albumId): + // Fetch album data separately + let album = await client.query("albums.get", albumId) + cache.updateEntity(album) +``` + +### After (Unified + Type Registry) +```rust +// Automatic emission via TransactionManager +pub async fn create_album( + tm: &TransactionManager, + library: Arc, + name: String, +) -> Result { + let model = albums::ActiveModel { /* ... */ }; + + // TM emits ResourceChanged automatically + let album = tm.commit::(library, model).await?; + + Ok(album) +} + +// Client: ZERO resource-specific code! +case .ResourceChanged(let resourceType, let json): + // ✅ Works for Album, File, Tag, Location, everything! + let resource = try ResourceTypeRegistry.decode(resourceType, json) + cache.updateEntity(resource) + // Add 100 new resources: this code never changes! +``` + +**Adding a 101st resource**: +- Rust: `impl Identifiable for NewResource` (3 lines) +- Client: Nothing! (codegen handles it) + +**Horizontal scaling achieved!** 🎉 + +## Event Size Considerations + +**Concern**: Sending full resources in events increases bandwidth + +**Mitigations**: +1. **Gzip compression**: Event bus can compress large payloads +2. **Client caching**: Only send if resource changed +3. **Delta events** (future): Send only changed fields +4. **Bulk events**: Don't send individual resources (just metadata) + +**Measurement**: +- File resource: ~500 bytes JSON +- Album resource: ~200 bytes JSON +- Tag resource: ~150 bytes JSON + +Even with 100 concurrent updates: 500 bytes × 100 = 50KB (negligible) + +## Alternative: Lightweight Events + +If bandwidth becomes an issue, use two-tier system: + +```rust +pub enum EventKind { + // Lightweight: just ID + ResourceChanged { + resource_type: String, + resource_id: Uuid, + // Client fetches if needed + }, + + // Rich: full data (opt-in) + ResourceChangedRich { + resource_type: String, + resource: serde_json::Value, + }, +} +``` + +But start with rich events (simpler, better cache consistency). + +## Conclusion + +This unified event system: +- ✅ Eliminates ~35 specialized event variants +- ✅ Makes TransactionManager sole event emitter +- ✅ Enables generic client handling +- ✅ Reduces boilerplate to zero +- ✅ Scales to infinite resource types +- ✅ Aligns perfectly with Identifiable/Syncable design + +**Next Step**: Implement `Event` refactor alongside TransactionManager in mini-spec. diff --git a/docs/core/design/sync/UNIFIED_TRANSACTIONAL_SYNC_AND_CACHE.md b/docs/core/design/sync/UNIFIED_TRANSACTIONAL_SYNC_AND_CACHE.md new file mode 100644 index 000000000..d25ecfb84 --- /dev/null +++ b/docs/core/design/sync/UNIFIED_TRANSACTIONAL_SYNC_AND_CACHE.md @@ -0,0 +1,2294 @@ +# Unified Architecture: Transactional Sync and Real-Time Caching + +**Version**: 1.0 +**Status**: RFC / Design Document +**Date**: 2025-10-07 +**Authors**: James Pine with AI Assistant +**Related**: SYNC_DESIGN.md, NORMALIZED_CACHE_DESIGN.md, INFRA_LAYER_SEPARATION.md + +## Executive Summary + +This document presents a **unified architectural design** that integrates: +1. **Transactional backend sync** for data persistence across devices +2. **Real-time normalized client cache** for instant UI updates + +The cornerstone is a new **`TransactionManager`** service that acts as the single point of truth for all write operations, guaranteeing atomic consistency across: +- ✅ Database writes +- ✅ Sync log creation +- ✅ Event emission to clients + +This replaces scattered, non-transactional database writes with a robust, traceable persistence pattern that serves as the foundation for both reliable sync and real-time caching. + +## Core Innovation: Dual Model Architecture + +### The Fundamental Separation + +```rust +// PERSISTENCE LAYER (Sync's domain) +pub struct Entry { + pub id: i32, // Database primary key + pub uuid: Option, // Sync identifier + pub name: String, + pub size: i64, + pub version: i64, // For Syncable + pub last_modified_at: DateTime, + // Lean, normalized, database-focused +} + +impl Syncable for Entry { /* ... */ } + +// ──────────────────────────────────────── + +// QUERY LAYER (Client cache's domain) +pub struct File { + pub id: Uuid, // Client identifier + pub name: String, + pub size: u64, + pub tags: Vec, // Denormalized, rich + pub content_identity: Option, + pub sd_path: SdPath, + // Rich, computed, client-focused +} + +impl Identifiable for File { /* ... */ } +``` + +### Why This Separation Matters + +| Aspect | Entry (Persistence) | File (Query) | +|--------|---------------------|--------------| +| **Purpose** | Database storage, sync transport | Client API, UI display | +| **Structure** | Normalized, lean | Denormalized, rich | +| **Computation** | Direct from DB | Computed via joins | +| **Traits** | `Syncable` | `Identifiable` | +| **Identity** | i32 (DB), Uuid (sync) | Uuid (client cache) | +| **Mutability** | Mutable, versioned | Immutable snapshot | +| **Relationships** | Foreign keys (id) | Nested objects (full data) | + +**Key Insight**: Don't force one model to serve both purposes. Let each model excel at its job. + +## The TransactionManager: Unified Orchestration + +### Architecture + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Action Layer (Business Logic) │ +│ • Determines WHAT to change │ +│ • Creates ActiveModel instances │ +│ • Calls TransactionManager │ +└────────────────────┬─────────────────────────────────────────┘ + │ + ↓ +┌──────────────────────────────────────────────────────────────┐ +│ TransactionManager (Single Point of Write) │ +│ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Phase 1: ATOMIC TRANSACTION │ │ +│ │ BEGIN TRANSACTION │ │ +│ │ 1. Save persistence model (Entry) │ │ +│ │ 2. Create SyncLogEntry from Syncable trait │ │ +│ │ 3. Save SyncLogEntry │ │ +│ │ COMMIT │ │ +│ └────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Phase 2: POST-COMMIT (outside transaction) │ │ +│ │ 4. Compute query model (Entry → File) │ │ +│ │ 5. Emit event with query model │ │ +│ │ event_bus.emit(FileUpdated { file }) │ │ +│ └────────────────────────────────────────────────────────┘ │ +└────────────────────┬─────────────────────────────────────────┘ + │ + ┌──────────┴──────────┐ + │ │ + ↓ ↓ +┌─────────────────┐ ┌──────────────────┐ +│ Sync System │ │ Event Bus → │ +│ • SyncLogEntry │ │ Client Caches │ +│ • Followers │ │ • Normalized │ +│ • Replication │ │ • Real-time │ +└─────────────────┘ └──────────────────┘ +``` + +### Core Guarantees + +The `TransactionManager` provides **ironclad guarantees**: + +1. **Atomicity**: DB write + sync log = atomic or neither +2. **Ordering**: Sync log entries are sequential, ordered +3. **Completeness**: Every DB change has a sync log entry +4. **Reliability**: Events always fire after successful commits +5. **Traceability**: Every change is logged and auditable + +## Implementation Design + +### 1. TransactionManager Interface + +```rust +// core/src/infra/transaction/manager.rs + +use crate::{ + domain::{File, Tag, Location, Identifiable}, + infra::event::EventBus, + sync::{Syncable, SyncLogEntry, SyncChange}, +}; +use sea_orm::{DatabaseConnection, DatabaseTransaction, TransactionTrait}; +use std::sync::Arc; +use uuid::Uuid; + +/// Central service for all write operations +/// Guarantees atomic: DB + sync log + events +pub struct TransactionManager { + event_bus: Arc, + sync_sequence: Arc>>, // library_id → sequence +} + +impl TransactionManager { + pub fn new(event_bus: Arc) -> Self { + Self { + event_bus, + sync_sequence: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Core method: Commit a change with sync and events + pub async fn commit_entry_change( + &self, + library: Arc, + entry_model: entry::ActiveModel, + compute_file: F, + ) -> Result + where + F: FnOnce(&entry::Model) -> BoxFuture<'static, Result>, + { + let library_id = library.id(); + let db = library.db().conn(); + + // Phase 1: ATOMIC TRANSACTION + let saved_entry = db.transaction(|txn| async move { + // 1. Save entry to database + let saved_entry = entry_model.save(txn).await?; + + // 2. Create sync log entry from Syncable trait + let sync_entry = self.create_sync_log_entry( + library_id, + &saved_entry, + ChangeType::Upsert, + ).await?; + + // 3. Save sync log entry + sync_entry.insert(txn).await?; + + Ok::<_, TransactionError>(saved_entry) + }).await?; + + // Phase 2: POST-COMMIT (outside transaction) + + // 4. Compute rich File model from Entry + let file = compute_file(&saved_entry).await?; + + // 5. Emit event with File for client caches + self.event_bus.emit(Event::FileUpdated { + library_id, + file: file.clone(), + }); + + tracing::info!( + library_id = %library_id, + entry_id = %file.id, + "Transaction committed: DB + sync + event" + ); + + Ok(file) + } + + /// Batch commit for bulk operations + pub async fn commit_entry_batch( + &self, + library: Arc, + entries: Vec, + compute_files: F, + ) -> Result, TransactionError> + where + F: FnOnce(&[entry::Model]) -> BoxFuture<'static, Result, QueryError>>, + { + let library_id = library.id(); + let db = library.db().conn(); + + // Phase 1: ATOMIC BATCH TRANSACTION + let saved_entries = db.transaction(|txn| async move { + let mut saved = Vec::new(); + + for entry_model in entries { + // Save entry + let saved_entry = entry_model.save(txn).await?; + + // Create sync log entry + let sync_entry = self.create_sync_log_entry( + library_id, + &saved_entry, + ChangeType::Upsert, + ).await?; + + sync_entry.insert(txn).await?; + + saved.push(saved_entry); + } + + Ok::<_, TransactionError>(saved) + }).await?; + + // Phase 2: POST-COMMIT BATCH PROCESSING + + // Compute all Files in one go (single query with joins) + let files = compute_files(&saved_entries).await?; + + // Emit batch event + self.event_bus.emit(Event::FilesBatchUpdated { + library_id, + files: files.clone(), + }); + + tracing::info!( + library_id = %library_id, + count = files.len(), + "Batch transaction committed" + ); + + Ok(files) + } + + /// Create sync log entry from a Syncable model + fn create_sync_log_entry( + &self, + library_id: Uuid, + model: &S, + change_type: ChangeType, + ) -> Result { + let sequence = self.next_sequence(library_id); + + Ok(SyncLogEntryActiveModel { + sequence: Set(sequence), + library_id: Set(library_id), + model_type: Set(S::SYNC_ID.to_string()), + record_id: Set(model.id().to_string()), + version: Set(model.version()), + change_type: Set(change_type), + data: Set(serde_json::to_value(model)?), + timestamp: Set(model.last_modified_at()), + device_id: Set(self.get_device_id()), + ..Default::default() + }) + } + + fn next_sequence(&self, library_id: Uuid) -> u64 { + let mut sequences = self.sync_sequence.lock().unwrap(); + let seq = sequences.entry(library_id).or_insert(0); + *seq += 1; + *seq + } +} +``` + +### 2. Entry → File Conversion Service + +```rust +// core/src/domain/file_builder.rs + +/// Service for converting Entry persistence models to File query models +pub struct FileBuilder { + library: Arc, +} + +impl FileBuilder { + pub fn new(library: Arc) -> Self { + Self { library } + } + + /// Build a single File from an Entry with all relationships + pub async fn build_file_from_entry( + &self, + entry: &entry::Model, + ) -> QueryResult { + let db = self.library.db().conn(); + + // Single query with LEFT JOINs for all relationships + let file_data = self.fetch_file_data(entry.id, db).await?; + + Ok(File::from_construction_data(file_data)) + } + + /// Build multiple Files efficiently (single query) + pub async fn build_files_from_entries( + &self, + entries: &[entry::Model], + ) -> QueryResult> { + let db = self.library.db().conn(); + let entry_ids: Vec = entries.iter().map(|e| e.id).collect(); + + // Single query with joins for ALL entries + let files_data = self.fetch_batch_file_data(&entry_ids, db).await?; + + Ok(files_data.into_iter().map(File::from_construction_data).collect()) + } + + /// Optimized query with LEFT JOINs + async fn fetch_file_data( + &self, + entry_id: i32, + db: &DatabaseConnection, + ) -> QueryResult { + // SQL with joins: + // SELECT + // entry.*, + // content_identity.*, + // tags.*, + // sidecars.* + // FROM entry + // LEFT JOIN content_identity ON entry.content_id = content_identity.id + // LEFT JOIN entry_tags ON entry.id = entry_tags.entry_id + // LEFT JOIN tags ON entry_tags.tag_id = tags.id + // LEFT JOIN sidecars ON entry.content_id = sidecars.content_id + // WHERE entry.id = ? + + // (Implementation details...) + todo!() + } +} +``` + +### 3. Specialized TransactionManager Methods + +```rust +impl TransactionManager { + /// High-level method for renaming a file + pub async fn rename_entry( + &self, + library: Arc, + entry_id: Uuid, + new_name: String, + ) -> Result { + // Get current entry + let entry = self.find_entry_by_uuid(&library, entry_id).await?; + + // Create ActiveModel for update + let mut entry_model: entry::ActiveModel = entry.into(); + entry_model.name = Set(new_name); + entry_model.version = Set(entry_model.version.as_ref() + 1); + entry_model.last_modified_at = Set(Utc::now()); + + // Commit through manager + let file_builder = FileBuilder::new(library.clone()); + self.commit_entry_change( + library, + entry_model, + |saved_entry| { + Box::pin(async move { + file_builder.build_file_from_entry(saved_entry).await + }) + }, + ).await + } + + /// High-level method for applying a tag + pub async fn apply_tag_to_entry( + &self, + library: Arc, + entry_id: Uuid, + tag_id: Uuid, + ) -> Result { + let db = library.db().conn(); + + // Phase 1: Atomic transaction + let saved_entry = db.transaction(|txn| async move { + // 1. Create tag link + let tag_link = entry_tags::ActiveModel { + entry_id: Set(entry_id_i32), + tag_id: Set(tag_id_i32), + ..Default::default() + }; + tag_link.insert(txn).await?; + + // 2. Bump entry version (for sync) + let entry = self.find_entry_by_uuid_tx(txn, entry_id).await?; + let mut entry_model: entry::ActiveModel = entry.into(); + entry_model.version = Set(entry_model.version.as_ref() + 1); + let saved_entry = entry_model.update(txn).await?; + + // 3. Create sync log entries (for both models) + let entry_sync = self.create_sync_log_entry( + library.id(), + &saved_entry, + ChangeType::Update, + )?; + entry_sync.insert(txn).await?; + + let tag_link_sync = self.create_sync_log_entry( + library.id(), + &tag_link_model, + ChangeType::Insert, + )?; + tag_link_sync.insert(txn).await?; + + Ok::<_, TransactionError>(saved_entry) + }).await?; + + // Phase 2: Post-commit + let file_builder = FileBuilder::new(library.clone()); + let file = file_builder.build_file_from_entry(&saved_entry).await?; + + // Emit event with full File (includes new tag!) + self.event_bus.emit(Event::FileUpdated { + library_id: library.id(), + file: file.clone(), + }); + + Ok(file) + } + + /// Bulk indexing operation (optimized) + pub async fn index_entries_batch( + &self, + library: Arc, + entries: Vec, + ) -> Result, TransactionError> { + self.commit_entry_batch( + library.clone(), + entries, + |saved_entries| { + let file_builder = FileBuilder::new(library.clone()); + Box::pin(async move { + file_builder.build_files_from_entries(saved_entries).await + }) + }, + ).await + } +} +``` + +### 4. Integration with Existing Infrastructure + +#### Replace Direct Database Writes + +**Before** (scattered in indexer): +```rust +// ❌ Current pattern - no sync log, manual events, non-atomic +impl Indexer { + async fn process_file(&mut self, path: PathBuf) { + let entry = entry::ActiveModel { + name: Set(file_name), + size: Set(file_size), + // ... + }; + + // Direct write - bypasses sync! + entry.insert(self.db).await?; + + // Manual event - might not fire if code crashes here! + self.event_bus.emit(Event::EntryCreated { /* ... */ }); + } +} +``` + +**After** (using TransactionManager): +```rust +// ✅ New pattern - automatic sync log, guaranteed events, atomic +impl Indexer { + tx_manager: Arc, + + async fn process_file(&mut self, path: PathBuf) { + let entry = entry::ActiveModel { + name: Set(file_name), + size: Set(file_size), + // ... + }; + + // Single call handles everything atomically + let file = self.tx_manager.commit_entry_change( + self.library.clone(), + entry, + |saved| { + let file_builder = FileBuilder::new(self.library.clone()); + Box::pin(async move { + file_builder.build_file_from_entry(saved).await + }) + }, + ).await?; + + // That's it! Sync log created, event emitted automatically + } +} +``` + +## Benefits of Unified Architecture + +### For Sync System +- ✅ **Guaranteed consistency**: Sync log always matches database +- ✅ **No missed changes**: TransactionManager is the only write path +- ✅ **Atomic operations**: DB + sync log commit together or rollback together +- ✅ **Sequential ordering**: Sequence numbers assigned atomically +- ✅ **Centralized**: All sync log creation happens in one place + +### For Client Cache +- ✅ **Rich events**: Events contain full File objects, not just IDs +- ✅ **Guaranteed delivery**: Events always fire after successful commit +- ✅ **Atomic updates**: Cache receives complete, consistent data +- ✅ **No stale data**: Events reflect committed state, never in-progress +- ✅ **Type safety**: Identifiable trait ensures cache consistency + +### For Developers +- ✅ **Simple API**: One method call replaces multi-step process +- ✅ **Less error-prone**: Can't forget to create sync log or emit event +- ✅ **Testable**: Mock TransactionManager for tests +- ✅ **Traceable**: All writes go through one service +- ✅ **Maintainable**: Business logic separated from persistence mechanics + +## Data Flow Example: Complete Lifecycle + +### Scenario: User renames a file + +```rust +// 1. ACTION LAYER - Business logic +impl FileRenameAction { + async fn execute( + self, + library: Arc, + context: Arc, + ) -> ActionResult { + // Find entry by uuid + let entry = entry::Entity::find() + .filter(entry::Column::Uuid.eq(self.entry_id)) + .one(library.db().conn()) + .await? + .ok_or(ActionError::Internal("Entry not found".into()))?; + + // Prepare update + let mut entry_model: entry::ActiveModel = entry.into(); + entry_model.name = Set(self.new_name.clone()); + entry_model.version = Set(entry_model.version.as_ref() + 1); + entry_model.last_modified_at = Set(Utc::now()); + + // Commit through TransactionManager + let file = context + .transaction_manager() + .rename_entry(library, self.entry_id, self.new_name) + .await?; + + Ok(RenameOutput { + file, + success: true, + }) + } +} + +// 2. TRANSACTION MANAGER - Orchestration +// (See implementation above - handles all phases atomically) + +// 3. SYNC SYSTEM - Receives SyncLogEntry +// Leader device has new entry in sync log: +// SyncLogEntry { +// sequence: 1234, +// library_id: lib_uuid, +// model_type: "entry", +// record_id: entry_uuid, +// version: 5, +// change_type: Update, +// data: { "name": "new_name.jpg", ... }, +// timestamp: now, +// } + +// Followers pull this change and apply it + +// 4. EVENT BUS - Broadcasts to clients +// Event::FileUpdated { +// library_id: lib_uuid, +// file: File { +// id: entry_uuid, +// name: "new_name.jpg", +// tags: [...], // Full data +// // ... +// } +// } + +// 5. CLIENT CACHE - Atomic update +// Swift: +// cache.updateEntity(file) +// // UI updates instantly, no refetch! +``` + +## Critical Insight: Bulk Operations vs Transactional Operations + +### The Indexing Problem + +**Original design flaw**: Creating sync log entries for every file during indexing + +```rust +// ❌ PROBLEM: Indexer creates 1,000,000 entries +for entry in scanned_entries { + tx_manager.commit_entry_change(entry).await?; + // Creates 1,000,000 sync log entries! 😱 + // Each with its own transaction! + // Completely unnecessary - indexing is LOCAL +} +``` + +**Why this is wrong**: +1. **Indexing is not sync** - Each device indexes its own filesystem independently +2. **Sync log bloat** - Million entries for filesystem discovery +3. **Performance killer** - Million small transactions instead of one bulk insert +4. **Sync is for changes** - Initial index is not a "change" + +### The Solution: Context-Aware Commits + +The `TransactionManager` must differentiate between: + +| Context | Use Case | Sync Log? | Event? | Transaction Size | +|---------|----------|-----------|--------|------------------| +| **Transactional** | User renames file | ✅ Per entry | ✅ Rich (FileUpdated) | Single, small | +| **Bulk** | Indexer scans location | ✅ ONE metadata entry | ✅ Summary (LibraryIndexed) | Single, massive | +| **Silent** | Background maintenance | ❌ No | ❌ No | Varies | + +**Key distinction**: Bulk operations create **ONE sync log entry with metadata**, not millions of individual entries. + +## Refined TransactionManager Design + +### Core Methods + +```rust +// core/src/infra/transaction/manager.rs + +pub struct TransactionManager { + event_bus: Arc, + sync_sequence: Arc>>, +} + +impl TransactionManager { + /// Method 1: TRANSACTIONAL COMMIT + /// For user-driven, sync-worthy changes + /// Creates: DB write + sync log + rich event + pub async fn commit_transactional( + &self, + library: Arc, + entry_model: entry::ActiveModel, + ) -> Result { + let library_id = library.id(); + let db = library.db().conn(); + + // Phase 1: ATOMIC TRANSACTION + let saved_entry = db.transaction(|txn| async move { + // 1. Save entry + let saved = entry_model.save(txn).await?; + + // 2. Create & save sync log entry + let sync_entry = self.create_sync_log_entry( + library_id, + &saved, + ChangeType::Upsert, + )?; + sync_entry.insert(txn).await?; + + Ok::<_, TransactionError>(saved) + }).await?; + + // Phase 2: POST-COMMIT + let file = self.build_file_from_entry(&library, &saved_entry).await?; + + // Emit rich event for client cache + self.event_bus.emit(Event::FileUpdated { + library_id, + file: file.clone(), + }); + + tracing::info!( + entry_id = %file.id, + "Transactional commit: DB + sync + event" + ); + + Ok(file) + } + + /// Method 2: BULK COMMIT + /// For system operations like indexing + /// Creates: DB write + ONE summary sync log entry + pub async fn commit_bulk( + &self, + library: Arc, + entries: Vec, + operation_type: BulkOperation, + ) -> Result { + let library_id = library.id(); + let db = library.db().conn(); + + tracing::info!( + count = entries.len(), + operation = ?operation_type, + "Starting bulk commit" + ); + + // Phase 1: SINGLE BULK TRANSACTION + let saved_count = db.transaction(|txn| async move { + // 1. Bulk insert entries - highly optimized by database + let result = entry::Entity::insert_many(entries) + .exec(txn) + .await?; + + // 2. Create ONE sync log entry with metadata (not individual entries!) + let bulk_sync_entry = SyncLogEntryActiveModel { + sequence: Set(self.next_sequence(library_id)), + library_id: Set(library_id), + model_type: Set("bulk_operation".to_string()), + record_id: Set(Uuid::new_v4().to_string()), // Unique ID for this operation + version: Set(1), + change_type: Set(ChangeType::BulkInsert), + data: Set(json!({ + "operation": operation_type, + "affected_count": entries.len(), + "summary": "Bulk indexing operation", + // NO individual entry data! + })), + timestamp: Set(Utc::now()), + device_id: Set(self.get_device_id()), + ..Default::default() + }; + + bulk_sync_entry.insert(txn).await?; + + Ok::<_, TransactionError>(result.last_insert_id) + }).await?; + + // Phase 2: SUMMARY EVENT + // Don't compute 1M File objects! + self.event_bus.emit(Event::BulkOperationCompleted { + library_id, + operation: operation_type, + affected_count: entries.len(), + completed_at: Utc::now(), + }); + + tracing::info!( + count = entries.len(), + "Bulk commit: 1 sync log entry (metadata only), {} DB entries", + entries.len() + ); + + Ok(BulkCommitResult { + affected_count: entries.len(), + }) + } + + /// Method 3: SILENT COMMIT + /// For internal operations that don't need sync or events + /// Creates: DB write only + pub async fn commit_silent( + &self, + library: Arc, + entry_model: entry::ActiveModel, + ) -> Result { + let db = library.db().conn(); + + // Just save, no sync log, no event + let saved = entry_model.save(db).await?; + + Ok(saved) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum BulkOperation { + /// Initial indexing of a location + InitialIndex { location_id: Uuid }, + + /// Re-indexing after changes + ReIndex { location_id: Uuid }, + + /// Bulk import from external source + Import { source: String }, + + /// Background maintenance (cleanup, optimization) + Maintenance, +} + +#[derive(Debug, Clone)] +pub struct BulkCommitResult { + pub affected_count: usize, +} +``` + +### When to Use Each Method + +```rust +// USER ACTIONS → commit_transactional +// ✅ Rename file +// ✅ Tag file +// ✅ Move file +// ✅ Delete file (user-initiated) +// ✅ Update file metadata +// ✅ Create/update location (user action) + +// SYSTEM OPERATIONS → commit_bulk +// ✅ Initial indexing (1M files) +// ✅ Re-indexing after watcher events +// ✅ Bulk imports +// ✅ Background content identification + +// INTERNAL OPERATIONS → commit_silent +// ✅ Temp file cleanup +// ✅ Statistics updates +// ✅ Cache invalidation markers +// ✅ Internal state tracking +``` + +## Refined Sync Strategy + +### Index Sync: Watcher-Driven, Not Indexer-Driven + +**Key Realization**: The indexer creates the **initial** state, but sync tracks **changes** + +``` +Device A Device B +─────────────────────────────── ─────────────────────────────── +1. Indexer runs (bulk commit) + → 1M entries created + → ONE sync log entry ✅ + (metadata only: location_id, + count, operation type) + +2. User renames file + → Transactional commit + → Sync log entry ✅ + (full entry data) + → Event: FileUpdated ✅ + + 3. Sync service pulls changes + → Gets bulk operation metadata + → Sees: "Device A indexed location X" + → Triggers local indexing of same location + + 4. Sync service pulls rename + → Gets full entry data + → Applies to local DB + → Emits FileUpdated event + + 5. Indexer runs (bulk commit) + → 1M entries created locally + → ONE sync log entry ✅ +``` + +**Sync strategy per operation**: + +| Operation | Sync Log? | What's in Sync Log? | +|-----------|-----------|---------------------| +| Initial indexing | ✅ ONE metadata entry | `{ operation: "InitialIndex", location_id, count }` | +| Watcher: file created | ✅ Per-entry | Full entry data for each file | +| Watcher: file modified | ✅ Per-entry | Full entry data for each file | +| Watcher: file deleted | ✅ Per-entry | Entry ID + deletion marker | +| User: rename file | ✅ Per-entry | Full updated entry data | +| User: tag file | ✅ Per-entry | Updated entry + tag relationship | +| Background: thumbnail gen | ❌ No | N/A - derived data | + +### Indexer Integration + +```rust +// core/src/indexer/mod.rs + +impl Indexer { + tx_manager: Arc, + + /// Initial scan of a location (bulk operation) + pub async fn index_location_initial( + &mut self, + location_id: Uuid, + ) -> Result { + let mut entries = Vec::new(); + + // Scan filesystem + for path in self.scan_directory(&location_path) { + let metadata = fs::metadata(&path).await?; + let entry = self.create_entry_model(path, metadata); + entries.push(entry); + + // Batch in memory, don't write yet + } + + tracing::info!( + location_id = %location_id, + count = entries.len(), + "Scanned {} entries, starting bulk commit", + entries.len() + ); + + // ✅ Single bulk commit - no sync log + let result = self.tx_manager.commit_bulk( + self.library.clone(), + entries, + BulkOperation::InitialIndex { location_id }, + ).await?; + + // Client receives: Event::BulkOperationCompleted + // Client reaction: Invalidate "directory:/location_path" queries + + Ok(IndexResult { + location_id, + indexed_count: result.affected_count, + }) + } + + /// Process watcher event (transactional operation) + pub async fn handle_watcher_event( + &mut self, + event: WatcherEvent, + ) -> Result<(), IndexerError> { + match event { + WatcherEvent::Created(path) => { + let entry = self.create_entry_from_path(path).await?; + + // ✅ Transactional commit - creates sync log + let file = self.tx_manager.commit_transactional( + self.library.clone(), + entry, + ).await?; + + // Client receives: Event::FileUpdated { file } + // Client reaction: Update cache atomically + + tracing::info!(file_id = %file.id, "File created via watcher"); + } + + WatcherEvent::Modified(path) => { + // Similar - transactional commit + } + + WatcherEvent::Deleted(path) => { + // Similar - transactional commit + } + } + + Ok(()) + } +} +``` + +## Design Analysis: Why This is Brilliant + +### 1. Aligns with Domain Semantics ✅ + +Your insight about **"indexing is not sync"** is **architecturally correct**: + +- **Indexing** = Local filesystem discovery (each device does independently) +- **Sync** = Replicating changes between devices (coordination required) + +**Example**: +``` +Device A has /photos with 10,000 images +Device B has /documents with 5,000 PDFs + +When paired: +- Device A does NOT sync its 10K images to Device B +- Device B does NOT sync its 5K PDFs to Device A +- Each device keeps its own index + +BUT: +- User tags a photo on Device A → Sync to Device B ✅ +- User renames PDF on Device B → Sync to Device A ✅ +``` + +This matches the **Index Sync domain** from SYNC_DESIGN.md: "Mirror each device's local filesystem index" not "replicate all files". + +### 2. Performance is Critical ✅ + +**Bulk operations are** the bottleneck: +- Initial indexing: 1M+ files per location +- Re-indexing: 100K+ files after mount +- Imports: 10K+ files from external source + +**With per-entry sync logs**: +``` +1,000,000 files × (1 DB write + 1 sync log write + 1 event) = 3M operations +Time: ~10 minutes on SSD +Sync log size: ~500MB for just the index +``` + +**With bulk commits** (ONE sync log entry with metadata): +``` +1,000,000 files × (1 DB write) + 1 sync log entry = 1M + 1 operations +Time: ~1 minute on SSD (10x faster!) +Sync log size: ~500 bytes (just metadata, not 500MB!) + +Sync log entry contains: +{ + "sequence": 1234, + "model_type": "bulk_operation", + "operation": "InitialIndex", + "location_id": "uuid-123", + "affected_count": 1000000, + "device_id": "device-abc", + "timestamp": "2025-10-07T..." +} +``` + +### 3. Client Behavior is Appropriate ✅ + +**Client reaction to bulk event**: +```swift +case .BulkOperationCompleted(let libraryId, let operation, let count): + switch operation { + case .InitialIndex(let locationId): + print("📦 Indexed \(count) files in location \(locationId)") + + // Invalidate queries for this location + cache.invalidateQueriesMatching { query in + query.contains("directory:") && query.contains(locationId.uuidString) + } + + // Show UI notification + showToast("Indexed \(count) files") + + // Don't try to update 1M entities! + // Just invalidate and let queries refetch lazily + } +``` + +This is **correct** because: +- Users don't have 1M files loaded in memory anyway +- UI typically shows 50-100 files at once +- Lazy loading handles the rest +- Cache invalidation + refetch is the right pattern + +### 4. Watcher Integration is Perfect ✅ + +**Watcher creates individual sync entries** - exactly right: + +```rust +// Watcher detects: user created file in watched directory +WatcherEvent::Created("/photos/new_photo.jpg") + +// Indexer processes it TRANSACTIONALLY +tx_manager.commit_transactional(entry).await? +// → Sync log entry created ✅ +// → Other devices see the new file ✅ +// → Clients update cache atomically ✅ +``` + +This is **semantically correct**: +- File created **after** initial index = incremental change +- Incremental changes are sync-worthy +- Other devices should reflect this change + +## Additional Refinements + +### Refinement 1: Micro-Batch for Watcher Events + +**Problem**: Watcher emits 100 events in rapid succession (user copies folder) + +**Solution**: Micro-batching within transactional context + +```rust +impl TransactionManager { + /// Commit multiple entries in a single transaction with sync logs + /// Good for: Watcher batch events (10-100 files) + pub async fn commit_transactional_batch( + &self, + library: Arc, + entries: Vec, + ) -> Result, TransactionError> { + let library_id = library.id(); + let db = library.db().conn(); + + // Phase 1: Single transaction with sync logs + let saved_entries = db.transaction(|txn| async move { + let mut saved = Vec::new(); + + for entry_model in entries { + // Save entry + let saved_entry = entry_model.save(txn).await?; + + // Create sync log (sync-worthy!) + let sync_entry = self.create_sync_log_entry( + library_id, + &saved_entry, + ChangeType::Upsert, + )?; + sync_entry.insert(txn).await?; + + saved.push(saved_entry); + } + + Ok::<_, TransactionError>(saved) + }).await?; + + // Phase 2: Batch File construction (single query!) + let files = self.build_files_from_entries_batch( + &library, + &saved_entries, + ).await?; + + // Emit batch event + self.event_bus.emit(Event::FilesBatchUpdated { + library_id, + files: files.clone(), + operation: BatchOperation::WatcherBatch, + }); + + tracing::info!( + count = files.len(), + "Transactional batch commit: {} entries with sync logs", + files.len() + ); + + Ok(files) + } +} + +// Watcher integration +impl Indexer { + async fn handle_watcher_batch( + &mut self, + events: Vec, + ) -> Result<(), IndexerError> { + let mut entries = Vec::new(); + + for event in events { + if let Some(entry) = self.process_watcher_event_to_model(event).await? { + entries.push(entry); + } + } + + if !entries.is_empty() { + // Batch commit with sync logs + self.tx_manager.commit_transactional_batch( + self.library.clone(), + entries, + ).await?; + } + + Ok(()) + } +} +``` + +### Refinement 2: Selective Sync Log Fields + +**Problem**: Sync log stores full Entry JSON - wasteful for large models + +**Solution**: Only store sync-relevant fields + +```rust +impl Syncable for entry::ActiveModel { + fn sync_fields() -> Vec<&'static str> { + vec![ + "uuid", + "name", + "size", + "version", + "content_id", + "location_id", + "parent_id", + "metadata_id", + // Exclude: inode, file_id (platform-specific) + // Exclude: cached_thumbnail_path (derived data) + ] + } + + fn to_sync_json(&self) -> serde_json::Value { + // Only serialize sync-relevant fields + json!({ + "uuid": self.uuid, + "name": self.name, + "size": self.size, + // ... + }) + } +} + +impl TransactionManager { + fn create_sync_log_entry( + &self, + library_id: Uuid, + model: &S, + ) -> Result { + Ok(SyncLogEntryActiveModel { + // ... + data: Set(model.to_sync_json()), // ✅ Only sync fields + // ... + }) + } +} +``` + +### Refinement 3: Bulk Operation Sync Protocol + +**The Complete Picture**: What happens when Device B syncs from Device A + +#### Device A (Leader) - Creates Bulk Sync Entry + +```rust +// Device A: Indexes location with 1M files +tx_manager.commit_bulk( + library, + entries, // 1M entries + BulkOperation::InitialIndex { location_id } +).await?; + +// Sync log now contains ONE entry: +SyncLogEntry { + sequence: 1234, + library_id: lib_uuid, + model_type: "bulk_operation", + record_id: operation_uuid, + change_type: BulkInsert, + data: json!({ + "operation": "InitialIndex", + "location_id": location_uuid, + "location_path": "/Users/alice/Photos", + "affected_count": 1_000_000, + "index_statistics": { + "total_size": 50_000_000_000, + "file_count": 980_000, + "directory_count": 20_000, + } + }), + timestamp: now, + device_id: device_a_id, +} +``` + +#### Device B (Follower) - Processes Bulk Sync Entry + +```rust +impl SyncFollowerService { + async fn apply_sync_log_entry( + &mut self, + entry: SyncLogEntry, + ) -> Result<()> { + match entry.model_type.as_str() { + "bulk_operation" => { + // Parse bulk operation metadata + let operation: BulkOperationMetadata = serde_json::from_value(entry.data)?; + + self.handle_bulk_operation(operation).await?; + } + _ => { + // Regular sync log entry - apply normally + self.apply_regular_change(entry).await?; + } + } + + Ok(()) + } + + async fn handle_bulk_operation( + &mut self, + operation: BulkOperationMetadata, + ) -> Result<()> { + match operation.operation { + BulkOperation::InitialIndex { location_id, location_path } => { + tracing::info!( + location_id = %location_id, + count = operation.affected_count, + "Peer completed bulk index - checking if we need to index locally" + ); + + // Check if we have a matching location + // (same path, or user has linked it) + if let Some(local_location) = self.find_matching_location(&location_path).await? { + // We have this location too! Trigger our own index + tracing::info!( + local_location_id = %local_location.id, + "Triggering local indexing job" + ); + + self.job_manager.queue(IndexerJob { + location_id: local_location.id, + mode: IndexMode::Full, + }).await?; + } else { + // We don't have this location - that's fine + tracing::debug!( + "Peer indexed location we don't have - no action needed" + ); + } + + // Mark operation as processed + self.update_sync_position(operation.sequence).await?; + } + + _ => {} + } + + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BulkOperationMetadata { + pub sequence: u64, + pub operation: BulkOperation, + pub affected_count: usize, + pub index_statistics: Option, +} +``` + +#### Key Insight: Bulk Operations Don't Transfer Data + +**Important**: When Device B sees Device A's bulk index operation: +- ✅ Device B **triggers its own local indexing** job +- ❌ Device B does **NOT** pull 1M entries over the network +- ✅ Device B reads its own filesystem (fast, local) +- ❌ Device B does **NOT** try to replicate Device A's filesystem + +**Why this works**: +- Both devices are indexing **their own** filesystems +- Each device's index is independent +- Sync log entry is just a **notification**: "I indexed this location" +- Useful for UI ("Your library is being indexed on your other devices") + +**Example**: +``` +Device A (Laptop): /Users/alice/Photos → 1M images +Device B (Phone): /storage/DCIM → 500 photos + +Device A indexes /Users/alice/Photos: +→ Sync log: "Indexed location: /Users/alice/Photos, count: 1M" + +Device B receives sync entry: +→ Checks: Do I have /Users/alice/Photos? NO +→ Action: Nothing (I don't have that location) + +Device B indexes /storage/DCIM: +→ Sync log: "Indexed location: /storage/DCIM, count: 500" + +Device A receives sync entry: +→ Checks: Do I have /storage/DCIM? NO +→ Action: Nothing (I don't have that location) +``` + +**Each device maintains its own index. The sync log just tracks "what indexing happened."** + +#### What Actually Syncs Between Devices + +**Index data (entries)**: ❌ NOT synced via sync log during bulk indexing +- Each device indexes its own filesystem +- Sync log contains metadata notification only +- Result: Efficient, no network bottleneck + +**Metadata & changes**: ✅ Synced via sync log +- User tags a file → Sync log entry with full data +- User renames a file → Sync log entry with full data +- Location settings updated → Sync log entry with full data + +**Example flow**: +``` +Device A: Initial index → 1 sync log entry (metadata) +Device A: User tags photo → 1 sync log entry (full entry + tag data) + +Device B receives sync: +→ Sync entry 1: "Device A indexed /photos with 1M files" + Action: Trigger my own /photos index (if I have it) + +→ Sync entry 2: "Entry uuid-123 tagged with 'vacation'" + Action: Apply tag to my local entry uuid-123 +``` + +This is why the design is so efficient: +- Filesystem discovery: Local operation, metadata sync only +- Metadata changes: Full sync with complete data +- Best of both worlds! +``` + +## Performance Optimization 1: Lazy File Construction + +**Problem**: Computing File for every write is expensive + +**Solution**: Only compute when clients are listening + +```rust +impl TransactionManager { + pub async fn commit_entry_change_lazy( + &self, + library: Arc, + entry_model: entry::ActiveModel, + ) -> Result { + let library_id = library.id(); + let db = library.db().conn(); + + // Phase 1: Transaction (same as before) + let saved_entry = /* ... */; + + // Phase 2: Conditional File construction + if self.event_bus.has_subscribers() { + // Clients are connected - compute File + let file = FileBuilder::new(library.clone()) + .build_file_from_entry(&saved_entry) + .await?; + + self.event_bus.emit(Event::FileUpdated { + library_id, + file, + }); + } else { + // No clients - emit lightweight event + self.event_bus.emit(Event::EntryModified { + library_id, + entry_id: saved_entry.uuid.unwrap(), + }); + } + + Ok(saved_entry) + } +} +``` + +### Optimization 2: Batch File Construction + +**Problem**: Indexer creates 1000 entries, computing 1000 Files individually is slow + +**Solution**: Bulk join query + +```rust +impl FileBuilder { + /// Build multiple Files with a single query + pub async fn build_files_from_entries( + &self, + entries: &[entry::Model], + ) -> QueryResult> { + let db = self.library.db().conn(); + let entry_ids: Vec = entries.iter().map(|e| e.id).collect(); + + // Single query with all joins: + // SELECT * FROM entry + // LEFT JOIN content_identity ON ... + // LEFT JOIN entry_tags ON ... + // LEFT JOIN tags ON ... + // WHERE entry.id IN (?, ?, ?, ...) + + let rows = db.query_all(Statement::from_sql_and_values( + DbBackend::Sqlite, + r#" + SELECT + entry.*, + content_identity.uuid as ci_uuid, + content_identity.hash as ci_hash, + tags.uuid as tag_uuid, + tags.name as tag_name + FROM entry + LEFT JOIN content_identity ON entry.content_id = content_identity.id + LEFT JOIN entry_tags ON entry.id = entry_tags.entry_id + LEFT JOIN tags ON entry_tags.tag_id = tags.id + WHERE entry.id IN (?) + "#, + vec![entry_ids.into()], + )).await?; + + // Parse rows into FileConstructionData, group by entry_id + let files = self.parse_joined_rows(rows)?; + + Ok(files) + } +} +``` + +### Optimization 3: Event Batching + +**Problem**: 1000 FileUpdated events flood clients + +**Solution**: Batch events + +```rust +// Instead of: +for file in files { + event_bus.emit(Event::FileUpdated { file }); +} + +// Do: +event_bus.emit(Event::FilesBatchUpdated { + library_id, + files, // Vec + operation: BatchOperation::Index, +}); + +// Client handles batch: +for file in batch.files { + cache.updateEntity(file); // Still atomic per entity +} +``` + +## Error Handling Strategy + +### Transaction Failures + +```rust +impl TransactionManager { + pub async fn commit_entry_change( + // ... + ) -> Result { + // Phase 1: Transaction + let saved_entry = match db.transaction(|txn| async move { + // Save entry + sync log + }).await { + Ok(entry) => entry, + Err(e) => { + // Transaction rolled back automatically + tracing::error!( + library_id = %library_id, + error = %e, + "Transaction failed - rolled back" + ); + return Err(TransactionError::DatabaseError(e)); + } + }; + + // Phase 2: Post-commit (can't rollback!) + match compute_file(&saved_entry).await { + Ok(file) => { + // Success - emit event + self.event_bus.emit(Event::FileUpdated { /* ... */ }); + Ok(file) + } + Err(e) => { + // File construction failed, but DB committed! + tracing::error!( + entry_id = %saved_entry.uuid.unwrap(), + error = %e, + "File construction failed after commit - emitting lightweight event" + ); + + // Fallback: emit lightweight event + self.event_bus.emit(Event::EntryModified { + library_id: library.id(), + entry_id: saved_entry.uuid.unwrap(), + }); + + // Return error to action (partial success) + Err(TransactionError::FileConstructionFailed(e)) + } + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum TransactionError { + #[error("Database error: {0}")] + DatabaseError(#[from] sea_orm::DbErr), + + #[error("File construction failed: {0}")] + FileConstructionFailed(#[from] QueryError), + + #[error("Sync log creation failed: {0}")] + SyncLogError(String), + + #[error("Event emission failed: {0}")] + EventError(String), +} +``` + +## Integration Points + +### 1. CoreContext Extension + +```rust +// core/src/context.rs + +pub struct CoreContext { + // ... existing fields + + /// Central transaction manager for all writes + transaction_manager: Arc, + + /// File builder service for Entry → File conversion + file_builder_pool: Arc, // Pool for performance +} + +impl CoreContext { + pub fn transaction_manager(&self) -> &Arc { + &self.transaction_manager + } + + pub fn file_builder(&self, library: Arc) -> FileBuilder { + self.file_builder_pool.get(library) + } +} +``` + +### 2. Event Enum Extensions + +```rust +// core/src/infra/event/mod.rs + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum Event { + // ... existing events + + // ✅ Rich events with full Identifiable models + FileUpdated { + library_id: Uuid, + file: File, // Full File domain object + }, + + FilesBatchUpdated { + library_id: Uuid, + files: Vec, + operation: BatchOperation, + }, + + TagUpdated { + library_id: Uuid, + tag: Tag, + }, + + LocationUpdated { + library_id: Uuid, + location: Location, + }, + + JobUpdated { + library_id: Uuid, + job: JobInfo, // Implements Identifiable + }, + + // Relationship events (lightweight) + TagApplied { + library_id: Uuid, + file_id: Uuid, + tag_id: Uuid, + }, + + TagRemoved { + library_id: Uuid, + file_id: Uuid, + tag_id: Uuid, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum BatchOperation { + Index, + Search, + BulkUpdate, +} +``` + +### 3. Refactoring Checklist + +**Services to update**: +- [ ] Indexer - Replace all `entry.insert()` with `tx_manager.commit_entry_change()` +- [ ] VolumeManager - Use TransactionManager for location updates +- [ ] TagService - Use TransactionManager for tag operations +- [ ] FileOperations - Use TransactionManager for rename/move/delete + +**Pattern to find and replace**: +```rust +// Find: +entry_model.insert(db).await?; +// or +entry_model.update(db).await?; + +// Replace with: +tx_manager.commit_entry_change(library, entry_model, |saved| { + // File construction closure +}).await?; +``` + +## Advanced: Handling Edge Cases + +### Edge Case 1: File Construction Fails + +**Scenario**: Database commits, but computing File fails (e.g., corrupt data) + +**Handling**: +1. Transaction has committed - can't rollback +2. Sync log is created - followers will get the change +3. Emit lightweight event as fallback: `EntryModified { entry_id }` +4. Client invalidates affected queries, refetches on next access +5. Log error for investigation + +### Edge Case 2: Event Bus is Down + +**Scenario**: No clients connected, event bus has no subscribers + +**Handling**: +1. Check `event_bus.has_subscribers()` before computing File +2. If no subscribers, skip File construction (expensive) +3. Emit lightweight event or skip event entirely +4. Clients will refetch on next connection + +### Edge Case 3: Bulk Operation Partial Failure + +**Scenario**: Indexing 1000 files, one fails mid-transaction + +**Handling**: +1. Use sub-transactions or batch commits +2. Log failures, continue with remaining files +3. Emit batch event for successful files +4. Queue retry for failed files + +```rust +impl TransactionManager { + pub async fn commit_entry_batch_resilient( + &self, + library: Arc, + entries: Vec, + ) -> Result { + let mut successful = Vec::new(); + let mut failed = Vec::new(); + + // Commit in sub-batches + for chunk in entries.chunks(100) { + match self.commit_entry_batch_internal(library.clone(), chunk).await { + Ok(files) => successful.extend(files), + Err(e) => { + failed.push(BatchFailure { + entries: chunk.to_vec(), + error: e, + }); + } + } + } + + // Emit events for successful commits + if !successful.is_empty() { + self.event_bus.emit(Event::FilesBatchUpdated { + library_id: library.id(), + files: successful.clone(), + operation: BatchOperation::Index, + }); + } + + Ok(BatchCommitResult { + successful, + failed, + }) + } +} +``` + +## Migration Strategy + +### Phase 1: Infrastructure (Week 1) +- [ ] Create `TransactionManager` service +- [ ] Create `FileBuilder` service +- [ ] Add `version` field to Entry model +- [ ] Extend Event enum with rich events +- [ ] Add TransactionManager to CoreContext + +### Phase 2: Core Integration (Week 2) +- [ ] Update Indexer to use TransactionManager +- [ ] Update VolumeManager +- [ ] Update FileOperations (rename, move, delete) +- [ ] Update TagService + +### Phase 3: Testing & Validation (Week 3) +- [ ] Unit tests for TransactionManager +- [ ] Integration tests for sync consistency +- [ ] Verify events fire correctly +- [ ] Performance benchmarking + +### Phase 4: Rollout (Week 4) +- [ ] Deploy to staging +- [ ] Monitor sync logs for consistency +- [ ] Monitor event delivery +- [ ] Roll out to production + +## Design Validation: Addressing Concerns + +### Concern 1: Performance Impact + +**Question**: Is computing File on every write too slow? + +**Analysis**: +- **Write operations are infrequent** compared to reads +- **Indexing**: Batch commits amortize cost (1 query for 100 entries) +- **User actions**: Single file rename is already slow (user perception) +- **Optimization available**: Lazy construction when no clients connected + +**Verdict**: ✅ Acceptable with batching and lazy evaluation + +### Concern 2: Transaction Scope + +**Question**: What if File construction needs to write to DB (circular deps)? + +**Analysis**: +- **File construction is read-only** by design +- If additional writes needed, split into multiple transactions +- Example: Create Entry first, then create related resources + +**Verdict**: ✅ File construction must remain read-only + +### Concern 3: Event Ordering + +**Question**: Do events maintain order with sync log? + +**Analysis**: +- **Sync log**: Sequentially ordered by sequence number +- **Events**: Emitted in order of transaction commits +- **Guarantee**: If sync entry A has seq < B, event A fires before event B + +**Verdict**: ✅ Ordering is maintained by design + +## Comparison to Alternatives + +### Alternative 1: ORM Hooks Only + +**Approach**: Use SeaORM `after_save` hooks for everything + +**Problems**: +- ❌ Hooks are synchronous, can't do async File construction +- ❌ No control over transaction boundaries +- ❌ Can't batch operations +- ❌ Hard to test + +### Alternative 2: Event Sourcing + +**Approach**: Store events as primary source of truth + +**Problems**: +- ❌ Major architectural shift +- ❌ Requires event replay for current state +- ❌ Complex to query (need projections) +- ❌ Doesn't fit Spacedrive's model + +### Alternative 3: Distributed Transactions (2PC) + +**Approach**: Two-phase commit across DB + event bus + +**Problems**: +- ❌ Overly complex for single-process system +- ❌ Event bus doesn't support transactions +- ❌ Performance overhead +- ❌ Not necessary for local operations + +**Our Approach** (TransactionManager): +- ✅ Simple: One service, clear responsibility +- ✅ Performant: Single transaction, batch-friendly +- ✅ Testable: Easy to mock +- ✅ Pragmatic: Fits Spacedrive's architecture + +## Conclusion + +This unified architecture provides **guaranteed consistency** across three critical systems: + +1. **Database** (source of truth for persistence) +2. **Sync Log** (source of truth for replication) +3. **Client Cache** (source of truth for UI) + +By centralizing all write operations in the `TransactionManager`, we eliminate an entire class of bugs (missed sync entries, missing events, inconsistent state) while providing a clean, maintainable API for developers. + +The dual-model approach (Entry for persistence, File for queries) allows each layer to excel at its purpose without compromise. The TransactionManager serves as the bridge, guaranteeing that changes flow atomically from persistence to sync to clients. + +**This is the foundation for reliable, real-time, multi-device Spacedrive.** + +--- + +## Appendix: Complete Code Example + +### Complete Action Implementation + +```rust +// core/src/ops/files/rename/action.rs + +use crate::{ + context::CoreContext, + domain::File, + infra::action::{LibraryAction, ActionError, ActionResult}, + infra::transaction::TransactionManager, + library::Library, +}; + +pub struct FileRenameAction { + pub entry_id: Uuid, + pub new_name: String, +} + +#[async_trait] +impl LibraryAction for FileRenameAction { + type Output = FileRenameOutput; + type Input = FileRenameInput; + + fn from_input(input: Self::Input) -> Result { + Ok(Self { + entry_id: input.entry_id, + new_name: input.new_name, + }) + } + + async fn validate( + &self, + library: Arc, + _context: Arc, + ) -> Result<(), ActionError> { + // Validate entry exists + let db = library.db().conn(); + let exists = entry::Entity::find() + .filter(entry::Column::Uuid.eq(self.entry_id)) + .count(db) + .await? > 0; + + if !exists { + return Err(ActionError::Internal("Entry not found".into())); + } + + Ok(()) + } + + async fn execute( + self, + library: Arc, + context: Arc, + ) -> ActionResult { + // Use TransactionManager for atomic write + sync + event + let file = context + .transaction_manager() + .rename_entry(library, self.entry_id, self.new_name) + .await + .map_err(|e| ActionError::Internal(e.to_string()))?; + + Ok(FileRenameOutput { + file, + success: true, + }) + } + + fn action_kind(&self) -> &'static str { + "files.rename" + } +} + +#[derive(Debug, Serialize, Deserialize, Type)] +pub struct FileRenameOutput { + pub file: File, + pub success: bool, +} +``` + +### Complete Client Integration + +```swift +// Client-side cache receives and applies the update + +class EventCacheUpdater { + let cache: NormalizedCache + + func handleEvent(_ event: Event) async { + switch event { + case .FileUpdated(let libraryId, let file): + // Atomic cache update + await cache.updateEntity(file) + + // All views observing this file update automatically + print("✅ Updated File:\(file.id) - \(file.name)") + + case .FilesBatchUpdated(let libraryId, let files, let operation): + // Batch update + for file in files { + await cache.updateEntity(file) + } + print("✅ Batch updated \(files.count) files") + + default: + break + } + } +} + +// SwiftUI view observes cache +struct FileListView: View { + @ObservedObject var cache: NormalizedCache + let queryKey: String + + var files: [File] { + cache.getQueryResult(queryKey: queryKey) ?? [] + } + + var body: some View { + List(files, id: \.id) { file in + FileRow(file: file) + // When FileUpdated event arrives: + // 1. Cache updates + // 2. This view re-renders + // 3. User sees new name instantly + } + } +} +``` + +## Summary: The Three Commit Patterns + +### Decision Matrix + +Use this matrix to determine which commit method to use: + +| Scenario | Method | Rationale | Example | +|----------|--------|-----------|---------| +| User action on single file | `commit_transactional` | Sync-worthy, needs cache update | Rename, tag, move | +| Watcher: 1-10 files | `commit_transactional` | Sync-worthy, real-time update | User creates files | +| Watcher: 10-1000 files | `commit_transactional_batch` | Sync-worthy, optimize with batch | User copies folder | +| Watcher: 1000+ files | `commit_bulk` | Too many for sync log | User moves large directory | +| Initial indexing | `commit_bulk` | Not sync-worthy, local operation | Indexer first run | +| Background tasks | `commit_silent` | Not sync-worthy, no UI impact | Stats update, cleanup | + +### When Sync Log is Created + +```rust +// ✅ CREATES SYNC LOG (sync-worthy changes): +- User renames file (commit_transactional) +- User tags file (commit_transactional) +- User moves file (commit_transactional) +- Watcher: file created/modified/deleted (commit_transactional_batch) +- User updates location settings (commit_transactional) + +// ❌ NO SYNC LOG (local operations): +- Initial indexing (commit_bulk) +- Bulk imports (commit_bulk) +- Re-indexing after mount (commit_bulk) +- Thumbnail generation (commit_silent) +- Statistics updates (commit_silent) +- Temp file cleanup (commit_silent) +``` + +### The Semantic Distinction + +**The key insight**: Distinguish between **discovery** and **change** + +- **Discovery** (Indexer): "Here's what exists on my filesystem" + - Not sync-worthy (each device discovers independently) + - Use `commit_bulk` + +- **Change** (Watcher/User): "Something changed from known state" + - Sync-worthy (other devices need to know) + - Use `commit_transactional` + +This aligns perfectly with the **Index Sync** concept from SYNC_DESIGN.md: +> "Index Sync mirrors each device's local filesystem index and file-specific metadata" + +The **index itself** is local. The **changes to the index** (after initial discovery) are synced. + +### Implementation Checklist + +**Phase 1: Build TransactionManager** +- [ ] Implement `commit_transactional` method +- [ ] Implement `commit_bulk` method +- [ ] Implement `commit_silent` method +- [ ] Implement `commit_transactional_batch` method +- [ ] Add FileBuilder service +- [ ] Add to CoreContext + +**Phase 2: Refactor Indexer** +- [ ] Replace initial scan writes with `commit_bulk` +- [ ] Replace watcher writes with `commit_transactional` or `commit_transactional_batch` +- [ ] Add batching logic for watcher events +- [ ] Benchmark: before vs after + +**Phase 3: Refactor User Actions** +- [ ] FileRenameAction → `commit_transactional` +- [ ] FileTagAction → `commit_transactional` +- [ ] FileMoveAction → `commit_transactional` +- [ ] FileDeleteAction → `commit_transactional` +- [ ] LocationUpdateAction → `commit_transactional` + +**Phase 4: Client Integration** +- [ ] Handle `Event::FileUpdated` → atomic cache update +- [ ] Handle `Event::FilesBatchUpdated` → batch cache update +- [ ] Handle `Event::BulkOperationCompleted` → invalidate queries +- [ ] Test real-time updates work +- [ ] Measure cache hit rate + +### Final Architecture Diagram + +``` +┌────────────────────────────────────────────────────────────────┐ +│ USER ACTION (e.g., rename file) │ +└─────────────────────┬──────────────────────────────────────────┘ + ↓ + ┌────────────────────────────┐ + │ commit_transactional() │ + └────────────────────────────┘ + ↓ + ┌────────────────────────────┐ + │ DB + Sync Log (atomic) │ ← Single transaction + └────────────────────────────┘ + ↓ + ┌────────────────────────────┐ + │ Build File (query) │ ← Outside transaction + └────────────────────────────┘ + ↓ + ┌────────────────────────────┐ + │ Event::FileUpdated │ ← Rich event + └─────────────┬──────────────┘ + │ + ┌───────────┴───────────┐ + ↓ ↓ + ┌─────────────┐ ┌──────────────┐ + │ Sync System │ │ Client Cache │ + │ (Followers) │ │ (Atomic │ + │ │ │ Update) │ + └─────────────┘ └──────────────┘ + + +┌────────────────────────────────────────────────────────────────┐ +│ SYSTEM OPERATION (e.g., index 1M files) │ +└─────────────────────┬──────────────────────────────────────────┘ + ↓ + ┌────────────────────────────┐ + │ commit_bulk() │ + └────────────────────────────┘ + ↓ + ┌────────────────────────────┐ + │ DB only (no sync log) │ ← Bulk insert + └────────────────────────────┘ + ↓ + ┌────────────────────────────┐ + │ Event::BulkOperation │ ← Summary event + │ Completed │ + └─────────────┬──────────────┘ + │ + ┌───────────┴───────────┐ + ↓ ↓ + ┌─────────────┐ ┌──────────────┐ + │ Sync System │ │ Client Cache │ + │ (Triggers │ │ (Invalidate │ + │ local │ │ Queries) │ + │ index) │ └──────────────┘ + └─────────────┘ +``` + +## Critical Design Decisions + +### Decision 1: Indexing is Local ✅ + +**Rationale**: Each device has different files +- Device A indexes /photos → 10K images +- Device B indexes /documents → 5K PDFs +- No need to sync the indexes themselves +- Sync the **metadata** and **changes** instead + +### Decision 2: Watcher Events are Sync-Worthy ✅ + +**Rationale**: Watcher captures real filesystem changes +- User creates file → Other devices should know +- User modifies file → Content may have changed, sync metadata +- User deletes file → Other devices should mark as deleted + +### Decision 3: Bulk Events Don't Need Individual Updates ✅ + +**Rationale**: Clients can't handle 1M updates anyway +- Invalidate affected queries +- Refetch on demand (lazy) +- Better UX than freezing UI with 1M updates + +### Decision 4: Three Methods, Not One ✅ + +**Rationale**: Different semantics require different handling +- Don't force one pattern to serve all use cases +- Each method is optimized for its scenario +- Clear separation of concerns + +## Why This Design is Production-Ready + +### 1. **Correct Semantics** ✅ +- Indexing ≠ Sync (domain separation) +- Discovery ≠ Change (operational separation) +- Bulk ≠ Transactional (performance separation) + +### 2. **Performance** ✅ +- Indexing: 10x faster (bulk insert) +- Sync log: 100x smaller (only changes) +- Events: Appropriate granularity + +### 3. **Maintainability** ✅ +- Clear API: developers know which method to use +- Self-documenting: method names describe purpose +- Easy to test: each method isolated + +### 4. **Extensibility** ✅ +- New bulk operations: add to BulkOperation enum +- New event types: extend Event enum +- New sync strategies: implement in TransactionManager + +## Potential Concerns & Mitigations + +### Concern 1: "What if watcher batch is 100K files?" + +**Answer**: Use heuristic threshold + +```rust +impl Indexer { + const TRANSACTIONAL_BATCH_THRESHOLD: usize = 1000; + + async fn handle_watcher_batch(&mut self, events: Vec) { + if events.len() > Self::TRANSACTIONAL_BATCH_THRESHOLD { + // Too large for individual sync logs - use bulk + self.tx_manager.commit_bulk( + self.library.clone(), + entries, + BulkOperation::WatcherLargeBatch, + ).await?; + } else { + // Small enough - create sync logs + self.tx_manager.commit_transactional_batch( + self.library.clone(), + entries, + ).await?; + } + } +} +``` + +### Concern 2: "Clients miss bulk operation event?" + +**Answer**: Clients invalidate on reconnect anyway + +```swift +func onReconnect(libraryId: UUID) async { + // Check if anything changed while offline + let lastEventId = cache.getLastEventId(libraryId) + + // Fetch event summary since disconnect + let missedEvents = try await client.query( + "query:events.since.v1", + input: EventsSinceInput(lastEventId: lastEventId) + ) + + // If bulk operation happened, invalidate and refetch + for event in missedEvents { + if case .BulkOperationCompleted = event { + cache.invalidateLibrary(libraryId) + break + } + } +} +``` + +### Concern 3: "How to handle 'in-between' sizes?" + +**Answer**: Use `commit_transactional_batch` with pragmatic limits + +```rust +// Heuristic thresholds +const SINGLE_THRESHOLD: usize = 1; // 1 file → commit_transactional +const BATCH_THRESHOLD: usize = 1000; // < 1K → commit_transactional_batch +const BULK_THRESHOLD: usize = 1000; // ≥ 1K → commit_bulk + +match entries.len() { + 0 => Ok(()), + 1 => self.commit_transactional(entries.pop().unwrap()).await, + n if n < BATCH_THRESHOLD => self.commit_transactional_batch(entries).await, + _ => self.commit_bulk(entries, operation_type).await, +} +``` + +## Conclusion: A Unified, Pragmatic Architecture + +This design achieves the **perfect balance**: + +- ✅ **Transactional safety** for user actions (sync + cache) +- ✅ **Bulk performance** for system operations (indexing) +- ✅ **Clear semantics** (discovery vs change, bulk vs transactional) +- ✅ **Client-appropriate events** (rich for changes, summary for bulk) + +The three-method approach (`transactional`, `bulk`, `silent`) provides the flexibility needed for real-world performance while maintaining the atomic guarantees required for data consistency. + +**This is production-ready and scales from single-file edits to million-file indexes.** + +--- + +This unified architecture provides a solid foundation for both reliable multi-device sync and instant, real-time UI updates, with the performance characteristics needed for Spacedrive's scale. diff --git a/docs/core/devices.md b/docs/core/devices.md new file mode 100644 index 000000000..74445cc08 --- /dev/null +++ b/docs/core/devices.md @@ -0,0 +1,1290 @@ +# Spacedrive Device System + +**Status**: Production +**Version**: 2.0 +**Last Updated**: 2025-10-08 + +## Overview + +Devices are the fundamental building blocks of Spacedrive's multi-device architecture. A **Device** represents a single machine (laptop, phone, server) running Spacedrive. Devices can pair with each other, share libraries, and synchronize data. + +This document covers the complete device lifecycle from initialization to pairing to sync participation. + +## Architecture: Three Layers + +Devices exist across three distinct layers: + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ 1. IDENTITY LAYER (device/manager.rs, device/config.rs) │ +│ • Device initialization and configuration │ +│ • Persistent device ID and metadata │ +│ • Master encryption key management │ +│ • Platform-specific detection (OS, hardware) │ +└──────────────────────────────────────────────────────────────────┘ + ↓ +┌──────────────────────────────────────────────────────────────────┐ +│ 2. DOMAIN LAYER (domain/device.rs, infra/db/entities/device.rs) │ +│ • Rich Device domain model │ +│ • Sync leadership per library │ +│ • Online/offline state │ +│ • Database persistence in each library │ +└──────────────────────────────────────────────────────────────────┘ + ↓ +┌──────────────────────────────────────────────────────────────────┐ +│ 3. NETWORK LAYER (service/network/device/) │ +│ • P2P discovery and connections │ +│ • Device pairing protocol │ +│ • Connection state management │ +│ • Session key management │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## Layer 1: Device Identity + +### DeviceManager + +The `DeviceManager` manages the **current device's** identity and configuration. + +**Location**: `core/src/device/manager.rs` + +```rust +pub struct DeviceManager { + config: Arc>, + device_key_manager: DeviceKeyManager, + data_dir: Option, +} + +impl DeviceManager { + /// Initialize device (creates new ID on first run) + pub fn init() -> Result; + + /// Initialize with custom data directory (for iOS/Android) + pub fn init_with_path_and_name( + data_dir: &PathBuf, + device_name: Option, + ) -> Result; + + /// Get the current device's UUID + pub fn device_id(&self) -> Result; + + /// Get device as domain model + pub fn to_device(&self) -> Result; + + /// Update device name + pub fn set_name(&self, name: String) -> Result<(), DeviceError>; + + /// Get master encryption key + pub fn master_key(&self) -> Result<[u8; 32], DeviceError>; +} +``` + +### DeviceConfig + +Persistent configuration stored on disk. + +**Location**: `core/src/device/config.rs` + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeviceConfig { + /// Unique device identifier (generated once, never changes) + pub id: Uuid, + + /// User-friendly device name (can be updated) + pub name: String, + + /// When this device was first initialized + pub created_at: DateTime, + + /// Hardware model (e.g., "MacBook Pro 16-inch 2023") + pub hardware_model: Option, + + /// Operating system + pub os: String, + + /// Spacedrive version that created this config + pub version: String, +} +``` + +**Storage Location**: +- macOS: `~/Library/Application Support/com.spacedrive/device.json` +- Linux: `~/.config/spacedrive/device.json` +- Windows: `%APPDATA%/Spacedrive/device.json` +- iOS/Android: Custom data directory (passed via `init_with_path`) + +### Global Device ID + +For performance, the device ID is cached globally. + +**Location**: `core/src/device/id.rs` + +```rust +/// Global reference to current device ID +pub static CURRENT_DEVICE_ID: Lazy> = Lazy::new(|| RwLock::new(Uuid::nil())); + +/// Initialize the current device ID (called during Core init) +pub fn set_current_device_id(id: Uuid); + +/// Get the current device ID (fast, no error handling) +pub fn get_current_device_id() -> Uuid; +``` + +**Usage**: +```rust +// During Core initialization +let device_manager = DeviceManager::init()?; +set_current_device_id(device_manager.device_id()?); + +// Anywhere in the codebase +let device_id = get_current_device_id(); +``` + +**Rationale**: +- Device ID accessed frequently (audit logs, sync entries, actions) +- Immutable once set (no concurrency concerns) +- Performance: Avoids Arc overhead on every access +- Convenience: No need to pass CoreContext everywhere + +## Layer 2: Domain & Database + +### Device Domain Model + +The rich domain model used in application logic and API responses. + +**Location**: `core/src/domain/device.rs` + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Device { + /// Unique identifier + pub id: Uuid, + + /// Human-readable name + pub name: String, + + /// Operating system + pub os: OperatingSystem, + + /// Hardware model (e.g., "MacBook Pro", "iPhone 15") + pub hardware_model: Option, + + /// Network addresses for P2P connections + pub network_addresses: Vec, + + /// Whether this device is currently online + pub is_online: bool, + + /// Sync leadership status per library + pub sync_leadership: HashMap, + + /// Last time this device was seen + pub last_seen_at: DateTime, + + /// When this device was first added + pub created_at: DateTime, + + /// When this device info was last updated + pub updated_at: DateTime, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)] +pub enum SyncRole { + /// This device maintains the sync log for the library + Leader, + + /// This device syncs from the leader + Follower, + + /// This device doesn't participate in sync for this library + Inactive, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)] +pub enum OperatingSystem { + MacOS, + Windows, + Linux, + IOs, + Android, + Other, +} +``` + +### Key Methods + +```rust +impl Device { + /// Create the current device + pub fn current() -> Self; + + /// Mark device as online/offline + pub fn mark_online(&mut self); + pub fn mark_offline(&mut self); + + /// Sync role management + pub fn set_sync_role(&mut self, library_id: Uuid, role: SyncRole); + pub fn sync_role(&self, library_id: &Uuid) -> SyncRole; + pub fn is_sync_leader(&self, library_id: &Uuid) -> bool; + pub fn leader_libraries(&self) -> Vec; +} +``` + +### Database Entity + +Devices are stored **per library** (not globally). + +**Location**: `core/src/infra/db/entities/device.rs` + +```rust +#[derive(Clone, Debug, DeriveEntityModel)] +#[sea_orm(table_name = "devices")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, // Database primary key + pub uuid: Uuid, // Global device identifier + pub name: String, + pub os: String, + pub os_version: Option, + pub hardware_model: Option, + pub network_addresses: Json, // Vec + pub is_online: bool, + pub last_seen_at: DateTimeUtc, + pub capabilities: Json, // DeviceCapabilities + pub sync_leadership: Json, // HashMap + pub created_at: DateTimeUtc, + pub updated_at: DateTimeUtc, +} +``` + +**Why per-library?** +- Different devices may have access to different libraries +- Sync role is library-specific (leader in Library A, follower in Library B) +- Library-specific device metadata (last_seen per library) + +## Layer 3: Network + +### DeviceRegistry + +Central state manager for all network-layer device interactions. + +**Location**: `core/src/service/network/device/registry.rs` + +```rust +pub struct DeviceRegistry { + device_manager: Arc, + devices: HashMap, + node_to_device: HashMap, + session_to_device: HashMap, + persistence: DevicePersistence, +} + +pub enum DeviceState { + /// Discovered via Iroh (not yet paired) + Discovered { + node_id: NodeId, + node_addr: NodeAddr, + discovered_at: DateTime, + }, + + /// Pairing in progress + Pairing { + node_id: NodeId, + session_id: Uuid, + started_at: DateTime, + }, + + /// Successfully paired (persisted) + Paired { + info: DeviceInfo, + session_keys: SessionKeys, + paired_at: DateTime, + }, + + /// Currently connected (active P2P connection) + Connected { + info: DeviceInfo, + session_keys: SessionKeys, + connection: ConnectionInfo, + connected_at: DateTime, + }, + + /// Disconnected (but still paired) + Disconnected { + info: DeviceInfo, + session_keys: SessionKeys, + last_seen: DateTime, + reason: DisconnectionReason, + }, +} +``` + +### Device Lifecycle + +``` +┌─────────────┐ +│ Unknown │ +└──────┬──────┘ + │ Iroh discovery + ↓ +┌─────────────┐ +│ Discovered │ ← Device found on network +└──────┬──────┘ + │ User initiates pairing + ↓ +┌─────────────┐ +│ Pairing │ ← Cryptographic handshake +└──────┬──────┘ + │ Challenge/response succeeds + ↓ +┌─────────────┐ +│ Paired │ ← Persisted, can reconnect +└──────┬──────┘ + │ P2P connection established + ↓ +┌─────────────┐ +│ Connected │ ← Active, can send messages +└──────┬──────┘ + │ Connection lost + ↓ +┌─────────────┐ +│ Disconnected│ ← Can reconnect +└─────────────┘ + │ Auto-reconnect + └─→ Connected +``` + +### DeviceInfo + +Metadata exchanged during pairing and stored in registry. + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeviceInfo { + pub device_id: Uuid, + pub device_name: String, + pub device_type: DeviceType, + pub os_version: String, + pub app_version: String, + pub network_fingerprint: NetworkFingerprint, + pub last_seen: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum DeviceType { + Desktop, + Laptop, + Mobile, + Server, + Other(String), +} +``` + +## Device Pairing Protocol + +Cryptographic authentication between two devices. + +**Location**: `core/src/service/network/protocol/pairing/` + +### Pairing Flow + +``` +Device A (Initiator) Device B (Joiner) +───────────────────── ────────────────── +1. Generate pairing code + → "ABCD-1234-EFGH" + +2. Display code to user + 3. User enters code + + 4. PairingRequest → + { device_info, public_key } + +5. Generate challenge + ← Challenge + { challenge_bytes } + + 6. Sign challenge with private key + + 7. Response → + { signature, device_info } + +8. Verify signature + +9. Derive shared secret + +10. Complete → + { success: true } + 11. Save as paired device + +12. Save as paired device + +13. Both devices now in "Paired" state + • Can establish P2P connections + • Can discover each other's libraries + • Can set up library sync +``` + +### Pairing Actions + +**Initiate pairing**: +```rust +client.action("network.pair.generate.v1", {}) → PairingCode +``` + +**Join pairing**: +```rust +client.action("network.pair.join.v1", { code: "ABCD-1234-EFGH" }) → Success +``` + +**Query pairing status**: +```rust +client.query("network.pair.status.v1", {}) → PairingStatus +``` + +## Device Discovery + +Devices discover each other via **Iroh's mDNS** on local networks. + +```rust +// Iroh automatically discovers nearby nodes +// NetworkingService listens for discovery events + +// When node discovered: +device_registry.add_discovered_node(device_id, node_id, node_addr); +// State: Discovered + +// User can now initiate pairing with this device +``` + +## Device Registration in Libraries + +After pairing, devices must be **registered in each other's libraries** to enable sync. + +**Process** (see `sync-setup.md`): +1. Pair devices (network layer) +2. Discover remote libraries +3. Register Device B in Library A's database +4. Register Device A in Library B's database +5. Elect sync leader +6. Start sync service + +**Database Entry**: +```sql +-- In Library A's database +INSERT INTO devices (uuid, name, os, sync_leadership, ...) +VALUES ('device-b-uuid', 'Bob's MacBook', 'macOS', '{"lib-a-uuid": "Follower"}', ...); + +-- In Library B's database +INSERT INTO devices (uuid, name, os, sync_leadership, ...) +VALUES ('device-a-uuid', 'Alice's iPhone', 'iOS', '{"lib-b-uuid": "Leader"}', ...); +``` + +## Sync Leadership + +Each library has **one leader device** that assigns sync log sequence numbers. + +### Leadership Model + +```rust +// Device domain model tracks leadership per library +pub struct Device { + pub sync_leadership: HashMap, // library_id → role +} + +impl Device { + pub fn set_sync_role(&mut self, library_id: Uuid, role: SyncRole); + pub fn is_sync_leader(&self, library_id: &Uuid) -> bool; + pub fn leader_libraries(&self) -> Vec; +} +``` + +### Election Strategy + +1. **Initial leader**: Device that creates the library +2. **Explicit assignment**: During library sync setup +3. **Failover** (future): Heartbeat-based re-election if leader goes offline + +### Usage in TransactionManager + +```rust +impl TransactionManager { + async fn next_sequence(&self, library_id: Uuid) -> Result { + // Check if current device is leader + if !self.is_leader(library_id).await { + return Err(TxError::NotLeader); + } + + // Assign next sequence number + let mut sequences = self.sync_sequence.lock().unwrap(); + let seq = sequences.entry(library_id).or_insert(0); + *seq += 1; + Ok(*seq) + } + + async fn is_leader(&self, library_id: Uuid) -> bool { + // Query device table in library database + let device = self.get_current_device(library_id).await?; + device.is_sync_leader(&library_id) + } +} +``` + +## Device Relationships + +Devices have relationships with other core entities: + +### Devices ↔ Libraries + +**Relationship**: Many-to-Many +- One device can access multiple libraries +- One library can be accessed by multiple devices +- Each device has a role (Leader/Follower/Inactive) per library + +**Implementation**: +- Devices stored in each library's database +- Global device registry managed by NetworkingService +- Library sync setup creates bidirectional registration + +### Devices ↔ Locations + +**Relationship**: One-to-Many +- Each location belongs to one device +- One device can have multiple locations + +**Schema**: +```sql +CREATE TABLE locations ( + id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL, + device_id INTEGER NOT NULL, -- Foreign key to devices.id + entry_id INTEGER NOT NULL, + -- ... + FOREIGN KEY (device_id) REFERENCES devices(id) +); +``` + +**Semantics**: +- `/Users/alice/Photos` on Device A is a different location from `/storage/DCIM` on Device B +- Each device indexes its own filesystem +- Location ownership never changes (location is tied to device) + +### Devices ↔ Volumes + +**Relationship**: One-to-Many +- Each volume belongs to one device +- One device can have multiple volumes (drives) + +**Schema**: +```sql +CREATE TABLE volumes ( + id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL, + device_id TEXT NOT NULL, -- Foreign key to devices.uuid + fingerprint TEXT NOT NULL, + -- ... +); +``` + +**Semantics**: +- Volumes are device-specific (external SSD on Device A) +- Volume fingerprints enable cross-device recognition (same SSD connected to Device B) +- Volume metadata syncs (name, capacity) but content does not (unless user configures sync conduit) + +## Queries and Actions + +### Query: List Paired Devices + +**Endpoint**: `query:network.devices.list.v1` + +**Location**: `core/src/ops/network/devices/query.rs` + +```rust +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct ListPairedDevicesInput { + /// Whether to include only connected devices + #[serde(default)] + pub connected_only: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct ListPairedDevicesOutput { + pub devices: Vec, + pub total: usize, + pub connected: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct PairedDeviceInfo { + pub id: Uuid, + pub name: String, + pub device_type: String, + pub os_version: String, + pub app_version: String, + pub is_connected: bool, + pub last_seen: DateTime, +} +``` + +**Usage**: +```rust +// Get all paired devices +let output = client.query("network.devices.list.v1", { + connected_only: false +}).await?; + +println!("Paired devices: {}", output.total); +println!("Connected: {}", output.connected); +``` + +### Action: Generate Pairing Code + +**Endpoint**: `action:network.pair.generate.v1` + +**Location**: `core/src/ops/network/pair/generate/action.rs` + +```rust +pub struct GeneratePairingCodeAction; + +impl CoreAction for GeneratePairingCodeAction { + type Output = PairingCodeOutput; + + async fn execute(self, context: Arc) -> ActionResult { + let networking = context.get_networking().await?; + let code = networking.start_pairing().await?; + + Ok(PairingCodeOutput { + code: code.to_string(), + expires_at: Utc::now() + Duration::seconds(300), // 5 minutes + }) + } +} +``` + +### Action: Join Pairing + +**Endpoint**: `action:network.pair.join.v1` + +```rust +pub struct JoinPairingAction { + pub code: String, +} + +impl CoreAction for JoinPairingAction { + type Output = JoinPairingOutput; + + async fn execute(self, context: Arc) -> ActionResult { + let networking = context.get_networking().await?; + let pairing_code = PairingCode::from_string(&self.code)?; + + networking.join_pairing(pairing_code).await?; + + Ok(JoinPairingOutput { success: true }) + } +} +``` + +## Device as an Identifiable Resource + +Devices should be **cacheable** on the client. + +### Implementation + +```rust +impl Identifiable for Device { + type Id = Uuid; + + fn resource_id(&self) -> Self::Id { + self.id + } + + fn resource_type() -> &'static str { + "device" + } +} +``` + +### Syncable Implementation + +Devices sync across libraries when registered. + +```rust +impl Syncable for entities::device::Model { + const SYNC_MODEL: &'static str = "device"; + + fn sync_id(&self) -> Uuid { + self.uuid + } + + fn version(&self) -> i64 { + // Devices use timestamp as version + self.updated_at.timestamp() + } + + fn exclude_fields() -> Option<&'static [&'static str]> { + Some(&[ + "id", // Database primary key + "is_online", // Ephemeral state + "network_addresses", // Network-specific + ]) + } +} +``` + +**What syncs**: +- ✅ Device name changes +- ✅ Hardware model updates +- ✅ Sync role assignments +- ❌ Online/offline status (ephemeral) +- ❌ Network addresses (connection-specific) + +## Device Events + +Using the unified event system: + +```rust +// When device connects +Event { + envelope: { id, timestamp, library_id: None }, + kind: ResourceChanged { + resource_type: "device", + resource: Device { id, name, is_online: true, ... } + } +} + +// When device disconnects +Event { + envelope: { id, timestamp, library_id: None }, + kind: ResourceChanged { + resource_type: "device", + resource: Device { id, name, is_online: false, ... } + } +} + +// When sync role changes +Event { + envelope: { id, timestamp, library_id: Some(lib_uuid) }, + kind: ResourceChanged { + resource_type: "device", + resource: Device { id, name, sync_leadership: { lib_uuid: Leader }, ... } + } +} +``` + +**Client handling** (automatic via type registry): +```swift +// NO device-specific code needed! +// Generic handler works automatically: +case .ResourceChanged("device", let json): + let device = try ResourceTypeRegistry.decode("device", from: json) + cache.updateEntity(device) + // UI showing device list updates instantly! +``` + +## Security + +### Cryptographic Identity + +Each device has a unique cryptographic identity managed by Iroh: +- **NodeId**: Derived from Ed25519 public key +- **Key pair**: Generated and stored securely by Iroh +- **NetworkFingerprint**: Combines NodeId + device UUID + +### Session Keys + +After pairing, devices derive session keys for encrypted communication: + +```rust +pub struct SessionKeys { + pub encrypt_key: [u8; 32], + pub decrypt_key: [u8; 32], + pub mac_key: [u8; 32], +} + +impl SessionKeys { + /// Derive from shared secret (via ECDH) + pub fn from_shared_secret(secret: Vec) -> Self; +} +``` + +### Trust Levels + +```rust +pub enum TrustLevel { + /// Cryptographically verified via pairing + Verified, + + /// User manually approved + Trusted, + + /// Pending verification + Pending, + + /// Explicitly untrusted + Blocked, +} +``` + +## Persistence + +### Network Layer Persistence + +Paired devices persisted to survive app restarts. + +**Location**: `~/.spacedrive/paired_devices.json` + +```rust +pub struct PersistedPairedDevice { + pub device_id: Uuid, + pub device_info: DeviceInfo, + pub session_keys: SessionKeys, + pub trust_level: TrustLevel, + pub paired_at: DateTime, + pub auto_reconnect: bool, +} +``` + +### Library Database Persistence + +Devices registered in each library's database. + +**Table**: `devices` (per library) + +```sql +CREATE TABLE devices ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + uuid TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + os TEXT NOT NULL, + os_version TEXT, + hardware_model TEXT, + network_addresses TEXT NOT NULL, -- JSON array + is_online BOOLEAN NOT NULL DEFAULT 0, + last_seen_at TEXT NOT NULL, + capabilities TEXT NOT NULL, -- JSON object + sync_leadership TEXT NOT NULL, -- JSON: { "lib-uuid": "Leader" } + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE INDEX idx_devices_uuid ON devices(uuid); +``` + +## API Examples + +### Swift Client + +```swift +// List paired devices +let devices = try await client.query( + "network.devices.list.v1", + input: ListPairedDevicesInput(connectedOnly: false) +) + +print("Paired devices: \(devices.total)") +for device in devices.devices { + print("\(device.name) - \(device.isConnected ? "Connected" : "Offline")") +} + +// Start pairing +let pairingCode = try await client.action( + "network.pair.generate.v1", + input: EmptyInput() +) + +print("Pairing code: \(pairingCode.code)") +print("Show this code to the other device") + +// Join pairing +let result = try await client.action( + "network.pair.join.v1", + input: JoinPairingInput(code: "ABCD-1234-EFGH") +) + +if result.success { + print("Successfully paired!") +} +``` + +### TypeScript Client + +```typescript +// List paired devices +const devices = await client.query('network.devices.list.v1', { + connectedOnly: false +}); + +console.log(`Paired devices: ${devices.total}`); +devices.devices.forEach(device => { + console.log(`${device.name} - ${device.isConnected ? 'Connected' : 'Offline'}`); +}); + +// Generate pairing code +const pairing = await client.action('network.pair.generate.v1', {}); +console.log(`Pairing code: ${pairing.code}`); + +// Join pairing +const result = await client.action('network.pair.join.v1', { + code: 'ABCD-1234-EFGH' +}); +``` + +## Device State Queries + +### Get Current Device + +```rust +// Get the device running this code +let device_manager = context.device_manager(); +let device = device_manager.to_device()?; + +println!("Device ID: {}", device.id); +println!("Device name: {}", device.name); +println!("OS: {}", device.os); +``` + +### Get Device by ID + +```rust +// Query device from library database +let device = entities::device::Entity::find() + .filter(entities::device::Column::Uuid.eq(device_id)) + .one(library.db().conn()) + .await? + .ok_or(QueryError::DeviceNotFound(device_id))?; + +let domain_device = Device::try_from(device)?; +``` + +### Get Network State + +```rust +// Get device network state (from registry) +let networking = context.get_networking().await?; +let registry = networking.device_registry(); +let registry_lock = registry.read().await; + +if let Some(state) = registry_lock.get_device_state(device_id) { + match state { + DeviceState::Connected { connection, .. } => { + println!("Device connected with {} addresses", connection.addresses.len()); + } + DeviceState::Paired { .. } => { + println!("Device paired but not connected"); + } + _ => {} + } +} +``` + +## Platform-Specific Considerations + +### iOS/Android + +Mobile platforms require special handling: + +```rust +// iOS: UIDevice.name from Swift passed to Rust +let device_manager = DeviceManager::init_with_path_and_name( + &app_data_dir, + Some(ui_device_name), // From UIDevice.current.name +)?; + +// Device name updates when user changes it in Settings +// (On next app launch, name is updated in config) +``` + +### Desktop vs Mobile + +```rust +fn detect_device_type() -> DeviceType { + if cfg!(target_os = "ios") || cfg!(target_os = "android") { + DeviceType::Mobile + } else if cfg!(target_os = "macos") { + // Could detect MacBook vs iMac vs Mac Pro + DeviceType::Laptop + } else { + DeviceType::Desktop + } +} +``` + +## Integration with Core Systems + +### With Libraries + +```rust +// Get all devices in a library +let devices = entities::device::Entity::find() + .all(library.db().conn()) + .await?; + +// Check if device has access to library +let has_access = devices.iter().any(|d| d.uuid == device_id); + +// Get sync leader for library +let leader = devices.iter() + .find(|d| { + let sync_leadership: HashMap = + serde_json::from_value(d.sync_leadership.clone()).unwrap(); + matches!(sync_leadership.get(&library_id), Some(SyncRole::Leader)) + }); +``` + +### With Locations + +```rust +// Get all locations on a device +let locations = entities::location::Entity::find() + .filter(entities::location::Column::DeviceId.eq(device_db_id)) + .all(library.db().conn()) + .await?; + +// Location indexing is device-local +// Each device indexes its own filesystem independently +``` + +### With Volumes + +```rust +// Get all volumes on a device +let volumes = entities::volume::Entity::find() + .filter(entities::volume::Column::DeviceId.eq(device_uuid)) + .all(library.db().conn()) + .await?; + +// Volumes follow devices (USB drive connected to Device A) +// Volume fingerprints enable cross-device recognition +``` + +### With Sync System + +```rust +// Check if this device should sync +if device.is_sync_leader(&library_id) { + // This device creates sync logs + tm.commit(library, model).await?; +} else { + // This device is a follower - apply sync entries + sync_follower.sync_iteration().await?; +} +``` + +## Testing + +### Unit Tests + +```rust +#[test] +fn test_device_creation() { + let device = Device::current(); + assert!(!device.id.is_nil()); + assert!(!device.name.is_empty()); +} + +#[test] +fn test_sync_role_management() { + let mut device = Device::new("Test Device".into()); + let library_id = Uuid::new_v4(); + + // Initially inactive + assert_eq!(device.sync_role(&library_id), SyncRole::Inactive); + + // Set as leader + device.set_sync_role(library_id, SyncRole::Leader); + assert!(device.is_sync_leader(&library_id)); + + // Get leader libraries + let leaders = device.leader_libraries(); + assert_eq!(leaders.len(), 1); +} +``` + +### Integration Tests + +```rust +#[tokio::test] +async fn test_device_pairing_flow() { + // Device A generates code + let code = device_a.generate_pairing_code().await?; + + // Device B joins + device_b.join_pairing(code).await?; + + // Verify both in Paired state + let a_state = device_a.get_device_state(device_b.id()); + assert!(matches!(a_state, DeviceState::Paired { .. })); + + let b_state = device_b.get_device_state(device_a.id()); + assert!(matches!(b_state, DeviceState::Paired { .. })); +} +``` + +## Performance + +### Device Lookups + +- **By UUID**: Indexed, O(1) lookup +- **By NodeId**: HashMap in DeviceRegistry, O(1) +- **By session**: HashMap in DeviceRegistry, O(1) +- **All devices**: O(n) scan, but typically <10 devices per library + +### Network State + +- DeviceRegistry holds in-memory state (fast) +- Persistence updates are async (no blocking) +- Auto-reconnect on startup (loads paired devices from disk) + +## Monitoring + +### Device Status + +```rust +// Query core status includes device info +let status = client.query("core.status.v1", {}).await?; +println!("Current device: {}", status.device_name); +println!("Device ID: {}", status.device_id); + +// List paired devices with connection status +let devices = client.query("network.devices.list.v1", {}).await?; +println!("{} of {} devices connected", devices.connected, devices.total); +``` + +### Events + +```rust +// Device connected event +Event { + kind: ResourceChanged { + resource_type: "device", + resource: Device { is_online: true, ... } + } +} + +// Device disconnected event +Event { + kind: ResourceChanged { + resource_type: "device", + resource: Device { is_online: false, ... } + } +} +``` + +## Troubleshooting + +### Device Not Pairing + +**Symptom**: Pairing fails or times out + +**Checks**: +1. Both devices on same network? +2. mDNS discovery working? (check Iroh logs) +3. Firewall blocking connections? +4. Pairing code entered correctly? +5. Pairing code expired? (5 minute TTL) + +**Debug**: +```bash +# Check device discovery +RUST_LOG=iroh=debug,sd_core::service::network=debug cargo run + +# Look for: +# - "Node discovered via mDNS" +# - "Pairing request received" +# - "Challenge/response exchange" +``` + +### Device Showing as Offline + +**Symptom**: Paired device shows offline but is actually running + +**Checks**: +1. Connection lost? (network change, sleep) +2. Auto-reconnect disabled? +3. Device behind NAT/firewall? + +**Resolution**: +- Devices auto-reconnect every 30 seconds +- Manual reconnect: Close and reopen app +- Check relay connection if direct P2P fails + +### Sync Not Working + +**Symptom**: Changes not syncing between devices + +**Checks**: +1. Devices registered in each library? +2. Sync leader elected? +3. Follower sync service running? +4. Check sync log sequence numbers + +**Debug**: +```rust +// Check if device is registered in library +let device = entities::device::Entity::find() + .filter(entities::device::Column::Uuid.eq(device_id)) + .one(library.db().conn()) + .await?; + +if device.is_none() { + println!("Device not registered in library!"); + // Run library sync setup +} + +// Check sync role +let device = device.unwrap(); +let sync_leadership: HashMap = + serde_json::from_value(device.sync_leadership)?; + +match sync_leadership.get(&library_id) { + Some(SyncRole::Leader) => println!("This is the leader"), + Some(SyncRole::Follower) => println!("This is a follower"), + _ => println!("Not participating in sync for this library!"), +} +``` + +## Future Enhancements + +### Multi-Leader Support + +Current design: Single leader per library +Future: Multiple leaders with conflict-free sequence assignment + +### Device Capabilities + +```rust +pub struct DeviceCapabilities { + pub can_index: bool, // Has filesystem access + pub can_generate_thumbnails: bool, + pub can_transcode_video: bool, + pub has_gpu: bool, + pub storage_capacity: Option, +} +``` + +Usage: Job dispatch optimization (assign thumbnail generation to device with GPU) + +### Device Groups + +```rust +pub struct DeviceGroup { + pub id: Uuid, + pub name: String, + pub device_ids: Vec, + pub sync_policy: SyncPolicy, +} +``` + +Usage: "Sync all my personal devices" vs "Work devices only" + +## References + +- **Sync System**: `docs/core/sync.md` (sync leadership and follower service) +- **Sync Setup**: `docs/core/sync-setup.md` (library registration flow) +- **Pairing Protocol**: `docs/core/design/DEVICE_PAIRING_PROTOCOL.md` (crypto details) +- **Networking**: `docs/core/design/NETWORKING_SYSTEM_DESIGN.md` (P2P architecture) +- **Implementation**: + - `core/src/device/` (identity layer) + - `core/src/domain/device.rs` (domain model) + - `core/src/service/network/device/` (network layer) diff --git a/docs/core/events.md b/docs/core/events.md new file mode 100644 index 000000000..10594ead1 --- /dev/null +++ b/docs/core/events.md @@ -0,0 +1,688 @@ +# Unified Resource Event System + +**Status**: Implementation Ready +**Version**: 2.0 +**Last Updated**: 2025-10-08 + +## Overview + +Spacedrive's event system broadcasts real-time updates to all connected clients. This document specifies the **unified resource event architecture** that eliminates ~40 specialized event variants in favor of generic, horizontally-scalable events. + +## The Problem + +**Current system** (`core/src/infra/event/mod.rs` has 40+ variants): +```rust +Event::EntryCreated { library_id, entry_id } +Event::EntryModified { library_id, entry_id } +Event::VolumeAdded(Volume) +Event::LibraryCreated { id, name, path } +Event::JobStarted { job_id, job_type } +// ... 35 more variants +``` + +**Issues**: +- ❌ Manual emission scattered across codebase (easy to forget) +- ❌ Adding new resource = new event variant + client code changes +- ❌ No type safety between events and resources +- ❌ Clients must handle each variant specifically +- ❌ Horizontal scaling requires per-resource boilerplate + +## The Solution: Generic Resource Events + +All resources implementing `Identifiable` use a unified event structure. The TransactionManager emits these automatically. + +### Event Structure + +```rust +/// Unified event wrapper +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct Event { + /// Standard metadata for all events + pub envelope: EventEnvelope, + + /// The actual event payload + pub kind: EventKind, +} + +/// Standard envelope (addresses TODO on line 353 of current event/mod.rs) +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct EventEnvelope { + /// Event ID for deduplication + pub id: Uuid, + + /// When this event occurred + pub timestamp: DateTime, + + /// Library context (if applicable) + pub library_id: Option, + + /// Sequence number for ordering and gap detection + pub sequence: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +#[serde(tag = "type", content = "data")] +pub enum EventKind { + // ============================================ + // GENERIC RESOURCE EVENTS (for Identifiable) + // ============================================ + + /// A resource was created or updated + ResourceChanged { + /// Resource type from Identifiable::resource_type (e.g., "file", "album") + resource_type: String, + + /// The full resource as JSON (client deserializes generically) + #[specta(skip)] + resource: serde_json::Value, + }, + + /// Multiple resources changed in a batch + ResourceBatchChanged { + resource_type: String, + resources: Vec, + operation: BatchOperation, + }, + + /// A resource was deleted + ResourceDeleted { + resource_type: String, + resource_id: Uuid, + }, + + /// Bulk operation completed (notification only, no individual resources) + BulkOperationCompleted { + resource_type: String, + affected_count: usize, + operation_token: Uuid, + hints: serde_json::Value, // location_id, etc. + }, + + // ============================================ + // INFRASTRUCTURE EVENTS (not resources) + // ============================================ + + /// Core lifecycle + CoreStarted, + CoreShutdown, + + /// Library lifecycle + LibraryOpened { id: Uuid, name: String }, + LibraryClosed { id: Uuid }, + + /// Job status (not a domain resource) + Job { + job_id: String, + status: JobStatus, + progress: Option, + message: Option, + generic_progress: Option, + }, + + /// Raw filesystem changes (before DB resolution) + FsRawChange { kind: FsRawEventKind }, + + /// Log streaming + LogMessage { + timestamp: DateTime, + level: String, + target: String, + message: String, + job_id: Option, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum BatchOperation { + Index, + Search, + Update, + WatcherBatch, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub enum JobStatus { + Queued, + Started, + Progress, + Completed { output: JobOutput }, + Failed { error: String }, + Cancelled, + Paused, + Resumed, +} +``` + +## TransactionManager Integration + +The TM automatically emits events after successful commits: + +```rust +impl TransactionManager { + /// Emit resource changed event (automatic) + fn emit_resource_changed( + &self, + library_id: Uuid, + resource: &R, + ) { + let event = Event { + envelope: EventEnvelope { + id: Uuid::new_v4(), + timestamp: Utc::now(), + library_id: Some(library_id), + sequence: None, + }, + kind: EventKind::ResourceChanged { + resource_type: R::resource_type().to_string(), + resource: serde_json::to_value(resource).unwrap(), + }, + }; + + self.event_bus.emit(event); + } + + /// Commit automatically emits + pub async fn commit( + &self, + library: Arc, + model: M, + ) -> Result + where + M: Syncable + IntoActiveModel, + R: Identifiable + From, + { + // 1. Atomic transaction (DB + sync log) + let saved_model = /* ... */; + + // 2. Convert to client resource + let resource = R::from(saved_model); + + // 3. Emit automatically + self.emit_resource_changed(library.id(), &resource); + + Ok(resource) + } +} +``` + +**Result**: Application code never calls `event_bus.emit()` manually! + +## Client-Side: Zero-Friction Event Handling + +The true power is realized on the client through **type registries**. + +### Swift Implementation + +```swift +// =================================================== +// ONE-TIME SETUP: Type registry (auto-maintained) +// =================================================== + +protocol CacheableResource: Identifiable, Codable { + static var resourceType: String { get } +} + +class ResourceTypeRegistry { + private static var decoders: [String: (Data) throws -> any CacheableResource] = [:] + + static func register(_ type: T.Type) { + decoders[T.resourceType] = { data in + try JSONDecoder().decode(T.self, from: data) + } + } + + static func decode(resourceType: String, from data: Data) throws -> any CacheableResource { + guard let decoder = decoders[resourceType] else { + throw CacheError.unknownResourceType(resourceType) + } + return try decoder(data) + } +} + +// Register types (auto-generated via specta codegen) +extension File: CacheableResource { static let resourceType = "file" } +extension Album: CacheableResource { static let resourceType = "album" } +extension Tag: CacheableResource { static let resourceType = "tag" } +extension Location: CacheableResource { static let resourceType = "location" } +// Add 100 more: event handler NEVER changes! + +// =================================================== +// PERMANENT: Generic event handler +// THIS CODE NEVER CHANGES WHEN ADDING NEW RESOURCES! +// =================================================== + +actor EventCacheUpdater { + let cache: NormalizedCache + + func handleEvent(_ event: Event) async { + switch event.kind { + case .ResourceChanged(let resourceType, let resourceJSON): + do { + // ✅ Works for ALL current and future resources! + let resource = try ResourceTypeRegistry.decode( + resourceType: resourceType, + from: resourceJSON + ) + await cache.updateEntity(resource) + } catch { + print("Failed to decode \(resourceType): \(error)") + } + + case .ResourceBatchChanged(let resourceType, let resourcesJSON, _): + // ✅ Generic batch handling + let resources = resourcesJSON.compactMap { json in + try? ResourceTypeRegistry.decode(resourceType: resourceType, from: json) + } + for resource in resources { + await cache.updateEntity(resource) + } + + case .BulkOperationCompleted(let resourceType, let count, _, let hints): + // Invalidate affected queries + print("📦 Bulk operation: \(count) \(resourceType) items") + await cache.invalidateQueriesForResource(resourceType, hints: hints) + + case .ResourceDeleted(let resourceType, let resourceId): + // ✅ Generic deletion + await cache.deleteEntity(resourceType: resourceType, id: resourceId) + + // Infrastructure events + case .Job(let jobId, let status, let progress, _): + await cache.updateJobStatus(jobId: jobId, status: status, progress: progress) + + default: + break + } + } +} +``` + +**Key Achievement**: Adding a 101st resource requires **zero changes** to event handling! + +### TypeScript Implementation + +```typescript +// =================================================== +// Auto-generated by: cargo run --bin specta-gen +// =================================================== + +// src/bindings/resourceRegistry.ts +import { File } from './File'; +import { Album } from './Album'; +import { Tag } from './Tag'; +import { Location } from './Location'; + +export const resourceTypeMap = { + 'file': File, + 'album': Album, + 'tag': Tag, + 'location': Location, + // ... all Identifiable types +} as const; + +// =================================================== +// PERMANENT: Generic event handler (never changes!) +// =================================================== + +export class EventCacheUpdater { + constructor(private cache: NormalizedCache) {} + + handleEvent(event: Event) { + switch (event.kind.type) { + case 'ResourceChanged': { + const { resource_type, resource } = event.kind.data; + + // ✅ Generic decode via auto-generated registry! + const decoded = ResourceTypeRegistry.decode(resource_type, resource); + this.cache.updateEntity(resource_type, decoded); + break; + } + + case 'ResourceBatchChanged': { + const { resource_type, resources } = event.kind.data; + resources.forEach(r => { + const decoded = ResourceTypeRegistry.decode(resource_type, r); + this.cache.updateEntity(resource_type, decoded); + }); + break; + } + + case 'BulkOperationCompleted': { + const { resource_type, hints } = event.kind.data; + this.cache.invalidateQueries(resource_type, hints); + break; + } + + case 'ResourceDeleted': { + const { resource_type, resource_id } = event.kind.data; + this.cache.deleteEntity(resource_type, resource_id); + break; + } + } + } +} + +// =================================================== +// Generic type registry +// =================================================== + +class ResourceTypeRegistry { + private static validators = new Map any>(); + + static register(resourceType: string, validator: (data: unknown) => T) { + this.validators.set(resourceType, validator); + } + + static decode(resourceType: string, data: unknown): any { + const validator = this.validators.get(resourceType); + if (!validator) { + throw new Error(`Unknown resource type: ${resourceType}`); + } + return validator(data); + } +} + +// Auto-registration from generated map +Object.entries(resourceTypeMap).forEach(([type, TypeClass]) => { + ResourceTypeRegistry.register(type, (data) => data as InstanceType); +}); +``` + +## Specta Codegen Integration + +Spacedrive uses `specta` to generate TypeScript types from Rust. This extends to the event system: + +### Rust: Mark Types for Export + +```rust +#[derive(Debug, Clone, Serialize, Deserialize, Type)] // Type = specta +pub struct Album { + pub id: Uuid, + pub name: String, + pub cover_entry_uuid: Option, +} + +impl Identifiable for Album { + type Id = Uuid; + fn resource_id(&self) -> Self::Id { self.id } + fn resource_type() -> &'static str { "album" } +} +``` + +### Build Script + +```rust +// xtask/src/specta_gen.rs +fn main() { + let mut builder = TypeCollection::default(); + + // Export all Identifiable types + builder.register::(); + builder.register::(); + builder.register::(); + builder.register::(); + + // Auto-generate resourceRegistry.ts + generate_resource_registry(&builder); + + // Generate types + builder.export_ts("../packages/client/src/bindings/").unwrap(); +} +``` + +**Result**: Run `cargo xtask specta-gen` → TypeScript types + registry auto-update! + +## Migration Strategy + +### Phase 1: Additive (No Breaking Changes) +- Keep all existing Event variants +- Add new `ResourceChanged`, `ResourceBatchChanged`, etc. +- TransactionManager emits new events +- Old manual emissions still work +- Clients can gradually adopt new events + +### Phase 2: Parallel Systems +- New resources (Albums) use unified events only +- Existing resources (Files, Tags) emit both old and new +- Clients migrate to new event handlers one resource at a time +- Measure: No regressions in UI responsiveness + +### Phase 3: Deprecation +- Mark old variants as `#[deprecated]` +- Remove manual `event_bus.emit()` calls from ops +- Clients fully migrated to generic handlers + +### Phase 4: Cleanup +- Remove old event variants +- Codegen verification: All Identifiable types have registry entries +- Performance testing: Ensure no regression + +## Event Emission Guidelines + +### ✅ DO: Let TransactionManager Handle It + +```rust +// ✅ CORRECT: TM emits automatically +pub async fn create_album(tm: &TransactionManager, library: Arc, name: String) -> Result { + let model = albums::ActiveModel { /* ... */ }; + let album = tm.commit::(library, model).await?; + Ok(album) // Event emitted automatically! +} +``` + +### ❌ DON'T: Manual Event Emission + +```rust +// ❌ WRONG: Manual emission (error-prone, bypasses sync) +pub async fn create_album(library: Arc, name: String) -> Result { + let model = albums::ActiveModel { /* ... */ }; + model.insert(db).await?; // No sync log! + + event_bus.emit(Event::AlbumCreated { /* ... */ }); // Can forget this! + + Ok(album) +} +``` + +## Event Types by Category + +### Resource Events (Automatic via TM) +- `ResourceChanged` - Single create/update +- `ResourceBatchChanged` - Batch updates (10-1K items) +- `ResourceDeleted` - Single deletion +- `BulkOperationCompleted` - Bulk notification (1K+ items) + +**Resources**: +- File (from Entry persistence model) +- Album +- Tag +- Location +- Device +- ContentIdentity +- Sidecar +- Collection +- Label + +### Infrastructure Events (Manual, Specific) +- `CoreStarted`, `CoreShutdown` - Daemon lifecycle +- `LibraryOpened`, `LibraryClosed` - Library state +- `Job { status, progress }` - Job execution +- `FsRawChange` - Watcher events (pre-database) +- `LogMessage` - Log streaming + +## Complete Example: Album Creation + +### Rust Core + +```rust +// ops/albums/create/action.rs +impl LibraryAction for CreateAlbumAction { + async fn execute( + self, + library: Arc, + context: Arc, + ) -> Result { + let tm = context.transaction_manager(); + + let model = albums::ActiveModel { + id: NotSet, + uuid: Set(Uuid::new_v4()), + name: Set(self.input.name), + version: Set(1), + created_at: Set(Utc::now()), + updated_at: Set(Utc::now()), + }; + + // TM handles: DB write + sync log + event emission + let album = tm.commit::(library, model) + .await + .map_err(|e| ActionError::Internal(e.to_string()))?; + + Ok(CreateAlbumOutput { album }) + } +} + +// domain/album.rs +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct Album { + pub id: Uuid, + pub name: String, + pub cover_entry_uuid: Option, +} + +impl Identifiable for Album { + type Id = Uuid; + fn resource_id(&self) -> Self::Id { self.id } + fn resource_type() -> &'static str { "album" } +} + +impl From for Album { + fn from(model: albums::Model) -> Self { + Self { + id: model.uuid, + name: model.name, + cover_entry_uuid: model.cover_entry_uuid, + } + } +} +``` + +### Swift Client + +```swift +// NO RESOURCE-SPECIFIC CODE NEEDED! +// The generic handler already works: + +// User taps "Create Album" +try await client.action("albums.create.v1", input: ["name": "Vacation 2025"]) + +// Event arrives: +// Event { +// envelope: { id, timestamp, library_id, sequence }, +// kind: ResourceChanged { +// resource_type: "album", +// resource: { "id": "uuid-...", "name": "Vacation 2025" } +// } +// } + +// Generic handler processes it: +let resource = try ResourceTypeRegistry.decode("album", from: resourceJSON) +await cache.updateEntity(resource) + +// SwiftUI view updates automatically! +``` + +## Performance + +### Bandwidth + +**Single update**: +- Album resource: ~200 bytes JSON +- File resource: ~500 bytes JSON +- Overhead: ~100 bytes (envelope) +- Total: <1KB per event + +**Bulk operation**: +- Metadata only: ~500 bytes +- NO individual resources sent +- Client invalidates queries, refetches on demand + +### Event Bus Capacity + +- Tokio broadcast channel: 1024 event buffer (configurable) +- Slow subscribers lag detection +- No blocking on emit (fire-and-forget) + +## Testing + +### Unit Tests + +```rust +#[tokio::test] +async fn test_resource_changed_emission() { + let event_bus = EventBus::new(100); + let mut subscriber = event_bus.subscribe(); + + let tm = TransactionManager::new(event_bus.clone()); + + // Commit should emit ResourceChanged + let album = tm.commit::(library, model).await.unwrap(); + + let event = subscriber.recv().await.unwrap(); + match event.kind { + EventKind::ResourceChanged { resource_type, resource } => { + assert_eq!(resource_type, "album"); + let decoded: Album = serde_json::from_value(resource).unwrap(); + assert_eq!(decoded.id, album.id); + } + _ => panic!("Wrong event type"), + } +} +``` + +### Integration Tests + +- Create resource on Device A → Event emitted +- Device B receives sync entry → Event emitted locally +- Client cache updates → UI reflects change + +## Migration Checklist + +- [ ] Implement new Event enum with EventEnvelope +- [ ] Update TransactionManager to emit ResourceChanged +- [ ] Create ResourceTypeRegistry for Swift +- [ ] Create ResourceTypeRegistry for TypeScript +- [ ] Update specta codegen to generate registry +- [ ] Migrate Albums to unified events +- [ ] Migrate Tags to unified events +- [ ] Migrate Locations to unified events +- [ ] Migrate Files to unified events +- [ ] Remove old event variants +- [ ] Remove manual event_bus.emit() calls + +## Benefits Summary + +### Rust Core +- ✅ Zero manual event emission +- ✅ 40 variants → 5 generic variants +- ✅ Type-safe: Events always match resources +- ✅ Centralized: All emission in TransactionManager + +### Clients +- ✅ Zero switch statements per resource +- ✅ Type registry handles all deserialization +- ✅ Auto-generated via specta +- ✅ Add 100 resources: zero event handling changes + +### Maintenance +- ✅ Less code: ~2000 lines eliminated +- ✅ No forgetting: TM always emits +- ✅ Consistent: Same pattern everywhere +- ✅ Scalable: Horizontal scaling achieved + +## References + +- **Sync System**: `docs/core/sync.md` +- **Client Cache**: `docs/core/normalized_cache.md` +- **Current Implementation**: `core/src/infra/event/mod.rs` +- **Design Details**: `docs/core/design/sync/UNIFIED_RESOURCE_EVENTS.md` diff --git a/docs/core/normalized_cache.md b/docs/core/normalized_cache.md new file mode 100644 index 000000000..b7d24e603 --- /dev/null +++ b/docs/core/normalized_cache.md @@ -0,0 +1,933 @@ +# Normalized Client Cache + +**Status**: Implementation Ready +**Version**: 2.0 +**Last Updated**: 2025-10-08 + +## Overview + +The Normalized Client Cache is a client-side entity store that provides instant UI updates, offline support, and massive bandwidth savings. Inspired by Apollo Client, it normalizes all resources by unique ID and updates atomically when events arrive. + +## The Problem + +**Traditional approach**: +```swift +// Query returns files +let files = try await client.query("files.search", input: searchParams) + +// User renames file on Device B +// ... + +// UI doesn't update! Must manually refetch: +let files = try await client.query("files.search", input: searchParams) // Network call +``` + +**Issues**: +- ❌ Stale data in UI +- ❌ Manual refetch required (slow, bandwidth-heavy) +- ❌ No offline support +- ❌ Duplicate data (same file in multiple queries) + +## The Solution + +**Normalized cache** + **event-driven updates**: +```swift +// Query uses cache +let files = cache.query("files.search", input: searchParams) // Instant! + +// Device B renames file → Event arrives +// Event: ResourceChanged { resource_type: "file", resource: File { id, name: "new.jpg" } } + +// Cache updates automatically +cache.updateEntity(file) + +// UI updates instantly (ObservableObject/StateFlow) +// No refetch, no network, no user action! +``` + +## Cache Architecture + +### Two-Level Structure + +``` +┌─────────────────────────────────────────────────────────────┐ +│ LEVEL 1: Entity Store (normalized by ID) │ +│ │ +│ "file:uuid-1" → File { id: uuid-1, name: "photo.jpg" } │ +│ "file:uuid-2" → File { id: uuid-2, name: "doc.pdf" } │ +│ "album:uuid-3" → Album { id: uuid-3, name: "Vacation" } │ +│ "tag:uuid-4" → Tag { id: uuid-4, name: "Important" } │ +│ │ +└─────────────────────────────────────────────────────────────┘ + ↑ + │ Atomic updates + │ +┌─────────────────────────────────────────────────────────────┐ +│ LEVEL 2: Query Index (maps queries to entity IDs) │ +│ │ +│ "search:photos" → ["file:uuid-1", "file:uuid-2"] │ +│ "directory:/vacation" → ["file:uuid-1"] │ +│ "albums.list" → ["album:uuid-3"] │ +│ │ +└─────────────────────────────────────────────────────────────┘ +``` + +**Key Insight**: When `file:uuid-1` updates, we find all queries referencing it and trigger UI updates for those views. + +### Swift Implementation + +```swift +/// Normalized entity cache with event-driven updates +actor NormalizedCache { + // LEVEL 1: Entity store + private var entityStore: [String: any Identifiable] = [:] + + // LEVEL 2: Query index + private var queryIndex: [String: QueryCacheEntry] = [:] + + // Observers for reactive UI updates + private var queryObservers: [String: Set] = [:] + + /// Update a single entity (called by event handler) + func updateEntity(_ resource: T) { + let cacheKey = "\(T.resourceType):\(resource.id.uuidString)" + + // 1. Update entity store + entityStore[cacheKey] = resource + + // 2. Find all queries containing this entity + let affectedQueries = queryIndex.filter { _, entry in + entry.entityKeys.contains(cacheKey) + } + + // 3. Notify observers (SwiftUI views re-render) + for (queryKey, _) in affectedQueries { + notifyObservers(for: queryKey) + } + } + + /// Execute a query (with caching) + func query( + _ method: String, + input: Encodable + ) async throws -> [T] { + let queryKey = generateQueryKey(method, input) + + // Check cache + if let cached = queryIndex[queryKey], !cached.isExpired { + // Cache hit! Return from entity store + return cached.entityKeys.compactMap { key in + entityStore[key] as? T + } + } + + // Cache miss - fetch from server + let results: [T] = try await client.query(method, input: input) + + // Store entities + for resource in results { + let cacheKey = "\(T.resourceType):\(resource.id.uuidString)" + entityStore[cacheKey] = resource + } + + // Store query index + let entityKeys = results.map { "\(T.resourceType):\($0.id.uuidString)" } + queryIndex[queryKey] = QueryCacheEntry( + entityKeys: Set(entityKeys), + fetchedAt: Date(), + ttl: 300 // 5 minutes + ) + + return results + } + + /// Delete entity (called by event handler) + func deleteEntity(resourceType: String, id: UUID) { + let cacheKey = "\(resourceType):\(id.uuidString)" + + // Remove from store + entityStore.removeValue(forKey: cacheKey) + + // Remove from query indices + for (queryKey, var entry) in queryIndex { + if entry.entityKeys.remove(cacheKey) != nil { + queryIndex[queryKey] = entry + notifyObservers(for: queryKey) + } + } + } + + /// Invalidate queries (called by bulk operation events) + func invalidateQueriesForResource(_ resourceType: String, hints: [String: Any]) { + // Invalidate all queries matching hints (e.g., location_id) + let keysToInvalidate = queryIndex.keys.filter { queryKey in + if let locationId = hints["location_id"] as? String { + return queryKey.contains(locationId) + } + return queryKey.contains(resourceType) + } + + for key in keysToInvalidate { + queryIndex.removeValue(forKey: key) + notifyObservers(for: key) + } + } +} + +struct QueryCacheEntry { + var entityKeys: Set // References to entity store + let fetchedAt: Date + let ttl: TimeInterval // Time to live + + var isExpired: Bool { + Date().timeIntervalSince(fetchedAt) > ttl + } +} +``` + +### TypeScript Implementation + +```typescript +/** + * Normalized entity cache with reactive updates + */ +export class NormalizedCache { + // LEVEL 1: Entity store + private entityStore = new Map(); + + // LEVEL 2: Query index + private queryIndex = new Map(); + + // Reactive subscriptions (for React hooks) + private querySubscriptions = new Map void>>(); + + /** + * Update entity (called by event handler) + */ + updateEntity(resourceType: string, resource: any) { + const cacheKey = `${resourceType}:${resource.id}`; + + // 1. Update entity + this.entityStore.set(cacheKey, resource); + + // 2. Find affected queries + for (const [queryKey, entry] of this.queryIndex.entries()) { + if (entry.entityKeys.has(cacheKey)) { + this.notifySubscribers(queryKey); + } + } + } + + /** + * Query with caching + */ + async query(method: string, input: any): Promise { + const queryKey = this.generateQueryKey(method, input); + + // Check cache + const cached = this.queryIndex.get(queryKey); + if (cached && !cached.isExpired()) { + // Cache hit! + return Array.from(cached.entityKeys) + .map(key => this.entityStore.get(key)) + .filter(Boolean) as T[]; + } + + // Cache miss - fetch + const results: T[] = await this.client.query(method, input); + + // Store entities + const entityKeys = new Set(); + for (const resource of results) { + const cacheKey = `${(resource as any).__resourceType}:${(resource as any).id}`; + this.entityStore.set(cacheKey, resource); + entityKeys.add(cacheKey); + } + + // Store query + this.queryIndex.set(queryKey, { + entityKeys, + fetchedAt: Date.now(), + ttl: 300000, // 5 minutes + }); + + return results; + } + + /** + * Subscribe to query changes (for React hooks) + */ + subscribe(queryKey: string, callback: () => void): () => void { + if (!this.querySubscriptions.has(queryKey)) { + this.querySubscriptions.set(queryKey, new Set()); + } + this.querySubscriptions.get(queryKey)!.add(callback); + + // Return unsubscribe function + return () => { + this.querySubscriptions.get(queryKey)?.delete(callback); + }; + } + + private notifySubscribers(queryKey: string) { + const subscribers = this.querySubscriptions.get(queryKey); + if (subscribers) { + subscribers.forEach(callback => callback()); + } + } +} +``` + +## React Integration + +### useCachedQuery Hook + +```typescript +/** + * React hook for cached queries with automatic updates + */ +export function useCachedQuery( + method: string, + input: any, + options?: { enabled?: boolean } +): { data: T[] | null; loading: boolean; error: Error | null } { + const cache = useContext(CacheContext); + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + if (options?.enabled === false) return; + + const queryKey = cache.generateQueryKey(method, input); + + // Subscribe to cache changes + const unsubscribe = cache.subscribe(queryKey, () => { + // Query result changed - re-read from cache + const result = cache.getQueryResult(queryKey); + setData(result); + }); + + // Initial fetch + (async () => { + try { + const result = await cache.query(method, input); + setData(result); + } catch (e) { + setError(e as Error); + } finally { + setLoading(false); + } + })(); + + return unsubscribe; + }, [method, JSON.stringify(input), options?.enabled]); + + return { data, loading, error }; +} + +// Usage in component +function AlbumList() { + const { data: albums, loading } = useCachedQuery('albums.list', {}); + + if (loading) return ; + + // When ResourceChanged event arrives for an album: + // 1. Cache updates + // 2. This component re-renders + // 3. User sees new data instantly! + return ( +
+ {albums?.map(album => )} +
+ ); +} +``` + +## SwiftUI Integration + +### ObservableObject Pattern + +```swift +/// Observable cache for SwiftUI +@MainActor +class CachedQueryClient: ObservableObject { + private let cache: NormalizedCache + @Published private var queryResults: [String: Any] = [:] + + init(cache: NormalizedCache) { + self.cache = cache + + // Subscribe to cache changes + Task { + for await notification in cache.changeStream { + // Update published results + queryResults[notification.queryKey] = notification.newValue + } + } + } + + func query(_ method: String, input: Encodable) async throws -> [T] { + let results = try await cache.query(method, input: input) + + // Store in published results for observation + let queryKey = cache.generateQueryKey(method, input) + queryResults[queryKey] = results + + return results + } + + func getQueryResult(_ queryKey: String) -> [T]? { + queryResults[queryKey] as? [T] + } +} + +// Usage in SwiftUI view +struct AlbumListView: View { + @ObservedObject var client: CachedQueryClient + @State private var albums: [Album] = [] + + var body: some View { + List(albums, id: \.id) { album in + Text(album.name) + } + .task { + albums = try await client.query("albums.list", input: EmptyInput()) + } + // When ResourceChanged event arrives: + // 1. Cache updates + // 2. client publishes change + // 3. SwiftUI re-renders + // 4. User sees update instantly! + } +} +``` + +## Memory Management + +### LRU Eviction + +```swift +actor NormalizedCache { + private let maxEntities: Int = 10_000 + private var accessOrder: [String] = [] // LRU tracking + + func updateEntity(_ resource: T) { + let cacheKey = "\(T.resourceType):\(resource.id.uuidString)" + + // Update store + entityStore[cacheKey] = resource + + // Update access order (LRU) + if let index = accessOrder.firstIndex(of: cacheKey) { + accessOrder.remove(at: index) + } + accessOrder.append(cacheKey) + + // Evict if over limit + if entityStore.count > maxEntities { + evictLRU() + } + } + + private func evictLRU() { + // Evict oldest unreferenced entities + let referencedKeys = Set(queryIndex.values.flatMap { $0.entityKeys }) + + for key in accessOrder { + if !referencedKeys.contains(key) { + // Not in any active query - safe to evict + entityStore.removeValue(forKey: key) + accessOrder.removeAll { $0 == key } + + if entityStore.count <= maxEntities * 9 / 10 { + break // Evicted 10% - done + } + } + } + } +} +``` + +### TTL (Time-To-Live) + +```swift +struct QueryCacheEntry { + var entityKeys: Set + let fetchedAt: Date + let ttl: TimeInterval = 300 // 5 minutes default + + var isExpired: Bool { + Date().timeIntervalSince(fetchedAt) > ttl + } +} + +// Different TTLs per query type +func getTTL(for method: String) -> TimeInterval { + switch method { + case "files.search": return 60 // 1 minute (changes frequently) + case "albums.list": return 300 // 5 minutes (changes rarely) + case "core.status": return 10 // 10 seconds (real-time) + default: return 300 + } +} +``` + +### Reference Counting + +```swift +// Track which queries reference each entity +private var entityRefCounts: [String: Int] = [:] + +func removeQuery(_ queryKey: String) { + guard let entry = queryIndex[queryKey] else { return } + + // Decrement ref counts + for entityKey in entry.entityKeys { + entityRefCounts[entityKey, default: 0] -= 1 + + // If no longer referenced, can evict + if entityRefCounts[entityKey] == 0 { + entityStore.removeValue(forKey: entityKey) + entityRefCounts.removeValue(forKey: entityKey) + } + } + + queryIndex.removeValue(forKey: queryKey) +} +``` + +## Event-Driven Updates + +### Integration with Event System + +```swift +actor EventCacheUpdater { + let cache: NormalizedCache + + func start(eventStream: AsyncStream) async { + for await event in eventStream { + await handleEvent(event) + } + } + + func handleEvent(_ event: Event) async { + switch event.kind { + case .ResourceChanged(let resourceType, let resourceJSON): + // Decode resource + guard let resource = try? ResourceTypeRegistry.decode( + resourceType: resourceType, + from: resourceJSON + ) else { + print("Failed to decode \(resourceType)") + return + } + + // Update cache (triggers UI updates) + await cache.updateEntity(resource) + + case .ResourceBatchChanged(let resourceType, let resourcesJSON, _): + // Batch update + for json in resourcesJSON { + if let resource = try? ResourceTypeRegistry.decode(resourceType: resourceType, from: json) { + await cache.updateEntity(resource) + } + } + + case .ResourceDeleted(let resourceType, let resourceId): + // Remove from cache + await cache.deleteEntity(resourceType: resourceType, id: resourceId) + + case .BulkOperationCompleted(let resourceType, _, _, let hints): + // Invalidate affected queries + await cache.invalidateQueriesMatching { queryKey in + // Match by location_id or other hints + if let locationId = hints["location_id"] as? String { + return queryKey.contains(locationId) + } + return queryKey.contains(resourceType) + } + + default: + break + } + } +} +``` + +### Gap Detection + +When events have sequence numbers, detect gaps caused by network issues: + +```swift +actor NormalizedCache { + private var lastEventSequence: [UUID: UInt64] = [:] // library_id → sequence + + func processEvent(_ event: Event) async { + guard let libraryId = event.envelope.library_id, + let sequence = event.envelope.sequence else { + return + } + + let lastSeq = lastEventSequence[libraryId] ?? 0 + + if sequence > lastSeq + 1 { + // Gap detected! Missed events + print("⚠️ Gap detected: expected \(lastSeq + 1), got \(sequence)") + await reconcileState(libraryId: libraryId, fromSequence: lastSeq + 1) + } + + // Update sequence tracker + lastEventSequence[libraryId] = sequence + + // Process event normally + await handleEvent(event) + } + + /// Reconcile state after detecting missed events + func reconcileState(libraryId: UUID, fromSequence: UInt64) async { + print("🔄 Reconciling state from sequence \(fromSequence)") + + // Option 1: Fetch missed events + if let missedEvents = try? await client.query( + "events.since.v1", + input: ["library_id": libraryId, "sequence": fromSequence] + ) { + for event in missedEvents { + await processEvent(event) + } + } + + // Option 2: Full cache invalidation (fallback) + invalidateLibrary(libraryId) + } +} +``` + +## Cache Persistence (Offline Support) + +### SQLite Storage + +```swift +import SQLite + +actor NormalizedCache { + private let db: Connection + + init() { + // SQLite database for cache persistence + let path = FileManager.default + .urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent("spacedrive_cache.db") + + db = try! Connection(path.path) + createTables() + } + + func createTables() { + try! db.run(""" + CREATE TABLE IF NOT EXISTS entities ( + cache_key TEXT PRIMARY KEY, + resource_type TEXT NOT NULL, + resource_data TEXT NOT NULL, + updated_at INTEGER NOT NULL + ) + """) + + try! db.run(""" + CREATE TABLE IF NOT EXISTS queries ( + query_key TEXT PRIMARY KEY, + entity_keys TEXT NOT NULL, + fetched_at INTEGER NOT NULL, + ttl INTEGER NOT NULL + ) + """) + } + + func updateEntity(_ resource: T) async { + let cacheKey = "\(T.resourceType):\(resource.id.uuidString)" + let json = try! JSONEncoder().encode(resource) + + // Update memory + entityStore[cacheKey] = resource + + // Persist to disk + let stmt = try! db.prepare(""" + INSERT OR REPLACE INTO entities (cache_key, resource_type, resource_data, updated_at) + VALUES (?, ?, ?, ?) + """) + try! stmt.run(cacheKey, T.resourceType, String(data: json, encoding: .utf8)!, Date().timeIntervalSince1970) + } + + /// Load cache from disk on startup + func loadFromDisk() async { + let stmt = try! db.prepare("SELECT cache_key, resource_data FROM entities") + + for row in stmt { + let cacheKey = row[0] as! String + let jsonString = row[1] as! String + + // Deserialize using type registry + let parts = cacheKey.split(separator: ":") + let resourceType = String(parts[0]) + + if let data = jsonString.data(using: .utf8), + let resource = try? ResourceTypeRegistry.decode(resourceType: resourceType, from: data) { + entityStore[cacheKey] = resource + } + } + + print("✅ Loaded \(entityStore.count) entities from disk cache") + } +} +``` + +## Optimistic Updates + +```swift +actor NormalizedCache { + private var optimisticUpdates: [UUID: any Identifiable] = [:] // pending_id → resource + + /// Apply optimistic update immediately + func updateOptimistically(pendingId: UUID, resource: T) { + let cacheKey = "\(T.resourceType):\(resource.id.uuidString)" + + // Store in both places + entityStore[cacheKey] = resource + optimisticUpdates[pendingId] = resource + + // Notify observers (UI updates instantly!) + notifyAffectedQueries(cacheKey) + } + + /// Commit optimistic update when server confirms + func commitOptimisticUpdate(pendingId: UUID, confirmedResource: any Identifiable) { + optimisticUpdates.removeValue(forKey: pendingId) + updateEntity(confirmedResource) // Final update + } + + /// Rollback optimistic update on error + func rollbackOptimisticUpdate(pendingId: UUID) { + guard let resource = optimisticUpdates.removeValue(forKey: pendingId) else { + return + } + + let cacheKey = "\(type(of: resource).resourceType):\(resource.id.uuidString)" + entityStore.removeValue(forKey: cacheKey) + notifyAffectedQueries(cacheKey) + } +} + +// Usage example +func renameAlbum(id: UUID, newName: String) async throws { + let pendingId = UUID() + + // 1. Optimistic update (instant UI) + let optimisticAlbum = Album(id: id, name: newName, cover: nil) + await cache.updateOptimistically(pendingId: pendingId, resource: optimisticAlbum) + + do { + // 2. Send action to server + let confirmed = try await client.action("albums.rename.v1", input: ["id": id, "name": newName]) + + // 3. Commit (replace optimistic with confirmed) + await cache.commitOptimisticUpdate(pendingId: pendingId, confirmedResource: confirmed) + } catch { + // 4. Rollback on error + await cache.rollbackOptimisticUpdate(pendingId: pendingId) + throw error + } +} +``` + +## Query Invalidation + +### Manual Invalidation + +```swift +// After bulk operations +cache.invalidateQuery("files.search", input: searchParams) + +// After mutations +cache.invalidateQueriesMatching { queryKey in + queryKey.contains("albums.list") +} + +// Clear entire library +cache.invalidateLibrary(libraryId) +``` + +### Automatic Invalidation + +```swift +// ResourceBatchChanged with hints +case .ResourceBatchChanged(_, _, let operation): + switch operation { + case .Index: + // Invalidate directory listings + cache.invalidateQueriesMatching { $0.contains("directory:") } + case .WatcherBatch: + // Keep cache (events contain full data) + break + } +``` + +## Memory Budget + +```swift +struct CacheConfig { + // Entity store limits + let maxEntities: Int = 10_000 // ~10MB at 1KB/entity + let evictionThreshold: Int = 9_000 // Start evicting at 90% + + // Query limits + let maxQueries: Int = 100 + let defaultTTL: TimeInterval = 300 // 5 minutes + + // Persistence + let persistToDisk: Bool = true + let maxDiskSize: Int64 = 50_000_000 // 50MB +} +``` + +## Testing + +### Unit Tests + +```swift +func testCacheUpdate() async { + let cache = NormalizedCache() + + // Store entity + let album = Album(id: UUID(), name: "Test", cover: nil) + await cache.updateEntity(album) + + // Verify stored + let retrieved = await cache.getEntity(Album.self, id: album.id) + XCTAssertEqual(retrieved?.name, "Test") +} + +func testQueryInvalidation() async { + let cache = NormalizedCache() + + // Query and cache + let albums = try await cache.query("albums.list", input: EmptyInput()) + XCTAssertEqual(albums.count, 5) + + // Invalidate + await cache.invalidateQuery("albums.list", input: EmptyInput()) + + // Verify cache miss + let cached = await cache.getQueryResult("albums.list", input: EmptyInput()) + XCTAssertNil(cached) +} +``` + +### Integration Tests + +1. **Real-time update**: Create album on Device A → Event → Device B cache updates +2. **Offline resilience**: Disconnect → Queue writes → Reconnect → Sync +3. **Memory limits**: Load 20K entities → Verify LRU eviction +4. **Gap detection**: Miss events → Detect gap → Reconcile + +## Performance Metrics + +### Cache Hit Rates (Target) +- File queries: >90% hit rate +- Album/Tag queries: >95% hit rate +- Search queries: >70% hit rate (more volatile) + +### Memory Usage (Typical) +- Entity store: 5-10MB (5K-10K entities) +- Query index: 1-2MB (100 queries) +- Total: <15MB + +### Update Latency +- Event received → Cache updated: <1ms +- Cache updated → UI re-renders: <16ms (1 frame) +- Total: <20ms from server to UI + +## Implementation Checklist + +### Swift +- [ ] Create `NormalizedCache` actor +- [ ] Implement entity store + query index +- [ ] Implement `EventCacheUpdater` +- [ ] Create `ResourceTypeRegistry` +- [ ] Add LRU eviction +- [ ] Add SQLite persistence +- [ ] Create `CachedQueryClient` (ObservableObject) +- [ ] Create SwiftUI view integration +- [ ] Unit tests +- [ ] Integration tests + +### TypeScript/React +- [ ] Create `NormalizedCache` class +- [ ] Implement entity store + query index +- [ ] Create `EventCacheUpdater` +- [ ] Create `ResourceTypeRegistry` +- [ ] Add LRU eviction +- [ ] Add IndexedDB persistence +- [ ] Create `useCachedQuery` hook +- [ ] Create React integration examples +- [ ] Unit tests +- [ ] Integration tests + +## Migration Strategy + +### Phase 1: Parallel Systems +- New cache runs alongside existing query system +- No breaking changes +- Opt-in per view/component + +### Phase 2: Gradual Adoption +- Migrate high-traffic views first (file browser, search) +- Measure: Cache hit rate, UI responsiveness +- Iterate on memory management + +### Phase 3: Full Migration +- All queries use cache +- Remove old query caching logic +- Cleanup legacy code + +## Edge Cases + +### Circular References + +```swift +// File references Album, Album references Files (cover) +// Solution: Store by ID, resolve lazily + +struct Album { + let id: UUID + let name: String + let coverFileId: UUID? // Just ID, not full File object +} + +// UI resolves when needed: +let coverFile = cache.getEntity(File.self, id: album.coverFileId) +``` + +### Large Resources + +```swift +// File with 1000 tags (rare but possible) +// Solution: Paginate relationships or use lazy loading + +struct File { + let id: UUID + let name: String + let tagIds: [UUID] // Just IDs + // NOT: tags: [Tag] // Would explode memory +} + +// Load tags on demand: +let tags = album.tagIds.compactMap { cache.getEntity(Tag.self, id: $0) } +``` + +## References + +- **Sync System**: `docs/core/sync.md` +- **Event System**: `docs/core/events.md` +- **Design Details**: `docs/core/design/sync/NORMALIZED_CACHE_DESIGN.md` (2674 lines, comprehensive) +- **Client Architecture**: `docs/core/design/sync/SYNC_TX_CACHE_MINI_SPEC.md` diff --git a/docs/core/sync.md b/docs/core/sync.md new file mode 100644 index 000000000..3c1046238 --- /dev/null +++ b/docs/core/sync.md @@ -0,0 +1,561 @@ +# Spacedrive Sync System + +**Status**: Implementation Ready +**Version**: 2.0 +**Last Updated**: 2025-10-08 + +## Overview + +Spacedrive's sync system enables real-time, multi-device synchronization of library metadata, ensuring that changes made on one device are reflected across all paired devices. This document provides the definitive specification for implementing sync. + +## Core Architecture + +### The Three Pillars + +1. **TransactionManager (TM)**: Sole gatekeeper for all syncable database writes, ensuring atomic DB commits + sync log creation +2. **Sync Log**: Append-only, sequentially-ordered log of all state changes per library, maintained only by the leader device +3. **Sync Service**: Replicates sync log entries between paired devices using pull-based synchronization + +### Data Flow + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Device A │ +│ │ +│ User Action (e.g., create album) │ +│ ↓ │ +│ [ Action Layer ] │ +│ ↓ │ +│ [ TransactionManager ] │ +│ ↓ │ +│ ┌─────────────────────────────┐ │ +│ │ ATOMIC TRANSACTION │ │ +│ │ 1. Write to database │ │ +│ │ 2. Create sync log entry │ │ +│ │ COMMIT │ │ +│ └─────────────────────────────┘ │ +│ ↓ │ +│ [ Event Bus ] → Client cache updates │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ + ↓ Sync replication +┌─────────────────────────────────────────────────────────────────────┐ +│ Device B │ +│ │ +│ [ Sync Service ] │ +│ ↓ (polls for new entries) │ +│ Fetch sync log from Device A │ +│ ↓ │ +│ [ Apply Sync Entry ] │ +│ ↓ │ +│ [ TransactionManager ] (applies change) │ +│ ↓ │ +│ Database updated + Event emitted │ +│ ↓ │ +│ Client cache updates → UI reflects change │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +## Syncable Trait + +All database models that need to sync implement the `Syncable` trait: + +```rust +/// Enables automatic sync log creation for database models +pub trait Syncable { + /// Stable model identifier used in sync logs (e.g., "album", "tag", "entry") + const SYNC_MODEL: &'static str; + + /// Globally unique ID for this resource across all devices + fn sync_id(&self) -> Uuid; + + /// Version number for optimistic concurrency control + fn version(&self) -> i64; + + /// Optional: Exclude platform-specific or derived fields from sync + fn exclude_fields() -> Option<&'static [&'static str]> { + None + } + + /// Optional: Convert to sync-safe JSON (default: full serialization) + fn to_sync_json(&self) -> serde_json::Value where Self: Serialize { + serde_json::to_value(self).unwrap_or(serde_json::json!({})) + } +} +``` + +**Example Implementation**: +```rust +// Database model +#[derive(Clone, Debug, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "albums")] +pub struct Model { + pub id: i32, // Database primary key + pub uuid: Uuid, // Sync identifier + pub name: String, + pub version: i64, // For conflict resolution + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl Syncable for albums::Model { + const SYNC_MODEL: &'static str = "album"; + + fn sync_id(&self) -> Uuid { + self.uuid + } + + fn version(&self) -> i64 { + self.version + } + + fn exclude_fields() -> Option<&'static [&'static str]> { + // Don't sync database IDs or timestamps (platform-specific) + Some(&["id", "created_at", "updated_at"]) + } +} +``` + +## TransactionManager + +The TM is the **only** component that performs state-changing writes. It guarantees atomicity and automatic sync log creation. + +### Core API + +```rust +pub struct TransactionManager { + event_bus: Arc, + sync_sequence: Arc>>, // library_id → sequence +} + +impl TransactionManager { + /// Commit single resource change (creates sync log) + pub async fn commit( + &self, + library: Arc, + model: M, + ) -> Result + where + M: Syncable + IntoActiveModel, + R: Identifiable + From; + + /// Commit batch of changes (10-1K items, creates per-item sync logs) + pub async fn commit_batch( + &self, + library: Arc, + models: Vec, + ) -> Result, TxError> + where + M: Syncable + IntoActiveModel, + R: Identifiable + From; + + /// Commit bulk operation (1K+ items, creates ONE metadata sync log) + pub async fn commit_bulk( + &self, + library: Arc, + changes: ChangeSet, + ) -> Result + where + M: Syncable + IntoActiveModel; +} +``` + +### Commit Strategies + +| Method | Use Case | Sync Log | Event | Example | +|--------|----------|----------|-------|---------| +| `commit()` | Single user action | 1 per item | Rich resource | User renames file | +| `commit_batch()` | Watcher events (10-1K) | 1 per item | Batch | User copies folder | +| `commit_bulk()` | Initial indexing (1K+) | 1 metadata only | Summary | Index 1M files | + +### Critical: Bulk Operations + +**Problem**: Indexing 1M files shouldn't create 1M sync log entries. + +**Solution**: Bulk operations create **ONE** metadata sync log: + +```json +{ + "sequence": 1234, + "model_type": "bulk_operation", + "operation": "InitialIndex", + "location_id": "uuid-...", + "affected_count": 1000000, + "hints": { + "location_path": "/Users/alice/Photos" + } +} +``` + +**Why**: Each device indexes its own filesystem independently. The sync log just says "I indexed location X" — it does NOT replicate 1M entries. Other devices trigger their own local indexing jobs when they see this notification. + +**Performance Impact**: +- With per-entry sync logs: ~500MB, 10 minutes, 3M operations +- With bulk metadata: ~500 bytes, 1 minute, 1M operations (10x faster!) + +### Usage Example + +```rust +// Before: Manual DB write + event emission (error-prone) +let model = albums::ActiveModel { /* ... */ }; +model.insert(db).await?; +event_bus.emit(Event::AlbumCreated { /* ... */ }); // Can forget this! + +// After: TransactionManager (atomic, automatic) +let model = albums::ActiveModel { /* ... */ }; +let album = tm.commit::(library, model).await?; +// ✅ DB write + sync log + event — all atomic! +``` + +## Sync Log Schema + +```sql +CREATE TABLE sync_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sequence INTEGER NOT NULL, -- Monotonic per library + library_id TEXT NOT NULL, + device_id TEXT NOT NULL, -- Device that created this entry + timestamp TEXT NOT NULL, + + -- Change details + model_type TEXT NOT NULL, -- "album", "tag", "entry", "bulk_operation" + record_id TEXT NOT NULL, -- UUID of changed record + change_type TEXT NOT NULL, -- "insert", "update", "delete", "bulk_insert" + version INTEGER NOT NULL DEFAULT 1, -- Optimistic concurrency + + -- Data payload (JSON) + data TEXT NOT NULL, + + UNIQUE(library_id, sequence) +); + +CREATE INDEX idx_sync_log_library_sequence ON sync_log(library_id, sequence); +CREATE INDEX idx_sync_log_device ON sync_log(device_id); +CREATE INDEX idx_sync_log_model_record ON sync_log(model_type, record_id); +``` + +## Leader Election + +Each library requires a **single leader device** responsible for assigning sync log sequence numbers. This prevents sequence collisions. + +### Election Strategy + +1. **Initial Leader**: Device that creates the library +2. **Heartbeat**: Leader sends heartbeat every 30 seconds +3. **Re-election**: If leader offline >60s, devices elect new leader (highest device_id wins) +4. **Lease**: Leader holds exclusive write lease + +### Implementation + +```rust +pub struct SyncLeader { + library_id: Uuid, + leader_device_id: Uuid, + lease_expires_at: DateTime, +} + +impl TransactionManager { + pub async fn request_leadership(&self, library_id: Uuid) -> Result { + // Check if current leader is still valid + // If not, attempt to become leader + // Update leadership table with lease + } + + pub async fn is_leader(&self, library_id: Uuid) -> bool { + // Check if this device holds valid lease + } + + async fn next_sequence(&self, library_id: Uuid) -> Result { + if !self.is_leader(library_id).await { + return Err(TxError::NotLeader); + } + + let mut sequences = self.sync_sequence.lock().unwrap(); + let seq = sequences.entry(library_id).or_insert(0); + *seq += 1; + Ok(*seq) + } +} +``` + +## Sync Service (Follower) + +Devices that are not the leader pull sync log entries and apply them locally. + +```rust +pub struct SyncFollowerService { + library_id: Uuid, + leader_device_id: Uuid, + last_synced_sequence: Arc>, + tx_manager: Arc, +} + +impl SyncFollowerService { + /// Poll for new sync entries (called every 5 seconds) + pub async fn sync_iteration(&mut self) -> Result { + let last_seq = *self.last_synced_sequence.lock().unwrap(); + + // Fetch entries from leader since last_seq + let entries = self.fetch_entries_from_leader(last_seq).await?; + + if entries.is_empty() { + return Ok(SyncResult::NoChanges); + } + + // Apply each entry + for entry in entries { + self.apply_sync_entry(entry).await?; + } + + Ok(SyncResult::Applied { count: entries.len() }) + } + + async fn apply_sync_entry(&mut self, entry: SyncLogEntry) -> Result<(), SyncError> { + match entry.model_type.as_str() { + "bulk_operation" => { + // Parse metadata + let metadata: BulkOperationMetadata = serde_json::from_value(entry.data)?; + self.handle_bulk_operation(metadata).await?; + } + _ => { + // Regular sync entry - deserialize and apply + let model = self.deserialize_model(&entry)?; + self.apply_model_change(model, entry.change_type).await?; + } + } + + // Update last synced sequence + *self.last_synced_sequence.lock().unwrap() = entry.sequence; + Ok(()) + } + + async fn handle_bulk_operation(&mut self, metadata: BulkOperationMetadata) -> Result<(), SyncError> { + match metadata.operation { + BulkOperation::InitialIndex { location_id, location_path } => { + tracing::info!( + "Peer indexed location {} with {} entries", + location_id, metadata.affected_count + ); + + // Check if we have this location locally + if let Some(local_location) = self.find_matching_location(&location_path).await? { + // Trigger our own indexing job + self.job_manager.queue(IndexerJob { + location_id: local_location.id, + mode: IndexMode::Full, + }).await?; + } + } + _ => {} + } + Ok(()) + } +} +``` + +## Library Sync Setup (Phase 1) + +Before devices can sync, they must: +1. **Pair** (cryptographic authentication) +2. **Discover** libraries on remote device +3. **Register** devices in each other's libraries + +See `sync-setup.md` for complete implementation details. + +### Setup Flow + +```rust +// 1. Discover remote libraries (after pairing) +let discovery = client.query( + "query:network.sync_setup.discover.v1", + DiscoverRemoteLibrariesInput { device_id: paired_device.id } +).await?; + +// 2. Setup library sync (RegisterOnly in Phase 1) +let setup_result = client.action( + "action:network.sync_setup.input.v1", + LibrarySyncSetupInput { + local_device_id: my_device_id, + remote_device_id: paired_device.id, + local_library_id: my_library.id, + remote_library_id: discovery.libraries[0].id, + action: LibrarySyncAction::RegisterOnly, + leader_device_id: my_device_id, // This device becomes leader + } +).await?; + +// 3. Ready for sync! +// Sync service starts polling for changes +``` + +## Sync Domains + +Spacedrive syncs different types of data with different strategies: + +| Domain | What Syncs | Strategy | +|--------|-----------|----------| +| **Index** | File/folder entries | Metadata only (each device indexes own filesystem) | +| **Metadata** | Tags, albums, collections | Full replication across devices | +| **Content** | File content (future) | User-configured sync conduits | +| **State** | UI state, preferences | Device-specific, no sync | + +## Conflict Resolution + +### Optimistic Concurrency + +All `Syncable` models have a `version` field. When applying a sync entry: + +```rust +async fn apply_model_change(&self, remote_model: Model, change_type: ChangeType) -> Result<()> { + match change_type { + ChangeType::Update => { + // Fetch current local version + let local_model = Model::find_by_uuid(remote_model.sync_id(), db).await?; + + if let Some(local) = local_model { + if local.version >= remote_model.version { + // Local is newer or same - skip update + tracing::debug!("Skipping sync entry: local version is newer"); + return Ok(()); + } + } + + // Remote is newer - apply update + remote_model.update(db).await?; + } + ChangeType::Insert => { + remote_model.insert(db).await?; + } + ChangeType::Delete => { + Model::delete_by_uuid(remote_model.sync_id(), db).await?; + } + } + Ok(()) +} +``` + +### Conflict Strategy + +- **Last-Write-Wins (LWW)**: Use `version` field to determine winner +- **No CRDTs**: Simpler, sufficient for metadata sync +- **User Metadata**: Tags, albums use union merge (both versions kept) + +## Raw SQL Compatibility + +**Reads**: Unrestricted. Use SeaORM query builder or raw SQL freely. + +**Writes**: Must go through TransactionManager. For advanced cases: + +```rust +tm.with_tx(library, |txn| async move { + // Raw SQL writes inside TM transaction + txn.execute(Statement::from_sql_and_values( + DbBackend::Sqlite, + "UPDATE albums SET name = ? WHERE uuid = ?", + vec![name.into(), uuid.into()], + )).await?; + + // Tell TM to log this change + tm.sync_log_for::(txn, uuid).await?; + + Ok(()) +}).await?; +``` + +## Implementation Roadmap + +### Phase 1: Foundation (Current) +- [x] Device pairing protocol +- [x] Library sync setup (RegisterOnly) +- [ ] TransactionManager core +- [ ] Syncable trait + derives +- [ ] Sync log schema +- [ ] Leader election + +### Phase 2: Basic Sync +- [ ] Sync follower service (pull-based) +- [ ] Apply sync entries +- [ ] Handle bulk operations +- [ ] Conflict resolution +- [ ] Album/Tag/Location sync + +### Phase 3: File Sync +- [ ] Entry sync (metadata only) +- [ ] Watcher integration +- [ ] Bulk indexing with metadata logs +- [ ] Cross-device file operations + +### Phase 4: Advanced Features +- [ ] Content sync (via sync conduits) +- [ ] Push-based sync (optional optimization) +- [ ] Multi-leader support +- [ ] Conflict resolution UI + +## Performance Considerations + +### Indexing Performance + +- **1M files, per-entry logs**: 10 minutes, 500MB sync log +- **1M files, bulk metadata**: 1 minute, 500 bytes sync log +- **Result**: 10x faster, 1 million times smaller sync log + +### Network Efficiency + +- Pull-based sync: Batch fetch (max 100 entries per request) +- Compression: Gzip sync log JSON (typically 5x reduction) +- Delta sync: Only fetch entries since last sequence + +### Database Optimization + +- Sync log: Append-only, no updates (fast writes) +- Indexes on (library_id, sequence) for efficient polling +- Vacuum old entries after successful sync (> 30 days) + +## Security + +### Encryption +- All sync data transmitted over encrypted Iroh streams +- Sync log contains full model data (no encryption at rest in Phase 1) +- Future: Library-level encryption (see `AT_REST_LIBRARY_ENCRYPTION.md`) + +### Access Control +- Only paired devices can sync +- Device pairing uses cryptographic challenge/response +- Leader election prevents unauthorized writes + +## Testing Strategy + +### Unit Tests +```rust +#[tokio::test] +async fn test_sync_log_creation() { + let tm = TransactionManager::new(event_bus); + let model = albums::Model { /* ... */ }; + + let album = tm.commit::(library, model).await.unwrap(); + + // Verify sync log entry created + let entry = sync_log::Entity::find() + .filter(sync_log::Column::RecordId.eq(album.id)) + .one(db) + .await + .unwrap() + .unwrap(); + + assert_eq!(entry.model_type, "album"); +} +``` + +### Integration Tests +- Two-device sync simulation +- Leader failover scenarios +- Bulk operation handling +- Conflict resolution + +## References + +- **Sync Setup**: `docs/core/sync-setup.md` +- **Event System**: `docs/core/events.md` +- **Client Cache**: `docs/core/normalized_cache.md` +- **Design Details**: `docs/core/design/sync/`