mirror of
https://github.com/spacedriveapp/spacedrive.git
synced 2026-05-18 21:36:56 -04:00
feat(sync): enhance foreign key handling and mime type synchronization
This commit introduces improvements to the synchronization process by adding foreign key conversion to UUIDs for cross-device compatibility. It also implements batch syncing of mime types before content identities to ensure proper dependency resolution. Additionally, the content identity model is updated to include mime type and kind ID handling, enhancing data integrity during sync operations. Debugging information is added to track registered models and their sync types, improving traceability in the sync process.
This commit is contained in:
@@ -252,7 +252,7 @@ impl Syncable for Model {
|
||||
continue;
|
||||
}
|
||||
|
||||
let json = match content.to_sync_json() {
|
||||
let mut json = match content.to_sync_json() {
|
||||
Ok(j) => j,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, content_hash = %content.content_hash, "Failed to serialize content_identity for sync");
|
||||
@@ -260,6 +260,21 @@ impl Syncable for Model {
|
||||
}
|
||||
};
|
||||
|
||||
// Convert FK to UUID for cross-device compatibility
|
||||
for fk in Self::foreign_key_mappings() {
|
||||
if let Err(e) =
|
||||
crate::infra::sync::fk_mapper::convert_fk_to_uuid(&mut json, &fk, db).await
|
||||
{
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
uuid = %content.uuid.unwrap(),
|
||||
fk_field = fk.local_field,
|
||||
"Failed to convert FK to UUID, skipping content_identity"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
sync_results.push((content.uuid.unwrap(), json, content.last_verified_at));
|
||||
}
|
||||
|
||||
@@ -272,7 +287,14 @@ impl Syncable for Model {
|
||||
) -> Result<(), sea_orm::DbErr> {
|
||||
match entry.change_type {
|
||||
ChangeType::Insert | ChangeType::Update => {
|
||||
let data = entry.data.as_object().ok_or_else(|| {
|
||||
// Map UUIDs to local IDs for FK fields
|
||||
use crate::infra::sync::fk_mapper;
|
||||
let data =
|
||||
fk_mapper::map_sync_json_to_local(entry.data, Self::foreign_key_mappings(), db)
|
||||
.await
|
||||
.map_err(|e| sea_orm::DbErr::Custom(format!("FK mapping failed: {}", e)))?;
|
||||
|
||||
let data = data.as_object().ok_or_else(|| {
|
||||
sea_orm::DbErr::Custom("ContentIdentity data is not an object".to_string())
|
||||
})?;
|
||||
|
||||
@@ -290,6 +312,20 @@ impl Syncable for Model {
|
||||
)
|
||||
.map_err(|e| sea_orm::DbErr::Custom(format!("Invalid content_hash: {}", e)))?;
|
||||
|
||||
let mime_type_id: Option<i32> = serde_json::from_value(
|
||||
data.get("mime_type_id")
|
||||
.cloned()
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
)
|
||||
.map_err(|e| sea_orm::DbErr::Custom(format!("Invalid mime_type_id: {}", e)))?;
|
||||
|
||||
let kind_id: i32 = serde_json::from_value(
|
||||
data.get("kind_id")
|
||||
.cloned()
|
||||
.unwrap_or(serde_json::Value::Number(0.into())),
|
||||
)
|
||||
.map_err(|e| sea_orm::DbErr::Custom(format!("Invalid kind_id: {}", e)))?;
|
||||
|
||||
let active = ActiveModel {
|
||||
id: NotSet,
|
||||
uuid: Set(Some(uuid)),
|
||||
@@ -300,18 +336,8 @@ impl Syncable for Model {
|
||||
)
|
||||
.unwrap()),
|
||||
content_hash: Set(content_hash),
|
||||
mime_type_id: Set(serde_json::from_value(
|
||||
data.get("mime_type_id")
|
||||
.cloned()
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
)
|
||||
.unwrap()),
|
||||
kind_id: Set(serde_json::from_value(
|
||||
data.get("kind_id")
|
||||
.cloned()
|
||||
.unwrap_or(serde_json::Value::Number(0.into())),
|
||||
)
|
||||
.unwrap()),
|
||||
mime_type_id: Set(mime_type_id),
|
||||
kind_id: Set(kind_id),
|
||||
text_content: Set(serde_json::from_value(
|
||||
data.get("text_content")
|
||||
.cloned()
|
||||
@@ -338,6 +364,8 @@ impl Syncable for Model {
|
||||
.update_columns([
|
||||
Column::IntegrityHash,
|
||||
Column::ContentHash,
|
||||
Column::MimeTypeId,
|
||||
Column::KindId,
|
||||
Column::TextContent,
|
||||
Column::TotalSize,
|
||||
Column::LastVerifiedAt,
|
||||
|
||||
@@ -113,7 +113,7 @@ impl MigrationTrait for Migration {
|
||||
.auto_increment()
|
||||
.primary_key(),
|
||||
)
|
||||
.col(ColumnDef::new(MimeTypes::Uuid).uuid().not_null())
|
||||
.col(ColumnDef::new(MimeTypes::Uuid).uuid().not_null().unique_key())
|
||||
.col(
|
||||
ColumnDef::new(MimeTypes::MimeType)
|
||||
.string()
|
||||
|
||||
@@ -917,6 +917,16 @@ mod tests {
|
||||
// Access registry to trigger initialization
|
||||
let registry = SYNCABLE_REGISTRY.read().await;
|
||||
|
||||
// Print all registered models for debugging
|
||||
let mut models: Vec<_> = registry.keys().cloned().collect();
|
||||
models.sort();
|
||||
println!("Registered syncable models ({}):", models.len());
|
||||
for model in &models {
|
||||
let reg = registry.get(model).unwrap();
|
||||
let sync_type = if reg.is_device_owned { "device-owned" } else { "shared" };
|
||||
println!(" - {} ({})", model, sync_type);
|
||||
}
|
||||
|
||||
// Verify location is registered as device-owned
|
||||
assert!(registry.contains_key("location"));
|
||||
let location_reg = registry.get("location").unwrap();
|
||||
@@ -934,6 +944,18 @@ mod tests {
|
||||
assert!(!tag_reg.is_device_owned);
|
||||
assert!(tag_reg.state_apply_fn.is_none());
|
||||
assert!(tag_reg.shared_apply_fn.is_some());
|
||||
|
||||
// Verify mime_type is registered as shared
|
||||
assert!(
|
||||
registry.contains_key("mime_type"),
|
||||
"mime_type should be registered but was not found. Registered models: {:?}",
|
||||
models
|
||||
);
|
||||
let mime_type_reg = registry.get("mime_type").unwrap();
|
||||
assert_eq!(mime_type_reg.model_type, "mime_type");
|
||||
assert_eq!(mime_type_reg.table_name, "mime_types");
|
||||
assert!(!mime_type_reg.is_device_owned);
|
||||
assert!(mime_type_reg.shared_apply_fn.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -133,6 +133,8 @@ pub struct ContentLinkResult {
|
||||
pub content_identity: entities::content_identity::Model,
|
||||
pub entry: entities::entry::Model,
|
||||
pub is_new_content: bool,
|
||||
pub mime_type: Option<entities::mime_type::Model>,
|
||||
pub is_new_mime_type: bool,
|
||||
}
|
||||
|
||||
impl DatabaseStorage {
|
||||
@@ -828,7 +830,7 @@ impl DatabaseStorage {
|
||||
.await
|
||||
.map_err(|e| JobError::execution(format!("Failed to query content identity: {}", e)))?;
|
||||
|
||||
let (content_model, is_new_content) = if let Some(existing) = existing {
|
||||
let (content_model, is_new_content, mime_type_model, is_new_mime_type) = if let Some(existing) = existing {
|
||||
let mut existing_active: entities::content_identity::ActiveModel = existing.into();
|
||||
existing_active.entry_count = Set(existing_active.entry_count.unwrap() + 1);
|
||||
existing_active.last_verified_at = Set(chrono::Utc::now());
|
||||
@@ -837,7 +839,8 @@ impl DatabaseStorage {
|
||||
JobError::execution(format!("Failed to update content identity: {}", e))
|
||||
})?;
|
||||
|
||||
(updated, false)
|
||||
// Content already exists, no new mime_type was created
|
||||
(updated, false, None, false)
|
||||
} else {
|
||||
let file_size = tokio::fs::symlink_metadata(path)
|
||||
.await
|
||||
@@ -852,11 +855,11 @@ impl DatabaseStorage {
|
||||
|
||||
let file_type_result = registry.identify(path).await;
|
||||
|
||||
let (kind_id, mime_type_id) = match file_type_result {
|
||||
let (kind_id, mime_type_id, mime_type_model, is_new_mime_type) = match file_type_result {
|
||||
Ok(result) => {
|
||||
let kind_id = result.file_type.category as i32;
|
||||
|
||||
let mime_type_id = if let Some(mime_str) = result.file_type.primary_mime_type()
|
||||
let (mime_type_id, mime_type_model, is_new_mime_type) = if let Some(mime_str) = result.file_type.primary_mime_type()
|
||||
{
|
||||
let existing = entities::mime_type::Entity::find()
|
||||
.filter(entities::mime_type::Column::MimeType.eq(mime_str))
|
||||
@@ -867,7 +870,7 @@ impl DatabaseStorage {
|
||||
})?;
|
||||
|
||||
match existing {
|
||||
Some(mime_record) => Some(mime_record.id),
|
||||
Some(mime_record) => (Some(mime_record.id), Some(mime_record), false),
|
||||
None => {
|
||||
let new_mime = entities::mime_type::ActiveModel {
|
||||
uuid: Set(Uuid::new_v4()),
|
||||
@@ -883,16 +886,16 @@ impl DatabaseStorage {
|
||||
))
|
||||
})?;
|
||||
|
||||
Some(mime_result.id)
|
||||
(Some(mime_result.id), Some(mime_result), true)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
(None, None, false)
|
||||
};
|
||||
|
||||
(kind_id, mime_type_id)
|
||||
(kind_id, mime_type_id, mime_type_model, is_new_mime_type)
|
||||
}
|
||||
Err(_) => (0, None),
|
||||
Err(_) => (0, None, None, false),
|
||||
};
|
||||
|
||||
let new_content = entities::content_identity::ActiveModel {
|
||||
@@ -913,7 +916,7 @@ impl DatabaseStorage {
|
||||
// content identity between our check and insert. Catch UNIQUE constraint violations
|
||||
// and use the existing record instead of failing.
|
||||
let result = match new_content.insert(db).await {
|
||||
Ok(model) => (model, true),
|
||||
Ok(model) => (model, true, mime_type_model, is_new_mime_type),
|
||||
Err(e) => {
|
||||
if e.to_string().contains("UNIQUE constraint failed") {
|
||||
let existing = entities::content_identity::Entity::find()
|
||||
@@ -932,7 +935,8 @@ impl DatabaseStorage {
|
||||
JobError::execution(format!("Failed to update content identity: {}", e))
|
||||
})?;
|
||||
|
||||
(updated, false)
|
||||
// Content already existed (race condition), but mime_type may still be new
|
||||
(updated, false, mime_type_model, is_new_mime_type)
|
||||
} else {
|
||||
return Err(JobError::execution(format!(
|
||||
"Failed to create content identity: {}",
|
||||
@@ -962,6 +966,8 @@ impl DatabaseStorage {
|
||||
content_identity: content_model,
|
||||
entry: updated_entry,
|
||||
is_new_content,
|
||||
mime_type: mime_type_model,
|
||||
is_new_mime_type,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -120,6 +120,7 @@ pub async fn run_content_phase(
|
||||
|
||||
let hash_results = futures::future::join_all(content_hash_futures).await;
|
||||
|
||||
let mut mime_types_to_sync = Vec::new();
|
||||
let mut content_identities_to_sync = Vec::new();
|
||||
let mut entries_to_sync = Vec::new();
|
||||
|
||||
@@ -144,6 +145,13 @@ pub async fn run_content_phase(
|
||||
content_hash
|
||||
));
|
||||
|
||||
// Collect mime_type if it was newly created
|
||||
if let Some(mime_type) = result.mime_type {
|
||||
if result.is_new_mime_type {
|
||||
mime_types_to_sync.push(mime_type);
|
||||
}
|
||||
}
|
||||
|
||||
content_identities_to_sync.push(result.content_identity);
|
||||
entries_to_sync.push(result.entry);
|
||||
|
||||
@@ -188,6 +196,36 @@ pub async fn run_content_phase(
|
||||
}
|
||||
}
|
||||
|
||||
// Sync mime_types first (content_identities depend on them via FK)
|
||||
if !mime_types_to_sync.is_empty() {
|
||||
let library = ctx.library();
|
||||
match library
|
||||
.sync_models_batch(
|
||||
&mime_types_to_sync,
|
||||
crate::infra::sync::ChangeType::Insert,
|
||||
ctx.library_db(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
ctx.log(format!(
|
||||
"Batch synced {} mime types",
|
||||
mime_types_to_sync.len()
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to batch sync {} mime types: {}",
|
||||
mime_types_to_sync.len(),
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Yield to let mime_type sync messages propagate before content_identity inserts
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
if !content_identities_to_sync.is_empty() {
|
||||
let library = ctx.library();
|
||||
match library
|
||||
|
||||
@@ -863,75 +863,120 @@ impl BackfillManager {
|
||||
};
|
||||
|
||||
let db = self.peer_sync.db().clone();
|
||||
if let Err(e) = crate::infra::sync::registry::apply_shared_change(entry.clone(), db).await {
|
||||
// Extract diagnostic information for FK errors
|
||||
let fk_mappings = crate::infra::sync::registry::get_fk_mappings(&model_type);
|
||||
let uuid_fields: Vec<String> = if let Some(obj) = data.as_object() {
|
||||
obj.keys()
|
||||
.filter(|k| k.ends_with("_uuid"))
|
||||
.map(|k| {
|
||||
let value = obj.get(k).and_then(|v| v.as_str()).unwrap_or("null");
|
||||
format!("{}={}", k, value)
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
match crate::infra::sync::registry::apply_shared_change(entry.clone(), db).await {
|
||||
Ok(()) => {
|
||||
// Resolve any state changes waiting for this shared resource
|
||||
let waiting_updates = self
|
||||
.peer_sync
|
||||
.dependency_tracker()
|
||||
.resolve(record_uuid)
|
||||
.await;
|
||||
|
||||
warn!(
|
||||
model_type = %model_type,
|
||||
uuid = %record_uuid,
|
||||
error = %e,
|
||||
fk_mappings = ?fk_mappings,
|
||||
uuid_fields = ?uuid_fields,
|
||||
"Failed to apply current_state snapshot record - likely missing FK dependency"
|
||||
);
|
||||
} else {
|
||||
// Resolve any state changes waiting for this shared resource
|
||||
let waiting_updates = self
|
||||
.peer_sync
|
||||
.dependency_tracker()
|
||||
.resolve(record_uuid)
|
||||
.await;
|
||||
if !waiting_updates.is_empty() {
|
||||
tracing::debug!(
|
||||
resolved_uuid = %record_uuid,
|
||||
model_type = %model_type,
|
||||
waiting_count = waiting_updates.len(),
|
||||
"Resolving dependencies after current_state snapshot"
|
||||
);
|
||||
|
||||
if !waiting_updates.is_empty() {
|
||||
tracing::debug!(
|
||||
resolved_uuid = %record_uuid,
|
||||
model_type = %model_type,
|
||||
waiting_count = waiting_updates.len(),
|
||||
"Resolving dependencies after current_state snapshot"
|
||||
);
|
||||
|
||||
for update in waiting_updates {
|
||||
match update {
|
||||
super::state::BufferedUpdate::StateChange(dependent_change) => {
|
||||
if let Err(e) = self
|
||||
.peer_sync
|
||||
.apply_state_change(dependent_change.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
record_uuid = %dependent_change.record_uuid,
|
||||
"Failed to apply dependent state change after current_state snapshot"
|
||||
);
|
||||
for update in waiting_updates {
|
||||
match update {
|
||||
super::state::BufferedUpdate::StateChange(dependent_change) => {
|
||||
if let Err(e) = self
|
||||
.peer_sync
|
||||
.apply_state_change(dependent_change.clone())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
record_uuid = %dependent_change.record_uuid,
|
||||
"Failed to apply dependent state change after current_state snapshot"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
super::state::BufferedUpdate::SharedChange(dependent_entry) => {
|
||||
// Retry the shared change now that its dependency exists
|
||||
let entry_clone = dependent_entry.clone();
|
||||
let db = self.peer_sync.db().clone();
|
||||
if let Err(e) = crate::infra::sync::registry::apply_shared_change(entry_clone, db).await {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
record_uuid = %dependent_entry.record_uuid,
|
||||
"Failed to apply dependent shared change after current_state snapshot"
|
||||
);
|
||||
super::state::BufferedUpdate::SharedChange(dependent_entry) => {
|
||||
// Retry the shared change now that its dependency exists
|
||||
let entry_clone = dependent_entry.clone();
|
||||
let db = self.peer_sync.db().clone();
|
||||
if let Err(e) = crate::infra::sync::registry::apply_shared_change(entry_clone, db).await {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
record_uuid = %dependent_entry.record_uuid,
|
||||
"Failed to apply dependent shared change after current_state snapshot"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let error_str = e.to_string();
|
||||
|
||||
// Check if this is a FK dependency error
|
||||
if error_str.contains("Sync dependency missing")
|
||||
|| error_str.contains("FOREIGN KEY constraint failed")
|
||||
{
|
||||
// Try to extract the missing UUID from the error message
|
||||
if let Some(missing_uuid) =
|
||||
super::dependency::extract_missing_dependency_uuid(&error_str)
|
||||
{
|
||||
tracing::debug!(
|
||||
record_uuid = %record_uuid,
|
||||
model_type = %model_type,
|
||||
missing_uuid = %missing_uuid,
|
||||
"Snapshot record has missing FK dependency, buffering for retry"
|
||||
);
|
||||
|
||||
// Buffer this shared change for retry when dependency arrives
|
||||
self.peer_sync
|
||||
.dependency_tracker()
|
||||
.add_dependency(
|
||||
missing_uuid,
|
||||
super::state::BufferedUpdate::SharedChange(
|
||||
entry.clone(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
continue; // Skip to next record
|
||||
} else {
|
||||
// FK error but can't extract UUID
|
||||
let fk_mappings = crate::infra::sync::registry::get_fk_mappings(&model_type);
|
||||
let uuid_fields: Vec<String> = if let Some(obj) = data.as_object() {
|
||||
obj.keys()
|
||||
.filter(|k| k.ends_with("_uuid"))
|
||||
.map(|k| {
|
||||
let value = obj.get(k).and_then(|v| v.as_str()).unwrap_or("null");
|
||||
format!("{}={}", k, value)
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
|
||||
warn!(
|
||||
model_type = %model_type,
|
||||
uuid = %record_uuid,
|
||||
error = %e,
|
||||
fk_mappings = ?fk_mappings,
|
||||
uuid_fields = ?uuid_fields,
|
||||
"Failed to apply current_state snapshot record - FK constraint failed but cannot extract missing UUID"
|
||||
);
|
||||
|
||||
continue; // Skip this record
|
||||
}
|
||||
}
|
||||
|
||||
// Non-dependency error - this is unexpected, log but continue
|
||||
warn!(
|
||||
model_type = %model_type,
|
||||
uuid = %record_uuid,
|
||||
error = %e,
|
||||
"Failed to apply current_state snapshot record - unexpected error"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,9 +307,11 @@ pub async fn wait_for_sync(
|
||||
let mut last_alice_entries = 0;
|
||||
let mut last_alice_content = 0;
|
||||
let mut last_bob_entries = 0;
|
||||
let mut last_bob_closure = 0;
|
||||
let mut stable_iterations = 0;
|
||||
let mut no_progress_iterations = 0;
|
||||
let mut alice_stable_iterations = 0;
|
||||
let mut closure_stable_iterations = 0;
|
||||
|
||||
while start.elapsed() < max_duration {
|
||||
let alice_entries = entities::entry::Entity::find()
|
||||
@@ -347,20 +349,46 @@ pub async fn wait_for_sync(
|
||||
no_progress_iterations = 0;
|
||||
}
|
||||
|
||||
// Check Bob's entry_closure table stability
|
||||
// The rebuild runs in multiple iterations, we need to wait for it to finish
|
||||
let bob_closure_count = entities::entry_closure::Entity::find()
|
||||
.count(library_bob.db().conn())
|
||||
.await?;
|
||||
|
||||
if bob_closure_count == last_bob_closure {
|
||||
closure_stable_iterations += 1;
|
||||
} else {
|
||||
closure_stable_iterations = 0;
|
||||
}
|
||||
|
||||
// Only check sync completion if Alice has stabilized first
|
||||
if alice_stable_iterations >= 5 {
|
||||
if alice_entries == bob_entries && alice_content == bob_content {
|
||||
stable_iterations += 1;
|
||||
if stable_iterations >= 5 {
|
||||
tracing::info!(
|
||||
duration_ms = start.elapsed().as_millis(),
|
||||
alice_entries = alice_entries,
|
||||
bob_entries = bob_entries,
|
||||
alice_content = alice_content,
|
||||
bob_content = bob_content,
|
||||
"Sync completed - Alice stable and Bob caught up"
|
||||
// Also check that Bob's entry_closure table has stabilized
|
||||
// This prevents race condition where we check integrity before rebuild completes
|
||||
// The rebuild runs many iterations, so we need several stable checks
|
||||
if closure_stable_iterations >= 3 {
|
||||
stable_iterations += 1;
|
||||
if stable_iterations >= 5 {
|
||||
tracing::info!(
|
||||
duration_ms = start.elapsed().as_millis(),
|
||||
alice_entries = alice_entries,
|
||||
bob_entries = bob_entries,
|
||||
alice_content = alice_content,
|
||||
bob_content = bob_content,
|
||||
bob_closure_count = bob_closure_count,
|
||||
"Sync completed - Alice stable, Bob caught up, and closure table rebuilt"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
} else {
|
||||
stable_iterations = 0;
|
||||
tracing::debug!(
|
||||
bob_closure_count = bob_closure_count,
|
||||
last_bob_closure = last_bob_closure,
|
||||
closure_stable_iters = closure_stable_iterations,
|
||||
"Waiting for entry_closure rebuild to stabilize"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
} else {
|
||||
stable_iterations = 0;
|
||||
@@ -408,6 +436,7 @@ pub async fn wait_for_sync(
|
||||
last_alice_entries = alice_entries;
|
||||
last_alice_content = alice_content;
|
||||
last_bob_entries = bob_entries;
|
||||
last_bob_closure = bob_closure_count;
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ use sd_core::{
|
||||
use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QuerySelect};
|
||||
use std::sync::Arc;
|
||||
use tokio::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
@@ -43,13 +44,16 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
|
||||
tracing::info!("=== Phase 1: Alice indexes location (Bob not connected yet) ===");
|
||||
|
||||
// Generate a shared library UUID for both devices
|
||||
let library_id = Uuid::new_v4();
|
||||
|
||||
let core_alice = Core::new(temp_dir_alice.clone())
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to create Alice core: {}", e))?;
|
||||
let device_alice_id = core_alice.device.device_id()?;
|
||||
let library_alice = core_alice
|
||||
.libraries
|
||||
.create_library_no_sync("Backfill Test Library", None, core_alice.context.clone())
|
||||
.create_library_with_id(library_id, "Backfill Test Library", None, core_alice.context.clone())
|
||||
.await?;
|
||||
|
||||
let device_record = entities::device::Entity::find()
|
||||
@@ -141,10 +145,14 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
let alice_content_after_index = entities::content_identity::Entity::find()
|
||||
.count(library_alice.db().conn())
|
||||
.await?;
|
||||
let alice_mime_types_after_index = entities::mime_type::Entity::find()
|
||||
.count(library_alice.db().conn())
|
||||
.await?;
|
||||
|
||||
tracing::info!(
|
||||
entries = alice_entries_after_index,
|
||||
content_identities = alice_content_after_index,
|
||||
mime_types = alice_mime_types_after_index,
|
||||
"Alice indexing complete"
|
||||
);
|
||||
|
||||
@@ -171,7 +179,7 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
let device_bob_id = core_bob.device.device_id()?;
|
||||
let library_bob = core_bob
|
||||
.libraries
|
||||
.create_library_no_sync("Backfill Test Library", None, core_bob.context.clone())
|
||||
.create_library_with_id(library_id, "Backfill Test Library", None, core_bob.context.clone())
|
||||
.await?;
|
||||
|
||||
register_device(&library_alice, device_bob_id, "Bob").await?;
|
||||
@@ -237,6 +245,9 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
let bob_content_final = entities::content_identity::Entity::find()
|
||||
.count(library_bob.db().conn())
|
||||
.await?;
|
||||
let bob_mime_types_final = entities::mime_type::Entity::find()
|
||||
.count(library_bob.db().conn())
|
||||
.await?;
|
||||
let alice_volumes_final = entities::volume::Entity::find()
|
||||
.count(library_alice.db().conn())
|
||||
.await?;
|
||||
@@ -249,6 +260,8 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
bob_entries = bob_entries_final,
|
||||
alice_content = alice_content_after_index,
|
||||
bob_content = bob_content_final,
|
||||
alice_mime_types = alice_mime_types_after_index,
|
||||
bob_mime_types = bob_mime_types_final,
|
||||
alice_volumes = alice_volumes_final,
|
||||
bob_volumes = bob_volumes_final,
|
||||
"=== Final counts ==="
|
||||
@@ -256,6 +269,7 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
|
||||
let entry_diff = (alice_entries_after_index as i64 - bob_entries_final as i64).abs();
|
||||
let content_diff = (alice_content_after_index as i64 - bob_content_final as i64).abs();
|
||||
let mime_type_diff = (alice_mime_types_after_index as i64 - bob_mime_types_final as i64).abs();
|
||||
|
||||
assert!(
|
||||
entry_diff <= 5,
|
||||
@@ -273,6 +287,41 @@ async fn test_initial_backfill_alice_indexes_first() -> anyhow::Result<()> {
|
||||
content_diff
|
||||
);
|
||||
|
||||
assert!(
|
||||
mime_type_diff == 0,
|
||||
"Mime type count mismatch after backfill: Alice {}, Bob {} (diff: {})",
|
||||
alice_mime_types_after_index,
|
||||
bob_mime_types_final,
|
||||
mime_type_diff
|
||||
);
|
||||
|
||||
// Verify mime types have valid UUIDs (required for sync)
|
||||
let alice_mime_types_with_uuid = entities::mime_type::Entity::find()
|
||||
.filter(entities::mime_type::Column::Uuid.is_not_null())
|
||||
.count(library_alice.db().conn())
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
alice_mime_types_with_uuid, alice_mime_types_after_index,
|
||||
"All mime types on Alice should have UUIDs for sync"
|
||||
);
|
||||
|
||||
let bob_mime_types_with_uuid = entities::mime_type::Entity::find()
|
||||
.filter(entities::mime_type::Column::Uuid.is_not_null())
|
||||
.count(library_bob.db().conn())
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
bob_mime_types_with_uuid, bob_mime_types_final,
|
||||
"All mime types on Bob should have UUIDs after sync"
|
||||
);
|
||||
|
||||
tracing::info!(
|
||||
alice_mime_types = alice_mime_types_after_index,
|
||||
bob_mime_types = bob_mime_types_final,
|
||||
"Mime type sync verification passed"
|
||||
);
|
||||
|
||||
// Verify volume sync
|
||||
assert_eq!(
|
||||
alice_volumes_final, bob_volumes_final,
|
||||
@@ -356,13 +405,16 @@ async fn test_bidirectional_volume_sync() -> anyhow::Result<()> {
|
||||
TestConfigBuilder::new(temp_dir_alice.clone()).build()?;
|
||||
TestConfigBuilder::new(temp_dir_bob.clone()).build()?;
|
||||
|
||||
// Generate a shared library UUID for both devices
|
||||
let library_id = Uuid::new_v4();
|
||||
|
||||
let core_alice = Core::new(temp_dir_alice.clone())
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to create Alice core: {}", e))?;
|
||||
let device_alice_id = core_alice.device.device_id()?;
|
||||
let library_alice = core_alice
|
||||
.libraries
|
||||
.create_library_no_sync("Volume Sync Test", None, core_alice.context.clone())
|
||||
.create_library_with_id(library_id, "Volume Sync Test", None, core_alice.context.clone())
|
||||
.await?;
|
||||
|
||||
let core_bob = Core::new(temp_dir_bob.clone())
|
||||
@@ -371,7 +423,7 @@ async fn test_bidirectional_volume_sync() -> anyhow::Result<()> {
|
||||
let device_bob_id = core_bob.device.device_id()?;
|
||||
let library_bob = core_bob
|
||||
.libraries
|
||||
.create_library_no_sync("Volume Sync Test", None, core_bob.context.clone())
|
||||
.create_library_with_id(library_id, "Volume Sync Test", None, core_bob.context.clone())
|
||||
.await?;
|
||||
|
||||
register_device(&library_alice, device_bob_id, "Bob").await?;
|
||||
|
||||
@@ -4,19 +4,23 @@ description: Infrastructure for multi-device computing
|
||||
sidebarTitle: Introduction
|
||||
---
|
||||
|
||||
<Warning>
|
||||
<Info icon="rocket">
|
||||
**v2.0.0-alpha.1 Released: December 26, 2025**
|
||||
|
||||
The complete ground-up rewrite is here. This is an alpha release for testing and feedback. macOS and Linux are supported now. Windows support coming in alpha.2.
|
||||
|
||||
Read the [full release notes](https://github.com/spacedriveapp/spacedrive/releases) for more details on the rewrite.
|
||||
|
||||
</Warning>
|
||||
</Info>
|
||||
|
||||
## What is Spacedrive?
|
||||
|
||||
Spacedrive is a local-first file manager that unifies data across devices without centralized cloud services. Files stay where they are. Spacedrive creates a content-aware index that makes them searchable, syncable, and manageable from anywhere.
|
||||
|
||||
<Frame>
|
||||
<img src="/public/SDColumnView.webp" alt="Spacedrive column view file explorer" />
|
||||
</Frame>
|
||||
|
||||
## Why Spacedrive Exists
|
||||
|
||||
Computing was designed for single devices. File managers like Finder and Explorer assume your data lives on the computer in front of you. The shift to multi-device forced us into cloud ecosystems. Convenience required giving up data ownership.
|
||||
@@ -160,6 +164,32 @@ Spacedrive v2 development began in June 2025. This is a ground-up rewrite addres
|
||||
|
||||
v2.0.0-alpha.1 released December 26, 2025 after six months of development. This alpha provides the core infrastructure for multi-device file management. Additional platforms and features will roll out in subsequent releases.
|
||||
|
||||
## Screenshots
|
||||
|
||||
<Frame>
|
||||
<img src="/public/SDColumnView.webp" alt="Column view - File explorer interface" />
|
||||
</Frame>
|
||||
|
||||
<Frame>
|
||||
<img src="/public/SDVideoPlayer.webp" alt="Video player" />
|
||||
</Frame>
|
||||
|
||||
<Frame>
|
||||
<img src="/public/SDGridView.webp" alt="Grid view - Browse files in a grid layout" />
|
||||
</Frame>
|
||||
|
||||
<Frame>
|
||||
<img src="/public/SDMediaView.webp" alt="Media view - Gallery and media browsing" />
|
||||
</Frame>
|
||||
|
||||
<Frame>
|
||||
<img src="/public/SDSizeView.webp" alt="Size view - Visualize storage usage" />
|
||||
</Frame>
|
||||
|
||||
<Frame>
|
||||
<img src="/public/SDSplatView.webp" alt="3D Gaussian Splat viewer" />
|
||||
</Frame>
|
||||
|
||||
## Get Involved
|
||||
|
||||
- **Test the code** - Build from source, report bugs, validate cross-platform
|
||||
|
||||
BIN
docs/public/SDColumnView.webp
Normal file
BIN
docs/public/SDColumnView.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 105 KiB |
BIN
docs/public/SDGridView.webp
Normal file
BIN
docs/public/SDGridView.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 109 KiB |
BIN
docs/public/SDMediaView.webp
Normal file
BIN
docs/public/SDMediaView.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 245 KiB |
BIN
docs/public/SDSizeView.webp
Normal file
BIN
docs/public/SDSizeView.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 78 KiB |
BIN
docs/public/SDSplatView.webp
Normal file
BIN
docs/public/SDSplatView.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 187 KiB |
BIN
docs/public/SDVideoPlayer.webp
Normal file
BIN
docs/public/SDVideoPlayer.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 60 KiB |
Reference in New Issue
Block a user