From 7d41e07ab9cc1bf242eaeee082e06b7d2a2ffe90 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Wed, 9 Sep 2026 18:25:30 -0700 Subject: [PATCH] Read files from the arena, not from entries Every query that answered from `entry` now answers from the volume index, which is the one that knows about the whole drive. Each of these had both paths already and picked between them at runtime; the entries branch was the one that only worked where someone had added a location. search collapsed hardest. IndexType existed to choose between an FTS5 index over entries and the in-memory one, and there is only the one now, so fast, normal and full modes were three routes to the same table. A library-wide search fans out over every partition, which is the search that was never possible before: it used to mean "every location you added". SearchScope loses Location. Nothing called it, and under the demotion a location is a subtree rather than a thing you search inside. files.media_listing walks the arena instead of joining four tables, and sorts DateTaken by birth time, which is the closest thing the arena knows and the one a photo library actually wants. files.alternate_instances asks content identity rather than content_id, so it spans drives: every source is asked about the same content uuid and they agree because the id comes from the bytes. Two new store queries back it, content_of and copies_of_content. Copy estimates read subtree_bytes and file_count off the arena. Those rollups are maintained as the arena is written, so the progress bar gets its numbers from a lookup rather than from a walk. files.unique_to_location had no callers and no meaning once a location stops owning records. Co-Authored-By: Claude Opus 5 (1M context) --- apps/cli/src/domains/search/args.rs | 6 - core/src/ops/files/copy/database.rs | 425 +---- core/src/ops/files/copy/job.rs | 2 +- .../ops/files/query/alternate_instances.rs | 336 +--- core/src/ops/files/query/directory_listing.rs | 636 +------ core/src/ops/files/query/file_by_id.rs | 304 +-- core/src/ops/files/query/file_by_path.rs | 261 +-- core/src/ops/files/query/media_listing.rs | 566 +----- core/src/ops/files/query/mod.rs | 2 - .../src/ops/files/query/unique_to_location.rs | 268 --- core/src/ops/indexing/ephemeral/store.rs | 29 + core/src/ops/search/ephemeral_search.rs | 197 +- core/src/ops/search/input.rs | 2 - core/src/ops/search/query.rs | 1625 +---------------- core/src/ops/search/tests.rs | 38 - core/src/ops/spaces/get_layout/query.rs | 130 +- crates/store/src/file.rs | 58 + crates/store/src/lib.rs | 6 +- packages/ts-client/src/generated/types.ts | 36 - 19 files changed, 438 insertions(+), 4489 deletions(-) delete mode 100644 core/src/ops/files/query/unique_to_location.rs diff --git a/apps/cli/src/domains/search/args.rs b/apps/cli/src/domains/search/args.rs index e190ce9b6..66b8a90f3 100644 --- a/apps/cli/src/domains/search/args.rs +++ b/apps/cli/src/domains/search/args.rs @@ -33,10 +33,6 @@ pub struct FileSearchArgs { #[arg(long)] pub exclude_tags: Option>, - /// Location filter - #[arg(long)] - pub location: Option, - /// Date field for filtering #[arg(long, value_enum, default_value = "modified")] pub date_field: DateFieldArg, @@ -156,8 +152,6 @@ impl From for FileSearchInput { SearchScope::Library } } - } else if let Some(location_id) = args.location { - SearchScope::Location { location_id } } else { SearchScope::Library }; diff --git a/core/src/ops/files/copy/database.rs b/core/src/ops/files/copy/database.rs index eb78c8268..e0a2e919c 100644 --- a/core/src/ops/files/copy/database.rs +++ b/core/src/ops/files/copy/database.rs @@ -1,405 +1,70 @@ -//! Database query support for copy operations +//! Instant size and file-count estimates for a copy. //! -//! Provides instant size and file count estimates by querying -//! Spacedrive's indexed data, enabling immediate progress feedback. +//! The arena maintains `subtree_bytes` and `file_count` along the ancestor +//! chain as it is written, so the numbers a copy needs for its progress bar are +//! already computed. Reading them costs a lookup rather than the walk the +//! filesystem would charge for the same answer. -use crate::{ - domain::addressing::SdPath, - infra::db::entities::{entry, location, Entry}, - ops::indexing::PathResolver, -}; +use crate::{context::CoreContext, domain::addressing::SdPath}; use anyhow::Result; -use sea_orm::{prelude::*, Condition, DatabaseConnection, QuerySelect}; -use std::path::{Path, PathBuf}; +use std::sync::Arc; -/// Database query engine for copy preparation +/// Estimate engine over the volume index. pub struct CopyDatabaseQuery { - db: DatabaseConnection, + context: Arc, } impl CopyDatabaseQuery { - pub fn new(db: DatabaseConnection) -> Self { - Self { db } + pub fn new(context: Arc) -> Self { + Self { context } } - /// Get instant estimates for multiple source paths + /// Estimates for several source paths at once. + /// + /// A path the arena does not cover contributes nothing and is counted as + /// unindexed, which is what `confidence` reports: the caller falls back to + /// walking the filesystem when the answer is partial. pub async fn get_estimates_for_paths(&self, sources: &[SdPath]) -> Result { - use crate::ops::indexing::PathResolver; - - let mut total_files = 0u64; - let mut total_bytes = 0u64; - let mut indexed_paths = 0u64; - - for source in sources { - // Use canonical PathResolver for all path types - match PathResolver::resolve_to_entry(&self.db, source).await { - Ok(Some(entry)) => { - let (file_count, total_size) = match entry.kind { - 0 => (1u64, entry.size as u64), // File - 1 => (entry.file_count as u64, entry.aggregate_size as u64), // Directory - _ => (0, 0), - }; - total_files += file_count; - total_bytes += total_size; - indexed_paths += 1; - } - Ok(None) | Err(_) => { - // Entry not found or query failed - will calculate from filesystem later - continue; - } - } - } - - Ok(PathEstimates { - file_count: total_files, - total_size: total_bytes, - indexed_paths, + let cache = self.context.ephemeral_cache(); + let mut estimates = PathEstimates { + file_count: 0, + total_size: 0, + indexed_paths: 0, total_paths: sources.len() as u64, - }) - } - - /// Get estimates for a single path - async fn get_path_estimates(&self, path: &Path) -> Result> { - let path_str = path.to_string_lossy().to_string(); - - // Get all locations to find which one contains this path - let locations = location::Entity::find().all(&self.db).await?; - - // Check each location to see if it contains this path - for location in locations { - // Skip locations without entry_id (not yet synced) - let Some(entry_id) = location.entry_id else { - continue; - }; - // Get the full path of the location's root entry - let location_path = match PathResolver::get_full_path(&self.db, entry_id).await { - Ok(path) => path, - Err(_) => continue, // Skip if we can't get the path - }; - - let location_path_str = location_path.to_string_lossy().to_string(); - - // Check if the target path is within this location - if path_str.starts_with(&location_path_str) { - // If querying the entire location root, use cached stats - if path == location_path { - return Ok(Some(SinglePathEstimate { - file_count: location.total_file_count as u64, - total_size: location.total_byte_size as u64, - })); - } - - // For paths within the location, we need to find the specific entry - // by traversing the hierarchy - let relative_path = match path.strip_prefix(&location_path) { - Ok(rel) => rel, - Err(_) => continue, - }; - - // Get path components - let components: Vec<&str> = relative_path - .components() - .filter_map(|c| c.as_os_str().to_str()) - .collect(); - - if components.is_empty() { - continue; - } - - // Start from the location's root entry and traverse down - let mut current_parent_id = location.entry_id; - let mut target_entry = None; - - for component in components { - if let Some(parent_id) = current_parent_id { - // Find child with matching name - let child = entry::Entity::find() - .filter(entry::Column::ParentId.eq(parent_id)) - .filter(entry::Column::Name.eq(component)) - .one(&self.db) - .await?; - - match child { - Some(c) => { - current_parent_id = Some(c.id); - target_entry = Some(c); - } - None => return Ok(None), // Path not indexed - } - } else { - return Ok(None); - } - } - - // Found the target entry - if let Some(entry) = target_entry { - let (file_count, total_size) = match entry.kind { - 0 => (1u64, entry.size as u64), // File - 1 => { - // Directory - use pre-calculated aggregate values - (entry.file_count as u64, entry.aggregate_size as u64) - } - _ => (0, 0), // Symlink or other - }; - - return Ok(Some(SinglePathEstimate { - file_count, - total_size, - })); - } - } - } - - Ok(None) // Path not in any indexed location - } - - /// Get estimates for a path on a specific device (for cross-device copies) - /// This queries the synced metadata for files on remote devices - async fn get_path_estimates_by_device( - &self, - device_slug: &str, - path: &PathBuf, - ) -> Result> { - use crate::infra::db::entities::{device, volume}; - - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Starting query for device '{}' path '{}'", - device_slug, - path.display() - ); - - // Find the device by slug - let device = device::Entity::find() - .filter(device::Column::Slug.eq(device_slug)) - .one(&self.db) - .await?; - - let Some(device) = device else { - tracing::warn!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Device not found: '{}'", - device_slug - ); - return Ok(None); }; - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Found device: id={} name={} uuid={}", - device.id, - device.name, - device.uuid - ); + for source in sources { + let SdPath::Physical { path, .. } = source else { + continue; + }; - // Find all volumes for this device (device_id references device.uuid, not device.id) - let volumes = volume::Entity::find() - .filter(volume::Column::DeviceId.eq(device.uuid)) - .all(&self.db) - .await?; + cache.ensure_restored(path).await; + let Some(index) = cache.get_for_search(path) else { + continue; + }; - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Found {} volumes for device", - volumes.len() - ); + let index = index.read().await; + let Some(metadata) = index.get_entry_ref(path) else { + continue; + }; - let path_str = path.to_string_lossy().to_string(); - - // Check each volume's locations - for volume in volumes { - let locations = location::Entity::find() - .filter(location::Column::VolumeId.eq(volume.id)) - .all(&self.db) - .await?; - - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Volume {} has {} locations", - volume.id, - locations.len() - ); - - for location in locations { - let Some(entry_id) = location.entry_id else { - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Location id={} has no entry_id, skipping", - location.id - ); + // A file is one file of its own size; a directory answers with the + // rollup the arena keeps for it. + if metadata.kind == crate::ops::indexing::state::EntryKind::Directory { + let Some(bytes) = index.subtree_size(path) else { continue; }; - - // Get the full path of the location's root entry - let location_path = match PathResolver::get_full_path(&self.db, entry_id).await { - Ok(path) => path, - Err(e) => { - tracing::warn!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Failed to get path for location id={}: {}", - location.id, - e - ); - continue; - } - }; - - let location_path_str = location_path.to_string_lossy().to_string(); - - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Checking location id={} with path '{}'", - location.id, - location_path_str - ); - - // Check if the target path is within this location - if path_str.starts_with(&location_path_str) { - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Path matches location! Target: '{}', Location: '{}'", - path_str, - location_path_str - ); - - // If querying the entire location root, use cached stats - if path == &location_path { - tracing::info!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] ✓ Using location stats: {} files, {} bytes", - location.total_file_count, - location.total_byte_size - ); - return Ok(Some(SinglePathEstimate { - file_count: location.total_file_count as u64, - total_size: location.total_byte_size as u64, - })); - } - - // For paths within the location, find the specific entry - let relative_path = match path.strip_prefix(&location_path) { - Ok(rel) => rel, - Err(e) => { - tracing::warn!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Failed to strip prefix: {}", - e - ); - continue; - } - }; - - let components: Vec<&str> = relative_path - .components() - .filter_map(|c| c.as_os_str().to_str()) - .collect(); - - if components.is_empty() { - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] No relative components, skipping" - ); - continue; - } - - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Traversing {} components: {:?}", - components.len(), - components - ); - - // Traverse hierarchy to find target entry - let mut current_parent_id = location.entry_id; - let mut target_entry = None; - - for component in components { - if let Some(parent_id) = current_parent_id { - // Remove extension for entry lookup (extensions stored separately) - let component_without_ext = if let Some(dot_pos) = component.rfind('.') - { - &component[..dot_pos] - } else { - component - }; - - // Normalize Unicode spaces (macOS uses special space characters) - // Replace narrow no-break space (\u{202f}) and other space variants with regular space - let normalized_name = component_without_ext - .replace('\u{202f}', " ") // Narrow no-break space - .replace('\u{00a0}', " ") // Non-breaking space - .replace('\u{2009}', " "); // Thin space - - let child = entry::Entity::find() - .filter(entry::Column::ParentId.eq(parent_id)) - .filter(entry::Column::Name.eq(normalized_name)) - .one(&self.db) - .await?; - - match child { - Some(c) => { - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Found component '{}' (id={})", - component, - c.id - ); - current_parent_id = Some(c.id); - target_entry = Some(c); - } - None => { - tracing::warn!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Component '{}' not found under parent_id={}", - component, - parent_id - ); - return Ok(None); - } - } - } else { - return Ok(None); - } - } - - // Found the target entry - if let Some(entry) = target_entry { - let (file_count, total_size) = match entry.kind { - 0 => (1u64, entry.size as u64), - 1 => (entry.file_count as u64, entry.aggregate_size as u64), - _ => (0, 0), - }; - - tracing::info!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] ✓ Found entry '{}': {} files, {} bytes", - entry.name, - file_count, - total_size - ); - - return Ok(Some(SinglePathEstimate { - file_count, - total_size, - })); - } - } else { - tracing::debug!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] Path mismatch - Target: '{}', Location: '{}'", - path_str, - location_path_str - ); - } + estimates.total_size += bytes; + estimates.file_count += index.subtree_file_count(path).unwrap_or(0) as u64; + } else { + estimates.total_size += metadata.size; + estimates.file_count += 1; } + + estimates.indexed_paths += 1; } - tracing::warn!( - target: "sd_core::copy_size_query", - "[SIZE QUERY] ✗ No matching entry found for '{}'", - path_str - ); - Ok(None) + Ok(estimates) } } diff --git a/core/src/ops/files/copy/job.rs b/core/src/ops/files/copy/job.rs index ae718f155..99a1f4606 100644 --- a/core/src/ops/files/copy/job.rs +++ b/core/src/ops/files/copy/job.rs @@ -237,7 +237,7 @@ impl JobHandler for FileCopyJob { // Try to get estimates from database let db_estimates = { - let db_query = CopyDatabaseQuery::new(ctx.library_db().clone()); + let db_query = CopyDatabaseQuery::new(ctx.library().core_context().clone()); match db_query.get_estimates_for_paths(&self.sources.paths).await { Ok(estimates) => { ctx.log(format!( diff --git a/core/src/ops/files/query/alternate_instances.rs b/core/src/ops/files/query/alternate_instances.rs index 2fb325a5b..15ab1c8a8 100644 --- a/core/src/ops/files/query/alternate_instances.rs +++ b/core/src/ops/files/query/alternate_instances.rs @@ -1,22 +1,21 @@ -//! Query to get all alternate instances of a file by entry ID +//! Every copy of one file's bytes, wherever this machine keeps them. //! -//! This query finds all other entries that share the same content_id and returns -//! them as complete File objects with all related data (tags, sidecars, media data). +//! Answers from content identity rather than from a path or a size, so a copy +//! renamed on the way to its second home is still the same file. The identity +//! is derived from the bytes, so the answer can span drives: each source is +//! asked about the same content uuid and they agree without comparing notes. +//! +//! A record whose bytes have not been hashed yet has no identity and therefore +//! no alternates, which is the honest answer rather than a guess from size. -use crate::infra::query::{QueryError, QueryResult}; use crate::{ context::CoreContext, - domain::{addressing::SdPath, content_identity::ContentIdentity, file::File}, - infra::db::entities::{ - content_identity, device, directory_paths, entry, location, sidecar, tag, user_metadata, - user_metadata_tag, video_media_data, - }, - infra::query::LibraryQuery, + domain::{addressing::SdPath, file::File}, + infra::query::{LibraryQuery, QueryResult}, }; -use sea_orm::{ColumnTrait, ConnectionTrait, DatabaseConnection, EntityTrait, QueryFilter}; use serde::{Deserialize, Serialize}; use specta::Type; -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::{collections::HashSet, sync::Arc}; use uuid::Uuid; /// Input for alternate instances query @@ -60,302 +59,69 @@ impl LibraryQuery for AlternateInstancesQuery { async fn execute( self, context: Arc, - session: crate::infra::api::SessionContext, + _session: crate::infra::api::SessionContext, ) -> QueryResult { - let library_id = session - .current_library_id - .ok_or_else(|| QueryError::Internal("No library in session".to_string()))?; + let cache = context.ephemeral_cache(); - let library = context - .libraries() - .await - .get_library(library_id) - .await - .ok_or_else(|| QueryError::Internal("Library not found".to_string()))?; + // A drive in a drawer still holds records, and its copies are real + // even though they cannot be opened right now, so detached sources + // are asked too. What matters is that the record exists somewhere. + let mut stores = Vec::new(); + for source in cache.sources() { + if let Some(store) = cache.store_for(&source.root).await { + stores.push(store); + } + } - let db = library.db(); + let mut content = None; + for store in &stores { + if let Some(uuid) = store.content_of(self.input.entry_uuid).await { + content = Some(uuid); + break; + } + } - // Find the original entry - let original_entry = entry::Entity::find() - .filter(entry::Column::Uuid.eq(self.input.entry_uuid)) - .one(db.conn()) - .await? - .ok_or_else(|| QueryError::Internal("Entry not found".to_string()))?; - - // Get the content_id - let content_id = original_entry.content_id.ok_or_else(|| { - QueryError::Internal( - "Entry has no content identity, cannot find alternates".to_string(), - ) - })?; - - // Find all entries with the same content_id - let alternate_entries = entry::Entity::find() - .filter(entry::Column::ContentId.eq(content_id)) - .all(db.conn()) - .await?; - - if alternate_entries.is_empty() { + let Some(content) = content else { return Ok(AlternateInstancesOutput { instances: Vec::new(), total_count: 0, }); - } - - // Batch load content identity - let content_identity_model = content_identity::Entity::find_by_id(content_id) - .one(db.conn()) - .await? - .ok_or_else(|| QueryError::Internal("Content identity not found".to_string()))?; - - let content_uuid = content_identity_model.uuid; - - // Batch load sidecars - let sidecars = if let Some(ci_uuid) = content_uuid { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.eq(ci_uuid)) - .all(db.conn()) - .await? - .into_iter() - .map(|s| crate::domain::file::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }) - .collect() - } else { - Vec::new() }; - // Batch load tags for all entries - let entry_uuids: Vec = alternate_entries.iter().filter_map(|e| e.uuid).collect(); - - let mut tags_by_entry: HashMap> = HashMap::new(); - - if !entry_uuids.is_empty() || content_uuid.is_some() { - // Load user_metadata for entries and content - let mut filter = user_metadata::Column::EntryUuid.is_in(entry_uuids.clone()); - if let Some(ci_uuid) = content_uuid { - filter = filter.or(user_metadata::Column::ContentIdentityUuid.eq(ci_uuid)); - } - - let metadata_records = user_metadata::Entity::find() - .filter(filter) - .all(db.conn()) - .await?; - - if !metadata_records.is_empty() { - let metadata_ids: Vec = metadata_records.iter().map(|m| m.id).collect(); - - // Load user_metadata_tag records - let metadata_tags = user_metadata_tag::Entity::find() - .filter(user_metadata_tag::Column::UserMetadataId.is_in(metadata_ids)) - .all(db.conn()) - .await?; - - if !metadata_tags.is_empty() { - let tag_ids: Vec = metadata_tags.iter().map(|mt| mt.tag_id).collect(); - - // Load tag entities - let tag_models = tag::Entity::find() - .filter(tag::Column::Id.is_in(tag_ids)) - .all(db.conn()) - .await?; - - // Build tag_id -> Tag mapping - let tag_map: HashMap = tag_models - .into_iter() - .filter_map(|t| { - let db_id = t.id; - crate::ops::tags::manager::model_to_domain(t) - .ok() - .map(|tag| (db_id, tag)) - }) - .collect(); - - // Build metadata_id -> Vec mapping - let mut tags_by_metadata: HashMap> = - HashMap::new(); - for mt in metadata_tags { - if let Some(tag) = tag_map.get(&mt.tag_id) { - tags_by_metadata - .entry(mt.user_metadata_id) - .or_insert_with(Vec::new) - .push(tag.clone()); - } - } - - // Map tags to entries (prioritize entry-scoped, fall back to content-scoped) - for metadata in &metadata_records { - if let Some(tags) = tags_by_metadata.get(&metadata.id) { - // Entry-scoped metadata (higher priority) - if let Some(entry_uuid) = metadata.entry_uuid { - tags_by_entry.insert(entry_uuid, tags.clone()); - } - // Content-scoped metadata (applies to all entries with this content) - else if let Some(_content_uuid) = metadata.content_identity_uuid { - // Apply to all entries - for entry_uuid in &entry_uuids { - tags_by_entry - .entry(*entry_uuid) - .or_insert_with(|| tags.clone()); - } - } - } - } - } - } - } - - // Build content identity domain object - let content_identity_domain = ContentIdentity { - uuid: content_uuid.unwrap_or_else(Uuid::new_v4), - kind: crate::domain::ContentKind::from_id(content_identity_model.kind_id), - content_hash: content_identity_model.content_hash, - integrity_hash: content_identity_model.integrity_hash, - mime_type_id: content_identity_model.mime_type_id, - text_content: content_identity_model.text_content, - total_size: content_identity_model.total_size, - entry_count: content_identity_model.entry_count, - first_seen_at: content_identity_model.first_seen_at, - last_verified_at: content_identity_model.last_verified_at, - }; - - // Load media data if available - let video_media_data = if let Some(video_id) = content_identity_model.video_media_data_id { - video_media_data::Entity::find_by_id(video_id) - .one(db.conn()) - .await? - .map(Into::into) - } else { - None - }; - - // Convert each entry to a complete File object let mut instances = Vec::new(); - for entry_model in alternate_entries { - // Resolve full path for this entry - let sd_path = match self.resolve_entry_path(&entry_model, db.conn()).await { - Ok(path) => path, - Err(e) => { - tracing::warn!("Failed to resolve path for entry {}: {}", entry_model.id, e); + let mut seen: HashSet = HashSet::new(); + + for store in &stores { + for copy in store.copies_of_content(content).await { + if !seen.insert(copy.record_uuid) { continue; } - }; - // Create File from entry model - let mut file = File::from_entity_model(entry_model.clone(), sd_path); + let index = cache.resolve_index(©.path); + let mut index = index.write().await; + let Some(metadata) = index.get_entry_ref(©.path) else { + continue; + }; + let kind = index.get_content_kind(©.path); + drop(index); - // Add content identity, sidecars, and media data - file.content_identity = Some(content_identity_domain.clone()); - file.sidecars = sidecars.clone(); - file.video_media_data = video_media_data.clone(); - file.content_kind = content_identity_domain.kind; - file.duration_seconds = video_media_data.as_ref().and_then(|v| v.duration_seconds); - - // Add tags for this specific entry - if let Some(entry_uuid) = entry_model.uuid { - if let Some(tags) = tags_by_entry.get(&entry_uuid) { - file.tags = tags.clone(); - } + let mut file = File::from_ephemeral( + copy.record_uuid, + &metadata, + SdPath::local(copy.path.clone()), + ); + file.content_kind = kind; + instances.push(file); } - - instances.push(file); } - let total_count = instances.len() as u32; + instances.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase())); Ok(AlternateInstancesOutput { + total_count: instances.len() as u32, instances, - total_count, }) } } -impl AlternateInstancesQuery { - /// Resolve the full absolute SdPath for an entry - async fn resolve_entry_path( - &self, - entry: &entry::Model, - db: &DatabaseConnection, - ) -> QueryResult { - // Walk up the entry hierarchy to build the full path - let mut path_components = Vec::new(); - - // Add the file name with extension - let file_name = if let Some(ext) = &entry.extension { - format!("{}.{}", entry.name, ext) - } else { - entry.name.clone() - }; - path_components.push(file_name); - - // Walk up parent chain - let mut current_parent_id = entry.parent_id; - let mut location_entry_id = None; - - while let Some(parent_id) = current_parent_id { - let parent = entry::Entity::find_by_id(parent_id) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Parent entry not found".to_string()))?; - - // Check if this is the location root (no parent) - if parent.parent_id.is_none() { - location_entry_id = Some(parent.id); - break; - } - - // Add parent directory name to path - path_components.push(parent.name.clone()); - current_parent_id = parent.parent_id; - } - - // Reverse to get correct order (root -> file) - path_components.reverse(); - - // Get location info - let location_entry_id = location_entry_id - .ok_or_else(|| QueryError::Internal("Could not find location root".to_string()))?; - - let location_model = location::Entity::find() - .filter(location::Column::EntryId.eq(location_entry_id)) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Location not found for entry".to_string()))?; - - // Get device slug - let device_model = device::Entity::find_by_id(location_model.device_id) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Device not found".to_string()))?; - - // Get location root absolute path - let location_root_path = directory_paths::Entity::find() - .filter(directory_paths::Column::EntryId.eq(location_entry_id)) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Location root path not found".to_string()))?; - - // Build absolute path: location_root + relative components - let mut absolute_path = PathBuf::from(&location_root_path.path); - for component in path_components { - absolute_path.push(component); - } - - Ok(SdPath::Physical { - device_slug: device_model.slug, - path: absolute_path.into(), - }) - } -} - -// Register the query crate::register_library_query!(AlternateInstancesQuery, "files.alternate_instances"); diff --git a/core/src/ops/files/query/directory_listing.rs b/core/src/ops/files/query/directory_listing.rs index df65c23de..7f050f6cf 100644 --- a/core/src/ops/files/query/directory_listing.rs +++ b/core/src/ops/files/query/directory_listing.rs @@ -130,493 +130,12 @@ impl LibraryQuery for DirectoryListingQuery { .await .ok_or_else(|| QueryError::Internal("Library not found".to_string()))?; - let db = library.db(); - - // Check if this path's location has IndexMode::None - if let Some(should_use_ephemeral) = self.check_location_index_mode(db.conn()).await { - if should_use_ephemeral { - tracing::debug!("IndexMode::None, using ephemeral indexing"); - return self - .query_ephemeral_directory_impl(context, library_id) - .await; - } - } - - // Try to find the parent directory entry - match self.find_parent_directory(db.conn()).await { - Ok(parent_entry) => { - // Path is indexed - query from database - let parent_id = parent_entry.id; - tracing::debug!("Indexed path, parent_id={}", parent_id); - self.query_indexed_directory_impl(parent_id, db.conn()) - .await - } - Err(e) => { - // Path not indexed - trigger ephemeral indexing - tracing::debug!("Path not indexed, using ephemeral (err={:?})", e); - self.query_ephemeral_directory_impl(context, library_id) - .await - } - } + self.query_ephemeral_directory_impl(context, library_id) + .await } } impl DirectoryListingQuery { - /// Query indexed directory from database - async fn query_indexed_directory_impl( - &self, - parent_id: i32, - db: &DatabaseConnection, - ) -> QueryResult { - // Build efficient SQL query to get all data in one go - let mut sql_query = r#" - SELECT - e.id as entry_id, - e.uuid as entry_uuid, - e.name as entry_name, - e.kind as entry_kind, - e.extension as entry_extension, - e.size as entry_size, - e.aggregate_size as entry_aggregate_size, - e.child_count as entry_child_count, - e.file_count as entry_file_count, - e.created_at as entry_created_at, - e.modified_at as entry_modified_at, - e.accessed_at as entry_accessed_at, - e.inode as entry_inode, - e.parent_id as entry_parent_id, - ci.id as content_identity_id, - ci.uuid as content_identity_uuid, - ci.content_hash as content_hash, - ci.integrity_hash as integrity_hash, - ci.mime_type_id as mime_type_id, - ci.text_content as text_content, - ci.total_size as total_size, - ci.entry_count as entry_count, - ci.first_seen_at as first_seen_at, - ci.last_verified_at as last_verified_at, - ck.id as content_kind_id, - ck.name as content_kind_name, - vmd.uuid as video_media_uuid, - vmd.duration_seconds as video_duration_seconds, - vmd.width as video_width, - vmd.height as video_height - FROM entries e - LEFT JOIN content_identities ci ON e.content_id = ci.id - LEFT JOIN content_kinds ck ON ci.kind_id = ck.id - LEFT JOIN video_media_data vmd ON ci.video_media_data_id = vmd.id - WHERE e.parent_id = ?1 - "# - .to_string(); - - // Apply hidden file filter - if !self.input.include_hidden.unwrap_or(false) { - sql_query.push_str(" AND e.name NOT LIKE '.%'"); - } - - // Apply sorting - let folders_first = self.input.folders_first.unwrap_or(false); - sql_query.push_str(" ORDER BY "); - - if folders_first { - // Always show directories before files - sql_query.push_str("e.kind DESC, "); - } - - match self.input.sort_by { - DirectorySortBy::Name => sql_query.push_str("e.name ASC"), - DirectorySortBy::Modified => sql_query.push_str("e.modified_at DESC"), - DirectorySortBy::Size => sql_query.push_str("e.size DESC"), - DirectorySortBy::Type => { - if !folders_first { - // Only add kind sorting if folders_first isn't already set - sql_query.push_str("e.kind DESC, "); - } - sql_query.push_str("e.name ASC"); - } - } - - // Apply limit - if let Some(limit) = self.input.limit { - sql_query.push_str(&format!(" LIMIT {}", limit)); - } - - // Execute the query - let rows = db - .query_all(sea_orm::Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Sqlite, - &sql_query, - [parent_id.into()], - )) - .await?; - - tracing::debug!(" Query executed, found {} entries", rows.len()); - - if rows.is_empty() { - tracing::debug!(" Directory is empty"); - return Ok(DirectoryListingOutput { - files: Vec::new(), - total_count: 0, - has_more: false, - }); - } - - // Get total count for pagination - use the count from the already fetched entries - let total_count = rows.len() as u32; - - // Collect all content UUIDs for batch sidecar query - let content_uuids: Vec = rows - .iter() - .filter_map(|row| { - row.try_get::>("", "content_identity_uuid") - .ok() - .flatten() - }) - .collect(); - - // Batch fetch all sidecars for these content UUIDs - let all_sidecars = if !content_uuids.is_empty() { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.is_in(content_uuids.clone())) - .all(db) - .await? - } else { - Vec::new() - }; - - // Group sidecars by content_uuid for fast lookup - let mut sidecars_by_content: HashMap> = - HashMap::new(); - for s in all_sidecars { - sidecars_by_content - .entry(s.content_uuid) - .or_insert_with(Vec::new) - .push(crate::domain::file::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }); - } - - // Collect entry UUIDs for tag lookup - let entry_uuids: Vec = rows - .iter() - .filter_map(|row| row.try_get::>("", "entry_uuid").ok().flatten()) - .collect(); - - // Batch fetch tags for these entries (both entry-scoped and content-scoped) - let mut tags_by_entry: HashMap> = HashMap::new(); - - if !entry_uuids.is_empty() || !content_uuids.is_empty() { - use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; - - tracing::debug!( - "Loading tags for {} entries and {} content identities", - entry_uuids.len(), - content_uuids.len() - ); - - // Load user_metadata for entries and content - let mut metadata_records = user_metadata::Entity::find() - .filter( - user_metadata::Column::EntryUuid - .is_in(entry_uuids.clone()) - .or(user_metadata::Column::ContentIdentityUuid.is_in(content_uuids.clone())), - ) - .all(db) - .await?; - - tracing::debug!("Found {} metadata records", metadata_records.len()); - - if !metadata_records.is_empty() { - let metadata_ids: Vec = metadata_records.iter().map(|m| m.id).collect(); - - // Load user_metadata_tag records - let metadata_tags = user_metadata_tag::Entity::find() - .filter(user_metadata_tag::Column::UserMetadataId.is_in(metadata_ids)) - .all(db) - .await?; - - // Get all unique tag IDs - let tag_ids: Vec = metadata_tags.iter().map(|mt| mt.tag_id).collect(); - let unique_tag_ids: std::collections::HashSet = - tag_ids.iter().cloned().collect(); - - tracing::debug!( - "Found {} user_metadata_tag records with {} unique tags", - metadata_tags.len(), - unique_tag_ids.len() - ); - - // Load tag entities - let tag_models = tag::Entity::find() - .filter(tag::Column::Id.is_in(tag_ids)) - .all(db) - .await?; - - tracing::debug!("Loaded {} tag models", tag_models.len()); - - // Build tag_db_id -> Tag mapping - let tag_map: HashMap = tag_models - .into_iter() - .filter_map(|t| { - let db_id = t.id; - crate::ops::tags::manager::model_to_domain(t) - .ok() - .map(|tag| (db_id, tag)) - }) - .collect(); - - tracing::debug!("Built tag map with {} entries", tag_map.len()); - - // Build metadata_id -> Vec mapping - let mut tags_by_metadata: HashMap> = - HashMap::new(); - for mt in metadata_tags { - if let Some(tag) = tag_map.get(&mt.tag_id) { - tags_by_metadata - .entry(mt.user_metadata_id) - .or_insert_with(Vec::new) - .push(tag.clone()); - } - } - - // Map tags to entries (prioritize entry-scoped, fall back to content-scoped) - for metadata in metadata_records { - if let Some(tags) = tags_by_metadata.get(&metadata.id) { - // Entry-scoped metadata (higher priority) - if let Some(entry_uuid) = metadata.entry_uuid { - tags_by_entry.insert(entry_uuid, tags.clone()); - } - // Content-scoped metadata (applies to all entries with this content) - else if let Some(content_uuid) = metadata.content_identity_uuid { - // Apply to all entries with this content_uuid - for row in &rows { - if let Some(ci_uuid) = row - .try_get::>("", "content_identity_uuid") - .ok() - .flatten() - { - if ci_uuid == content_uuid { - if let Some(entry_uuid) = row - .try_get::>("", "entry_uuid") - .ok() - .flatten() - { - // Only set if not already set by entry-scoped metadata - tags_by_entry - .entry(entry_uuid) - .or_insert_with(|| tags.clone()); - } - } - } - } - } - } - } - } - } - - // Convert to File objects - let mut files = Vec::new(); - for row in rows { - // Extract data from SQL row - let entry_id: i32 = row.try_get("", "entry_id").unwrap_or(0); - let entry_uuid: Option = row.try_get("", "entry_uuid").ok(); - let entry_name: String = row.try_get("", "entry_name").unwrap_or_default(); - let entry_kind: i32 = row.try_get("", "entry_kind").unwrap_or(0); - let entry_extension: Option = row.try_get("", "entry_extension").ok(); - let entry_size: i64 = row.try_get("", "entry_size").unwrap_or(0); - let entry_aggregate_size: i64 = row.try_get("", "entry_aggregate_size").unwrap_or(0); - let entry_child_count: i32 = row.try_get("", "entry_child_count").unwrap_or(0); - let entry_file_count: i32 = row.try_get("", "entry_file_count").unwrap_or(0); - let entry_created_at: chrono::DateTime = row - .try_get("", "entry_created_at") - .unwrap_or_else(|_| chrono::Utc::now()); - let entry_modified_at: chrono::DateTime = row - .try_get("", "entry_modified_at") - .unwrap_or_else(|_| chrono::Utc::now()); - let entry_accessed_at: Option> = - row.try_get("", "entry_accessed_at").ok(); - - // Content identity data - let content_identity_uuid: Option = row.try_get("", "content_identity_uuid").ok(); - let content_hash: Option = row.try_get("", "content_hash").ok(); - let integrity_hash: Option = row.try_get("", "integrity_hash").ok(); - let mime_type_id: Option = row.try_get("", "mime_type_id").ok(); - let text_content: Option = row.try_get("", "text_content").ok(); - let total_size: Option = row.try_get("", "total_size").ok(); - let entry_count: Option = row.try_get("", "entry_count").ok(); - let first_seen_at: Option> = - row.try_get("", "first_seen_at").ok(); - let last_verified_at: Option> = - row.try_get("", "last_verified_at").ok(); - - // Content kind data - let content_kind_name: Option = row.try_get("", "content_kind_name").ok(); - - // Video media data - let video_media_uuid: Option = row.try_get("", "video_media_uuid").ok(); - let video_duration_seconds: Option = - row.try_get("", "video_duration_seconds").ok(); - let video_width: Option = row.try_get("", "video_width").ok(); - let video_height: Option = row.try_get("", "video_height").ok(); - - // Build SdPath for this entry (child of the parent path) - // IMPORTANT: Include extension in the path for files - let full_name = if let Some(ext) = &entry_extension { - format!("{}.{}", entry_name, ext) - } else { - entry_name.clone() - }; - - let entry_sd_path = match &self.input.path { - SdPath::Physical { device_slug, path } => SdPath::Physical { - device_slug: device_slug.clone(), - path: path.join(&full_name).into(), - }, - SdPath::Cloud { - service, - identifier, - path, - } => SdPath::Cloud { - service: *service, - identifier: identifier.clone(), - path: format!("{}/{}", path, full_name), - }, - SdPath::Content { content_id } => { - // This shouldn't happen since we error on Content paths earlier - SdPath::Content { - content_id: *content_id, - } - } - SdPath::Sidecar { .. } => { - // This shouldn't happen since we error on Sidecar paths earlier - return Err(QueryError::Internal( - "Sidecar paths not supported for directory listing".to_string(), - )); - } - }; - - // Create entity model for conversion - let entity_model = entry::Model { - id: entry_id, - uuid: entry_uuid, - name: entry_name, - kind: entry_kind, - extension: entry_extension, - metadata_id: None, - content_id: None, - size: entry_size, - aggregate_size: entry_aggregate_size, - child_count: entry_child_count, - file_count: entry_file_count, - created_at: entry_created_at, - modified_at: entry_modified_at, - accessed_at: entry_accessed_at, - indexed_at: None, // Not needed for query result conversion - permissions: None, - inode: None, - parent_id: None, - volume_id: None, - }; - - // Convert to File using from_entity_model - let mut file = File::from_entity_model(entity_model, entry_sd_path); - - // Add content identity if available - if let (Some(ci_uuid), Some(ci_hash), Some(ci_first_seen), Some(ci_last_verified)) = ( - content_identity_uuid, - content_hash, - first_seen_at, - last_verified_at, - ) { - // Convert content_kind name to ContentKind enum - let kind = content_kind_name - .as_ref() - .map(|name| crate::domain::ContentKind::from(name.as_str())) - .unwrap_or(crate::domain::ContentKind::Unknown); - - file.content_identity = Some(ContentIdentity { - uuid: ci_uuid, - kind, - content_hash: ci_hash, - integrity_hash, - mime_type_id, - text_content, - total_size: total_size.unwrap_or(0), - entry_count: entry_count.unwrap_or(0), - first_seen_at: ci_first_seen, - last_verified_at: ci_last_verified, - }); - file.content_kind = kind; - - // Add sidecars from batch lookup - if let Some(sidecars) = sidecars_by_content.get(&ci_uuid) { - file.sidecars = sidecars.clone(); - } - } - - // Add video media data if available - if let Some(vmd_uuid) = video_media_uuid { - file.video_media_data = Some(crate::domain::VideoMediaData { - uuid: vmd_uuid, - width: video_width.unwrap_or(0) as u32, - height: video_height.unwrap_or(0) as u32, - duration_seconds: video_duration_seconds, - bit_rate: None, - codec: None, - pixel_format: None, - color_space: None, - color_range: None, - color_primaries: None, - color_transfer: None, - fps_num: None, - fps_den: None, - audio_codec: None, - audio_channels: None, - audio_sample_rate: None, - audio_bit_rate: None, - title: None, - artist: None, - album: None, - creation_time: None, - date_captured: None, - blurhash: None, - }); - } - - // Add tags from batch lookup - if let Some(entry_uuid_val) = entry_uuid { - if let Some(tags) = tags_by_entry.get(&entry_uuid_val) { - tracing::debug!("Adding {} tags to entry {}", tags.len(), entry_uuid_val); - file.tags = tags.clone(); - } - } - - files.push(file); - } - - let has_more = if let Some(limit) = self.input.limit { - total_count > limit - } else { - false - }; - - Ok(DirectoryListingOutput { - files, - total_count, - has_more, - }) - } - /// Query ephemeral directory (not indexed) - check cache first, then trigger on-demand indexing async fn query_ephemeral_directory_impl( &self, @@ -877,155 +396,4 @@ impl DirectoryListingQuery { } } -impl DirectoryListingQuery { - /// Check if the location containing this path has IndexMode::None - /// Returns Some(true) if should use ephemeral, Some(false) if indexed, None if location not found - async fn check_location_index_mode(&self, db: &DatabaseConnection) -> Option { - use crate::infra::db::entities::location; - - match &self.input.path { - SdPath::Physical { device_slug, path } => { - // Find all locations for this device - let path_str = path.to_string_lossy().to_string(); - - // Get all locations and find the one that is a parent of this path - if let Ok(locations) = location::Entity::find().all(db).await { - for loc in locations { - // Get the location's root path - if let Some(entry_id) = loc.entry_id { - if let Ok(Some(dir_path)) = - directory_paths::Entity::find_by_id(entry_id).one(db).await - { - // Check if this location's path is a parent of the requested path - if path_str.starts_with(&dir_path.path) { - // Check if index_mode is "none" - return Some(loc.index_mode == "none"); - } - } - } - } - } - None - } - SdPath::Cloud { .. } => { - // Similar logic for cloud paths - // For now, assume cloud paths should check their location too - None - } - _ => None, - } - } - - /// Find the parent directory entry for the given SdPath - async fn find_parent_directory(&self, db: &DatabaseConnection) -> QueryResult { - tracing::debug!( - " find_parent_directory called with path: {:?}", - self.input.path - ); - - match &self.input.path { - SdPath::Physical { device_slug, path } => { - // For directory browsing, we need to find the directory entry - // by matching the path in the directory_paths table - let path_str = path.to_string_lossy().to_string(); - tracing::debug!(" Looking for directory path: '{}'", path_str); - - // Find directory entry by path - tracing::debug!(" Querying directory_paths table..."); - let directory_path = directory_paths::Entity::find() - .filter(directory_paths::Column::Path.eq(&path_str)) - .one(db) - .await?; - tracing::debug!(" Directory path query result: {:?}", directory_path); - - match directory_path { - Some(dp) => { - tracing::debug!(" Found directory path entry: {:?}", dp); - tracing::debug!(" Looking for entry with ID: {}", dp.entry_id); - - // Get the entry for this directory - let entry_result = entry::Entity::find_by_id(dp.entry_id).one(db).await?; - tracing::debug!(" Entry query result: {:?}", entry_result); - - entry_result.ok_or_else(|| { - QueryError::Internal(format!( - "Entry not found for directory: {}", - dp.entry_id - )) - }) - } - None => { - tracing::debug!(" Directory not found in directory_paths table"); - // Directory not found in directory_paths table - // This means the directory hasn't been indexed yet - // Return a helpful error message - Err(QueryError::Internal( - format!("Directory '{}' has not been indexed yet. Please add this location to Spacedrive and wait for indexing to complete.", path_str) - )) - } - } - } - SdPath::Cloud { - service, - identifier, - path, - } => { - // Cloud storage directory browsing - tracing::debug!( - " Looking for cloud directory: service={}, identifier={}, path='{}'", - service.scheme(), - identifier, - path - ); - - // Find directory entry by path in directory_paths table - // Cloud paths are stored the same way as physical paths - tracing::debug!(" Querying directory_paths table..."); - let directory_path = directory_paths::Entity::find() - .filter(directory_paths::Column::Path.eq(path)) - .one(db) - .await?; - tracing::debug!(" Directory path query result: {:?}", directory_path); - - match directory_path { - Some(dp) => { - tracing::debug!(" Found directory path entry: {:?}", dp); - tracing::debug!(" Looking for entry with ID: {}", dp.entry_id); - - // Get the entry for this directory - let entry_result = entry::Entity::find_by_id(dp.entry_id).one(db).await?; - tracing::debug!(" Entry query result: {:?}", entry_result); - - entry_result.ok_or_else(|| { - QueryError::Internal(format!( - "Entry not found for cloud directory: {}", - dp.entry_id - )) - }) - } - None => { - tracing::debug!(" Cloud directory not found in directory_paths table"); - Err(QueryError::Internal( - format!("Cloud directory '{}' has not been indexed yet. Please ensure the cloud volume is connected and indexing is complete.", path) - )) - } - } - } - SdPath::Sidecar { .. } => { - // Sidecar paths are not supported for directory browsing - Err(QueryError::Internal( - "Sidecar paths not supported for directory browsing".to_string(), - )) - } - SdPath::Content { .. } => { - // Content-addressed paths are not supported for directory browsing - Err(QueryError::Internal( - "Content-addressed paths not supported for directory browsing".to_string(), - )) - } - } - } -} - -// Register the query crate::register_library_query!(DirectoryListingQuery, "files.directory_listing"); diff --git a/core/src/ops/files/query/file_by_id.rs b/core/src/ops/files/query/file_by_id.rs index 757597f28..23015b8c3 100644 --- a/core/src/ops/files/query/file_by_id.rs +++ b/core/src/ops/files/query/file_by_id.rs @@ -44,205 +44,9 @@ impl LibraryQuery for FileByIdQuery { context: Arc, session: crate::infra::api::SessionContext, ) -> QueryResult { - // Check database first (primary source of truth) - let library_id = session - .current_library_id - .ok_or_else(|| QueryError::Internal("No library in session".to_string()))?; - let library = context - .libraries() - .await - .get_library(library_id) - .await - .ok_or_else(|| QueryError::Internal("Library not found".to_string()))?; + let _ = &session; - let db = library.db(); - - // Get the entry with all related data in one query using JOINs - let entry_with_relations = entry::Entity::find() - .filter(entry::Column::Uuid.eq(self.file_id)) - .find_also_related(content_identity::Entity) - .one(db.conn()) - .await?; - - // If found in database, process and return it - if let Some(entry_with_relations) = entry_with_relations { - let (entry_model, content_identity_model_opt) = entry_with_relations; - - // Resolve the full absolute path for this file - let sd_path = self.resolve_file_path(&entry_model, db.conn()).await?; - - // Process content identity and load media data - let (content_identity_domain, sidecars, image_media, video_media, audio_media) = - if let Some(content_identity_model) = content_identity_model_opt { - let content_uuid = content_identity_model.uuid; - - // Load media data in parallel using JOINs - let (image_media_opt, video_media_opt, audio_media_opt) = tokio::join!( - async { - if let Some(image_id) = content_identity_model.image_media_data_id { - image_media_data::Entity::find_by_id(image_id) - .one(db.conn()) - .await - .ok() - .flatten() - .map(Into::into) - } else { - None - } - }, - async { - if let Some(video_id) = content_identity_model.video_media_data_id { - video_media_data::Entity::find_by_id(video_id) - .one(db.conn()) - .await - .ok() - .flatten() - .map(Into::into) - } else { - None - } - }, - async { - if let Some(audio_id) = content_identity_model.audio_media_data_id { - audio_media_data::Entity::find_by_id(audio_id) - .one(db.conn()) - .await - .ok() - .flatten() - .map(Into::into) - } else { - None - } - } - ); - - // Fetch sidecars for this content UUID - let sidecars = if let Some(uuid) = content_uuid { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.eq(uuid)) - .all(db.conn()) - .await? - .into_iter() - .map(|s| crate::domain::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }) - .collect() - } else { - Vec::new() - }; - - // Convert content_identity to domain type - let content_identity = crate::domain::ContentIdentity { - uuid: content_identity_model - .uuid - .unwrap_or_else(|| Uuid::new_v4()), - kind: crate::domain::ContentKind::from_id(content_identity_model.kind_id), - content_hash: content_identity_model.content_hash, - integrity_hash: content_identity_model.integrity_hash, - mime_type_id: content_identity_model.mime_type_id, - text_content: content_identity_model.text_content, - total_size: content_identity_model.total_size, - entry_count: content_identity_model.entry_count, - first_seen_at: content_identity_model.first_seen_at, - last_verified_at: content_identity_model.last_verified_at, - }; - - ( - Some(content_identity), - sidecars, - image_media_opt, - video_media_opt, - audio_media_opt, - ) - } else { - (None, Vec::new(), None, None, None) - }; - - // Convert to File using from_entity_model - let mut file = File::from_entity_model(entry_model.clone(), sd_path.clone()); - file.sidecars = sidecars; - file.content_identity = content_identity_domain; - file.image_media_data = image_media; - file.video_media_data = video_media.clone(); - file.audio_media_data = audio_media; - file.duration_seconds = video_media.as_ref().and_then(|v| v.duration_seconds); - if let Some(ref ci) = file.content_identity { - file.content_kind = ci.kind; - } - - // Populate alternate paths (other instances of same content) - if let Some(content_id) = entry_model.content_id { - file.alternate_paths = self - .get_alternate_paths(content_id, entry_model.id, db.conn()) - .await?; - } - - // Load tags for this entry - if let Some(entry_uuid) = entry_model.uuid { - use std::collections::HashMap; - - // Load user_metadata for this entry (both entry-scoped and content-scoped) - let mut metadata_filter = user_metadata::Column::EntryUuid.eq(entry_uuid); - - // Also check for content-scoped metadata if content identity exists - if let Some(ref ci) = file.content_identity { - metadata_filter = - metadata_filter.or(user_metadata::Column::ContentIdentityUuid.eq(ci.uuid)); - } - - let metadata_records = user_metadata::Entity::find() - .filter(metadata_filter) - .all(db.conn()) - .await?; - - if !metadata_records.is_empty() { - let metadata_ids: Vec = metadata_records.iter().map(|m| m.id).collect(); - - // Load user_metadata_tag records - let metadata_tags = user_metadata_tag::Entity::find() - .filter(user_metadata_tag::Column::UserMetadataId.is_in(metadata_ids)) - .all(db.conn()) - .await?; - - if !metadata_tags.is_empty() { - let tag_ids: Vec = metadata_tags.iter().map(|mt| mt.tag_id).collect(); - - // Load tag entities - let tag_models = tag::Entity::find() - .filter(tag::Column::Id.is_in(tag_ids)) - .all(db.conn()) - .await?; - - // Convert to domain tags - let mut tags = Vec::new(); - for tag_model in tag_models { - if let Ok(domain_tag) = - crate::ops::tags::manager::model_to_domain(tag_model) - { - tags.push(domain_tag); - } - } - - file.tags = tags; - tracing::debug!("Loaded {} tags for entry {}", file.tags.len(), entry_uuid); - } - } - } - - return Ok(Some(file)); - } - - // Fall back to the ephemeral partitions if not found in the database; - // a uuid gives no path to route by, so every partition is checked. + // A uuid gives no path to route by, so every partition is checked. let ephemeral_cache = context.ephemeral_cache(); for index in ephemeral_cache.all_indexes() { let index_read = index.read().await; @@ -264,109 +68,5 @@ impl LibraryQuery for FileByIdQuery { } } -impl FileByIdQuery { - /// Get alternate paths for all other entries with the same content_id - async fn get_alternate_paths( - &self, - content_id: i32, - current_entry_id: i32, - db: &DatabaseConnection, - ) -> QueryResult> { - // Find all entries with the same content_id (excluding current entry) - let alternate_entries = entry::Entity::find() - .filter(entry::Column::ContentId.eq(content_id)) - .filter(entry::Column::Id.ne(current_entry_id)) - .all(db) - .await?; - - let mut alternate_paths = Vec::new(); - - // Resolve path for each alternate entry - for alt_entry in alternate_entries { - if let Ok(alt_path) = self.resolve_file_path(&alt_entry, db).await { - alternate_paths.push(alt_path); - } - } - - Ok(alternate_paths) - } - - /// Resolve the full absolute SdPath for a file entry - async fn resolve_file_path( - &self, - entry: &entry::Model, - db: &DatabaseConnection, - ) -> QueryResult { - // Walk up the entry hierarchy to build the full path - let mut path_components = Vec::new(); - - // Add the file name with extension - let file_name = if let Some(ext) = &entry.extension { - format!("{}.{}", entry.name, ext) - } else { - entry.name.clone() - }; - path_components.push(file_name); - - // Walk up parent chain - let mut current_parent_id = entry.parent_id; - let mut location_entry_id = None; - - while let Some(parent_id) = current_parent_id { - let parent = entry::Entity::find_by_id(parent_id) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Parent entry not found".to_string()))?; - - // Check if this is the location root (no parent) - if parent.parent_id.is_none() { - location_entry_id = Some(parent.id); - break; - } - - // Add parent directory name to path - path_components.push(parent.name.clone()); - current_parent_id = parent.parent_id; - } - - // Reverse to get correct order (root -> file) - path_components.reverse(); - - // Get location info - let location_entry_id = location_entry_id - .ok_or_else(|| QueryError::Internal("Could not find location root".to_string()))?; - - let location_model = location::Entity::find() - .filter(location::Column::EntryId.eq(location_entry_id)) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Location not found for entry".to_string()))?; - - // Get device slug - let device_model = device::Entity::find_by_id(location_model.device_id) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Device not found".to_string()))?; - - // Get location root absolute path - let location_root_path = directory_paths::Entity::find() - .filter(directory_paths::Column::EntryId.eq(location_entry_id)) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Location root path not found".to_string()))?; - - // Build absolute path: location_root + relative components - let mut absolute_path = PathBuf::from(&location_root_path.path); - for component in path_components { - absolute_path.push(component); - } - - Ok(SdPath::Physical { - device_slug: device_model.slug, - path: absolute_path.into(), - }) - } -} - // Register the query crate::register_library_query!(FileByIdQuery, "files.by_id"); diff --git a/core/src/ops/files/query/file_by_path.rs b/core/src/ops/files/query/file_by_path.rs index 9c4ae06c6..f5f7c5db5 100644 --- a/core/src/ops/files/query/file_by_path.rs +++ b/core/src/ops/files/query/file_by_path.rs @@ -44,162 +44,10 @@ impl LibraryQuery for FileByPathQuery { context: Arc, session: crate::infra::api::SessionContext, ) -> QueryResult { - // Check database first (primary source of truth) - let library_id = session - .current_library_id - .ok_or_else(|| QueryError::Internal("No library in session".to_string()))?; - let library = context - .libraries() - .await - .get_library(library_id) - .await - .ok_or_else(|| QueryError::Internal("Library not found".to_string()))?; + let _ = &session; - let db = library.db(); - - // Convert the local path to SdPath internally - let sd_path = SdPath::local(self.path.clone()); - - // Find the entry by SdPath - let entry_result = self.find_entry_by_sd_path(&sd_path, db.conn()).await; - - // If found in database, process and return it - if let Ok(entry_model) = entry_result { - // Only proceed if this is actually a file (not a directory) - // if entry_model.kind == 1 { - // return Ok(None); - // } - - // Fetch content identity, sidecars, and media data if file has content_id - let (content_identity_domain, sidecars, image_media, video_media, audio_media) = - if let Some(content_id) = entry_model.content_id { - // Get content identity - if let Some(content_identity_model) = - content_identity::Entity::find_by_id(content_id) - .one(db.conn()) - .await? - { - let content_uuid = content_identity_model.uuid; - - // Load media data in parallel - let (image_media_opt, video_media_opt, audio_media_opt) = tokio::join!( - async { - if let Some(image_id) = content_identity_model.image_media_data_id { - image_media_data::Entity::find_by_id(image_id) - .one(db.conn()) - .await - .ok() - .flatten() - .map(Into::into) - } else { - None - } - }, - async { - if let Some(video_id) = content_identity_model.video_media_data_id { - video_media_data::Entity::find_by_id(video_id) - .one(db.conn()) - .await - .ok() - .flatten() - .map(Into::into) - } else { - None - } - }, - async { - if let Some(audio_id) = content_identity_model.audio_media_data_id { - audio_media_data::Entity::find_by_id(audio_id) - .one(db.conn()) - .await - .ok() - .flatten() - .map(Into::into) - } else { - None - } - } - ); - - // Fetch sidecars for this content UUID - let sidecars = if let Some(uuid) = content_uuid { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.eq(uuid)) - .all(db.conn()) - .await? - .into_iter() - .map(|s| crate::domain::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }) - .collect() - } else { - Vec::new() - }; - - // Convert content_identity to domain type - let content_identity = crate::domain::ContentIdentity { - uuid: content_identity_model - .uuid - .unwrap_or_else(|| Uuid::new_v4()), - kind: crate::domain::ContentKind::from_id( - content_identity_model.kind_id, - ), - content_hash: content_identity_model.content_hash, - integrity_hash: content_identity_model.integrity_hash, - mime_type_id: content_identity_model.mime_type_id, - text_content: content_identity_model.text_content, - total_size: content_identity_model.total_size, - entry_count: content_identity_model.entry_count, - first_seen_at: content_identity_model.first_seen_at, - last_verified_at: content_identity_model.last_verified_at, - }; - - ( - Some(content_identity), - sidecars, - image_media_opt, - video_media_opt, - audio_media_opt, - ) - } else { - (None, Vec::new(), None, None, None) - } - } else { - (None, Vec::new(), None, None, None) - }; - - // Convert to File using from_entity_model - let mut file = File::from_entity_model(entry_model.clone(), sd_path); - file.sidecars = sidecars; - file.content_identity = content_identity_domain; - file.image_media_data = image_media; - file.video_media_data = video_media.clone(); - file.audio_media_data = audio_media; - file.duration_seconds = video_media.as_ref().and_then(|v| v.duration_seconds); - if let Some(ref ci) = file.content_identity { - file.content_kind = ci.kind; - } - - // Populate alternate paths (other instances of same content) - if let Some(content_id) = entry_model.content_id { - file.alternate_paths = self - .get_alternate_paths(content_id, entry_model.id, db.conn()) - .await?; - } - - return Ok(Some(file)); - } - - // Fall back to ephemeral index if not found in database + // The arena is the index for every drive; the store is its durable + // half rather than a second answer. let ephemeral_cache = context.ephemeral_cache(); ephemeral_cache.ensure_restored(&self.path).await; let index = ephemeral_cache.resolve_index(&self.path); @@ -221,107 +69,4 @@ impl LibraryQuery for FileByPathQuery { } } -impl FileByPathQuery { - /// Get alternate paths for all other entries with the same content_id - async fn get_alternate_paths( - &self, - content_id: i32, - current_entry_id: i32, - db: &DatabaseConnection, - ) -> QueryResult> { - use crate::infra::db::entities::{device, directory_paths, location}; - - // Find all entries with the same content_id (excluding current entry) - let alternate_entries = entry::Entity::find() - .filter(entry::Column::ContentId.eq(content_id)) - .filter(entry::Column::Id.ne(current_entry_id)) - .all(db) - .await?; - - let mut alternate_paths = Vec::new(); - - // Resolve path for each alternate entry - for alt_entry in alternate_entries { - // Build the full path for this entry - let mut path_components = Vec::new(); - - // Add the file name with extension - let file_name = if let Some(ext) = &alt_entry.extension { - format!("{}.{}", alt_entry.name, ext) - } else { - alt_entry.name.clone() - }; - path_components.push(file_name); - - // Walk up parent chain - let mut current_parent_id = alt_entry.parent_id; - let mut location_entry_id = None; - - while let Some(parent_id) = current_parent_id { - if let Some(parent) = entry::Entity::find_by_id(parent_id).one(db).await? { - if parent.parent_id.is_none() { - location_entry_id = Some(parent.id); - break; - } - path_components.push(parent.name.clone()); - current_parent_id = parent.parent_id; - } else { - break; - } - } - - if let Some(location_entry_id) = location_entry_id { - path_components.reverse(); - - // Get location and device info - if let Some(location_model) = location::Entity::find() - .filter(location::Column::EntryId.eq(location_entry_id)) - .one(db) - .await? - { - if let Some(device_model) = device::Entity::find_by_id(location_model.device_id) - .one(db) - .await? - { - if let Some(location_root_path) = directory_paths::Entity::find() - .filter(directory_paths::Column::EntryId.eq(location_entry_id)) - .one(db) - .await? - { - // Build absolute path - let mut absolute_path = PathBuf::from(&location_root_path.path); - for component in path_components { - absolute_path.push(component); - } - - alternate_paths.push(SdPath::Physical { - device_slug: device_model.slug, - path: absolute_path.into(), - }); - } - } - } - } - } - - Ok(alternate_paths) - } - - /// Find entry by SdPath using canonical PathResolver - async fn find_entry_by_sd_path( - &self, - sd_path: &SdPath, - db: &DatabaseConnection, - ) -> QueryResult { - use crate::ops::indexing::PathResolver; - - PathResolver::resolve_to_entry(db, sd_path) - .await - .map_err(|e| QueryError::Internal(format!("Database error: {}", e)))? - .ok_or_else(|| { - QueryError::Internal(format!("Entry not found for path: {}", sd_path.display())) - }) - } -} - crate::register_library_query!(FileByPathQuery, "files.by_path"); diff --git a/core/src/ops/files/query/media_listing.rs b/core/src/ops/files/query/media_listing.rs index 39113c7ef..81a552713 100644 --- a/core/src/ops/files/query/media_listing.rs +++ b/core/src/ops/files/query/media_listing.rs @@ -1,23 +1,19 @@ -//! Query to list media content (images/videos) for gallery views +//! Media in a directory, for the gallery views. //! -//! This query is optimized for gallery/camera roll UI patterns. -//! It returns media files (images and videos) from a directory and optionally -//! includes all descendants, making any directory browsable as a media gallery. +//! Reads the volume index rather than walking the filesystem: the arena +//! already knows every entry under the drive and what kind each one is, so a +//! camera roll over a deep tree costs a traversal of memory rather than of +//! disk. -use crate::infra::query::{QueryError, QueryResult}; +use crate::infra::query::{LibraryQuery, QueryError, QueryResult}; use crate::{ context::CoreContext, - domain::{addressing::SdPath, content_identity::ContentIdentity, file::File, ContentKind}, - infra::db::entities::{ - content_identity, directory_paths, entry, image_media_data, sidecar, video_media_data, - }, - infra::query::LibraryQuery, + domain::{addressing::SdPath, file::File, ContentKind}, + ops::indexing::state::EntryKind, }; -use sea_orm::{ColumnTrait, ConnectionTrait, DatabaseConnection, EntityTrait, QueryFilter}; use serde::{Deserialize, Serialize}; use specta::Type; -use std::{collections::HashMap, sync::Arc}; -use uuid::Uuid; +use std::{collections::VecDeque, path::PathBuf, sync::Arc}; /// Input for media listing #[derive(Debug, Clone, Serialize, Deserialize, Type)] @@ -104,418 +100,86 @@ impl LibraryQuery for MediaListingQuery { type Output = MediaListingOutput; fn from_input(input: Self::Input) -> QueryResult { - tracing::info!( - "MediaListingQuery::from_input called with input: {:?}", - input - ); Ok(Self { input }) } async fn execute( self, context: Arc, - session: crate::infra::api::SessionContext, + _session: crate::infra::api::SessionContext, ) -> QueryResult { - tracing::info!( - "MediaListingQuery::execute called with path: {:?}", - self.input.path - ); + let SdPath::Physical { device_slug, path } = &self.input.path else { + return Err(QueryError::Internal( + "Media listing needs a physical path".to_string(), + )); + }; - let library_id = session - .current_library_id - .ok_or_else(|| QueryError::Internal("No library in session".to_string()))?; - tracing::info!("Library ID: {}", library_id); - - let library = context - .libraries() - .await - .get_library(library_id) - .await - .ok_or_else(|| QueryError::Internal("Library not found".to_string()))?; - - let db = library.db(); - - // First, find the parent directory entry - let parent_entry = self.find_parent_directory(db.conn()).await?; - let parent_id = parent_entry.id; - - // Get media types to filter (default to Image and Video) - let media_types = self + let wanted = self .input .media_types - .as_ref() - .cloned() + .clone() .unwrap_or_else(|| vec![ContentKind::Image, ContentKind::Video]); - // Convert ContentKind enum to database IDs - let media_type_ids: Vec = media_types.iter().map(|k| *k as i32).collect(); - let media_type_ids_str = media_type_ids - .iter() - .map(|id| id.to_string()) - .collect::>() - .join(", "); + let cache = context.ephemeral_cache(); + cache.ensure_restored(path).await; - // Build efficient SQL query - let mut sql_query = format!( - r#" - SELECT - e.id as entry_id, - e.uuid as entry_uuid, - e.name as entry_name, - e.kind as entry_kind, - e.extension as entry_extension, - e.size as entry_size, - e.created_at as entry_created_at, - e.modified_at as entry_modified_at, - e.accessed_at as entry_accessed_at, - e.inode as entry_inode, - e.parent_id as entry_parent_id, - ci.id as content_identity_id, - ci.uuid as content_identity_uuid, - ci.content_hash as content_hash, - ci.integrity_hash as integrity_hash, - ci.mime_type_id as mime_type_id, - ci.text_content as text_content, - ci.total_size as total_size, - ci.entry_count as entry_count, - ci.first_seen_at as first_seen_at, - ci.last_verified_at as last_verified_at, - ck.id as content_kind_id, - ck.name as content_kind_name, - dp.path as directory_path, - vmd.uuid as video_media_uuid, - vmd.duration_seconds as video_duration_seconds, - vmd.width as video_width, - vmd.height as video_height, - vmd.date_captured as video_date_captured, - imd.date_taken as image_date_taken - FROM entries e - INNER JOIN content_identities ci ON e.content_id = ci.id - LEFT JOIN content_kinds ck ON ci.kind_id = ck.id - LEFT JOIN directory_paths dp ON e.parent_id = dp.entry_id - LEFT JOIN video_media_data vmd ON ci.video_media_data_id = vmd.id - LEFT JOIN image_media_data imd ON ci.image_media_data_id = imd.id - WHERE ci.kind_id IN ({}) - "#, - media_type_ids_str - ); - - // Apply directory scope filter - if self.input.include_descendants.unwrap_or(false) { - // Include all descendants: match entries where directory path starts with parent path - sql_query.push_str(" AND (e.parent_id = ?1 OR dp.path LIKE ?2)"); - } else { - // Only direct children - sql_query.push_str(" AND e.parent_id = ?1"); - } - - // Apply sorting - match self.input.sort_by { - MediaSortBy::Modified => sql_query.push_str(" ORDER BY e.modified_at DESC"), - MediaSortBy::Created => sql_query.push_str(" ORDER BY e.created_at DESC"), - MediaSortBy::DateTaken => { - // Use date_taken for images, date_captured for videos, fall back to modified_at - sql_query.push_str( - " ORDER BY COALESCE(imd.date_taken, vmd.date_captured, e.modified_at) DESC", - ) - } - MediaSortBy::Name => sql_query.push_str(" ORDER BY e.name ASC"), - MediaSortBy::Size => sql_query.push_str(" ORDER BY e.size DESC"), - } - - // Apply limit - if let Some(limit) = self.input.limit { - sql_query.push_str(&format!(" LIMIT {}", limit)); - } - - // Build query parameters - let parent_path_pattern = if self.input.include_descendants.unwrap_or(false) { - // Get the directory path for the parent to construct LIKE pattern - let parent_path = self.get_parent_directory_path(db.conn(), parent_id).await?; - Some(format!("{}/%", parent_path)) - } else { - None - }; - - // Execute the query - let rows = if let Some(path_pattern) = parent_path_pattern { - db.conn() - .query_all(sea_orm::Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Sqlite, - &sql_query, - [parent_id.into(), path_pattern.into()], - )) - .await? - } else { - db.conn() - .query_all(sea_orm::Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Sqlite, - &sql_query, - [parent_id.into()], - )) - .await? - }; - - tracing::debug!("Query executed, found {} media files", rows.len()); - - if rows.is_empty() { - tracing::debug!("No media files found"); + let Some(index) = cache.get_for_search(path) else { return Ok(MediaListingOutput { files: Vec::new(), total_count: 0, has_more: false, }); - } - - let total_count = rows.len() as u32; - - // Collect all content UUIDs for batch sidecar query - let content_uuids: Vec = rows - .iter() - .filter_map(|row| { - row.try_get::>("", "content_identity_uuid") - .ok() - .flatten() - }) - .collect(); - - // Batch fetch all sidecars for these content UUIDs - let all_sidecars = if !content_uuids.is_empty() { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.is_in(content_uuids.clone())) - .all(db.conn()) - .await? - } else { - Vec::new() }; - // Group sidecars by content_uuid for fast lookup - let mut sidecars_by_content: HashMap> = - HashMap::new(); - for s in all_sidecars { - sidecars_by_content - .entry(s.content_uuid) - .or_insert_with(Vec::new) - .push(crate::domain::file::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }); - } - - // Convert to File objects + let recurse = self.input.include_descendants.unwrap_or(false); let mut files = Vec::new(); - for row in rows { - // Extract data from SQL row - let entry_id: i32 = row.try_get("", "entry_id").unwrap_or(0); - let entry_uuid: Option = row.try_get("", "entry_uuid").ok(); - let entry_name: String = row.try_get("", "entry_name").unwrap_or_default(); - let entry_kind: i32 = row.try_get("", "entry_kind").unwrap_or(0); - let entry_extension: Option = row.try_get("", "entry_extension").ok(); - let entry_size: i64 = row.try_get("", "entry_size").unwrap_or(0); - let entry_created_at: chrono::DateTime = row - .try_get("", "entry_created_at") - .unwrap_or_else(|_| chrono::Utc::now()); - let entry_modified_at: chrono::DateTime = row - .try_get("", "entry_modified_at") - .unwrap_or_else(|_| chrono::Utc::now()); - let entry_accessed_at: Option> = - row.try_get("", "entry_accessed_at").ok(); + let mut queue = VecDeque::from([path.clone()]); + let mut index = index.write().await; - // Content identity data - let content_identity_uuid: Option = row.try_get("", "content_identity_uuid").ok(); - let content_hash: Option = row.try_get("", "content_hash").ok(); - let integrity_hash: Option = row.try_get("", "integrity_hash").ok(); - let mime_type_id: Option = row.try_get("", "mime_type_id").ok(); - let text_content: Option = row.try_get("", "text_content").ok(); - let total_size: Option = row.try_get("", "total_size").ok(); - let entry_count: Option = row.try_get("", "entry_count").ok(); - let first_seen_at: Option> = - row.try_get("", "first_seen_at").ok(); - let last_verified_at: Option> = - row.try_get("", "last_verified_at").ok(); - - // Content kind data - let content_kind_name: Option = row.try_get("", "content_kind_name").ok(); - - // Directory path - let directory_path: Option = row.try_get("", "directory_path").ok(); - - // Video media data - let video_media_uuid: Option = row.try_get("", "video_media_uuid").ok(); - let video_duration_seconds: Option = - row.try_get("", "video_duration_seconds").ok(); - let video_width: Option = row.try_get("", "video_width").ok(); - let video_height: Option = row.try_get("", "video_height").ok(); - - // Build full path with extension - let full_name = if let Some(ext) = &entry_extension { - format!("{}.{}", entry_name, ext) - } else { - entry_name.clone() + while let Some(dir) = queue.pop_front() { + let Some(children) = index.list_directory(&dir) else { + continue; }; - // Construct full file path - let entry_sd_path = if let Some(dir_path) = directory_path { - let full_path = if dir_path.ends_with('/') { - format!("{}{}", dir_path, full_name) - } else { - format!("{}/{}", dir_path, full_name) + for child in children { + let Some(metadata) = index.get_entry_ref(&child) else { + continue; }; - match &self.input.path { - SdPath::Physical { device_slug, .. } => SdPath::Physical { - device_slug: device_slug.clone(), - path: full_path.into(), - }, - SdPath::Cloud { - service, - identifier, - .. - } => SdPath::Cloud { - service: *service, - identifier: identifier.clone(), - path: full_path, - }, - SdPath::Content { content_id } => SdPath::Content { - content_id: *content_id, - }, - SdPath::Sidecar { .. } => { - return Err(QueryError::Internal( - "Sidecar paths not supported for media listing".to_string(), - )); + if metadata.kind == EntryKind::Directory { + if recurse { + queue.push_back(child); } + continue; } - } else { - // Fallback to constructing path from parent - match &self.input.path { - SdPath::Physical { device_slug, path } => SdPath::Physical { - device_slug: device_slug.clone(), - path: path.join(&full_name).into(), - }, - SdPath::Cloud { - service, - identifier, - path, - } => SdPath::Cloud { - service: *service, - identifier: identifier.clone(), - path: format!("{}/{}", path, full_name), - }, - SdPath::Content { content_id } => SdPath::Content { - content_id: *content_id, - }, - SdPath::Sidecar { .. } => { - return Err(QueryError::Internal( - "Sidecar paths not supported for media listing".to_string(), - )); - } + + // A summarised directory holds a count and no children, so + // recursing into one yields nothing rather than yielding wrong + // numbers. + let kind = index.get_content_kind(&child); + if !wanted.contains(&kind) { + continue; } - }; - // Create entity model for conversion - let entity_model = entry::Model { - id: entry_id, - uuid: entry_uuid, - name: entry_name, - kind: entry_kind, - extension: entry_extension, - metadata_id: None, - content_id: None, - size: entry_size, - aggregate_size: 0, - child_count: 0, - file_count: 0, - created_at: entry_created_at, - modified_at: entry_modified_at, - accessed_at: entry_accessed_at, - indexed_at: None, - permissions: None, - inode: None, - parent_id: None, - volume_id: None, - }; + let uuid = index.get_or_assign_uuid(&child); + let sd_path = SdPath::Physical { + device_slug: device_slug.clone(), + path: child.clone(), + }; - // Convert to File using from_entity_model - let mut file = File::from_entity_model(entity_model, entry_sd_path); - - // Add content identity if available - if let (Some(ci_uuid), Some(ci_hash), Some(ci_first_seen), Some(ci_last_verified)) = ( - content_identity_uuid, - content_hash, - first_seen_at, - last_verified_at, - ) { - // Convert content_kind name to ContentKind enum - let kind = content_kind_name - .as_ref() - .map(|name| ContentKind::from(name.as_str())) - .unwrap_or(ContentKind::Unknown); - - file.content_identity = Some(ContentIdentity { - uuid: ci_uuid, - kind, - content_hash: ci_hash, - integrity_hash, - mime_type_id, - text_content, - total_size: total_size.unwrap_or(0), - entry_count: entry_count.unwrap_or(0), - first_seen_at: ci_first_seen, - last_verified_at: ci_last_verified, - }); + let mut file = File::from_ephemeral(uuid, &metadata, sd_path); file.content_kind = kind; - - // Add sidecars from batch lookup - if let Some(sidecars) = sidecars_by_content.get(&ci_uuid) { - file.sidecars = sidecars.clone(); - } + files.push(file); } - - // Add video media data if available - if let Some(vmd_uuid) = video_media_uuid { - file.video_media_data = Some(crate::domain::VideoMediaData { - uuid: vmd_uuid, - width: video_width.unwrap_or(0) as u32, - height: video_height.unwrap_or(0) as u32, - duration_seconds: video_duration_seconds, - bit_rate: None, - codec: None, - pixel_format: None, - color_space: None, - color_range: None, - color_primaries: None, - color_transfer: None, - fps_num: None, - fps_den: None, - audio_codec: None, - audio_channels: None, - audio_sample_rate: None, - audio_bit_rate: None, - title: None, - artist: None, - album: None, - creation_time: None, - date_captured: None, - blurhash: None, - }); - } - - files.push(file); } + drop(index); - let has_more = if let Some(limit) = self.input.limit { - total_count > limit - } else { - false - }; + sort_media(&mut files, &self.input.sort_by); + + let total_count = files.len() as u32; + let limit = self.input.limit.unwrap_or(1000) as usize; + let has_more = files.len() > limit; + files.truncate(limit); Ok(MediaListingOutput { files, @@ -525,118 +189,20 @@ impl LibraryQuery for MediaListingQuery { } } -impl MediaListingQuery { - /// Find the parent directory entry for the given SdPath - async fn find_parent_directory(&self, db: &DatabaseConnection) -> QueryResult { - tracing::debug!( - "find_parent_directory called with path: {:?}", - self.input.path - ); - - match &self.input.path { - SdPath::Physical { device_slug, path } => { - let path_str = path.to_string_lossy().to_string(); - tracing::debug!("Looking for directory path: '{}'", path_str); - - let directory_path = directory_paths::Entity::find() - .filter(directory_paths::Column::Path.eq(&path_str)) - .one(db) - .await?; - tracing::debug!("Directory path query result: {:?}", directory_path); - - match directory_path { - Some(dp) => { - tracing::debug!("Found directory path entry: {:?}", dp); - tracing::debug!("Looking for entry with ID: {}", dp.entry_id); - - let entry_result = entry::Entity::find_by_id(dp.entry_id).one(db).await?; - tracing::debug!("Entry query result: {:?}", entry_result); - - entry_result.ok_or_else(|| { - QueryError::Internal(format!( - "Entry not found for directory: {}", - dp.entry_id - )) - }) - } - None => { - tracing::debug!("Directory not found in directory_paths table"); - Err(QueryError::Internal( - format!("Directory '{}' has not been indexed yet. Please add this location to Spacedrive and wait for indexing to complete.", path_str) - )) - } - } - } - SdPath::Cloud { - service, - identifier, - path, - } => { - tracing::debug!( - "Looking for cloud directory: service={}, identifier={}, path='{}'", - service.scheme(), - identifier, - path - ); - - let directory_path = directory_paths::Entity::find() - .filter(directory_paths::Column::Path.eq(path)) - .one(db) - .await?; - tracing::debug!("Directory path query result: {:?}", directory_path); - - match directory_path { - Some(dp) => { - tracing::debug!("Found directory path entry: {:?}", dp); - tracing::debug!("Looking for entry with ID: {}", dp.entry_id); - - let entry_result = entry::Entity::find_by_id(dp.entry_id).one(db).await?; - tracing::debug!("Entry query result: {:?}", entry_result); - - entry_result.ok_or_else(|| { - QueryError::Internal(format!( - "Entry not found for cloud directory: {}", - dp.entry_id - )) - }) - } - None => { - tracing::debug!("Cloud directory not found in directory_paths table"); - Err(QueryError::Internal( - format!("Cloud directory '{}' has not been indexed yet. Please ensure the cloud volume is connected and indexing is complete.", path) - )) - } - } - } - SdPath::Sidecar { .. } => Err(QueryError::Internal( - "Sidecar paths not supported for media listing".to_string(), - )), - SdPath::Content { .. } => Err(QueryError::Internal( - "Content-addressed paths not supported for media listing".to_string(), - )), +/// Newest first for every date order, since a camera roll reads backwards. +fn sort_media(files: &mut [File], sort_by: &MediaSortBy) { + match sort_by { + MediaSortBy::Modified => files.sort_by(|a, b| b.modified_at.cmp(&a.modified_at)), + // The arena carries no capture time; birth time is the closest thing + // it knows, and it is the one a photo library sorts by. + MediaSortBy::Created | MediaSortBy::DateTaken => { + files.sort_by(|a, b| b.created_at.cmp(&a.created_at)) } - } - - /// Get the directory path for a given entry ID - async fn get_parent_directory_path( - &self, - db: &DatabaseConnection, - entry_id: i32, - ) -> QueryResult { - let directory_path = directory_paths::Entity::find() - .filter(directory_paths::Column::EntryId.eq(entry_id)) - .one(db) - .await? - .ok_or_else(|| { - QueryError::Internal(format!( - "Directory path not found for entry_id: {}", - entry_id - )) - })?; - - Ok(directory_path.path) + MediaSortBy::Name => { + files.sort_by(|a, b| a.name.to_lowercase().cmp(&b.name.to_lowercase())) + } + MediaSortBy::Size => files.sort_by(|a, b| b.size.cmp(&a.size)), } } -// Register the query crate::register_library_query!(MediaListingQuery, "files.media_listing"); diff --git a/core/src/ops/files/query/mod.rs b/core/src/ops/files/query/mod.rs index 7131815a4..a5b621291 100644 --- a/core/src/ops/files/query/mod.rs +++ b/core/src/ops/files/query/mod.rs @@ -9,7 +9,6 @@ pub mod file_by_id; pub mod file_by_path; pub mod media_listing; pub mod size_tree; -pub mod unique_to_location; pub use alternate_instances::*; pub use content_kind_stats::*; @@ -18,4 +17,3 @@ pub use duplicates::*; pub use file_by_id::*; pub use file_by_path::*; pub use media_listing::*; -pub use unique_to_location::*; diff --git a/core/src/ops/files/query/unique_to_location.rs b/core/src/ops/files/query/unique_to_location.rs deleted file mode 100644 index fae7edabc..000000000 --- a/core/src/ops/files/query/unique_to_location.rs +++ /dev/null @@ -1,268 +0,0 @@ -//! Query to find files that exist only in a specific location (not backed up elsewhere) -//! -//! This query uses content identity to identify files that are unique to a given location, -//! meaning their content hash doesn't appear in any other location. This is useful for -//! backup purposes to identify files that need to be backed up. - -use crate::infra::query::{QueryError, QueryResult}; -use crate::{ - context::CoreContext, - domain::{addressing::SdPath, File}, - infra::db::entities::{content_identity, entry, location}, - infra::query::LibraryQuery, -}; -use sea_orm::{ - ColumnTrait, ConnectionTrait, DatabaseConnection, EntityTrait, JoinType, QueryFilter, - QuerySelect, RelationTrait, Statement, -}; -use serde::{Deserialize, Serialize}; -use specta::Type; -use std::sync::Arc; -use uuid::Uuid; - -/// Input for finding files unique to a location -#[derive(Debug, Clone, Serialize, Deserialize, Type)] -pub struct UniqueToLocationInput { - /// The location ID to find unique files for - pub location_id: Uuid, - /// Optional limit on number of results - pub limit: Option, -} - -/// Output containing files that are unique to the specified location -#[derive(Debug, Clone, Serialize, Deserialize, Type)] -pub struct UniqueToLocationOutput { - /// Files that exist only in the specified location - pub unique_files: Vec, - /// Total count of unique files - pub total_count: u32, - /// Total size of unique files in bytes - pub total_size: u64, -} - -/// Query to find files unique to a location -#[derive(Debug, Clone, Serialize, Deserialize, Type)] -pub struct UniqueToLocationQuery { - pub input: UniqueToLocationInput, -} - -impl UniqueToLocationQuery { - pub fn new(location_id: Uuid) -> Self { - Self { - input: UniqueToLocationInput { - location_id, - limit: None, - }, - } - } - - pub fn with_limit(location_id: Uuid, limit: u32) -> Self { - Self { - input: UniqueToLocationInput { - location_id, - limit: Some(limit), - }, - } - } -} - -impl LibraryQuery for UniqueToLocationQuery { - type Input = UniqueToLocationInput; - type Output = UniqueToLocationOutput; - - fn from_input(input: Self::Input) -> QueryResult { - Ok(Self { input }) - } - - async fn execute( - self, - context: Arc, - session: crate::infra::api::SessionContext, - ) -> QueryResult { - let library_id = session - .current_library_id - .ok_or_else(|| QueryError::Internal("No library in session".to_string()))?; - let library = context - .libraries() - .await - .get_library(library_id) - .await - .ok_or_else(|| QueryError::Internal("Library not found".to_string()))?; - - let db = library.db().conn(); - - // Execute the query to find unique files - let (unique_files, total_size) = self.find_unique_files_in_location(db).await?; - - let total_count = unique_files.len() as u32; - - Ok(UniqueToLocationOutput { - unique_files, - total_count, - total_size, - }) - } -} - -impl UniqueToLocationQuery { - /// Find files that exist only in the specified location - async fn find_unique_files_in_location( - &self, - db: &DatabaseConnection, - ) -> QueryResult<(Vec, u64)> { - // First, get the location's root entry ID - let location_model = location::Entity::find() - .filter(location::Column::Uuid.eq(self.input.location_id)) - .one(db) - .await? - .ok_or_else(|| QueryError::Internal("Location not found".to_string()))?; - - let location_root_entry_id = location_model.entry_id; - - // Query to find content identities that exist only in this location - // This uses a subquery to find content_hash values that appear in only one location - let unique_content_hashes_query = r#" - WITH location_content AS ( - -- Get all content hashes from the specified location - SELECT DISTINCT ci.content_hash - FROM content_identities ci - JOIN entries e ON e.content_id = ci.id - JOIN entry_closure ec ON ec.descendant_id = e.id - WHERE ec.ancestor_id = ?1 - AND e.kind = 0 -- Files only - ), - other_locations_content AS ( - -- Get all content hashes from other locations - SELECT DISTINCT ci.content_hash - FROM content_identities ci - JOIN entries e ON e.content_id = ci.id - JOIN entry_closure ec ON ec.descendant_id = e.id - WHERE ec.ancestor_id != ?1 - AND e.kind = 0 -- Files only - ) - -- Find content hashes that are in our location but not in others - SELECT lc.content_hash - FROM location_content lc - LEFT JOIN other_locations_content olc ON lc.content_hash = olc.content_hash - WHERE olc.content_hash IS NULL - "#; - - let unique_content_hashes: Vec = db - .query_all(Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Sqlite, - unique_content_hashes_query, - [location_root_entry_id.into()], - )) - .await? - .into_iter() - .map(|row| row.try_get("", "content_hash").unwrap_or_default()) - .collect(); - - if unique_content_hashes.is_empty() { - return Ok((Vec::new(), 0)); - } - - // Now get all files from this location that have these unique content hashes - let files_query = r#" - SELECT - e.id as entry_id, - e.name, - e.size, - e.created_at, - e.updated_at, - ci.content_hash, - ci.uuid as content_uuid - FROM entries e - JOIN content_identities ci ON e.content_id = ci.id - JOIN entry_closure ec ON ec.descendant_id = e.id - WHERE ec.ancestor_id = ?1 - AND e.kind = 0 -- Files only - AND ci.content_hash IN ( - SELECT DISTINCT ci2.content_hash - FROM content_identities ci2 - JOIN entries e2 ON e2.content_id = ci2.id - JOIN entry_closure ec2 ON ec2.descendant_id = e2.id - WHERE ec2.ancestor_id = ?1 - AND e2.kind = 0 - AND ci2.content_hash NOT IN ( - SELECT DISTINCT ci3.content_hash - FROM content_identities ci3 - JOIN entries e3 ON e3.content_id = ci3.id - JOIN entry_closure ec3 ON ec3.descendant_id = e3.id - WHERE ec3.ancestor_id != ?1 - AND e3.kind = 0 - ) - ) - ORDER BY e.name - "#; - - let limit_clause = if let Some(limit) = self.input.limit { - format!("LIMIT {}", limit) - } else { - String::new() - }; - - let final_query = if limit_clause.is_empty() { - files_query.to_string() - } else { - format!("{} {}", files_query, limit_clause) - }; - - let file_rows = db - .query_all(Statement::from_sql_and_values( - sea_orm::DatabaseBackend::Sqlite, - &final_query, - [location_root_entry_id.into()], - )) - .await?; - - let mut files = Vec::new(); - let mut total_size = 0u64; - - for row in file_rows { - let entry_id: i32 = row.try_get("", "entry_id").unwrap_or(0); - let name: String = row.try_get("", "name").unwrap_or_default(); - let size: i64 = row.try_get("", "size").unwrap_or(0); - let content_hash: String = row.try_get("", "content_hash").unwrap_or_default(); - - total_size += size as u64; - - // Create entity model for conversion - let entity_model = entry::Model { - id: entry_id, - uuid: None, - name: name.clone(), - kind: 0, // File - extension: name.split('.').last().map(|s| s.to_string()), - metadata_id: None, - content_id: None, - size, - aggregate_size: 0, - child_count: 0, - file_count: 0, - created_at: chrono::Utc::now(), - modified_at: chrono::Utc::now(), - accessed_at: None, - indexed_at: None, // Not needed for query result conversion - permissions: None, - inode: None, - parent_id: None, - volume_id: None, - }; - - // Create placeholder SdPath - let sd_path = SdPath::Physical { - device_slug: format!("placeholder-{}", Uuid::new_v4()), - path: format!("/unknown/path/{}", name).into(), - }; - - let file = File::from_entity_model(entity_model, sd_path); - files.push(file); - } - - Ok((files, total_size)) - } -} - -// Register the query -crate::register_library_query!(UniqueToLocationQuery, "files.unique_to_location"); diff --git a/core/src/ops/indexing/ephemeral/store.rs b/core/src/ops/indexing/ephemeral/store.rs index d19ea7ea9..0df042eec 100644 --- a/core/src/ops/indexing/ephemeral/store.rs +++ b/core/src/ops/indexing/ephemeral/store.rs @@ -292,6 +292,35 @@ impl SourceStore { } } + /// The identity of the bytes behind one of this source's records. + pub async fn content_of(&self, record_uuid: Uuid) -> Option { + sd_store::content_of(self.db.pool(), record_uuid) + .await + .unwrap_or_default() + } + + /// Every copy of the given bytes that this source holds. + /// + /// Absolute paths, for the same reason [`Self::duplicates`] gives them: + /// a caller is going to show them to someone or open them. + pub async fn copies_of_content(&self, content_uuid: Uuid) -> Vec { + match sd_store::copies_of_content(self.db.pool(), content_uuid).await { + Ok(copies) => copies + .into_iter() + .map(|copy| DuplicateCopy { + content_uuid: copy.content_uuid, + size: copy.size.unwrap_or(0).max(0) as u64, + record_uuid: copy.record_uuid, + path: self.root.join(©.external_id), + }) + .collect(), + Err(error) => { + tracing::warn!(source = %self.id, %error, "could not list copies"); + Vec::new() + } + } + } + /// How many files are still waiting to be identified. pub async fn files_needing_content_count(&self) -> u64 { sd_store::count_files_needing_content(self.db.pool()) diff --git a/core/src/ops/search/ephemeral_search.rs b/core/src/ops/search/ephemeral_search.rs index 771b51429..8bed51e07 100644 --- a/core/src/ops/search/ephemeral_search.rs +++ b/core/src/ops/search/ephemeral_search.rs @@ -1,7 +1,10 @@ -//! Ephemeral index search implementation +//! Search over the volume index. //! -//! This module provides search functionality for the in-memory ephemeral index, -//! enabling search in unindexed locations and external drives. +//! Every drive this machine knows about has a partition in the arena, so a +//! library-wide search is a fan-out over all of them and a scoped one resolves +//! the partition that covers the path. Nothing here reads the durable store: +//! names live in memory, which is what makes a search answer while someone is +//! still typing. use crate::domain::{File, SdPath}; use crate::filetype::FileTypeRegistry; @@ -15,7 +18,7 @@ use std::cmp::Ordering; use std::path::PathBuf; use uuid::Uuid; -/// Search the ephemeral index for files matching the query +/// Search one partition of the volume index, scoped to a path within it. pub async fn search_ephemeral_index( query: &str, path_scope: &SdPath, @@ -23,76 +26,118 @@ pub async fn search_ephemeral_index( cache: &EphemeralIndexCache, file_type_registry: &FileTypeRegistry, ) -> Result, QueryError> { - // Get local path from SdPath - let local_path = match path_scope { - SdPath::Physical { path, .. } => path.clone(), - _ => { - return Ok(Vec::new()); // Only physical paths supported for ephemeral - } + let SdPath::Physical { + path: local_path, + device_slug, + } = path_scope + else { + return Ok(Vec::new()); }; - // A registered source that hasn't been touched this session restores from - // its snapshot here — including detached drives, whose indexes serve + // A registered source that has not been touched this session restores from + // its snapshot here, including detached drives, whose indexes serve // read-only. - cache.ensure_restored(&local_path).await; + cache.ensure_restored(local_path).await; - // Get ephemeral index (use get_for_search to check parent paths) - let index_arc = cache - .get_for_search(&local_path) - .ok_or_else(|| QueryError::Internal("Ephemeral index not found".to_string()))?; + let Some(index_arc) = cache.get_for_search(local_path) else { + return Ok(Vec::new()); + }; - // Perform name-based search with read lock let matching_paths = { let index = index_arc.read().await; - - if query.is_empty() { - // Empty query: return all files in scope - index.list_directory(&local_path).unwrap_or_default() - } else { - // Use registry for substring search - let query_lower = query.to_lowercase(); - - // Try exact name match first - let mut paths = index.find_by_name(&query_lower); - tracing::debug!("Exact match for '{}': {} paths", query_lower, paths.len()); - - // If no exact matches, try prefix search - if paths.is_empty() { - paths = index.find_by_prefix(&query_lower); - tracing::debug!("Prefix match for '{}': {} paths", query_lower, paths.len()); - } - - // If still no matches, try substring search - if paths.is_empty() { - paths = index.find_containing(&query_lower); - tracing::debug!( - "Substring match for '{}': {} paths", - query_lower, - paths.len() - ); - } - - tracing::debug!("Total paths before scope filter: {}", paths.len()); - tracing::debug!("Scope path: {:?}", local_path); - - // Filter to only paths within scope - let filtered: Vec = paths - .into_iter() - .filter(|path| path.starts_with(&local_path)) - .collect(); - - tracing::debug!("Paths after scope filter: {}", filtered.len()); - filtered - } + matches_in(&index, query, Some(local_path)) }; - tracing::debug!( - "Converting {} matching paths to results", - matching_paths.len() - ); + collect_results( + &index_arc, + matching_paths, + query, + device_slug, + filters, + file_type_registry, + ) + .await +} - // Convert to FileSearchResult with lazy UUID assignment - // Acquire write lock once for the entire batch instead of per-entry +/// Search every partition, which is what a library-wide search is now that +/// every attached drive is mapped. +pub async fn search_every_index( + query: &str, + filters: &SearchFilters, + cache: &EphemeralIndexCache, + file_type_registry: &FileTypeRegistry, +) -> Result, QueryError> { + let mut results = Vec::new(); + + for index_arc in cache.all_indexes() { + let matching_paths = { + let index = index_arc.read().await; + matches_in(&index, query, None) + }; + + results.extend( + collect_results( + &index_arc, + matching_paths, + query, + "", + filters, + file_type_registry, + ) + .await?, + ); + } + + rank(&mut results); + Ok(results) +} + +/// Paths in one partition whose name matches, narrowed to a scope if given. +/// +/// Exact, then prefix, then substring: the first tier that answers wins, so a +/// query that names a file exactly is not buried under everything containing it. +fn matches_in( + index: &crate::ops::indexing::ephemeral::EphemeralIndex, + query: &str, + scope: Option<&PathBuf>, +) -> Vec { + if query.is_empty() { + return match scope { + Some(path) => index.list_directory(path).unwrap_or_default(), + None => Vec::new(), + }; + } + + let query = query.to_lowercase(); + let mut paths = index.find_by_name(&query); + if paths.is_empty() { + paths = index.find_by_prefix(&query); + } + if paths.is_empty() { + paths = index.find_containing(&query); + } + + match scope { + Some(root) => paths + .into_iter() + .filter(|path| path.starts_with(root)) + .collect(), + None => paths, + } +} + +async fn collect_results( + index_arc: &std::sync::Arc< + tokio::sync::RwLock, + >, + matching_paths: Vec, + query: &str, + device_slug: &str, + filters: &SearchFilters, + file_type_registry: &FileTypeRegistry, +) -> Result, QueryError> { + // One write lock for the batch rather than one per entry: the lazy uuid + // assignment needs it and the lock is the expensive part. let mut index = index_arc.write().await; let mut results = Vec::new(); @@ -112,16 +157,11 @@ pub async fn search_ephemeral_index( // Get or assign UUID (lazy generation) let uuid = index.get_or_assign_uuid(&path); - // Build SdPath - let sd_path = match path_scope { - SdPath::Physical { device_slug, .. } => SdPath::Physical { - device_slug: device_slug.clone(), - path: path.clone(), - }, - _ => continue, + let sd_path = SdPath::Physical { + device_slug: device_slug.to_string(), + path: path.clone(), }; - // Get content kind let content_kind = index.get_content_kind(&path); // Convert to File @@ -141,11 +181,14 @@ pub async fn search_ephemeral_index( } } - // Sort by score and limit + rank(&mut results); + Ok(results) +} + +/// Best match first, capped at what a person will scroll through. +fn rank(results: &mut Vec) { results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal)); results.truncate(200); - - Ok(results) } /// Check if metadata passes ephemeral filters @@ -219,8 +262,8 @@ fn passes_ephemeral_filters( } } - // Tags and locations are not available in ephemeral - // These filters are simply ignored for ephemeral searches + // Tags have no arena representation, so a tag filter is ignored rather + // than silently excluding everything. true } diff --git a/core/src/ops/search/input.rs b/core/src/ops/search/input.rs index 28d2f28dc..bd7977986 100644 --- a/core/src/ops/search/input.rs +++ b/core/src/ops/search/input.rs @@ -34,8 +34,6 @@ pub struct FileSearchInput { pub enum SearchScope { /// Search the entire library (default) Library, - /// Restrict search to a specific location by its ID - Location { location_id: Uuid }, /// Restrict search to a specific directory path and all its descendants Path { path: SdPath }, } diff --git a/core/src/ops/search/query.rs b/core/src/ops/search/query.rs index 4b21397e5..4622d6150 100644 --- a/core/src/ops/search/query.rs +++ b/core/src/ops/search/query.rs @@ -48,1516 +48,58 @@ impl LibraryQuery for FileSearchQuery { async fn execute( self, context: Arc, - session: crate::infra::api::SessionContext, + _session: crate::infra::api::SessionContext, ) -> QueryResult { let start_time = std::time::Instant::now(); - // Validate input - self.input.validate().map_err(|e| { - QueryError::Internal(format!("Invalid search input: {}", e.to_string())) - })?; + self.input + .validate() + .map_err(|e| QueryError::Internal(format!("Invalid search input: {e}")))?; - let library_id = session - .current_library_id - .ok_or_else(|| QueryError::Internal("No library in session".to_string()))?; - let library = context - .libraries() - .await - .get_library(library_id) - .await - .ok_or_else(|| QueryError::Internal("Library not found".to_string()))?; - - let db = library.db(); let search_id = Uuid::new_v4(); - // Determine which index to use (ephemeral or persistent) - let index_type = self.determine_index_type(&context, db.conn()).await?; - tracing::info!( - "Search query: '{}', scope: {:?}, index: {:?}, mode: {:?}, limit: {}, offset: {}", + "Search query: '{}', scope: {:?}, mode: {:?}, limit: {}, offset: {}", self.input.query, self.input.scope, - index_type, self.input.mode, self.input.pagination.limit, self.input.pagination.offset ); - match index_type { - crate::ops::search::IndexType::Persistent => { - // Build device slug lookup map from database - use std::collections::HashMap; - let devices = crate::infra::db::entities::device::Entity::find() - .all(db.conn()) - .await - .map_err(QueryError::SeaOrm)?; - let device_slug_map: HashMap = devices - .into_iter() - .map(|device| (device.uuid, device.slug)) - .collect(); - - // Perform the search based on mode - let results = match self.input.mode { - crate::ops::search::input::SearchMode::Fast => { - self.execute_fast_search(db.conn(), &device_slug_map) - .await? - } - crate::ops::search::input::SearchMode::Normal => { - self.execute_normal_search(db.conn(), &device_slug_map) - .await? - } - crate::ops::search::input::SearchMode::Full => { - self.execute_full_search(db.conn(), &device_slug_map) - .await? - } - }; - - let execution_time = start_time.elapsed().as_millis() as u64; - - // Get actual total count for pagination - let total_count = self - .get_total_count(db.conn(), context.file_type_registry()) - .await - .unwrap_or(0); - - // Create output with persistent index type - let output = FileSearchOutput::new_persistent( - results, - total_count, - search_id, - execution_time, - ); - - Ok(output) - } - crate::ops::search::IndexType::Ephemeral => { - // Use ephemeral search - self.execute_ephemeral_search(context, search_id, start_time) - .await - } - crate::ops::search::IndexType::Hybrid => { - // Future: search both and merge results - Err(QueryError::Internal( - "Hybrid search not yet implemented".to_string(), - )) - } - } + self.search_index(context, search_id, start_time).await } } impl FileSearchQuery { - /// Construct full path for an entry by joining with directory_paths - async fn construct_full_path( - &self, - entry_model: &entry::Model, - db: &DatabaseConnection, - ) -> QueryResult { - // If this is a root entry (no parent), return just the name - if entry_model.parent_id.is_none() { - return Ok(format!("/{}", entry_model.name)); - } - - // Find the parent directory entry - if let Some(parent_id) = entry_model.parent_id { - // Look up the directory path in directory_paths table - let directory_path = directory_paths::Entity::find() - .filter(directory_paths::Column::EntryId.eq(parent_id)) - .one(db) - .await? - .ok_or_else(|| { - QueryError::Internal(format!( - "Directory path not found for parent_id: {}", - parent_id.to_string() - )) - })?; - - // Construct full path: directory_path + "/" + filename - let full_path = if directory_path.path.ends_with('/') { - format!("{}{}", directory_path.path, entry_model.name) - } else { - format!("{}/{}", directory_path.path, entry_model.name) - }; - - Ok(full_path) - } else { - // Fallback for entries without parent - Ok(format!("/{}", entry_model.name)) - } - } - - /// Execute fast search using FTS5 with efficient batch joins - pub async fn execute_fast_search( - &self, - db: &DatabaseConnection, - _device_slug_map: &std::collections::HashMap, - ) -> QueryResult> { - use sea_orm::Statement; - - // For empty queries (recents view), skip FTS and query entries directly - if self.input.query.trim().is_empty() { - return self.execute_fast_search_no_fts(db).await; - } - - // Use FTS5 for high-performance text search - let fts_query = self.build_fts5_query(); - let mut fts_results = self.execute_fts5_search(db, &fts_query).await?; - - let fts_count = fts_results.len(); - tracing::info!( - "FTS5 search returned {} results for query '{}'", - fts_count, - self.input.query - ); - - if fts_results.is_empty() { - return Ok(Vec::new()); - } - - // Apply tag filter on FTS results - if let Some(tag_filter) = &self.input.filters.tags { - let (include_ids, exclude_ids) = self.resolve_tag_filter(db, tag_filter).await?; - let include_set: Option> = include_ids.map(|v| v.into_iter().collect()); - let exclude_set: HashSet = exclude_ids.into_iter().collect(); - fts_results.retain(|(id, _)| { - include_set.as_ref().map_or(true, |s| s.contains(id)) && !exclude_set.contains(id) - }); - if fts_results.is_empty() { - return Ok(Vec::new()); - } - } - - // Build a map of entry_id -> bm25_score for later lookup - let score_map: std::collections::HashMap = fts_results.iter().cloned().collect(); - let entry_ids: Vec = fts_results.iter().map(|(id, _)| *id).collect(); - - // Build single efficient query with all joins - let entry_ids_str = entry_ids - .iter() - .map(|id| id.to_string()) - .collect::>() - .join(","); - - // Join directory_paths on parent_id directly - every directory has its full - // absolute path stored, so the parent of any file gives us the containing folder path. - let sql_query = format!( - r#" - SELECT - e.id as entry_id, - e.uuid as entry_uuid, - e.name as entry_name, - e.kind as entry_kind, - e.extension as entry_extension, - e.size as entry_size, - e.aggregate_size as entry_aggregate_size, - e.child_count as entry_child_count, - e.file_count as entry_file_count, - e.created_at as entry_created_at, - e.modified_at as entry_modified_at, - e.accessed_at as entry_accessed_at, - e.content_id as entry_content_id, - dp.path as parent_path, - d.slug as device_slug, - ck.name as content_kind_name, - ci.uuid as content_identity_uuid, - ci.content_hash as content_hash, - ci.integrity_hash as integrity_hash, - ci.mime_type_id as mime_type_id, - ci.text_content as text_content, - ci.total_size as ci_total_size, - ci.entry_count as ci_entry_count, - ci.first_seen_at as first_seen_at, - ci.last_verified_at as last_verified_at - FROM entries e - LEFT JOIN directory_paths dp ON dp.entry_id = e.parent_id - LEFT JOIN volumes v ON e.volume_id = v.id - LEFT JOIN devices d ON v.device_id = d.uuid - LEFT JOIN content_identities ci ON e.content_id = ci.id - LEFT JOIN content_kinds ck ON ci.kind_id = ck.id - WHERE e.id IN ({}) - "#, - entry_ids_str - ); - - let rows = db - .query_all(Statement::from_string( - sea_orm::DatabaseBackend::Sqlite, - sql_query, - )) - .await?; - - // Collect all content UUIDs for batch sidecar query - let content_uuids: Vec = rows - .iter() - .filter_map(|row| { - row.try_get::>("", "content_identity_uuid") - .ok() - .flatten() - }) - .collect(); - - // Batch fetch all sidecars for these content UUIDs - let all_sidecars = if !content_uuids.is_empty() { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.is_in(content_uuids.clone())) - .all(db) - .await? - } else { - Vec::new() - }; - - // Group sidecars by content_uuid for fast lookup - let mut sidecars_by_content: std::collections::HashMap< - Uuid, - Vec, - > = std::collections::HashMap::new(); - for s in all_sidecars { - sidecars_by_content - .entry(s.content_uuid) - .or_insert_with(Vec::new) - .push(crate::domain::file::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }); - } - - // Convert results to FileSearchResult objects - let mut results = Vec::new(); - let relevance_calc = - crate::ops::search::sorting::RelevanceCalculator::new(self.input.query.clone()); - - for row in rows { - let entry_id: i32 = row.try_get("", "entry_id").unwrap_or(0); - let entry_uuid: Option = row.try_get("", "entry_uuid").ok(); - let entry_name: String = row.try_get("", "entry_name").unwrap_or_default(); - let entry_kind: i32 = row.try_get("", "entry_kind").unwrap_or(0); - let entry_extension: Option = row.try_get("", "entry_extension").ok(); - let entry_size: i64 = row.try_get("", "entry_size").unwrap_or(0); - let entry_aggregate_size: i64 = row.try_get("", "entry_aggregate_size").unwrap_or(0); - let entry_child_count: i32 = row.try_get("", "entry_child_count").unwrap_or(0); - let entry_file_count: i32 = row.try_get("", "entry_file_count").unwrap_or(0); - let entry_created_at: chrono::DateTime = row - .try_get("", "entry_created_at") - .unwrap_or_else(|_| chrono::Utc::now()); - let entry_modified_at: chrono::DateTime = row - .try_get("", "entry_modified_at") - .unwrap_or_else(|_| chrono::Utc::now()); - let entry_accessed_at: Option> = - row.try_get("", "entry_accessed_at").ok(); - - // Get joined data - let parent_path: Option = row.try_get("", "parent_path").ok(); - let device_slug: Option = row.try_get("", "device_slug").ok(); - let content_kind_name: Option = row.try_get("", "content_kind_name").ok(); - let content_identity_uuid: Option = - row.try_get("", "content_identity_uuid").ok().flatten(); - - // Content identity fields for building ContentIdentity object - let content_hash: Option = row.try_get("", "content_hash").ok(); - let integrity_hash: Option = row.try_get("", "integrity_hash").ok(); - let mime_type_id: Option = row.try_get("", "mime_type_id").ok(); - let text_content: Option = row.try_get("", "text_content").ok(); - let ci_total_size: Option = row.try_get("", "ci_total_size").ok(); - let ci_entry_count: Option = row.try_get("", "ci_entry_count").ok(); - let first_seen_at: Option> = - row.try_get("", "first_seen_at").ok(); - let last_verified_at: Option> = - row.try_get("", "last_verified_at").ok(); - - // Build full path: parent directory path + filename (with extension) - let file_path = if let Some(dir_path) = parent_path { - let file_name = if let Some(ext) = &entry_extension { - format!("{}.{}", entry_name, ext) - } else { - entry_name.clone() - }; - if dir_path.ends_with('/') { - format!("{}{}", dir_path, file_name) - } else { - format!("{}/{}", dir_path, file_name) - } - } else { - // Entry has no parent (is a root entry) - use name directly - format!("/{}", entry_name) - }; - - // Create SdPath with device_slug from join - let sd_path = SdPath::Physical { - device_slug: device_slug - .unwrap_or_else(|| crate::device::get_current_device_slug()), - path: file_path.into(), - }; - - // Create entity model for File conversion - let entity_model = entry::Model { - id: entry_id, - uuid: entry_uuid, - name: entry_name.clone(), - kind: entry_kind, - extension: entry_extension.clone(), - metadata_id: None, - content_id: None, - size: entry_size, - aggregate_size: entry_aggregate_size, - child_count: entry_child_count, - file_count: entry_file_count, - created_at: entry_created_at, - modified_at: entry_modified_at, - accessed_at: entry_accessed_at, - indexed_at: None, - permissions: None, - inode: None, - parent_id: None, - volume_id: None, - }; - - // Convert to File - let mut file = File::from_entity_model(entity_model, sd_path); - - // Build and set content identity if we have the required fields - if let (Some(ci_uuid), Some(ci_hash), Some(ci_first_seen), Some(ci_last_verified)) = ( - content_identity_uuid, - content_hash, - first_seen_at, - last_verified_at, - ) { - // Convert content_kind name to ContentKind enum - let kind = content_kind_name - .as_ref() - .map(|name| crate::domain::ContentKind::from(name.as_str())) - .unwrap_or(crate::domain::ContentKind::Unknown); - - file.content_identity = Some(crate::domain::content_identity::ContentIdentity { - uuid: ci_uuid, - kind, - content_hash: ci_hash, - integrity_hash, - mime_type_id, - text_content, - total_size: ci_total_size.unwrap_or(0), - entry_count: ci_entry_count.unwrap_or(0), - first_seen_at: ci_first_seen, - last_verified_at: ci_last_verified, - }); - file.content_kind = kind; - - // Add sidecars from batch lookup - if let Some(sidecars) = sidecars_by_content.get(&ci_uuid) { - file.sidecars = sidecars.clone(); - } - } else if let Some(kind_name) = content_kind_name { - // Fallback: set content kind even without full content identity - file.content_kind = crate::domain::ContentKind::from(kind_name.as_str()); - } - - // Get BM25 score and calculate final score - let bm25_score = score_map.get(&entry_id).copied().unwrap_or(0.0); - let recency_boost = relevance_calc.calculate_recency_boost(file.modified_at); - let user_preference_boost = relevance_calc.calculate_user_preference_boost(entry_id); - let final_score = bm25_score as f32 + recency_boost + user_preference_boost; - - let result = crate::ops::search::output::FileSearchResult { - file, - score: final_score, - score_breakdown: crate::ops::search::output::ScoreBreakdown::new( - bm25_score as f32, - None, - 0.0, - recency_boost, - user_preference_boost, - ), - highlights: self.extract_highlights(&fts_query, &entry_name, &entry_extension), - matched_content: None, - }; - - results.push(result); - } - - tracing::info!( - "Built {} FileSearchResult objects from {} FTS5 results", - results.len(), - fts_count - ); - - // Sort by final score - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - Ok(results) - } - - /// Execute normal search with FTS5 + enhanced ranking - async fn execute_normal_search( - &self, - db: &DatabaseConnection, - device_slug_map: &std::collections::HashMap, - ) -> QueryResult> { - // Use FTS5 as base, then enhance with additional ranking factors - let mut results = self.execute_fast_search(db, device_slug_map).await?; - - // Enhanced ranking for normal search - for result in &mut results { - let mut enhanced_score = result.score; - - // Add metadata-based scoring - let size = result.file.size; - // Slightly boost files with reasonable sizes - if size > 1024 && size < 10_000_000 { - // 1KB to 10MB - enhanced_score += 0.1; - } - - // Boost files with extensions that match common document types - if let Some(ref extension) = result.file.extension { - match extension.as_str() { - "pdf" | "doc" | "docx" | "txt" | "md" => enhanced_score += 0.2, - "jpg" | "png" | "gif" | "webp" => enhanced_score += 0.1, - _ => {} - } - } - - // Update the score - result.score = enhanced_score; - result.score_breakdown.metadata_score = enhanced_score - - result.score_breakdown.temporal_score - - result.score_breakdown.recency_boost - - result.score_breakdown.user_preference_boost; - } - - // Re-sort with enhanced scores - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - Ok(results) - } - - /// Execute full search with FTS5 + content analysis - async fn execute_full_search( - &self, - db: &DatabaseConnection, - device_slug_map: &std::collections::HashMap, - ) -> QueryResult> { - // Start with normal search results - let mut results = self.execute_normal_search(db, device_slug_map).await?; - - // For full search, we would add content analysis here - // This is a placeholder for future implementation - for result in &mut results { - // TODO: Add content extraction and analysis - // For now, just add a small boost for files that might have content - let size = result.file.size; - if size > 0 && size < 100_000_000 { - // Up to 100MB - result.score += 0.05; - result.score_breakdown.metadata_score += 0.05; - } - } - - // Re-sort with content analysis scores - results.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - Ok(results) - } - - /// Apply scope filters to the query condition - fn apply_scope_filter(&self, mut condition: Condition) -> Condition { - match &self.input.scope { - crate::ops::search::input::SearchScope::Library => { - // No additional filtering needed for library-wide search - condition - } - crate::ops::search::input::SearchScope::Location { location_id } => { - // TODO: Add location filtering when location_id is available in entry table - condition - } - crate::ops::search::input::SearchScope::Path { path } => { - // Path-based filtering is handled in the main query with directory_paths join - // No additional condition needed here as it's applied in the query building - condition - } - } - } - - /// Apply additional filters to the query condition - fn apply_filters(&self, mut condition: Condition, registry: &FileTypeRegistry) -> Condition { - // File type filter - if let Some(file_types) = &self.input.filters.file_types { - if !file_types.is_empty() { - let mut file_type_condition = Condition::any(); - for file_type in file_types { - file_type_condition = - file_type_condition.add(entry::Column::Extension.eq(file_type)); - } - condition = condition.add(file_type_condition); - } - } - - // Date range filter - if let Some(date_range) = &self.input.filters.date_range { - let date_column = match date_range.field { - crate::ops::search::input::DateField::CreatedAt => entry::Column::CreatedAt, - crate::ops::search::input::DateField::ModifiedAt => entry::Column::ModifiedAt, - crate::ops::search::input::DateField::AccessedAt => entry::Column::AccessedAt, - crate::ops::search::input::DateField::IndexedAt => entry::Column::IndexedAt, - }; - - if let Some(start) = date_range.start { - condition = condition.add(date_column.gte(start)); - } - if let Some(end) = date_range.end { - condition = condition.add(date_column.lte(end)); - } - } - - // Size range filter - if let Some(size_range) = &self.input.filters.size_range { - if let Some(min) = size_range.min { - condition = condition.add(entry::Column::Size.gte(min as i64)); - } - if let Some(max) = size_range.max { - condition = condition.add(entry::Column::Size.lte(max as i64)); - } - } - - // Content type filter using file type registry - if let Some(content_types) = &self.input.filters.content_types { - if !content_types.is_empty() { - let mut content_condition = Condition::any(); - for content_type in content_types { - let extensions = registry.get_extensions_for_category(*content_type); - for extension in extensions { - content_condition = - content_condition.add(entry::Column::Extension.eq(extension)); - } - } - condition = condition.add(content_condition); - } - } - - // Location filter - join with locations table - if let Some(locations) = &self.input.filters.locations { - if !locations.is_empty() { - let mut location_condition = Condition::any(); - for location_id in locations { - // We need to join with locations table to filter by location UUID - // This will be handled in the main query with a join - location_condition = location_condition - .add(crate::infra::db::entities::location::Column::Uuid.eq(*location_id)); - } - condition = condition.add(location_condition); - } - } - - // Include hidden filter - if let Some(include_hidden) = self.input.filters.include_hidden { - if !include_hidden { - // TODO: Add hidden field to entry table - // condition = condition.add(entry::Column::Hidden.eq(false)); - } - } - - condition - } - - /// Get device UUID for an entry by traversing parent chain to find location root - async fn get_device_uuid_for_entry( - &self, - entry_model: &entry::Model, - db: &DatabaseConnection, - ) -> Option { - use sea_orm::{sea_query::Expr, Statement}; - - // Traverse parent chain to find the location root using recursive CTE - let query = format!( - r#" - WITH RECURSIVE parent_chain AS ( - SELECT id, parent_id FROM entries WHERE id = {} - UNION ALL - SELECT e.id, e.parent_id FROM entries e - INNER JOIN parent_chain pc ON e.id = pc.parent_id - ) - SELECT l.device_id - FROM parent_chain pc - INNER JOIN locations l ON l.entry_id = pc.id - LIMIT 1 - "#, - entry_model.id - ); - - let result = db - .query_one(Statement::from_string( - sea_orm::DatabaseBackend::Sqlite, - query, - )) - .await - .ok()??; - - let device_db_id: i32 = result.try_get("", "device_id").ok()?; - - // Now get the device UUID from the device_id - if let Ok(Some(device)) = - crate::infra::db::entities::device::Entity::find_by_id(device_db_id) - .one(db) - .await - { - return Some(device.uuid); - } - - None - } - - /// Get location UUID for an entry - async fn get_location_uuid_for_entry( - &self, - entry_model: &entry::Model, - db: &DatabaseConnection, - ) -> Option { - // Look up location by entry_id - if let Ok(location) = crate::infra::db::entities::location::Entity::find() - .filter(crate::infra::db::entities::location::Column::EntryId.eq(entry_model.id)) - .one(db) - .await - { - location.map(|l| l.uuid) - } else { - None - } - } - - /// Get metadata UUID for an entry - async fn get_metadata_uuid_for_entry( - &self, - metadata_id: i32, - db: &DatabaseConnection, - ) -> Option { - if let Ok(metadata) = - crate::infra::db::entities::user_metadata::Entity::find_by_id(metadata_id) - .one(db) - .await - { - metadata.map(|m| m.uuid) - } else { - None - } - } - - /// Get content UUID for an entry - async fn get_content_uuid_for_entry( - &self, - content_id: i32, - db: &DatabaseConnection, - ) -> Option { - if let Ok(content) = - crate::infra::db::entities::content_identity::Entity::find_by_id(content_id) - .one(db) - .await - { - content.and_then(|c| c.uuid) - } else { - None - } - } - - /// Get parent UUID for an entry - async fn get_parent_uuid_for_entry( - &self, - parent_id: i32, - db: &DatabaseConnection, - ) -> Option { - if let Ok(parent) = entry::Entity::find_by_id(parent_id).one(db).await { - parent.and_then(|p| p.uuid) - } else { - None - } - } - - /// Get content kind for an entry by looking up content identity - async fn get_content_kind_for_entry( - &self, - content_id: i32, - db: &DatabaseConnection, - ) -> Option { - use sea_orm::Statement; - - let query = format!( - r#" - SELECT ck.name - FROM content_identities ci - INNER JOIN content_kinds ck ON ci.kind_id = ck.id - WHERE ci.id = {} - "#, - content_id - ); - - let result = db - .query_one(Statement::from_string( - sea_orm::DatabaseBackend::Sqlite, - query, - )) - .await - .ok()??; - - let kind_name: String = result.try_get("", "name").ok()?; - Some(crate::domain::ContentKind::from(kind_name.as_str())) - } - - /// Get total count of matching entries for pagination - async fn get_total_count( - &self, - db: &DatabaseConnection, - registry: &FileTypeRegistry, - ) -> QueryResult { - let mut condition = Condition::any() - .add(entry::Column::Name.contains(&self.input.query)) - .add(entry::Column::Extension.contains(&self.input.query)); - - // Apply scope filters - condition = self.apply_scope_filter(condition); - - // Apply additional filters - condition = self.apply_filters(condition, registry); - - // Build count query - let mut query = entry::Entity::find() - .filter(condition) - .filter(entry::Column::Kind.eq(0)); // Only files - - // Add location join if location filtering is needed - if self.input.filters.locations.is_some() { - query = query.join( - JoinType::LeftJoin, - crate::infra::db::entities::location::Relation::Entry.def(), - ); - } - - // Apply SD path filtering if specified in scope - if let SearchScope::Path { path } = &self.input.scope { - if let Some(device_id) = path.device_id() { - if let Some(path_str) = path.path() { - // Join with directory_paths to filter by path - query = query - .join(JoinType::LeftJoin, directory_paths::Relation::Entry.def()) - .filter( - directory_paths::Column::Path - .like(&format!("{}%", path_str.to_string_lossy())), - ); - } - } - } - - // For queries with joins, we need to use a different approach - // We'll execute the query and count the results - let entries = query.all(db).await?; - Ok(entries.len() as u64) - } - - /// Convert an entry model to a FileSearchResult with device and path resolution - async fn entry_to_search_result( - &self, - entry_model: entry::Model, - db: &DatabaseConnection, - score: f32, - ) -> QueryResult> { - // Skip entries without volume_id (can't determine device slug) - let Some(volume_id) = entry_model.volume_id else { - return Ok(None); - }; - - // Get volume to find device info - let Some(volume) = crate::infra::db::entities::volume::Entity::find_by_id(volume_id) - .one(db) - .await? - else { - return Ok(None); - }; - - // Get device to find slug - let Some(device) = crate::infra::db::entities::device::Entity::find() - .filter(crate::infra::db::entities::device::Column::Uuid.eq(volume.device_id)) - .one(db) - .await? - else { - return Ok(None); - }; - - // Construct full path - let full_path = self.construct_full_path(&entry_model, db).await?; - - // Build SD path - let sd_path = crate::domain::addressing::SdPath::Physical { - device_slug: device.slug, - path: full_path.into(), - }; - - // Use File::from_entity_model to properly construct the file - let file = crate::domain::File::from_entity_model(entry_model, sd_path); - - Ok(Some(crate::ops::search::output::FileSearchResult { - file, - score, - score_breakdown: crate::ops::search::output::ScoreBreakdown::new( - score, None, 0.0, 0.0, 0.0, - ), - highlights: Vec::new(), - matched_content: None, - })) - } - - /// Execute fast search without FTS (for empty queries like recents view) - async fn execute_fast_search_no_fts( - &self, - db: &DatabaseConnection, - ) -> QueryResult> { - use crate::ops::search::filters::FilterBuilder; - use crate::ops::search::sorting::SortBuilder; - - tracing::info!( - "Executing fast search without FTS (empty query), sorting by {:?}", - self.input.sort.field - ); - - // Build query with filters and sorting - let mut query = entry::Entity::find(); - - // Apply filters - let filter_builder = FilterBuilder::new() - .file_types(&self.input.filters.file_types) - .date_range(&self.input.filters.date_range) - .size_range(&self.input.filters.size_range) - .at_risk(&self.input.filters.at_risk) - .on_volumes(&self.input.filters.on_volumes) - .not_on_volumes(&self.input.filters.not_on_volumes) - .min_volume_count(&self.input.filters.min_volume_count) - .max_volume_count(&self.input.filters.max_volume_count); - - query = query.filter(filter_builder.build()); - - // Apply tag filter - if let Some(tag_filter) = &self.input.filters.tags { - let (include_ids, exclude_ids) = self.resolve_tag_filter(db, tag_filter).await?; - if let Some(ids) = include_ids { - query = query.filter(entry::Column::Id.is_in(ids)); - } - if !exclude_ids.is_empty() { - query = query.filter(entry::Column::Id.is_not_in(exclude_ids)); - } - } - - // Apply sorting - let sort_builder = SortBuilder::new().apply_sort(&self.input.sort); - for (column, order) in sort_builder.build() { - match column.as_str() { - "name" => query = query.order_by(entry::Column::Name, order), - "size" => query = query.order_by(entry::Column::Size, order), - "modified_at" => query = query.order_by(entry::Column::ModifiedAt, order), - "created_at" => query = query.order_by(entry::Column::CreatedAt, order), - "indexed_at" => query = query.order_by(entry::Column::IndexedAt, order), - _ => {} - } - } - - // Apply pagination - query = query - .limit(self.input.pagination.limit as u64) - .offset(self.input.pagination.offset as u64); - - // Execute query - let entries = query.all(db).await?; - - tracing::info!("Fast search without FTS returned {} entries", entries.len()); - - // Batch-hydrate content_identity + content_kind + sidecars so the - // frontend gets real file metadata (mime, thumbs, dedup UUID) instead - // of "Unknown" placeholders. This mirrors the FTS path's hydration. - let content_ids: Vec = entries - .iter() - .filter_map(|e| e.content_id) - .collect::>() - .into_iter() - .collect(); - - let content_identities = if !content_ids.is_empty() { - content_identity::Entity::find() - .filter(content_identity::Column::Id.is_in(content_ids.clone())) - .all(db) - .await? - } else { - Vec::new() - }; - - // Build id -> ContentIdentity (with mapped ContentKind) lookup - let kind_ids: Vec = content_identities - .iter() - .map(|ci| ci.kind_id) - .collect::>() - .into_iter() - .collect(); - - let kinds_by_id: std::collections::HashMap = if !kind_ids.is_empty() { - crate::infra::db::entities::content_kind::Entity::find() - .filter(crate::infra::db::entities::content_kind::Column::Id.is_in(kind_ids)) - .all(db) - .await? - .into_iter() - .map(|k| (k.id, k.name)) - .collect() - } else { - std::collections::HashMap::new() - }; - - // Batch sidecars by content UUID - let content_uuids: Vec = content_identities.iter().filter_map(|ci| ci.uuid).collect(); - - let all_sidecars = if !content_uuids.is_empty() { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.is_in(content_uuids.clone())) - .all(db) - .await? - } else { - Vec::new() - }; - - let mut sidecars_by_content: std::collections::HashMap< - Uuid, - Vec, - > = std::collections::HashMap::new(); - for s in all_sidecars { - sidecars_by_content - .entry(s.content_uuid) - .or_insert_with(Vec::new) - .push(crate::domain::file::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }); - } - - // Index content identities by entry content_id for per-entry lookup - let identities_by_content_id: std::collections::HashMap = - content_identities.iter().map(|ci| (ci.id, ci)).collect(); - - // Convert entries to FileSearchResult, hydrating each with content_identity - let mut results = Vec::new(); - for entry_model in entries { - let content_id = entry_model.content_id; - if let Some(mut result) = self.entry_to_search_result(entry_model, db, 1.0).await? { - if let Some(cid) = content_id { - if let Some(ci) = identities_by_content_id.get(&cid) { - let kind = kinds_by_id - .get(&ci.kind_id) - .map(|name| crate::domain::ContentKind::from(name.as_str())) - .unwrap_or(crate::domain::ContentKind::Unknown); - - if let Some(ci_uuid) = ci.uuid { - result.file.content_identity = - Some(crate::domain::content_identity::ContentIdentity { - uuid: ci_uuid, - kind, - content_hash: ci.content_hash.clone(), - integrity_hash: ci.integrity_hash.clone(), - mime_type_id: ci.mime_type_id, - text_content: ci.text_content.clone(), - total_size: ci.total_size, - entry_count: ci.entry_count, - first_seen_at: ci.first_seen_at, - last_verified_at: ci.last_verified_at, - }); - if let Some(sidecars) = sidecars_by_content.get(&ci_uuid) { - result.file.sidecars = sidecars.clone(); - } - } - result.file.content_kind = kind; - } - } - results.push(result); - } - } - - Ok(results) - } - - pub fn build_fts5_query(&self) -> String { - // Escape special FTS5 characters and build query - let escaped_query = self - .input - .query - .replace('"', r#"\""#) - .replace('\'', r#"\'"#) - .replace('*', r#"\*"#) - .replace('(', r#"\("#) - .replace(')', r#"\)"#); - - // Add prefix matching for autocomplete if query is long enough - if self.input.query.len() > 2 { - format!("{}*", escaped_query) - } else { - escaped_query - } - } - - /// Execute FTS5 search with BM25 ranking - async fn execute_fts5_search( - &self, - db: &DatabaseConnection, - query: &str, - ) -> QueryResult> { - let sql = match &self.input.scope { - SearchScope::Path { path } => { - if let Some(path_str) = path.path() { - // FTS5 search with path filtering - r#" - WITH fts AS ( - SELECT rowid, bm25(search_index) AS rank - FROM search_index - WHERE search_index MATCH ? - ORDER BY rank - LIMIT 5000 - ) - SELECT e.id, fts.rank - FROM fts - JOIN entries e ON e.id = fts.rowid - JOIN directory_paths dp ON dp.entry_id = e.parent_id - WHERE dp.path LIKE ? - ORDER BY fts.rank - LIMIT ? OFFSET ? - "# - } else { - // Basic FTS5 search - r#" - SELECT e.id, bm25(search_index) as rank - FROM search_index - JOIN entries e ON e.id = search_index.rowid - WHERE search_index MATCH ? - ORDER BY rank - LIMIT ? OFFSET ? - "# - } - } - _ => { - // Basic FTS5 search - r#" - SELECT e.id, bm25(search_index) as rank - FROM search_index - JOIN entries e ON e.id = search_index.rowid - WHERE search_index MATCH ? - ORDER BY rank - LIMIT ? OFFSET ? - "# - } - }; - - let statement = Statement::from_string(db.get_database_backend(), sql.to_string()); - - let params = match &self.input.scope { - SearchScope::Path { path } if path.path().is_some() => { - let path_str = path.path().unwrap().to_string_lossy(); - let like_pattern = format!("{}%", path_str); - tracing::info!( - "Path scope FTS5: query='{}', LIKE pattern='{}'", - query, - like_pattern - ); - vec![ - query.into(), - like_pattern.into(), - self.input.pagination.limit.to_string().into(), - self.input.pagination.offset.to_string().into(), - ] - } - _ => { - vec![ - query.into(), - self.input.pagination.limit.to_string().into(), - self.input.pagination.offset.to_string().into(), - ] - } - }; - - let results = db - .query_all(Statement::from_sql_and_values( - db.get_database_backend(), - &statement.sql, - params, - )) - .await?; - - let mut fts_results = Vec::new(); - for row in results { - let entry_id: i32 = row.try_get("", "id")?; - let rank: f64 = row.try_get("", "rank")?; - fts_results.push((entry_id, rank)); - } - - Ok(fts_results) - } - - /// Check if an entry passes additional (non-text) filters - async fn passes_additional_filters( - &self, - entry_model: &entry::Model, - db: &DatabaseConnection, - registry: &FileTypeRegistry, - ) -> QueryResult { - // File type filter - if let Some(file_types) = &self.input.filters.file_types { - if !file_types.is_empty() { - if let Some(ref extension) = entry_model.extension { - if !file_types.contains(extension) { - return Ok(false); - } - } else { - return Ok(false); - } - } - } - - // Date range filter - if let Some(date_range) = &self.input.filters.date_range { - let date_to_check = match date_range.field { - crate::ops::search::input::DateField::CreatedAt => Some(entry_model.created_at), - crate::ops::search::input::DateField::ModifiedAt => Some(entry_model.modified_at), - crate::ops::search::input::DateField::AccessedAt => entry_model.accessed_at, - crate::ops::search::input::DateField::IndexedAt => entry_model.indexed_at, - }; - - if let Some(date) = date_to_check { - if let Some(start) = date_range.start { - if date < start { - return Ok(false); - } - } - if let Some(end) = date_range.end { - if date > end { - return Ok(false); - } - } - } - } - - // Size range filter - if let Some(size_range) = &self.input.filters.size_range { - if let Some(min) = size_range.min { - if (entry_model.size as u64) < min { - return Ok(false); - } - } - if let Some(max) = size_range.max { - if (entry_model.size as u64) > max { - return Ok(false); - } - } - } - - // Content type filter using file type registry - if let Some(content_types) = &self.input.filters.content_types { - if !content_types.is_empty() { - let mut matches_content_type = false; - - for content_type in content_types { - let extensions = registry.get_extensions_for_category(*content_type); - if let Some(ref extension) = entry_model.extension { - if extensions.contains(&extension.as_str()) { - matches_content_type = true; - break; - } - } - } - - if !matches_content_type { - return Ok(false); - } - } - } - - // Location filter - if let Some(locations) = &self.input.filters.locations { - if !locations.is_empty() { - // Check if entry belongs to one of the specified locations - if let Ok(Some(location)) = crate::infra::db::entities::location::Entity::find() - .filter( - crate::infra::db::entities::location::Column::EntryId.eq(entry_model.id), - ) - .one(db) - .await - { - if !locations.contains(&location.uuid) { - return Ok(false); - } - } else { - return Ok(false); - } - } - } - - Ok(true) - } - - /// Extract text highlights from FTS5 results - pub fn extract_highlights( - &self, - query: &str, - name: &str, - extension: &Option, - ) -> Vec { - let mut highlights = Vec::new(); - - // Highlight matches in filename - let name_lower = name.to_lowercase(); - let query_lower = query.replace('*', "").to_lowercase(); - - if let Some(start) = name_lower.find(&query_lower) { - highlights.push(crate::ops::search::output::TextHighlight { - field: "name".to_string(), - text: name.to_string(), - start, - end: start + query_lower.len(), - }); - } - - // Highlight matches in extension - if let Some(ref ext) = extension { - let ext_lower = ext.to_lowercase(); - if let Some(start) = ext_lower.find(&query_lower) { - highlights.push(crate::ops::search::output::TextHighlight { - field: "extension".to_string(), - text: ext.clone(), - start, - end: start + query_lower.len(), - }); - } - } - - highlights - } - - /// Execute search and return File objects with joined data - /// This method uses SQL joins to efficiently load all related data in one query - pub async fn execute_with_files( - &self, - db: &DatabaseConnection, - ) -> QueryResult> { - // Build device slug lookup map from database - use std::collections::HashMap; - let devices = crate::infra::db::entities::device::Entity::find() - .all(db) - .await - .map_err(QueryError::SeaOrm)?; - let device_slug_map: HashMap = devices - .into_iter() - .map(|device| (device.uuid, device.slug)) - .collect(); - - // First get the basic search results - let entry_results = match self.input.mode { - crate::ops::search::input::SearchMode::Fast => { - self.execute_fast_search(db, &device_slug_map).await? - } - crate::ops::search::input::SearchMode::Normal => { - self.execute_normal_search(db, &device_slug_map).await? - } - crate::ops::search::input::SearchMode::Full => { - self.execute_full_search(db, &device_slug_map).await? - } - }; - - if entry_results.is_empty() { - return Ok(Vec::new()); - } - - // Convert to enhanced results (already have File objects) - let enhanced_results = entry_results - .into_iter() - .map(|result| EnhancedFileSearchResult { - file: result.file, - score: result.score, - score_breakdown: result.score_breakdown, - highlights: result.highlights, - matched_content: result.matched_content, - }) - .collect(); - - Ok(enhanced_results) - } - - /// Determine which index type to use for this search - async fn determine_index_type( - &self, - context: &Arc, - db: &DatabaseConnection, - ) -> QueryResult { - use crate::ops::search::IndexType; - - match &self.input.scope { - SearchScope::Path { path } => { - // Check if location has IndexMode::None - if let Some(should_use_ephemeral) = self.check_location_index_mode(path, db).await { - if should_use_ephemeral { - return Ok(IndexType::Ephemeral); - } - } - - // Try to find path in database - match self.find_parent_directory(path, db).await { - Ok(_) => Ok(IndexType::Persistent), - Err(_) => { - // Path not indexed - check if ephemeral cache has it - let cache = context.ephemeral_cache(); - let local_path = match path { - SdPath::Physical { path, .. } => path.clone(), - _ => return Ok(IndexType::Persistent), // Default to persistent for non-physical - }; - - if cache.is_indexed(&local_path) { - Ok(IndexType::Ephemeral) - } else { - // Not indexed anywhere yet - will use ephemeral - // (directory listing would trigger indexing here) - tracing::debug!( - "Path not in any index, defaulting to ephemeral: {}", - local_path.display() - ); - Ok(IndexType::Ephemeral) - } - } - } - } - SearchScope::Location { .. } => { - // Locations are always persistent (by definition) - Ok(IndexType::Persistent) - } - SearchScope::Library => { - // Global search only searches persistent - Ok(IndexType::Persistent) - } - } - } - - /// Check if a location has IndexMode::None (should use ephemeral) - async fn check_location_index_mode( - &self, - path: &SdPath, - db: &DatabaseConnection, - ) -> Option { - use crate::infra::db::entities::location; - - match path { - SdPath::Physical { - device_slug: _, - path, - } => { - let path_str = path.to_string_lossy().to_string(); - - // Get all locations and find the one that is a parent of this path - if let Ok(locations) = location::Entity::find().all(db).await { - for loc in locations { - // Get the location's root path - if let Some(entry_id) = loc.entry_id { - if let Ok(Some(dir_path)) = - directory_paths::Entity::find_by_id(entry_id).one(db).await - { - // Check if this location's path is a parent of the requested path - if path_str.starts_with(&dir_path.path) { - // Check if index_mode is "none" - return Some(loc.index_mode == "none"); - } - } - } - } - } - None - } - _ => None, - } - } - - /// Find parent directory entry for a given path - async fn find_parent_directory( - &self, - path: &SdPath, - db: &DatabaseConnection, - ) -> QueryResult { - match path { - SdPath::Physical { path, .. } => { - // Get the directory path string - let path_str = path.to_string_lossy().to_string(); - - // Look up in directory_paths table - let dir_path = directory_paths::Entity::find() - .filter(directory_paths::Column::Path.eq(&path_str)) - .one(db) - .await?; - - if let Some(dir_path) = dir_path { - // Get the entry - let entry = entry::Entity::find_by_id(dir_path.entry_id) - .one(db) - .await? - .ok_or_else(|| { - QueryError::Internal("Entry not found for directory path".to_string()) - })?; - - Ok(entry) - } else { - Err(QueryError::Internal( - "Path not found in database".to_string(), - )) - } - } - _ => Err(QueryError::Internal( - "Only physical paths supported for directory lookup".to_string(), - )), - } - } - - /// Execute search using ephemeral index - async fn execute_ephemeral_search( + /// A search reads the volume index: one partition when the scope names a + /// path, every partition when it does not. + async fn search_index( &self, context: Arc, search_id: Uuid, start_time: std::time::Instant, ) -> QueryResult { - let path = match &self.input.scope { - SearchScope::Path { path } => path, - _ => { - return Err(QueryError::Internal( - "Ephemeral search requires Path scope".to_string(), - )) - } - }; + use crate::ops::search::ephemeral_search::{search_ephemeral_index, search_every_index}; let cache = context.ephemeral_cache(); - let results = crate::ops::search::ephemeral_search::search_ephemeral_index( - &self.input.query, - path, - &self.input.filters, - cache, - context.file_type_registry(), - ) - .await?; + let registry = context.file_type_registry(); + + let results = match &self.input.scope { + SearchScope::Path { path } => { + search_ephemeral_index( + &self.input.query, + path, + &self.input.filters, + cache, + registry, + ) + .await? + } + SearchScope::Library => { + search_every_index(&self.input.query, &self.input.filters, cache, registry).await? + } + }; let execution_time = start_time.elapsed().as_millis() as u64; let total = results.len() as u64; @@ -1569,119 +111,6 @@ impl FileSearchQuery { execution_time, )) } - - /// Find all entry IDs that are tagged with the given tag UUID. - /// Handles both entry-scoped and content-scoped user_metadata. - async fn find_entry_ids_for_tag( - &self, - db: &DatabaseConnection, - tag_uuid: Uuid, - ) -> QueryResult> { - use crate::infra::db::entities::{ - content_identity::Entity as ContentIdentity, entry::Entity as Entry, - tag::Entity as Tag, user_metadata::Entity as UserMetadata, - user_metadata_tag::Entity as UserMetadataTag, - }; - - // 1. Find tag by UUID - let Some(tag_model) = Tag::find() - .filter(tag::Column::Uuid.eq(tag_uuid)) - .one(db) - .await? - else { - return Ok(vec![]); - }; - - // 2. Find all user_metadata_tag records for this tag - let umt_records = UserMetadataTag::find() - .filter(user_metadata_tag::Column::TagId.eq(tag_model.id)) - .all(db) - .await?; - - if umt_records.is_empty() { - return Ok(vec![]); - } - - let um_ids: Vec = umt_records.iter().map(|r| r.user_metadata_id).collect(); - - // 3. Fetch all user_metadata records in batch - let um_records = UserMetadata::find() - .filter(user_metadata::Column::Id.is_in(um_ids)) - .all(db) - .await?; - - let entry_uuids: Vec = um_records.iter().filter_map(|um| um.entry_uuid).collect(); - let ci_uuids: Vec = um_records - .iter() - .filter_map(|um| um.content_identity_uuid) - .collect(); - - let mut entry_ids: HashSet = HashSet::new(); - - // 4a. Entries directly linked via entry_uuid - if !entry_uuids.is_empty() { - let entries = Entry::find() - .filter(entry::Column::Uuid.is_in(entry_uuids)) - .all(db) - .await?; - entry_ids.extend(entries.iter().map(|e| e.id)); - } - - // 4b. Entries linked via content_identity_uuid - if !ci_uuids.is_empty() { - let cis = ContentIdentity::find() - .filter(content_identity::Column::Uuid.is_in(ci_uuids.into_iter().map(Some))) - .all(db) - .await?; - if !cis.is_empty() { - let ci_ids: Vec = cis.iter().map(|ci| ci.id).collect(); - let entries = Entry::find() - .filter(entry::Column::ContentId.is_in(ci_ids.into_iter().map(Some))) - .all(db) - .await?; - entry_ids.extend(entries.iter().map(|e| e.id)); - } - } - - Ok(entry_ids.into_iter().collect()) - } - - /// Resolve a TagFilter to (include_ids, exclude_ids) entry ID lists. - /// include_ids = None means no include constraint. - /// include_ids = Some(ids) means the entry must be in this set (AND logic across tags). - async fn resolve_tag_filter( - &self, - db: &DatabaseConnection, - tag_filter: &crate::ops::search::input::TagFilter, - ) -> QueryResult<(Option>, Vec)> { - // Include: AND logic — entry must have ALL listed tags (intersection) - let include_ids = if !tag_filter.include.is_empty() { - let mut result_set: Option> = None; - for tag_uuid in &tag_filter.include { - let ids: HashSet = self - .find_entry_ids_for_tag(db, *tag_uuid) - .await? - .into_iter() - .collect(); - result_set = Some(match result_set { - None => ids, - Some(existing) => existing.intersection(&ids).copied().collect(), - }); - } - Some(result_set.unwrap_or_default().into_iter().collect()) - } else { - None - }; - - // Exclude: OR logic — entry must have NONE of the listed tags (union) - let mut exclude_ids: HashSet = HashSet::new(); - for tag_uuid in &tag_filter.exclude { - let ids = self.find_entry_ids_for_tag(db, *tag_uuid).await?; - exclude_ids.extend(ids); - } - - Ok((include_ids, exclude_ids.into_iter().collect())) - } } crate::register_library_query!(FileSearchQuery, "search.files"); diff --git a/core/src/ops/search/tests.rs b/core/src/ops/search/tests.rs index 8c1d658c6..218ad5447 100644 --- a/core/src/ops/search/tests.rs +++ b/core/src/ops/search/tests.rs @@ -89,42 +89,4 @@ mod tests { assert!(image_exts.len() > 5); // Should have more than basic hardcoded list assert!(code_exts.len() > 10); // Should have comprehensive code extensions } - - #[test] - fn test_fts5_query_building() { - use crate::ops::search::query::FileSearchQuery; - - let search_input = FileSearchInput::simple("test query".to_string()); - let query = FileSearchQuery::new(search_input); - - let fts_query = query.build_fts5_query(); - assert!(fts_query.contains("test")); - assert!(fts_query.contains("query")); - - // Test escaping - let search_input_special = FileSearchInput::simple("test*query".to_string()); - let query_special = FileSearchQuery::new(search_input_special); - let fts_query_special = query_special.build_fts5_query(); - assert!(fts_query_special.contains("test\\*query")); - } - - #[test] - fn test_highlight_extraction() { - use crate::ops::search::query::FileSearchQuery; - - let search_input = FileSearchInput::simple("test".to_string()); - let query = FileSearchQuery::new(search_input); - - let highlights = - query.extract_highlights("test", "test_file.txt", &Some("test".to_string())); - - assert_eq!(highlights.len(), 2); // Should match in both name and extension - assert_eq!(highlights[0].field, "name"); - assert_eq!(highlights[0].start, 0); - assert_eq!(highlights[0].end, 4); - - assert_eq!(highlights[1].field, "extension"); - assert_eq!(highlights[1].start, 0); - assert_eq!(highlights[1].end, 4); // "test" extension - } } diff --git a/core/src/ops/spaces/get_layout/query.rs b/core/src/ops/spaces/get_layout/query.rs index 5bc887ef6..4b47b9482 100644 --- a/core/src/ops/spaces/get_layout/query.rs +++ b/core/src/ops/spaces/get_layout/query.rs @@ -1,12 +1,12 @@ use super::output::SpaceLayoutOutput; use crate::domain::{ - addressing::SdPath, ContentKind, File, GroupType, ItemType, Space, SpaceGroup, - SpaceGroupWithItems, SpaceItem, SpaceLayout, + addressing::SdPath, File, GroupType, ItemType, Space, SpaceGroup, SpaceGroupWithItems, + SpaceItem, SpaceLayout, }; -use crate::infra::db::entities::{content_identity, entry, sidecar, space_item}; +use crate::infra::db::entities::space_item; use crate::infra::query::{QueryError, QueryResult}; use crate::{context::CoreContext, infra::query::LibraryQuery}; -use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter, QueryOrder}; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder}; use serde::{Deserialize, Serialize}; use specta::Type; use std::collections::HashMap; @@ -89,19 +89,9 @@ impl LibraryQuery for SpaceLayoutQuery { item_model.uuid, entry_uuid ); - if let Ok(Some(entry_model)) = entry::Entity::find() - .filter(entry::Column::Uuid.eq(entry_uuid)) - .one(db) + pinned_file(entry_uuid, &item_type, &context) .await - { - let file = build_file_from_entry(entry_model, &item_type, db) - .await - .map(Box::new); - file - } else { - tracing::warn!("Entry {} not found for space item", entry_uuid); - None - } + .map(Box::new) } else { tracing::debug!("Space item {} has no entry_uuid", item_model.uuid); None @@ -168,24 +158,9 @@ impl LibraryQuery for SpaceLayoutQuery { item_model.uuid, entry_uuid ); - if let Ok(Some(entry_model)) = entry::Entity::find() - .filter(entry::Column::Uuid.eq(entry_uuid)) - .one(db) + pinned_file(entry_uuid, &item_type, &context) .await - { - tracing::debug!("Found entry: name={}", entry_model.name); - let file = build_file_from_entry(entry_model, &item_type, db) - .await - .map(Box::new); - tracing::info!( - "Built file for group item: {:?}", - file.as_ref().map(|f| &f.name) - ); - file - } else { - tracing::warn!("Entry {} not found for group item", entry_uuid); - None - } + .map(Box::new) } else { None }; @@ -217,77 +192,34 @@ impl LibraryQuery for SpaceLayoutQuery { crate::register_library_query!(SpaceLayoutQuery, "spaces.get_layout"); -/// Build a minimal File object from an entry model (for sidebar display) -async fn build_file_from_entry( - entry_model: entry::Model, +/// The file a space item points at, for the sidebar. +/// +/// A space item stores the path it was pinned at and the uuid the entry had at +/// the time. The path is what makes it findable: the arena is keyed by path, so +/// a pinned file that has since been renamed resolves to nothing rather than to +/// whatever now sits at that path. +async fn pinned_file( + entry_uuid: Uuid, item_type: &ItemType, - db: &DatabaseConnection, + context: &Arc, ) -> Option { - // Get the SdPath from item_type - let sd_path = match item_type { - ItemType::Path { sd_path } => sd_path.clone(), - _ => return None, + let ItemType::Path { sd_path } = item_type else { + return None; + }; + let SdPath::Physical { path, .. } = sd_path else { + return None; }; - // Get content identity if available - let content_identity = if let Some(content_id) = entry_model.content_id { - content_identity::Entity::find_by_id(content_id) - .one(db) - .await - .ok() - .flatten() - .map(|ci| crate::domain::ContentIdentity { - uuid: ci.uuid.unwrap_or_else(Uuid::new_v4), - kind: ContentKind::from_id(ci.kind_id), - content_hash: ci.content_hash, - integrity_hash: ci.integrity_hash, - mime_type_id: ci.mime_type_id, - text_content: ci.text_content, - total_size: ci.total_size, - entry_count: ci.entry_count, - first_seen_at: ci.first_seen_at, - last_verified_at: ci.last_verified_at, - }) - } else { - None - }; + let cache = context.ephemeral_cache(); + cache.ensure_restored(path).await; - // Get sidecars for thumbnails - let sidecars = if let Some(ref ci) = content_identity { - if let Some(uuid) = Some(ci.uuid) { - sidecar::Entity::find() - .filter(sidecar::Column::ContentUuid.eq(uuid)) - .all(db) - .await - .ok() - .unwrap_or_default() - .into_iter() - .map(|s| crate::domain::Sidecar { - id: s.id, - content_uuid: s.content_uuid, - kind: s.kind, - variant: s.variant, - format: s.format, - status: s.status, - size: s.size, - version: s.version, - created_at: s.created_at, - updated_at: s.updated_at, - }) - .collect() - } else { - Vec::new() - } - } else { - Vec::new() - }; - - let mut file = File::from_entity_model(entry_model, sd_path); - file.content_identity = content_identity; - file.sidecars = sidecars; - if let Some(ref ci) = file.content_identity { - file.content_kind = ci.kind; - } + let index = cache.resolve_index(path); + let mut index = index.write().await; + let metadata = index.get_entry_ref(path)?; + let kind = index.get_content_kind(path); + drop(index); + let mut file = File::from_ephemeral(entry_uuid, &metadata, sd_path.clone()); + file.content_kind = kind; Some(file) } diff --git a/crates/store/src/file.rs b/crates/store/src/file.rs index 889d84734..a5d13206e 100644 --- a/crates/store/src/file.rs +++ b/crates/store/src/file.rs @@ -804,6 +804,64 @@ pub async fn duplicate_copies( .collect()) } +/// The identity of the bytes behind a record, if they have been identified. +/// +/// None is ordinary rather than exceptional: a record exists from the moment +/// the walk sees it, and its content row arrives later when the hash job gets +/// to it. +pub async fn content_of(pool: &sqlx::SqlitePool, record_uuid: Uuid) -> Result> { + Ok(sqlx::query_scalar( + "SELECT c.uuid FROM record r JOIN content c ON c.id = r.content_id WHERE r.uuid = ?", + ) + .bind(record_uuid) + .fetch_optional(pool) + .await?) +} + +/// Every record in this source holding the given bytes. +/// +/// Keyed by content rather than by record, so a caller can ask each source the +/// same question and get every copy on the machine. A copy under a different +/// name in a different directory is still found, because the identity is the +/// bytes. +pub async fn copies_of_content( + pool: &sqlx::SqlitePool, + content_uuid: Uuid, +) -> Result> { + let rows: Vec<( + Uuid, + Option, + Uuid, + Option, + Option, + Option, + )> = sqlx::query_as( + "SELECT c.uuid, c.size, r.uuid, r.parent_uuid, d.path, r.title + FROM content c + JOIN record r ON r.content_id = c.id + LEFT JOIN directory_path d ON d.record_uuid = r.parent_uuid + WHERE c.uuid = ? + ORDER BY r.title", + ) + .bind(content_uuid) + .fetch_all(pool) + .await?; + + Ok(rows + .into_iter() + .filter_map( + |(content_uuid, size, record_uuid, parent_uuid, parent_path, title)| { + Some(ContentCopy { + content_uuid, + size, + record_uuid, + external_id: address(parent_uuid, parent_path, title)?, + }) + }, + ) + .collect()) +} + /// A record's path, rebuilt from its parent and its name. /// /// Only directories store a path. A file is addressed through the one above it, diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 14aacd899..bd52a1d4f 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -52,9 +52,9 @@ pub use content::{uuid_for, ContentId, CONTENT_NAMESPACE}; pub use db::{ItemRow, OverlayEvidence, SourceDb, Stamp}; pub use error::{Error, Result}; pub use file::{ - count_files_needing_content, duplicate_copies, files_needing_content, filesystem_schema, - ContentCopy, FileKind, FileWrite, Ledger, Observation, PendingContent, Resolution, - SubtreeRename, Watermark, + content_of, copies_of_content, count_files_needing_content, duplicate_copies, + files_needing_content, filesystem_schema, ContentCopy, FileKind, FileWrite, Ledger, + Observation, PendingContent, Resolution, SubtreeRename, Watermark, }; pub use record::{ContentIdentity, Record, RECORD_SCHEMA}; pub use schema::{DataTypeSchema, FieldType, ModelDef}; diff --git a/packages/ts-client/src/generated/types.ts b/packages/ts-client/src/generated/types.ts index 8a904c115..460d94c82 100644 --- a/packages/ts-client/src/generated/types.ts +++ b/packages/ts-client/src/generated/types.ts @@ -3835,10 +3835,6 @@ export type SearchScope = * Search the entire library (default) */ "Library" | -/** - * Restrict search to a specific location by its ID - */ -{ Location: { location_id: string } } | /** * Restrict search to a specific directory path and all its descendants */ @@ -4714,36 +4710,6 @@ idle_seconds: number; */ memory_breakdown?: MemoryBreakdownStats | null }; -/** - * Input for finding files unique to a location - */ -export type UniqueToLocationInput = { -/** - * The location ID to find unique files for - */ -location_id: string; -/** - * Optional limit on number of results - */ -limit: number | null }; - -/** - * Output containing files that are unique to the specified location - */ -export type UniqueToLocationOutput = { -/** - * Files that exist only in the specified location - */ -unique_files: File[]; -/** - * Total count of unique files - */ -total_count: number; -/** - * Total size of unique files in bytes - */ -total_size: number }; - export type UpdateAdapterInput = { adapter_id: string }; export type UpdateAdapterOutput = { adapter_id: string; old_version: string; new_version: string; schema_changed: boolean }; @@ -5493,7 +5459,6 @@ export type LibraryQuery = | { type: 'files.content_kind_stats'; input: ContentKindStatsInput; output: ContentKindStatsOutput } | { type: 'files.directory_listing'; input: DirectoryListingInput; output: DirectoryListingOutput } | { type: 'files.media_listing'; input: MediaListingInput; output: MediaListingOutput } - | { type: 'files.unique_to_location'; input: UniqueToLocationInput; output: UniqueToLocationOutput } | { type: 'jobs.active'; input: ActiveJobsInput; output: ActiveJobsOutput } | { type: 'jobs.get_copy_metadata'; input: CopyMetadataQueryInput; output: CopyMetadataOutput } | { type: 'jobs.info'; input: JobInfoQueryInput; output: JobInfoOutput } @@ -5649,7 +5614,6 @@ export const WIRE_METHODS = { 'files.content_kind_stats': 'query:files.content_kind_stats', 'files.directory_listing': 'query:files.directory_listing', 'files.media_listing': 'query:files.media_listing', - 'files.unique_to_location': 'query:files.unique_to_location', 'jobs.active': 'query:jobs.active', 'jobs.get_copy_metadata': 'query:jobs.get_copy_metadata', 'jobs.info': 'query:jobs.info',