diff --git a/.vscode/settings.json b/.vscode/settings.json index 1fcc13707..4180530b4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -84,5 +84,9 @@ ".eslintrc.js": ".eslintcache", "package.json": "package-lock.json, yarn.lock, pnpm-lock.yaml, pnpm-workspace.yaml, .pnp.cjs, .pnp.loader.mjs", "tsconfig.json": "tsconfig.*.json" - } + }, + "rust-analyzer.linkedProjects": [], + "rust-analyzer.cargo.extraEnv": {}, + "rust-analyzer.check.targets": null, + "rust-analyzer.showUnlinkedFileNotification": false } diff --git a/Cargo.lock b/Cargo.lock index 61c385188..05f22ca60 100644 Binary files a/Cargo.lock and b/Cargo.lock differ diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 5c0283368..cdb2cc0d3 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -11,7 +11,6 @@ edition = { workspace = true } [dependencies] sd-core = { path = "../../../core", features = [ "ffmpeg", - "location-watcher", "heif", ] } sd-fda = { path = "../../../crates/fda" } diff --git a/apps/mobile/scripts/cleanTarget.sh b/apps/mobile/scripts/cleanTarget.sh new file mode 100755 index 000000000..c90d8243f --- /dev/null +++ b/apps/mobile/scripts/cleanTarget.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Check if the correct number of arguments is provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 [android|ios]" + exit 1 +fi + +# Set the target folder based on the first argument +TARGET_FOLDER="target" + +# Check if the target folder exists +if [ ! -d "$TARGET_FOLDER" ]; then + echo "Target folder '$TARGET_FOLDER' not found." + exit 1 +fi + +# Set the keyword based on the first argument +KEYWORD="" +if [ "$1" == "android" ]; then + KEYWORD="android" +elif [ "$1" == "ios" ]; then + KEYWORD="ios" +else + echo "Invalid argument. Please provide either 'android' or 'ios'." + exit 1 +fi + +# Delete files based on the target folder and keyword +echo "Deleting files in '$TARGET_FOLDER' with keyword '$KEYWORD' in folder names..." + +# Find and delete files in folders containing the specified keyword +find "$TARGET_FOLDER" -type d -name "*$KEYWORD*" -exec rm -r {} \; + +# End of the script +echo "Files deleted successfully." diff --git a/apps/mobile/src/navigation/TabNavigator.tsx b/apps/mobile/src/navigation/TabNavigator.tsx index d033ce859..7efba2af3 100644 --- a/apps/mobile/src/navigation/TabNavigator.tsx +++ b/apps/mobile/src/navigation/TabNavigator.tsx @@ -28,71 +28,71 @@ export default function TabNavigator() { labelStyle: Style; testID: string; }[] = [ - { - name: 'OverviewStack', - component: OverviewStack, - icon: ( - - ), - label: 'Overview', - labelStyle: tw`text-[10px] font-semibold`, - testID: 'overview-tab' - }, - { - name: 'NetworkStack', - component: NetworkStack, - icon: ( - - ), - label: 'Network', - labelStyle: tw`text-[10px] font-semibold`, - testID: 'network-tab' - }, - { - name: 'BrowseStack', - component: BrowseStack, - icon: ( - - ), - label: 'Browse', - labelStyle: tw`text-[10px] font-semibold`, - testID: 'browse-tab' - }, - { - name: 'SettingsStack', - component: SettingsStack, - icon: ( - - ), - label: 'Settings', - labelStyle: tw`text-[10px] font-semibold`, - testID: 'settings-tab' - } - ]; + { + name: 'OverviewStack', + component: OverviewStack, + icon: ( + + ), + label: 'Overview', + labelStyle: tw`text-[10px] font-semibold`, + testID: 'overview-tab' + }, + { + name: 'NetworkStack', + component: NetworkStack, + icon: ( + + ), + label: 'Network', + labelStyle: tw`text-[10px] font-semibold`, + testID: 'network-tab' + }, + { + name: 'BrowseStack', + component: BrowseStack, + icon: ( + + ), + label: 'Browse', + labelStyle: tw`text-[10px] font-semibold`, + testID: 'browse-tab' + }, + { + name: 'SettingsStack', + component: SettingsStack, + icon: ( + + ), + label: 'Settings', + labelStyle: tw`text-[10px] font-semibold`, + testID: 'settings-tab' + } + ]; return ( (); - // This is the main navigator we nest everything under. export default function RootNavigator() { return ( @@ -38,6 +37,6 @@ export type RootStackScreenProps = Stac declare global { // eslint-disable-next-line @typescript-eslint/no-namespace namespace ReactNavigation { - interface RootParamList extends RootStackParamList {} + interface RootParamList extends RootStackParamList { } } } diff --git a/apps/mobile/src/screens/Location.tsx b/apps/mobile/src/screens/Location.tsx index e5687ac2d..18e1d108b 100644 --- a/apps/mobile/src/screens/Location.tsx +++ b/apps/mobile/src/screens/Location.tsx @@ -25,7 +25,7 @@ export default function LocationScreen({ navigation, route }: BrowseStackScreenP take: 100 } ]); - + const pathsItemsReferences = useMemo(() => paths.data?.items ?? [], [paths.data]); useNodes(paths.data?.nodes); const pathsItems = useCache(pathsItemsReferences); @@ -52,5 +52,5 @@ export default function LocationScreen({ navigation, route }: BrowseStackScreenP getExplorerStore().path = path ?? ''; }, [id, path]); - return ; + return } diff --git a/apps/mobile/src/screens/settings/info/About.tsx b/apps/mobile/src/screens/settings/info/About.tsx index 4bf9dd403..bdb7d14a1 100644 --- a/apps/mobile/src/screens/settings/info/About.tsx +++ b/apps/mobile/src/screens/settings/info/About.tsx @@ -22,11 +22,10 @@ const AboutScreen = () => { Spacedrive{' '} - {`for ${ - Platform.OS === 'android' - ? Platform.OS[0]?.toUpperCase() + Platform.OS.slice(1) - : Platform.OS[0] + Platform.OS.slice(1).toUpperCase() - }`} + {`for ${Platform.OS === 'android' + ? Platform.OS[0]?.toUpperCase() + Platform.OS.slice(1) + : Platform.OS[0] + Platform.OS.slice(1).toUpperCase() + }`} The file manager from the future. diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index a3b1b541c..836defb71 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -13,7 +13,6 @@ ai-models = ["sd-core/ai"] [dependencies] sd-core = { path = "../../core", features = [ "ffmpeg", - "location-watcher", "heif", ] } diff --git a/core/Cargo.toml b/core/Cargo.toml index 8b16e2b92..e3b2717f0 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,7 +14,6 @@ default = [] mobile = [] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. ffmpeg = ["dep:sd-ffmpeg"] -location-watcher = ["dep:notify"] heif = ["sd-images/heif"] ai = ["dep:sd-ai"] @@ -48,7 +47,6 @@ sd-sync = { path = "../crates/sync" } sd-utils = { path = "../crates/utils" } sd-cloud-api = { version = "0.1.0", path = "../crates/cloud-api" } - # Workspace dependencies async-channel = { workspace = true } async-trait = { workspace = true } @@ -112,9 +110,9 @@ http-range = "0.1.5" int-enum = "0.5.0" itertools = "0.12.0" mini-moka = "0.10.2" -notify = { version = "=5.2.0", default-features = false, features = [ - "macos_fsevent", -], optional = true } +notify = { git="https://github.com/notify-rs/notify.git", rev="c3929ed114fbb0bc7457a9a498260461596b00ca", default-features = false, features = [ + "macos_fsevent", +] } rmpv = "^1.0.1" serde-hashkey = "0.4.5" serde_repr = "0.1" diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs index 11aa8f2cf..b0418c3c5 100644 --- a/core/src/api/locations.rs +++ b/core/src/api/locations.rs @@ -366,7 +366,6 @@ pub(crate) fn mount() -> AlphaRouter { pub location_id: location::id::Type, pub reidentify_objects: bool, } - R.with2(library()).mutation( |(node, library), FullRescanArgs { diff --git a/core/src/lib.rs b/core/src/lib.rs index 53ebdd3ca..fd6c89a3d 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -206,6 +206,8 @@ impl Node { "info" }; + // let level = "debug"; // Exists for now to debug the location manager + std::env::set_var( "RUST_LOG", format!("info,sd_core={level},sd_p2p=debug,sd_core::location::manager=info,sd_ai={level}"), diff --git a/core/src/location/manager/mod.rs b/core/src/location/manager/mod.rs index dfbda6199..466750dd4 100644 --- a/core/src/location/manager/mod.rs +++ b/core/src/location/manager/mod.rs @@ -20,16 +20,13 @@ use tokio::sync::{ broadcast::{self, Receiver}, oneshot, RwLock, }; -use tracing::error; +use tracing::{debug, error}; -#[cfg(feature = "location-watcher")] use tokio::sync::mpsc; use uuid::Uuid; -#[cfg(feature = "location-watcher")] mod watcher; -#[cfg(feature = "location-watcher")] mod helpers; #[derive(Clone, Copy, Debug)] @@ -67,22 +64,18 @@ pub struct WatcherManagementMessage { #[derive(Error, Debug)] pub enum LocationManagerError { - #[cfg(feature = "location-watcher")] #[error("Unable to send location management message to location manager actor: (error: {0})")] ActorSendLocationError(#[from] mpsc::error::SendError), - #[cfg(feature = "location-watcher")] #[error("Unable to send path to be ignored by watcher actor: (error: {0})")] ActorIgnorePathError(#[from] mpsc::error::SendError), - #[cfg(feature = "location-watcher")] #[error("Unable to watcher management message to watcher manager actor: (error: {0})")] ActorIgnorePathMessageError(#[from] mpsc::error::SendError), #[error("Unable to receive actor response: (error: {0})")] ActorResponseError(#[from] oneshot::error::RecvError), - #[cfg(feature = "location-watcher")] #[error("Watcher error: (error: {0})")] WatcherError(#[from] notify::Error), @@ -119,11 +112,10 @@ type OnlineLocations = BTreeSet>; #[must_use = "'LocationManagerActor::start' must be used to start the actor"] pub struct LocationManagerActor { - #[cfg(feature = "location-watcher")] location_management_rx: mpsc::Receiver, - #[cfg(feature = "location-watcher")] + watcher_management_rx: mpsc::Receiver, - #[cfg(feature = "location-watcher")] + stop_rx: oneshot::Receiver<()>, } @@ -178,25 +170,21 @@ impl LocationManagerActor { } }); - #[cfg(feature = "location-watcher")] tokio::spawn(Locations::run_locations_checker( self.location_management_rx, self.watcher_management_rx, self.stop_rx, node, )); - - #[cfg(not(feature = "location-watcher"))] - tracing::warn!("Location watcher is disabled, locations will not be checked"); } } pub struct Locations { online_locations: RwLock, pub online_tx: broadcast::Sender, - #[cfg(feature = "location-watcher")] + location_management_tx: mpsc::Sender, - #[cfg(feature = "location-watcher")] + watcher_management_tx: mpsc::Sender, stop_tx: Option>, } @@ -205,11 +193,11 @@ impl Locations { pub fn new() -> (Self, LocationManagerActor) { let online_tx = broadcast::channel(16).0; - #[cfg(feature = "location-watcher")] { let (location_management_tx, location_management_rx) = mpsc::channel(128); let (watcher_management_tx, watcher_management_rx) = mpsc::channel(128); let (stop_tx, stop_rx) = oneshot::channel(); + debug!("Starting location manager actor"); ( Self { @@ -226,19 +214,6 @@ impl Locations { }, ) } - - #[cfg(not(feature = "location-watcher"))] - { - tracing::warn!("Location watcher is disabled, locations will not be checked"); - ( - Self { - online_tx, - online_locations: Default::default(), - stop_tx: None, - }, - LocationManagerActor {}, - ) - } } #[inline] @@ -249,9 +224,9 @@ impl Locations { library: Arc, action: ManagementMessageAction, ) -> Result<(), LocationManagerError> { - #[cfg(feature = "location-watcher")] { let (tx, rx) = oneshot::channel(); + debug!("Sending location management message to location manager actor: {action:?}"); self.location_management_tx .send(LocationManagementMessage { @@ -264,9 +239,6 @@ impl Locations { rx.await? } - - #[cfg(not(feature = "location-watcher"))] - Ok(()) } #[inline] @@ -277,10 +249,11 @@ impl Locations { library: Arc, action: WatcherManagementMessageAction, ) -> Result<(), LocationManagerError> { - #[cfg(feature = "location-watcher")] { let (tx, rx) = oneshot::channel(); + debug!("Sending watcher management message to location manager actor: {action:?}"); + self.watcher_management_tx .send(WatcherManagementMessage { location_id, @@ -292,9 +265,6 @@ impl Locations { rx.await? } - - #[cfg(not(feature = "location-watcher"))] - Ok(()) } pub async fn add( @@ -377,7 +347,6 @@ impl Locations { }) } - #[cfg(feature = "location-watcher")] async fn run_locations_checker( mut location_management_rx: mpsc::Receiver, mut watcher_management_rx: mpsc::Receiver, @@ -388,7 +357,7 @@ impl Locations { use futures::stream::{FuturesUnordered, StreamExt}; use tokio::select; - use tracing::{info, warn}; + use tracing::warn; use helpers::{ check_online, drop_location, get_location, handle_ignore_path_request, @@ -430,6 +399,8 @@ impl Locations { (location_id, library.id), watcher ); + debug!("Location {location_id} is online, watching it"); + // info!("Locations watched: {:#?}", locations_watched); } else { locations_unwatched.insert( (location_id, library.id), @@ -578,7 +549,7 @@ impl Locations { } _ = &mut stop_rx => { - info!("Stopping location manager"); + debug!("Stopping location manager"); break; } } @@ -642,14 +613,12 @@ pub struct StopWatcherGuard<'m> { impl Drop for StopWatcherGuard<'_> { fn drop(&mut self) { - if cfg!(feature = "location-watcher") { - // FIXME: change this Drop to async drop in the future - if let Err(e) = block_on(self.manager.reinit_watcher( - self.location_id, - self.library.take().expect("library should be set"), - )) { - error!("Failed to reinit watcher on stop watcher guard drop: {e}"); - } + // FIXME: change this Drop to async drop in the future + if let Err(e) = block_on(self.manager.reinit_watcher( + self.location_id, + self.library.take().expect("library should be set"), + )) { + error!("Failed to reinit watcher on stop watcher guard drop: {e}"); } } } @@ -664,18 +633,16 @@ pub struct IgnoreEventsForPathGuard<'m> { impl Drop for IgnoreEventsForPathGuard<'_> { fn drop(&mut self) { - if cfg!(feature = "location-watcher") { - // FIXME: change this Drop to async drop in the future - if let Err(e) = block_on(self.manager.watcher_management_message( - self.location_id, - self.library.take().expect("library should be set"), - WatcherManagementMessageAction::IgnoreEventsForPath { - path: self.path.take().expect("path should be set"), - ignore: false, - }, - )) { - error!("Failed to un-ignore path on watcher guard drop: {e}"); - } + // FIXME: change this Drop to async drop in the future + if let Err(e) = block_on(self.manager.watcher_management_message( + self.location_id, + self.library.take().expect("library should be set"), + WatcherManagementMessageAction::IgnoreEventsForPath { + path: self.path.take().expect("path should be set"), + ignore: false, + }, + )) { + error!("Failed to un-ignore path on watcher guard drop: {e}"); } } } diff --git a/core/src/location/manager/watcher/android.rs b/core/src/location/manager/watcher/android.rs new file mode 100644 index 000000000..a72612a21 --- /dev/null +++ b/core/src/location/manager/watcher/android.rs @@ -0,0 +1,271 @@ +//! Android file system watcher implementation. +//! TODO: Still being worked on by @Rocky43007 on Branch Rocky43007:location-watcher-test-3 +//! DO NOT TOUCH FOR NOW + +use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; + +use sd_prisma::prisma::location; +use sd_utils::error::FileIOError; + +use std::{ + collections::{BTreeMap, HashMap}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use async_trait::async_trait; +use notify::{ + event::{CreateKind, DataChange, ModifyKind, RenameMode}, + Event, +}; +use tokio::{fs, time::Instant}; +use tracing::{error, info, trace}; + +use super::{ + utils::{create_dir, recalculate_directories_size, remove, rename, update_file}, + EventHandler, HUNDRED_MILLIS, ONE_SECOND, +}; + +#[derive(Debug)] +pub(super) struct AndroidEventHandler<'lib> { + location_id: location::id::Type, + library: &'lib Arc, + node: &'lib Arc, + last_events_eviction_check: Instant, + rename_from: HashMap, + recently_renamed_from: BTreeMap, + files_to_update: HashMap, + reincident_to_update_files: HashMap, + to_recalculate_size: HashMap, + path_and_instant_buffer: Vec<(PathBuf, Instant)>, +} + +#[async_trait] +impl<'lib> EventHandler<'lib> for AndroidEventHandler<'lib> { + fn new( + location_id: location::id::Type, + library: &'lib Arc, + node: &'lib Arc, + ) -> Self { + Self { + location_id, + library, + node, + last_events_eviction_check: Instant::now(), + rename_from: HashMap::new(), + recently_renamed_from: BTreeMap::new(), + files_to_update: HashMap::new(), + reincident_to_update_files: HashMap::new(), + to_recalculate_size: HashMap::new(), + path_and_instant_buffer: Vec::new(), + } + } + + async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> { + info!("Received Android event: {:#?}", event); + + // let Event { + // kind, mut paths, .. + // } = event; + + // match kind { + // EventKind::Create(CreateKind::File) + // | EventKind::Modify(ModifyKind::Data(DataChange::Any)) => { + // // When we receive a create, modify data or metadata events of the abore kinds + // // we just mark the file to be updated in a near future + // // each consecutive event of these kinds that we receive for the same file + // // we just store the path again in the map below, with a new instant + // // that effectively resets the timer for the file to be updated + // let path = paths.remove(0); + // if self.files_to_update.contains_key(&path) { + // if let Some(old_instant) = + // self.files_to_update.insert(path.clone(), Instant::now()) + // { + // self.reincident_to_update_files + // .entry(path) + // .or_insert(old_instant); + // } + // } else { + // self.files_to_update.insert(path, Instant::now()); + // } + // } + + // EventKind::Create(CreateKind::Folder) => { + // let path = &paths[0]; + + // // Don't need to dispatch a recalculate directory event as `create_dir` dispatches + // // a `scan_location_sub_path` function, which recalculates the size already + + // create_dir( + // self.location_id, + // path, + // &fs::metadata(path) + // .await + // .map_err(|e| FileIOError::from((path, e)))?, + // self.node, + // self.library, + // ) + // .await?; + // } + // EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { + // // Just in case we can't garantee that we receive the Rename From event before the + // // Rename Both event. Just a safeguard + // if self.recently_renamed_from.remove(&paths[0]).is_none() { + // self.rename_from.insert(paths.remove(0), Instant::now()); + // } + // } + + // EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { + // let from_path = &paths[0]; + // let to_path = &paths[1]; + + // self.rename_from.remove(from_path); + // rename( + // self.location_id, + // to_path, + // from_path, + // fs::metadata(to_path) + // .await + // .map_err(|e| FileIOError::from((to_path, e)))?, + // self.library, + // ) + // .await?; + // self.recently_renamed_from + // .insert(paths.swap_remove(0), Instant::now()); + // } + // EventKind::Remove(_) => { + // let path = paths.remove(0); + // if let Some(parent) = path.parent() { + // if parent != Path::new("") { + // self.to_recalculate_size + // .insert(parent.to_path_buf(), Instant::now()); + // } + // } + + // remove(self.location_id, &path, self.library).await?; + // } + // other_event_kind => { + // trace!("Other Linux event that we don't handle for now: {other_event_kind:#?}"); + // } + // } + + Ok(()) + } + + async fn tick(&mut self) { + if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS { + if let Err(e) = self.handle_to_update_eviction().await { + error!("Error while handling recently created or update files eviction: {e:#?}"); + } + + if let Err(e) = self.handle_rename_from_eviction().await { + error!("Failed to remove file_path: {e:#?}"); + } + + self.recently_renamed_from + .retain(|_, instant| instant.elapsed() < HUNDRED_MILLIS); + + if !self.to_recalculate_size.is_empty() { + if let Err(e) = recalculate_directories_size( + &mut self.to_recalculate_size, + &mut self.path_and_instant_buffer, + self.location_id, + self.library, + ) + .await + { + error!("Failed to recalculate directories size: {e:#?}"); + } + } + + self.last_events_eviction_check = Instant::now(); + } + } +} + +impl AndroidEventHandler<'_> { + async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> { + self.path_and_instant_buffer.clear(); + let mut should_invalidate = false; + + for (path, created_at) in self.files_to_update.drain() { + if created_at.elapsed() < HUNDRED_MILLIS * 5 { + self.path_and_instant_buffer.push((path, created_at)); + } else { + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + self.reincident_to_update_files.remove(&path); + update_file(self.location_id, &path, self.node, self.library).await?; + should_invalidate = true; + } + } + + self.files_to_update + .extend(self.path_and_instant_buffer.drain(..)); + + self.path_and_instant_buffer.clear(); + + // We have to check if we have any reincident files to update and update them after a bigger + // timeout, this way we keep track of files being update frequently enough to bypass our + // eviction check above + for (path, created_at) in self.reincident_to_update_files.drain() { + if created_at.elapsed() < ONE_SECOND * 10 { + self.path_and_instant_buffer.push((path, created_at)); + } else { + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + self.files_to_update.remove(&path); + update_file(self.location_id, &path, self.node, self.library).await?; + should_invalidate = true; + } + } + + if should_invalidate { + invalidate_query!(self.library, "search.paths"); + } + + self.reincident_to_update_files + .extend(self.path_and_instant_buffer.drain(..)); + + Ok(()) + } + + async fn handle_rename_from_eviction(&mut self) -> Result<(), LocationManagerError> { + self.path_and_instant_buffer.clear(); + let mut should_invalidate = false; + + for (path, instant) in self.rename_from.drain() { + if instant.elapsed() > HUNDRED_MILLIS { + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + remove(self.location_id, &path, self.library).await?; + should_invalidate = true; + trace!("Removed file_path due timeout: {}", path.display()); + } else { + self.path_and_instant_buffer.push((path, instant)); + } + } + + if should_invalidate { + invalidate_query!(self.library, "search.paths"); + } + + for (path, instant) in self.path_and_instant_buffer.drain(..) { + self.rename_from.insert(path, instant); + } + + Ok(()) + } +} diff --git a/core/src/location/manager/watcher/ios.rs b/core/src/location/manager/watcher/ios.rs new file mode 100644 index 000000000..a29925b96 --- /dev/null +++ b/core/src/location/manager/watcher/ios.rs @@ -0,0 +1,394 @@ +//! iOS file system watcher implementation. + +use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; + +use sd_file_path_helper::{check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData}; +use sd_prisma::prisma::location; +use sd_utils::error::FileIOError; + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; + +use async_trait::async_trait; +use notify::{ + event::{CreateKind, DataChange, MetadataKind, ModifyKind, RenameMode}, + Event, EventKind, +}; +use tokio::{fs, io, time::Instant}; +use tracing::{debug, error, trace, warn}; + +use super::{ + utils::{ + create_dir, create_file, extract_inode_from_path, extract_location_path, + recalculate_directories_size, remove, rename, update_file, + }, + EventHandler, INode, InstantAndPath, HUNDRED_MILLIS, ONE_SECOND, +}; + +#[derive(Debug)] +pub(super) struct IosEventHandler<'lib> { + location_id: location::id::Type, + library: &'lib Arc, + node: &'lib Arc, + files_to_update: HashMap, + reincident_to_update_files: HashMap, + last_events_eviction_check: Instant, + latest_created_dir: Option, + old_paths_map: HashMap, + new_paths_map: HashMap, + paths_map_buffer: Vec<(INode, InstantAndPath)>, + to_recalculate_size: HashMap, + path_and_instant_buffer: Vec<(PathBuf, Instant)>, +} + +#[async_trait] +impl<'lib> EventHandler<'lib> for IosEventHandler<'lib> { + fn new( + location_id: location::id::Type, + library: &'lib Arc, + node: &'lib Arc, + ) -> Self + where + Self: Sized, + { + Self { + location_id, + library, + node, + files_to_update: HashMap::new(), + reincident_to_update_files: HashMap::new(), + last_events_eviction_check: Instant::now(), + latest_created_dir: None, + old_paths_map: HashMap::new(), + new_paths_map: HashMap::new(), + paths_map_buffer: Vec::new(), + to_recalculate_size: HashMap::new(), + path_and_instant_buffer: Vec::new(), + } + } + + async fn handle_event(&mut self, event: Event) -> Result<(), LocationManagerError> { + let Event { + kind, mut paths, .. + } = event; + + match kind { + EventKind::Create(CreateKind::Folder) => { + let path = &paths[0]; + + create_dir( + self.location_id, + path, + &fs::metadata(path) + .await + .map_err(|e| FileIOError::from((path, e)))?, + self.node, + self.library, + ) + .await?; + self.latest_created_dir = Some(paths.remove(0)); + } + + EventKind::Create(CreateKind::File) + | EventKind::Modify(ModifyKind::Data(DataChange::Content)) + | EventKind::Modify(ModifyKind::Metadata( + MetadataKind::WriteTime | MetadataKind::Extended, + )) => { + // When we receive a create, modify data or metadata events of the abore kinds + // we just mark the file to be updated in a near future + // each consecutive event of these kinds that we receive for the same file + // we just store the path again in the map below, with a new instant + // that effectively resets the timer for the file to be updated <- Copied from macos.rs + let path = paths.remove(0); + if self.files_to_update.contains_key(&path) { + if let Some(old_instant) = + self.files_to_update.insert(path.clone(), Instant::now()) + { + self.reincident_to_update_files + .entry(path) + .or_insert(old_instant); + } + } else { + self.files_to_update.insert(path, Instant::now()); + } + } + EventKind::Modify(ModifyKind::Name(RenameMode::Any)) => { + self.handle_single_rename_event(paths.remove(0)).await?; + } + + // For some reason, iOS doesn't have a Delete Event, so the vent type comes up as this. + // Delete Event + EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) => { + debug!("File has been deleted: {:#?}", paths); + let path = paths.remove(0); + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + remove(self.location_id, &path, self.library).await?; //FIXME: Find out why this freezes the watcher + } + other_event_kind => { + trace!("Other iOS event that we don't handle for now: {other_event_kind:#?}"); + } + } + + Ok(()) + } + + async fn tick(&mut self) { + if self.last_events_eviction_check.elapsed() > HUNDRED_MILLIS { + if let Err(e) = self.handle_to_update_eviction().await { + error!("Error while handling recently created or update files eviction: {e:#?}"); + } + + // Cleaning out recently renamed files that are older than 100 milliseconds + if let Err(e) = self.handle_rename_create_eviction().await { + error!("Failed to create file_path on iOS : {e:#?}"); + } + + if let Err(e) = self.handle_rename_remove_eviction().await { + error!("Failed to remove file_path: {e:#?}"); + } + + if !self.to_recalculate_size.is_empty() { + if let Err(e) = recalculate_directories_size( + &mut self.to_recalculate_size, + &mut self.path_and_instant_buffer, + self.location_id, + self.library, + ) + .await + { + error!("Failed to recalculate directories size: {e:#?}"); + } + } + + self.last_events_eviction_check = Instant::now(); + } + } +} + +impl IosEventHandler<'_> { + async fn handle_to_update_eviction(&mut self) -> Result<(), LocationManagerError> { + self.path_and_instant_buffer.clear(); + let mut should_invalidate = false; + + for (path, created_at) in self.files_to_update.drain() { + if created_at.elapsed() < HUNDRED_MILLIS * 5 { + self.path_and_instant_buffer.push((path, created_at)); + } else { + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + self.reincident_to_update_files.remove(&path); + update_file(self.location_id, &path, self.node, self.library).await?; + should_invalidate = true; + } + } + + self.files_to_update + .extend(self.path_and_instant_buffer.drain(..)); + + self.path_and_instant_buffer.clear(); + + // We have to check if we have any reincident files to update and update them after a bigger + // timeout, this way we keep track of files being update frequently enough to bypass our + // eviction check above + for (path, created_at) in self.reincident_to_update_files.drain() { + if created_at.elapsed() < ONE_SECOND * 10 { + self.path_and_instant_buffer.push((path, created_at)); + } else { + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + self.files_to_update.remove(&path); + update_file(self.location_id, &path, self.node, self.library).await?; + should_invalidate = true; + } + } + + if should_invalidate { + invalidate_query!(self.library, "search.paths"); + } + + self.reincident_to_update_files + .extend(self.path_and_instant_buffer.drain(..)); + + Ok(()) + } + + async fn handle_rename_create_eviction(&mut self) -> Result<(), LocationManagerError> { + // Just to make sure that our buffer is clean + self.paths_map_buffer.clear(); + let mut should_invalidate = false; + + for (inode, (instant, path)) in self.new_paths_map.drain() { + if instant.elapsed() > HUNDRED_MILLIS { + if !self.files_to_update.contains_key(&path) { + let metadata = fs::metadata(&path) + .await + .map_err(|e| FileIOError::from((&path, e)))?; + + if metadata.is_dir() { + // Don't need to dispatch a recalculate directory event as `create_dir` dispatches + // a `scan_location_sub_path` function, which recalculates the size already + create_dir(self.location_id, &path, &metadata, self.node, self.library) + .await?; + } else { + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + create_file(self.location_id, &path, &metadata, self.node, self.library) + .await?; + } + + trace!("Created file_path due timeout: {}", path.display()); + should_invalidate = true; + } + } else { + self.paths_map_buffer.push((inode, (instant, path))); + } + } + + if should_invalidate { + invalidate_query!(self.library, "search.paths"); + } + + self.new_paths_map.extend(self.paths_map_buffer.drain(..)); + + Ok(()) + } + + async fn handle_rename_remove_eviction(&mut self) -> Result<(), LocationManagerError> { + // Just to make sure that our buffer is clean + self.paths_map_buffer.clear(); + let mut should_invalidate = false; + + for (inode, (instant, path)) in self.old_paths_map.drain() { + if instant.elapsed() > HUNDRED_MILLIS { + if let Some(parent) = path.parent() { + if parent != Path::new("") { + self.to_recalculate_size + .insert(parent.to_path_buf(), Instant::now()); + } + } + remove(self.location_id, &path, self.library).await?; + trace!("Removed file_path due timeout: {}", path.display()); + should_invalidate = true; + } else { + self.paths_map_buffer.push((inode, (instant, path))); + } + } + + if should_invalidate { + invalidate_query!(self.library, "search.paths"); + } + + self.old_paths_map.extend(self.paths_map_buffer.drain(..)); + + Ok(()) + } + + async fn handle_single_rename_event( + &mut self, + path: PathBuf, // this is used internally only once, so we can use just PathBuf + ) -> Result<(), LocationManagerError> { + match fs::metadata(&path).await { + Ok(meta) => { + // File or directory exists, so this can be a "new path" to an actual rename/move or a creation + trace!("Path exists: {}", path.display()); + + let inode = get_inode(&meta); + let location_path = extract_location_path(self.location_id, self.library).await?; + + if !check_file_path_exists::( + &IsolatedFilePathData::new( + self.location_id, + &location_path, + &path, + meta.is_dir(), + )?, + &self.library.db, + ) + .await? + { + if let Some((_, old_path)) = self.old_paths_map.remove(&inode) { + trace!( + "Got a match new -> old: {} -> {}", + path.display(), + old_path.display() + ); + + // We found a new path for this old path, so we can rename it + rename(self.location_id, &path, &old_path, meta, self.library).await?; + } else { + trace!("No match for new path yet: {}", path.display()); + self.new_paths_map.insert(inode, (Instant::now(), path)); + } + } else { + warn!( + "Received rename event for a file that already exists in the database: {}", + path.display() + ); + } + } + Err(e) if e.kind() == io::ErrorKind::NotFound => { + // File or directory does not exist in the filesystem, if it exists in the database, + // then we try pairing it with the old path from our map + + trace!("Path doesn't exists: {}", path.display()); + + let inode = + match extract_inode_from_path(self.location_id, &path, self.library).await { + Ok(inode) => inode, + Err(LocationManagerError::FilePath(FilePathError::NotFound(_))) => { + // temporary file, we can ignore it + return Ok(()); + } + Err(e) => return Err(e), + }; + + if let Some((_, new_path)) = self.new_paths_map.remove(&inode) { + trace!( + "Got a match old -> new: {} -> {}", + path.display(), + new_path.display() + ); + + // We found a new path for this old path, so we can rename it + rename( + self.location_id, + &new_path, + &path, + fs::metadata(&new_path) + .await + .map_err(|e| FileIOError::from((&new_path, e)))?, + self.library, + ) + .await?; + } else { + trace!("No match for old path yet: {}", path.display()); + // We didn't find a new path for this old path, so we store ir for later + self.old_paths_map.insert(inode, (Instant::now(), path)); + } + } + Err(e) => return Err(FileIOError::from((path, e)).into()), + } + + Ok(()) + } +} diff --git a/core/src/location/manager/watcher/mod.rs b/core/src/location/manager/watcher/mod.rs index 59810f216..d6d70b77f 100644 --- a/core/src/location/manager/watcher/mod.rs +++ b/core/src/location/manager/watcher/mod.rs @@ -24,6 +24,8 @@ use uuid::Uuid; use super::LocationManagerError; +mod android; +mod ios; mod linux; mod macos; mod windows; @@ -41,6 +43,12 @@ type Handler<'lib> = macos::MacOsEventHandler<'lib>; #[cfg(target_os = "windows")] type Handler<'lib> = windows::WindowsEventHandler<'lib>; +#[cfg(target_os = "android")] +type Handler<'lib> = android::AndroidEventHandler<'lib>; + +#[cfg(target_os = "ios")] +type Handler<'lib> = ios::IosEventHandler<'lib>; + pub(super) type IgnorePath = (PathBuf, bool); type INode = u64; @@ -142,12 +150,12 @@ impl LocationWatcher { let mut handler_interval = interval_at(Instant::now() + HUNDRED_MILLIS, HUNDRED_MILLIS); // In case of doubt check: https://docs.rs/tokio/latest/tokio/time/enum.MissedTickBehavior.html handler_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); - loop { select! { Some(event) = events_rx.recv() => { match event { Ok(event) => { + debug!("[Debug - handle_watch_events] Received event: {:#?}", event); if let Err(e) = Self::handle_single_event( location_id, location_pub_id, @@ -197,6 +205,7 @@ impl LocationWatcher { _library: &'lib Library, ignore_paths: &HashSet, ) -> Result<(), LocationManagerError> { + debug!("Event: {:#?}", event); if !check_event(&event, ignore_paths) { return Ok(()); } @@ -215,6 +224,8 @@ impl LocationWatcher { return Ok(()); } + // debug!("Handling event: {:#?}", event); + event_handler.handle_event(event).await } @@ -232,6 +243,7 @@ impl LocationWatcher { pub(super) fn watch(&mut self) { let path = &self.path; + debug!("Start watching location: (path: {path})"); if let Err(e) = self .watcher @@ -368,7 +380,7 @@ mod tests { use tracing::{debug, error}; // use tracing_test::traced_test; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "ios"))] use notify::event::DataChange; #[cfg(target_os = "linux")] @@ -447,7 +459,7 @@ mod tests { #[cfg(target_os = "windows")] expect_event(events_rx, &file_path, EventKind::Modify(ModifyKind::Any)).await; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "ios"))] expect_event( events_rx, &file_path, @@ -487,7 +499,7 @@ mod tests { #[cfg(target_os = "windows")] expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Any)).await; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "ios"))] expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Folder)).await; #[cfg(target_os = "linux")] @@ -528,7 +540,7 @@ mod tests { #[cfg(target_os = "windows")] expect_event(events_rx, &file_path, EventKind::Modify(ModifyKind::Any)).await; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "ios"))] expect_event( events_rx, &file_path, @@ -577,7 +589,7 @@ mod tests { ) .await; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "ios"))] expect_event( events_rx, &file_path, @@ -628,7 +640,7 @@ mod tests { ) .await; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "ios"))] expect_event( events_rx, &dir_path, @@ -676,6 +688,14 @@ mod tests { #[cfg(target_os = "linux")] expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::File)).await; + #[cfg(target_os = "ios")] + expect_event( + events_rx, + &file_path, + EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)), + ) + .await; + debug!("Unwatching root directory: {}", root_dir.path().display()); if let Err(e) = watcher.unwatch(root_dir.path()) { error!("Failed to unwatch root directory: {e:#?}"); @@ -723,6 +743,14 @@ mod tests { #[cfg(target_os = "linux")] expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Folder)).await; + #[cfg(target_os = "ios")] + expect_event( + events_rx, + &file_path, + EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)), + ) + .await; + debug!("Unwatching root directory: {}", root_dir.path().display()); if let Err(e) = watcher.unwatch(root_dir.path()) { error!("Failed to unwatch root directory: {e:#?}"); diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index 92246cf4f..9defd643f 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -21,7 +21,6 @@ use sd_utils::{ uuid_to_bytes, }; -#[cfg(feature = "location-watcher")] use sd_file_path_helper::IsolatedFilePathDataParts; use std::{ @@ -1037,7 +1036,6 @@ pub async fn get_location_path_from_location_id( }) } -#[cfg(feature = "location-watcher")] pub async fn create_file_path( crate::location::Library { db, sync, .. }: &crate::location::Library, IsolatedFilePathDataParts { diff --git a/core/src/object/media/mod.rs b/core/src/object/media/mod.rs index 09833ce1c..bfce2b4d0 100644 --- a/core/src/object/media/mod.rs +++ b/core/src/object/media/mod.rs @@ -28,7 +28,6 @@ pub fn media_data_image_to_query( }) } -#[cfg(feature = "location-watcher")] pub fn media_data_image_to_query_params( mdi: ImageMetadata, ) -> (Vec<(&'static str, serde_json::Value)>, Vec) {