diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4b34105de..68ebc37dd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,7 +22,7 @@ /apps/mobile/ @jamiepine @Brendonovich @oscartbeaumont @utkubakir # core logic -/core/ @jamiepine @Brendonovich @oscartbeaumont +/core/ @jamiepine @Brendonovich @oscartbeaumont @fogodev /packages/macos/ @jamiepine @Brendonovich @oscartbeaumont # server app diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5413e0e8a..67a28e164 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,6 +158,9 @@ jobs: - name: Cargo fetch run: cargo fetch + - name: Cargo test core + run: cargo test -p sd-core -F location-watcher + - name: Check core run: cargo check -p sd-core --release diff --git a/Cargo.lock b/Cargo.lock index e63a9d9e4..01aee7c75 100644 Binary files a/Cargo.lock and b/Cargo.lock differ diff --git a/apps/desktop/package.json b/apps/desktop/package.json index c6603ae9c..2ba90d52d 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -24,7 +24,6 @@ "devDependencies": { "@sd/config": "workspace:*", "@tauri-apps/cli": "1.1.1", - "@tauri-apps/tauricon": "github:tauri-apps/tauricon", "@types/babel-core": "^6.25.7", "@types/react": "^18.0.21", "@types/react-dom": "^18.0.6", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index c249d17a8..b979ce4aa 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -12,7 +12,7 @@ build = "build.rs" [dependencies] tauri = { version = "1.1.1", features = ["api-all", "macos-private-api"] } rspc = { workspace = true, features = ["tauri"] } -sd-core = { path = "../../../core", features = ["ffmpeg"] } +sd-core = { path = "../../../core", features = ["ffmpeg", "location-watcher"] } tokio = { version = "1.21.2", features = ["sync"] } window-shadows = "0.2.0" tracing = "0.1.36" diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index 12c835544..f94e75218 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -59,13 +59,13 @@ async fn main() -> Result<(), Box> { .setup(|app| { let app = app.handle(); app.windows().iter().for_each(|(_, window)| { - window.hide().unwrap(); + // window.hide().unwrap(); tokio::spawn({ let window = window.clone(); async move { sleep(Duration::from_secs(3)).await; - if window.is_visible().unwrap_or(true) == false { + if !window.is_visible().unwrap_or(true) { println!("Window did not emit `app_ready` event fast enough. Showing window..."); let _ = window.show(); } diff --git a/apps/landing/public/locations.webp b/apps/landing/public/locations.webp index 2d39d9369..73a7ce331 100644 Binary files a/apps/landing/public/locations.webp and b/apps/landing/public/locations.webp differ diff --git a/apps/landing/public/thumbnails.webp b/apps/landing/public/thumbnails.webp index 1633e0342..15d3318e7 100644 Binary files a/apps/landing/public/thumbnails.webp and b/apps/landing/public/thumbnails.webp differ diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index 115adbfa4..220bf4140 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -sd-core = { path = "../../core", features = [] } +sd-core = { path = "../../core", features = ["ffmpeg"] } rspc = { workspace = true, features = ["axum"] } axum = "0.5.16" tokio = { version = "1.21.2", features = ["sync", "rt-multi-thread", "signal"] } diff --git a/apps/server/package.json b/apps/server/package.json index 75d37d974..f1254391f 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -4,6 +4,6 @@ "main": "index.js", "license": "GPL-3.0-only", "scripts": { - "dev": "cargo watch -x 'run -p server'" + "dev": "RUST_LOG=\"sd_core=info\" cargo watch -x 'run -p server'" } } diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index 53d629323..a46e9c3cd 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -39,11 +39,12 @@ async fn main() { let app = axum::Router::new() .route("/", get(|| async { "Spacedrive Server!" })) .route("/health", get(|| async { "OK" })) - .route("/spacedrive/:id", { + .route("/spacedrive/*id", { let node = node.clone(); get(|extract::Path(path): extract::Path| async move { - let (status_code, content_type, body) = - node.handle_custom_uri(path.split('/').collect()).await; + let (status_code, content_type, body) = node + .handle_custom_uri(path.split('/').skip(1).collect()) + .await; ( StatusCode::from_u16(status_code).unwrap(), diff --git a/apps/web/public/favicon.ico b/apps/web/public/favicon.ico new file mode 100644 index 000000000..97e6ea933 Binary files /dev/null and b/apps/web/public/favicon.ico differ diff --git a/apps/web/public/logo-192x192.png b/apps/web/public/logo-192x192.png new file mode 100644 index 000000000..74e728884 Binary files /dev/null and b/apps/web/public/logo-192x192.png differ diff --git a/apps/web/public/logo-512x512.png b/apps/web/public/logo-512x512.png new file mode 100644 index 000000000..1afba9570 Binary files /dev/null and b/apps/web/public/logo-512x512.png differ diff --git a/apps/web/public/manifest.json b/apps/web/public/manifest.json index a5d9bf516..dd6d71798 100644 --- a/apps/web/public/manifest.json +++ b/apps/web/public/manifest.json @@ -8,18 +8,18 @@ "type": "image/x-icon" }, { - "src": "logo192.png", + "src": "logo-192x192.png", "type": "image/png", "sizes": "192x192" }, { - "src": "logo512.png", + "src": "logo-512x512.png", "type": "image/png", "sizes": "512x512" } ], "start_url": ".", "display": "standalone", - "theme_color": "#000000", - "background_color": "#ffffff" + "theme_color": "#101016", + "background_color": "#1C1D26" } diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 689392f04..b76625928 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -23,7 +23,9 @@ const client = hooks.createClient({ const platform: Platform = { platform: 'web', getThumbnailUrlById: (casId) => - `${import.meta.env.VITE_SDSERVER_BASE_URL}/spacedrive/thumbnail/${encodeURIComponent(casId)}`, + `${ + import.meta.env.VITE_SDSERVER_BASE_URL || 'http://localhost:8080' + }/spacedrive/thumbnail/${encodeURIComponent(casId)}.webp`, openLink: (url) => window.open(url, '_blank')?.focus(), demoMode: true }; diff --git a/apps/web/src/index.html b/apps/web/src/index.html index 0ec95cc2b..ce6f82f41 100644 --- a/apps/web/src/index.html +++ b/apps/web/src/index.html @@ -3,6 +3,7 @@ Spacedrive + diff --git a/core/Cargo.toml b/core/Cargo.toml index 70d45e0b9..94ce81be4 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -19,6 +19,9 @@ ffmpeg = [ "dep:ffmpeg-next", "dep:sd-ffmpeg", ] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. +location-watcher = [ + "dep:notify" +] [dependencies] hostname = "0.3.1" @@ -66,6 +69,7 @@ ctor = "0.1.23" globset = { version = "^0.4.9", features = ["serde1"] } itertools = "^0.10.5" enumflags2 = "0.7.5" +notify = { version = "5.0.0", default-features = false, features = ["macos_kqueue"], optional = true } [dev-dependencies] tempfile = "^3.3.0" diff --git a/core/build.rs b/core/build.rs index 03d2d9323..cf49cbcb2 100644 --- a/core/build.rs +++ b/core/build.rs @@ -2,7 +2,7 @@ use std::process::Command; fn main() { let output = Command::new("git") - .args(&["rev-parse", "--short", "HEAD"]) + .args(["rev-parse", "--short", "HEAD"]) .output() .expect("error getting git hash. Does `git rev-parse --short HEAD` work for you?"); let git_hash = String::from_utf8(output.stdout) diff --git a/core/prisma/migrations/20221004133318_init/migration.sql b/core/prisma/migrations/20221004133318_init/migration.sql index 5bf3a2f9d..709838f37 100644 --- a/core/prisma/migrations/20221004133318_init/migration.sql +++ b/core/prisma/migrations/20221004133318_init/migration.sql @@ -73,7 +73,7 @@ CREATE TABLE "object" ( "cas_id" TEXT NOT NULL, "integrity_checksum" TEXT, "name" TEXT, - "extension" TEXT, + "extension" TEXT COLLATE NOCASE, "kind" INTEGER NOT NULL DEFAULT 0, "size_in_bytes" TEXT NOT NULL, "key_id" INTEGER, @@ -98,7 +98,7 @@ CREATE TABLE "file_path" ( "location_id" INTEGER NOT NULL, "materialized_path" TEXT NOT NULL, "name" TEXT NOT NULL, - "extension" TEXT, + "extension" TEXT COLLATE NOCASE, "object_id" INTEGER, "parent_id" INTEGER, "key_id" INTEGER, @@ -107,7 +107,7 @@ CREATE TABLE "file_path" ( "date_indexed" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY ("location_id", "id"), - CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, CONSTRAINT "file_path_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT "file_path_key_id_fkey" FOREIGN KEY ("key_id") REFERENCES "key" ("id") ON DELETE SET NULL ON UPDATE CASCADE ); diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma index 9a7dc9b1f..6a5687d3e 100644 --- a/core/prisma/schema.prisma +++ b/core/prisma/schema.prisma @@ -142,9 +142,10 @@ model Object { model FilePath { id Int - is_dir Boolean @default(false) + is_dir Boolean @default(false) // location that owns this path location_id Int + location Location @relation(fields: [location_id], references: [id], onDelete: Cascade, onUpdate: Cascade) // a path generated from local file_path ids eg: "34/45/67/890" materialized_path String // the name and extension @@ -152,6 +153,7 @@ model FilePath { extension String? // the unique Object for this file path object_id Int? + object Object? @relation(fields: [object_id], references: [id], onDelete: Restrict) // the parent in the file tree parent_id Int? key_id Int? // replacement for encryption @@ -162,9 +164,6 @@ model FilePath { date_modified DateTime @default(now()) date_indexed DateTime @default(now()) - object Object? @relation(fields: [object_id], references: [id], onDelete: Cascade, onUpdate: Cascade) - location Location? @relation(fields: [location_id], references: [id], onDelete: Cascade, onUpdate: Cascade) - // NOTE: this self relation for the file tree was causing SQLite to go to forever bed, disabling until workaround // parent FilePath? @relation("directory_file_paths", fields: [parent_id], references: [id], onDelete: NoAction, onUpdate: NoAction) // children FilePath[] @relation("directory_file_paths") @@ -188,31 +187,31 @@ model FileConflict { // keys allow us to know exactly which files can be decrypted with a given key // they can be "mounted" to a client, and then used to decrypt files automatically model Key { - id Int @id @default(autoincrement()) + id Int @id @default(autoincrement()) // uuid to identify the key - uuid String @unique + uuid String @unique // the name that the user sets - name String? + name String? // is this key the default for encryption? // was not tagged as unique as i'm not too sure if PCR will handle it // can always be tagged as unique, the keys API will need updating to use `find_unique()` - default Boolean @default(false) + default Boolean @default(false) // nullable if concealed for security - date_created DateTime? @default(now()) + date_created DateTime? @default(now()) // encryption algorithm used to encrypt the key - algorithm Bytes + algorithm Bytes // hashing algorithm used for hashing the master password hashing_algorithm Bytes // salt used for encrypting data with this key - content_salt Bytes + content_salt Bytes // the *encrypted* master key (48 bytes) - master_key Bytes + master_key Bytes // the nonce used for encrypting the master key - master_key_nonce Bytes + master_key_nonce Bytes // the nonce used for encrypting the key - key_nonce Bytes + key_nonce Bytes // the *encrypted* key - key Bytes + key Bytes automount Boolean @default(false) @@ -236,8 +235,7 @@ model MediaData { codecs String? // eg: "h264,acc" streams Int? - // change this relation to Object after testing - objects Object? @relation(fields: [id], references: [id], onDelete: Cascade, onUpdate: Cascade) + object Object? @relation(fields: [id], references: [id], onDelete: Cascade, onUpdate: Cascade) @@map("media_data") } diff --git a/core/src/api/files.rs b/core/src/api/files.rs index 3bb93a599..8800925f5 100644 --- a/core/src/api/files.rs +++ b/core/src/api/files.rs @@ -16,10 +16,19 @@ use super::{utils::LibraryRequest, RouterBuilder}; pub(crate) fn mount() -> RouterBuilder { ::new() - .library_query("readMetadata", |t| { - t(|_, _id: i32, _| async move { - #[allow(unreachable_code)] - Ok(todo!()) + .library_query("get", |t| { + #[derive(Type, Deserialize)] + pub struct GetArgs { + pub id: i32, + } + t(|_, args: GetArgs, library| async move { + Ok(library + .db + .object() + .find_unique(object::id::equals(args.id)) + .include(object::include!({ file_paths media_data })) + .exec() + .await?) }) }) .library_mutation("setNote", |t| { @@ -41,6 +50,7 @@ pub(crate) fn mount() -> RouterBuilder { .await?; invalidate_query!(library, "locations.getExplorerData"); + invalidate_query!(library, "tags.getExplorerData"); Ok(()) }) @@ -64,6 +74,7 @@ pub(crate) fn mount() -> RouterBuilder { .await?; invalidate_query!(library, "locations.getExplorerData"); + invalidate_query!(library, "tags.getExplorerData"); Ok(()) }) @@ -94,9 +105,7 @@ pub(crate) fn mount() -> RouterBuilder { )); } - library - .spawn_job(Job::new(args, Box::new(FileEncryptorJob {}))) - .await; + library.spawn_job(Job::new(args, FileEncryptorJob {})).await; invalidate_query!(library, "locations.getExplorerData"); Ok(()) @@ -115,9 +124,7 @@ pub(crate) fn mount() -> RouterBuilder { )); } - library - .spawn_job(Job::new(args, Box::new(FileDecryptorJob {}))) - .await; + library.spawn_job(Job::new(args, FileDecryptorJob {})).await; invalidate_query!(library, "locations.getExplorerData"); Ok(()) diff --git a/core/src/api/jobs.rs b/core/src/api/jobs.rs index 76f679dbc..608aeb662 100644 --- a/core/src/api/jobs.rs +++ b/core/src/api/jobs.rs @@ -2,7 +2,7 @@ use crate::{ job::{Job, JobManager}, location::{fetch_location, LocationError}, object::{ - identifier_job::{FileIdentifierJob, FileIdentifierJobInit}, + identifier_job::full_identifier_job::{FullFileIdentifierJob, FullFileIdentifierJobInit}, preview::{ThumbnailJob, ThumbnailJobInit}, validation::validator_job::{ObjectValidatorJob, ObjectValidatorJobInit}, }, @@ -26,6 +26,12 @@ pub(crate) fn mount() -> RouterBuilder { .library_query("getHistory", |t| { t(|_, _: (), library| async move { Ok(JobManager::get_history(&library).await?) }) }) + .library_mutation("clearAll", |t| { + t(|_, _: (), library| async move { + JobManager::clear_all_jobs(&library).await?; + Ok(()) + }) + }) .library_mutation("generateThumbsForLocation", |t| { #[derive(Type, Deserialize)] pub struct GenerateThumbsForLocationArgs { @@ -49,10 +55,10 @@ pub(crate) fn mount() -> RouterBuilder { .spawn_job(Job::new( ThumbnailJobInit { location_id: args.id, - path: PathBuf::new(), - background: true, + root_path: PathBuf::new(), + background: false, }, - Box::new(ThumbnailJob {}), + ThumbnailJob {}, )) .await; @@ -82,7 +88,7 @@ pub(crate) fn mount() -> RouterBuilder { path: args.path, background: true, }, - Box::new(ObjectValidatorJob {}), + ObjectValidatorJob {}, )) .await; @@ -106,11 +112,11 @@ pub(crate) fn mount() -> RouterBuilder { library .spawn_job(Job::new( - FileIdentifierJobInit { + FullFileIdentifierJobInit { location_id: args.id, sub_path: Some(args.path), }, - Box::new(FileIdentifierJob {}), + FullFileIdentifierJob {}, )) .await; diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs index b8572f481..b5e185b79 100644 --- a/core/src/api/locations.rs +++ b/core/src/api/locations.rs @@ -1,17 +1,18 @@ use crate::{ - invalidate_query, location::{ - fetch_location, + delete_location, fetch_location, indexer::{indexer_job::indexer_job_location, rules::IndexerRuleCreateArgs}, - scan_location, LocationCreateArgs, LocationError, LocationUpdateArgs, + relink_location, scan_location, LocationCreateArgs, LocationError, LocationUpdateArgs, }, object::preview::THUMBNAIL_CACHE_DIR_NAME, prisma::{file_path, indexer_rule, indexer_rules_in_location, location, object, tag}, }; +use std::path::PathBuf; + use rspc::{self, internal::MiddlewareBuilderLike, ErrorCode, Type}; use serde::{Deserialize, Serialize}; -use tracing::info; +use tokio::{fs, io}; use super::{utils::LibraryRequest, Ctx, RouterBuilder}; @@ -76,7 +77,7 @@ pub(crate) fn mount() -> rspc::RouterBuilder< pub cursor: Option, } - t(|_, args: LocationExplorerArgs, library| async move { + t(|_, mut args: LocationExplorerArgs, library| async move { let location = library .db .location() @@ -87,6 +88,10 @@ pub(crate) fn mount() -> rspc::RouterBuilder< rspc::Error::new(ErrorCode::NotFound, "Location not found".into()) })?; + if !args.path.ends_with('/') { + args.path += "/"; + } + let directory = library .db .file_path() @@ -112,25 +117,42 @@ pub(crate) fn mount() -> rspc::RouterBuilder< .exec() .await?; + // library + // .queue_job(Job::new( + // ThumbnailJobInit { + // location_id: location.id, + // // recursive: false, // TODO: do this + // root_path: PathBuf::from(&directory.materialized_path), + // background: true, + // }, + // ThumbnailJob {}, + // )) + // .await; + + let mut items = Vec::with_capacity(file_paths.len()); + for mut file_path in file_paths { + if let Some(object) = &mut file_path.object.as_mut() { + // TODO: Use helper function to build this url as as the Rust file loading layer + let thumb_path = library + .config() + .data_directory() + .join(THUMBNAIL_CACHE_DIR_NAME) + .join(&object.cas_id) + .with_extension("webp"); + + object.has_thumbnail = (match fs::metadata(thumb_path).await { + Ok(_) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + }) + .map_err(LocationError::IOError)?; + } + items.push(ExplorerItem::Path(Box::new(file_path))); + } + Ok(ExplorerData { context: ExplorerContext::Location(location), - items: file_paths - .into_iter() - .map(|mut file_path| { - if let Some(object) = &mut file_path.object.as_mut() { - // TODO: Use helper function to build this url as as the Rust file loading layer - let thumb_path = library - .config() - .data_directory() - .join(THUMBNAIL_CACHE_DIR_NAME) - .join(&object.cas_id) - .with_extension("webp"); - - object.has_thumbnail = thumb_path.try_exists().unwrap(); - } - ExplorerItem::Path(Box::new(file_path)) - }) - .collect(), + items, }) }) }) @@ -148,38 +170,35 @@ pub(crate) fn mount() -> rspc::RouterBuilder< }) .library_mutation("delete", |t| { t(|_, location_id: i32, library| async move { + delete_location(&library, location_id) + .await + .map_err(Into::into) + }) + }) + .library_mutation("relink", |t| { + t(|_, location_path: PathBuf, library| async move { + relink_location(&library, location_path) + .await + .map_err(Into::into) + }) + }) + .library_mutation("addLibrary", |t| { + t(|_, args: LocationCreateArgs, library| async move { + let location = args.add_library(&library).await?; + scan_location(&library, location).await?; + Ok(()) + }) + }) + .library_mutation("fullRescan", |t| { + t(|_, location_id: i32, library| async move { + // remove existing paths library .db .file_path() .delete_many(vec![file_path::location_id::equals(location_id)]) .exec() .await?; - - library - .db - .indexer_rules_in_location() - .delete_many(vec![indexer_rules_in_location::location_id::equals( - location_id, - )]) - .exec() - .await?; - - library - .db - .location() - .delete(location::id::equals(location_id)) - .exec() - .await?; - - invalidate_query!(library, "locations.list"); - - info!("Location {} deleted", location_id); - - Ok(()) - }) - }) - .library_mutation("fullRescan", |t| { - t(|_, location_id: i32, library| async move { + // rescan location scan_location( &library, fetch_location(&library, location_id) diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs index c630abea3..8169332fc 100644 --- a/core/src/api/mod.rs +++ b/core/src/api/mod.rs @@ -78,7 +78,12 @@ pub(crate) fn mount() -> Arc { Ok(NodeState { config: ctx.config.get().await, // We are taking the assumption here that this value is only used on the frontend for display purposes - data_path: ctx.config.data_directory().to_string_lossy().into_owned(), + data_path: ctx + .config + .data_directory() + .to_str() + .expect("Found non-UTF-8 path") + .to_string(), }) }) }) @@ -101,7 +106,7 @@ pub(crate) fn mount() -> Arc { CoreEvent::InvalidateOperation(op) => yield op, CoreEvent::InvalidateOperationDebounced(op) => { let current = Instant::now(); - if current.duration_since(last) > Duration::from_millis(1000 / 60) { + if current.duration_since(last) > Duration::from_millis(1000 / 10) { last = current; yield op; } diff --git a/core/src/job/job_manager.rs b/core/src/job/job_manager.rs index ca69524b0..ff25e21b7 100644 --- a/core/src/job/job_manager.rs +++ b/core/src/job/job_manager.rs @@ -1,28 +1,33 @@ use crate::{ + invalidate_query, job::{worker::Worker, DynJob, Job, JobError}, library::LibraryContext, location::indexer::indexer_job::{IndexerJob, INDEXER_JOB_NAME}, object::{ - identifier_job::{FileIdentifierJob, IDENTIFIER_JOB_NAME}, + identifier_job::full_identifier_job::{FullFileIdentifierJob, FULL_IDENTIFIER_JOB_NAME}, preview::{ThumbnailJob, THUMBNAIL_JOB_NAME}, + validation::validator_job::{ObjectValidatorJob, VALIDATOR_JOB_NAME}, }, prisma::{job, node}, }; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + fmt::Debug, + fmt::{Display, Formatter}, + sync::Arc, + time::Duration, +}; + use int_enum::IntEnum; use prisma_client_rust::Direction; use rspc::Type; use serde::{Deserialize, Serialize}; -use std::{ - collections::{HashMap, VecDeque}, - fmt::Debug, - fmt::{Display, Formatter}, - sync::Arc, - time::Duration, +use tokio::{ + sync::{broadcast, mpsc, Mutex, RwLock}, + time::sleep, }; -use tokio::sync::{mpsc, Mutex, RwLock}; -use tokio::{sync::broadcast, time::sleep}; -use tracing::{error, info}; +use tracing::{debug, error, info}; use uuid::Uuid; // db is single threaded, nerd @@ -36,6 +41,7 @@ pub enum JobManagerEvent { /// Handling persisting JobReports to the database, pause/resuming, and /// pub struct JobManager { + current_jobs_hashes: RwLock>, job_queue: RwLock>>, running_workers: RwLock>>>, internal_sender: mpsc::UnboundedSender, @@ -47,6 +53,7 @@ impl JobManager { let (shutdown_tx, _shutdown_rx) = broadcast::channel(1); let (internal_sender, mut internal_receiver) = mpsc::unbounded_channel(); let this = Arc::new(Self { + current_jobs_hashes: RwLock::new(HashSet::new()), job_queue: RwLock::new(VecDeque::new()), running_workers: RwLock::new(HashMap::new()), internal_sender, @@ -58,7 +65,9 @@ impl JobManager { // FIXME: if this task crashes, the entire application is unusable while let Some(event) = internal_receiver.recv().await { match event { - JobManagerEvent::IngestJob(ctx, job) => this2.clone().ingest(&ctx, job).await, + JobManagerEvent::IngestJob(ctx, job) => { + this2.clone().dispatch_job(&ctx, job).await + } } } }); @@ -66,41 +75,45 @@ impl JobManager { this } - pub async fn ingest(self: Arc, ctx: &LibraryContext, mut job: Box) { - // create worker to process job - let mut running_workers = self.running_workers.write().await; - if running_workers.len() < MAX_WORKERS { - info!("Running job: {:?}", job.name()); + pub async fn ingest(self: Arc, ctx: &LibraryContext, job: Box) { + let job_hash = job.hash(); + debug!( + "Ingesting job: ", + job.name(), + job_hash + ); - let job_report = job - .report() - .take() - .expect("critical error: missing job on worker"); - - let job_id = job_report.id; - - let worker = Worker::new(job, job_report); - - let wrapped_worker = Arc::new(Mutex::new(worker)); - - if let Err(e) = - Worker::spawn(Arc::clone(&self), Arc::clone(&wrapped_worker), ctx.clone()).await - { - error!("Error spawning worker: {:?}", e); - } else { - running_workers.insert(job_id, wrapped_worker); - } + if !self.current_jobs_hashes.read().await.contains(&job_hash) { + self.current_jobs_hashes.write().await.insert(job_hash); + self.dispatch_job(ctx, job).await; } else { - self.job_queue.write().await.push_back(job); + debug!( + "Job already in queue: ", + job.name(), + job_hash + ); } } - pub async fn ingest_queue(&self, _ctx: &LibraryContext, job: Box) { - self.job_queue.write().await.push_back(job); + pub async fn ingest_queue(&self, job: Box) { + let job_hash = job.hash(); + debug!("Queueing job: ", job.name(), job_hash); + + if !self.current_jobs_hashes.read().await.contains(&job_hash) { + self.current_jobs_hashes.write().await.insert(job_hash); + self.job_queue.write().await.push_back(job); + } else { + debug!( + "Job already in queue: ", + job.name(), + job_hash + ); + } } - pub async fn complete(self: Arc, ctx: &LibraryContext, job_id: Uuid) { - // remove worker from running workers + pub async fn complete(self: Arc, ctx: &LibraryContext, job_id: Uuid, job_hash: u64) { + // remove worker from running workers and from current jobs hashes + self.current_jobs_hashes.write().await.remove(&job_hash); self.running_workers.write().await.remove(&job_id); // continue queue let job = self.job_queue.write().await.pop_front(); @@ -127,16 +140,26 @@ impl JobManager { pub async fn get_history( ctx: &LibraryContext, ) -> Result, prisma_client_rust::QueryError> { - let jobs = ctx + Ok(ctx .db .job() .find_many(vec![job::status::not(JobStatus::Running.int_value())]) .order_by(job::date_created::order(Direction::Desc)) .take(100) .exec() - .await?; + .await? + .into_iter() + .map(Into::into) + .collect()) + } - Ok(jobs.into_iter().map(Into::into).collect()) + pub async fn clear_all_jobs( + ctx: &LibraryContext, + ) -> Result<(), prisma_client_rust::QueryError> { + ctx.db.job().delete_many(vec![]).exec().await?; + + invalidate_query!(ctx, "jobs.getHistory"); + Ok(()) } pub fn shutdown_tx(&self) -> Arc> { @@ -176,20 +199,22 @@ impl JobManager { match paused_job.name.as_str() { THUMBNAIL_JOB_NAME => { Arc::clone(&self) - .ingest(ctx, Job::resume(paused_job, Box::new(ThumbnailJob {}))?) + .dispatch_job(ctx, Job::resume(paused_job, ThumbnailJob {})?) .await; } INDEXER_JOB_NAME => { Arc::clone(&self) - .ingest(ctx, Job::resume(paused_job, Box::new(IndexerJob {}))?) + .dispatch_job(ctx, Job::resume(paused_job, IndexerJob {})?) .await; } - IDENTIFIER_JOB_NAME => { + FULL_IDENTIFIER_JOB_NAME => { Arc::clone(&self) - .ingest( - ctx, - Job::resume(paused_job, Box::new(FileIdentifierJob {}))?, - ) + .dispatch_job(ctx, Job::resume(paused_job, FullFileIdentifierJob {})?) + .await; + } + VALIDATOR_JOB_NAME => { + Arc::clone(&self) + .dispatch_job(ctx, Job::resume(paused_job, ObjectValidatorJob {})?) .await; } _ => { @@ -204,6 +229,40 @@ impl JobManager { Ok(()) } + + async fn dispatch_job(self: Arc, ctx: &LibraryContext, mut job: Box) { + // create worker to process job + let mut running_workers = self.running_workers.write().await; + if running_workers.len() < MAX_WORKERS { + info!("Running job: {:?}", job.name()); + + let job_report = job + .report() + .take() + .expect("critical error: missing job on worker"); + + let job_id = job_report.id; + + let worker = Worker::new(job, job_report); + + let wrapped_worker = Arc::new(Mutex::new(worker)); + + if let Err(e) = + Worker::spawn(Arc::clone(&self), Arc::clone(&wrapped_worker), ctx.clone()).await + { + error!("Error spawning worker: {:?}", e); + } else { + running_workers.insert(job_id, wrapped_worker); + } + } else { + debug!( + "Queueing job: ", + job.name(), + job.hash() + ); + self.job_queue.write().await.push_back(job); + } + } } #[derive(Debug)] diff --git a/core/src/job/mod.rs b/core/src/job/mod.rs index 8919ba017..ad38524b3 100644 --- a/core/src/job/mod.rs +++ b/core/src/job/mod.rs @@ -1,10 +1,19 @@ -use crate::location::{indexer::IndexerError, LocationError}; -use sd_crypto::Error as CryptoError; +use crate::{ + location::{indexer::IndexerError, LocationError}, + object::{identifier_job::IdentifierJobError, preview::ThumbnailError}, +}; + +use std::{ + collections::{hash_map::DefaultHasher, VecDeque}, + fmt::Debug, + hash::{Hash, Hasher}, +}; use rmp_serde::{decode::Error as DecodeError, encode::Error as EncodeError}; +use sd_crypto::Error as CryptoError; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use std::{collections::VecDeque, fmt::Debug}; use thiserror::Error; +use tracing::warn; use uuid::Uuid; mod job_manager; @@ -15,14 +24,13 @@ pub use worker::*; #[derive(Error, Debug)] pub enum JobError { + // General errors #[error("Database error: {0}")] DatabaseError(#[from] prisma_client_rust::QueryError), - #[error("Location error: {0}")] - LocationError(#[from] LocationError), #[error("I/O error: {0}")] IOError(#[from] std::io::Error), #[error("Failed to join Tokio spawn blocking: {0}")] - JoinError(#[from] tokio::task::JoinError), + JoinTaskError(#[from] tokio::task::JoinError), #[error("Job state encode error: {0}")] StateEncode(#[from] EncodeError), #[error("Job state decode error: {0}")] @@ -35,8 +43,20 @@ pub enum JobError { "Tried to resume a job that doesn't have saved state data: job " )] MissingJobDataState(Uuid, String), + + // Specific job errors #[error("Indexer error: {0}")] IndexerError(#[from] IndexerError), + #[error("Location error: {0}")] + LocationError(#[from] LocationError), + #[error("Thumbnail error: {0}")] + ThumbnailError(#[from] ThumbnailError), + #[error("Identifier error: {0}")] + IdentifierError(#[from] IdentifierJobError), + + // Not errors + #[error("Job had a early finish: ")] + EarlyFinish { name: String, reason: String }, #[error("Crypto error: {0}")] CryptoError(#[from] CryptoError), #[error("Data needed for job execution not found: job ")] @@ -49,29 +69,21 @@ pub type JobResult = Result; pub type JobMetadata = Option; #[async_trait::async_trait] -pub trait StatefulJob: Send + Sync { - type Init: Serialize + DeserializeOwned + Send + Sync; +pub trait StatefulJob: Send + Sync + Sized { + type Init: Serialize + DeserializeOwned + Send + Sync + Hash; type Data: Serialize + DeserializeOwned + Send + Sync; type Step: Serialize + DeserializeOwned + Send + Sync; fn name(&self) -> &'static str; - async fn init( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> Result<(), JobError>; + async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError>; async fn execute_step( &self, ctx: WorkerContext, - state: &mut JobState, + state: &mut JobState, ) -> Result<(), JobError>; - async fn finalize( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> JobResult; + async fn finalize(&self, ctx: WorkerContext, state: &mut JobState) -> JobResult; } #[async_trait::async_trait] @@ -79,29 +91,17 @@ pub trait DynJob: Send + Sync { fn report(&mut self) -> &mut Option; fn name(&self) -> &'static str; async fn run(&mut self, ctx: WorkerContext) -> JobResult; + fn hash(&self) -> u64; } -pub struct Job -where - Init: Serialize + DeserializeOwned + Send + Sync, - Data: Serialize + DeserializeOwned + Send + Sync, - Step: Serialize + DeserializeOwned + Send + Sync, -{ +pub struct Job { report: Option, - state: JobState, - stateful_job: Box>, + state: JobState, + stateful_job: SJob, } -impl Job -where - Init: Serialize + DeserializeOwned + Send + Sync, - Data: Serialize + DeserializeOwned + Send + Sync, - Step: Serialize + DeserializeOwned + Send + Sync, -{ - pub fn new( - init: Init, - stateful_job: Box>, - ) -> Box { +impl Job { + pub fn new(init: SJob::Init, stateful_job: SJob) -> Box { Box::new(Self { report: Some(JobReport::new( Uuid::new_v4(), @@ -117,10 +117,7 @@ where }) } - pub fn resume( - mut report: JobReport, - stateful_job: Box>, - ) -> Result, JobError> { + pub fn resume(mut report: JobReport, stateful_job: SJob) -> Result, JobError> { let job_state_data = if let Some(data) = report.data.take() { data } else { @@ -135,21 +132,29 @@ where } } +impl Hash for Job { + fn hash(&self, state: &mut H) { + self.name().hash(state); + self.state.hash(state); + } +} + #[derive(Serialize, Deserialize)] -pub struct JobState { - pub init: Init, - pub data: Option, - pub steps: VecDeque, +pub struct JobState { + pub init: Job::Init, + pub data: Option, + pub steps: VecDeque, pub step_number: usize, } +impl Hash for JobState { + fn hash(&self, state: &mut H) { + self.init.hash(state); + } +} + #[async_trait::async_trait] -impl DynJob for Job -where - Init: Serialize + DeserializeOwned + Send + Sync, - Data: Serialize + DeserializeOwned + Send + Sync, - Step: Serialize + DeserializeOwned + Send + Sync, -{ +impl DynJob for Job { fn report(&mut self) -> &mut Option { &mut self.report } @@ -174,7 +179,12 @@ where ctx.clone(), &mut self.state, ) => { - step_result?; + if matches!(step_result, Err(JobError::EarlyFinish { .. })) { + warn!("{}", step_result.unwrap_err()); + break; + } else { + step_result?; + }; self.state.steps.pop_front(); } _ = &mut shutdown_rx_fut => { @@ -192,4 +202,10 @@ where .finalize(ctx.clone(), &mut self.state) .await } + + fn hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + Hash::hash(self, &mut hasher); + hasher.finish() + } } diff --git a/core/src/job/worker.rs b/core/src/job/worker.rs index dbbb8c679..fbf908c6a 100644 --- a/core/src/job/worker.rs +++ b/core/src/job/worker.rs @@ -29,7 +29,7 @@ pub enum WorkerEvent { #[derive(Clone)] pub struct WorkerContext { - library_ctx: LibraryContext, + pub library_ctx: LibraryContext, events_tx: UnboundedSender, shutdown_tx: Arc>, } @@ -52,10 +52,6 @@ impl WorkerContext { .expect("critical error: failed to send worker worker progress event updates"); } - pub fn library_ctx(&self) -> LibraryContext { - self.library_ctx.clone() - } - pub fn shutdown_rx(&self) -> broadcast::Receiver<()> { self.shutdown_tx.subscribe() } @@ -104,17 +100,23 @@ impl Worker { .take() .expect("critical error: missing job on worker"); + let job_hash = job.hash(); let job_id = worker.report.id; let old_status = worker.report.status; + worker.report.status = JobStatus::Running; + if matches!(old_status, JobStatus::Queued) { worker.report.create(&ctx).await?; + } else { + worker.report.update(&ctx).await?; } drop(worker); invalidate_query!(ctx, "jobs.isRunning"); - // spawn task to handle receiving events from the worker + let library_ctx = ctx.clone(); + // spawn task to handle receiving events from the worker tokio::spawn(Worker::track_progress( Arc::clone(&worker_mutex), worker_events_rx, @@ -178,7 +180,7 @@ impl Worker { if let Err(e) = done_rx.await { error!("failed to wait for worker completion: {:#?}", e); } - job_manager.complete(&ctx, job_id).await; + job_manager.complete(&ctx, job_id, job_hash).await; }); Ok(()) diff --git a/core/src/lib.rs b/core/src/lib.rs index 634f3de21..052a44303 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,7 +1,9 @@ use api::{CoreEvent, Ctx, Router}; use job::JobManager; use library::LibraryManager; +use location::{LocationManager, LocationManagerError}; use node::NodeConfigManager; + use std::{path::Path, sync::Arc}; use thiserror::Error; use tokio::{ @@ -27,6 +29,7 @@ pub(crate) mod prisma; pub struct NodeContext { pub config: Arc, pub jobs: Arc, + pub location_manager: Arc, pub event_bus_tx: broadcast::Sender, } @@ -52,7 +55,9 @@ impl Node { let data_dir = data_dir.as_ref(); #[cfg(debug_assertions)] let data_dir = data_dir.join("dev"); - let _ = fs::create_dir_all(&data_dir).await; // This error is ignore because it throwing on mobile despite the folder existing. + + // This error is ignored because it's throwing on mobile despite the folder existing. + let _ = fs::create_dir_all(&data_dir).await; // dbg!(get_object_kind_from_extension("png")); @@ -66,12 +71,12 @@ impl Node { EnvFilter::from_default_env() .add_directive("warn".parse().expect("Error invalid tracing directive!")) .add_directive( - "sd-core=debug" + "sd_core=debug" .parse() .expect("Error invalid tracing directive!"), ) .add_directive( - "sd-core-mobile=debug" + "sd_core_mobile=debug" .parse() .expect("Error invalid tracing directive!"), ) @@ -107,16 +112,39 @@ impl Node { let config = NodeConfigManager::new(data_dir.to_path_buf()).await?; let jobs = JobManager::new(); + let location_manager = LocationManager::new(); let library_manager = LibraryManager::new( data_dir.join("libraries"), NodeContext { config: Arc::clone(&config), jobs: Arc::clone(&jobs), + location_manager: Arc::clone(&location_manager), event_bus_tx: event_bus.0.clone(), }, ) .await?; + // Adding already existing locations for location management + for library_ctx in library_manager.get_all_libraries_ctx().await { + for location in library_ctx + .db + .location() + .find_many(vec![]) + .exec() + .await + .unwrap_or_else(|e| { + error!( + "Failed to get locations from database for location manager: {:#?}", + e + ); + vec![] + }) { + if let Err(e) = location_manager.add(location.id, library_ctx.clone()).await { + error!("Failed to add location to location manager: {:#?}", e); + } + } + } + // Trying to resume possible paused jobs let inner_library_manager = Arc::clone(&library_manager); let inner_jobs = Arc::clone(&jobs); @@ -136,6 +164,7 @@ impl Node { event_bus, }; + info!("Spacedrive online."); Ok((Arc::new(node), router)) } @@ -208,4 +237,6 @@ pub enum NodeError { FailedToInitializeConfig(#[from] node::NodeConfigError), #[error("Failed to initialize library manager: {0}")] FailedToInitializeLibraryManager(#[from] library::LibraryManagerError), + #[error("Location manager error: {0}")] + LocationManager(#[from] LocationManagerError), } diff --git a/core/src/library/library_ctx.rs b/core/src/library/library_ctx.rs index c54677997..758feb389 100644 --- a/core/src/library/library_ctx.rs +++ b/core/src/library/library_ctx.rs @@ -1,11 +1,17 @@ -use crate::job::DynJob; +use crate::{ + api::CoreEvent, job::DynJob, location::LocationManager, node::NodeConfigManager, + prisma::PrismaClient, NodeContext, +}; + +use std::{ + fmt::{Debug, Formatter}, + sync::Arc, +}; + use sd_crypto::keys::keymanager::KeyManager; -use std::sync::Arc; use tracing::warn; use uuid::Uuid; -use crate::{api::CoreEvent, node::NodeConfigManager, prisma::PrismaClient, NodeContext}; - use super::LibraryConfig; /// LibraryContext holds context for a library which can be passed around the application. @@ -25,25 +31,39 @@ pub struct LibraryContext { pub(super) node_context: NodeContext, } +impl Debug for LibraryContext { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Rolling out this implementation because `NodeContext` contains a DynJob which is + // troublesome to implement Debug trait + f.debug_struct("LibraryContext") + .field("id", &self.id) + .field("config", &self.config) + .field("db", &self.db) + .field("node_local_id", &self.node_local_id) + .finish() + } +} + impl LibraryContext { pub(crate) async fn spawn_job(&self, job: Box) { self.node_context.jobs.clone().ingest(self, job).await; } pub(crate) async fn queue_job(&self, job: Box) { - self.node_context.jobs.ingest_queue(self, job).await; + self.node_context.jobs.ingest_queue(job).await; } pub(crate) fn emit(&self, event: CoreEvent) { - match self.node_context.event_bus_tx.send(event) { - Ok(_) => (), - Err(err) => { - warn!("Error sending event to event bus: {:?}", err); - } + if let Err(e) = self.node_context.event_bus_tx.send(event) { + warn!("Error sending event to event bus: {e:?}"); } } pub(crate) fn config(&self) -> Arc { self.node_context.config.clone() } + + pub(crate) fn location_manager(&self) -> &Arc { + &self.node_context.location_manager + } } diff --git a/core/src/location/error.rs b/core/src/location/error.rs index 7a8cbf081..5558485bc 100644 --- a/core/src/location/error.rs +++ b/core/src/location/error.rs @@ -1,9 +1,14 @@ -use rspc::{self, ErrorCode}; +use crate::LocationManagerError; + use std::path::PathBuf; + +use rspc::{self, ErrorCode}; use thiserror::Error; use tokio::io; use uuid::Uuid; +use super::metadata::LocationMetadataError; + /// Error type for location related errors #[derive(Error, Debug)] pub enum LocationError { @@ -18,22 +23,31 @@ pub enum LocationError { // User errors #[error("Location not a directory (path: {0:?})")] NotDirectory(PathBuf), + #[error("Could not find directory in Location (path: {0:?})")] + DirectoryNotFound(String), #[error("Missing local_path (id: {0})")] MissingLocalPath(i32), + #[error("Library exists in the location metadata file, must relink: (old_path: {old_path:?}, new_path: {new_path:?})")] + NeedRelink { + old_path: PathBuf, + new_path: PathBuf, + }, + #[error("Exist a different library in the location metadata file, must add a new library: (path: {0:?})")] + AddLibraryToMetadata(PathBuf), + #[error("Location metadata file not found: (path: {0:?})")] + MetadataNotFound(PathBuf), #[error("Location already exists (path: {0:?})")] LocationAlreadyExists(PathBuf), // Internal Errors - #[error("Failed to create location (uuid {uuid:?})")] - CreateFailure { uuid: Uuid }, - #[error("Failed to read location dotfile (path: {1:?}); (error: {0:?})")] - DotfileReadFailure(io::Error, PathBuf), - #[error("Failed to serialize dotfile for location (at path: {1:?}); (error: {0:?})")] - DotfileSerializeFailure(serde_json::Error, PathBuf), - #[error("Dotfile location is read only (at path: {0:?})")] - ReadonlyDotFileLocationFailure(PathBuf), - #[error("Failed to write dotfile (path: {1:?}); (error: {0:?})")] - DotfileWriteFailure(io::Error, PathBuf), + #[error("Location metadata error (error: {0:?})")] + LocationMetadataError(#[from] LocationMetadataError), + #[error("Failed to read location path metadata info (path: {1:?}); (error: {0:?})")] + LocationPathFilesystemMetadataAccess(io::Error, PathBuf), + #[error("Location is read only (at path: {0:?})")] + ReadonlyLocationFailure(PathBuf), + #[error("Missing metadata file for location (path: {0:?})")] + MissingMetadataFile(PathBuf), #[error("Failed to open file from local os (error: {0:?})")] FileReadError(io::Error), #[error("Failed to read mounted volumes from local os (error: {0:?})")] @@ -42,18 +56,25 @@ pub enum LocationError { IOError(io::Error), #[error("Database error (error: {0:?})")] DatabaseError(#[from] prisma_client_rust::QueryError), + #[error("Location manager error (error: {0:?})")] + LocationManagerError(#[from] LocationManagerError), } impl From for rspc::Error { fn from(err: LocationError) -> Self { match err { + // Not found errors LocationError::PathNotFound(_) | LocationError::UuidNotFound(_) | LocationError::IdNotFound(_) => { rspc::Error::with_cause(ErrorCode::NotFound, err.to_string(), err) } - LocationError::NotDirectory(_) | LocationError::MissingLocalPath(_) => { + // User's fault errors + LocationError::NotDirectory(_) + | LocationError::MissingLocalPath(_) + | LocationError::NeedRelink { .. } + | LocationError::AddLibraryToMetadata(_) => { rspc::Error::with_cause(ErrorCode::BadRequest, err.to_string(), err) } diff --git a/core/src/location/file_path_helper.rs b/core/src/location/file_path_helper.rs new file mode 100644 index 000000000..a70a8217f --- /dev/null +++ b/core/src/location/file_path_helper.rs @@ -0,0 +1,141 @@ +use crate::{library::LibraryContext, prisma::file_path}; + +use std::sync::atomic::{AtomicI32, Ordering}; + +use chrono::{DateTime, Utc}; +use prisma_client_rust::{Direction, QueryError}; + +static LAST_FILE_PATH_ID: AtomicI32 = AtomicI32::new(0); + +file_path::select!(file_path_id_only { id }); + +pub async fn get_max_file_path_id(library_ctx: &LibraryContext) -> Result { + let mut last_id = LAST_FILE_PATH_ID.load(Ordering::Acquire); + if last_id == 0 { + last_id = fetch_max_file_path_id(library_ctx).await?; + LAST_FILE_PATH_ID.store(last_id, Ordering::Release); + } + + Ok(last_id) +} + +pub fn set_max_file_path_id(id: i32) { + LAST_FILE_PATH_ID.store(id, Ordering::Relaxed); +} + +async fn fetch_max_file_path_id(library_ctx: &LibraryContext) -> Result { + Ok(library_ctx + .db + .file_path() + .find_first(vec![]) + .order_by(file_path::id::order(Direction::Desc)) + .select(file_path_id_only::select()) + .exec() + .await? + .map(|r| r.id) + .unwrap_or(0)) +} + +#[cfg(feature = "location-watcher")] +pub async fn create_file_path( + library_ctx: &LibraryContext, + location_id: i32, + mut materialized_path: String, + name: String, + extension: Option, + parent_id: Option, + is_dir: bool, +) -> Result { + use crate::prisma::location; + + let mut last_id = LAST_FILE_PATH_ID.load(Ordering::Acquire); + if last_id == 0 { + last_id = fetch_max_file_path_id(library_ctx).await?; + } + + // If this new file_path is a directory, materialized_path must end with "/" + if is_dir && !materialized_path.ends_with('/') { + materialized_path += "/"; + } + + let next_id = last_id + 1; + + let created_path = library_ctx + .db + .file_path() + .create( + next_id, + location::id::equals(location_id), + materialized_path, + name, + vec![ + file_path::parent_id::set(parent_id), + file_path::is_dir::set(is_dir), + file_path::extension::set(extension), + ], + ) + .exec() + .await?; + + LAST_FILE_PATH_ID.store(next_id, Ordering::Release); + + Ok(created_path) +} + +pub struct FilePathBatchCreateEntry { + pub id: i32, + pub location_id: i32, + pub materialized_path: String, + pub name: String, + pub extension: Option, + pub parent_id: Option, + pub is_dir: bool, + pub created_at: DateTime, +} + +pub async fn create_many_file_paths( + library_ctx: &LibraryContext, + entries: Vec, +) -> Result { + library_ctx + .db + .file_path() + .create_many( + entries + .into_iter() + .map( + |FilePathBatchCreateEntry { + id, + location_id, + mut materialized_path, + name, + extension, + parent_id, + is_dir, + created_at, + }| { + // If this new file_path is a directory, materialized_path must end with "/" + if is_dir && !materialized_path.ends_with('/') { + materialized_path += "/"; + } + + file_path::create_unchecked( + id, + location_id, + materialized_path, + name, + vec![ + file_path::is_dir::set(is_dir), + file_path::parent_id::set(parent_id), + file_path::extension::set(extension), + file_path::date_created::set(created_at.into()), + ], + ) + }, + ) + .collect(), + ) + .skip_duplicates() + .exec() + .await +} diff --git a/core/src/location/indexer/indexer_job.rs b/core/src/location/indexer/indexer_job.rs index 6133a2ca7..37dc5e83c 100644 --- a/core/src/location/indexer/indexer_job.rs +++ b/core/src/location/indexer/indexer_job.rs @@ -3,16 +3,26 @@ use crate::{ prisma::{file_path, location}, }; +use std::{ + collections::HashMap, + ffi::OsStr, + hash::{Hash, Hasher}, + path::PathBuf, + time::Duration, +}; + use chrono::{DateTime, Utc}; use itertools::Itertools; -use prisma_client_rust::Direction; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, ffi::OsStr, path::PathBuf, time::Duration}; use tokio::time::Instant; use tracing::info; use super::{ - rules::IndexerRule, + super::file_path_helper::{ + create_many_file_paths, get_max_file_path_id, set_max_file_path_id, + FilePathBatchCreateEntry, + }, + rules::{IndexerRule, RuleKind}, walk::{walk, WalkEntry}, }; @@ -35,6 +45,7 @@ pub struct IndexerJob; location::include!(indexer_job_location { indexer_rules: select { indexer_rule } }); +file_path::select!(file_path_id_only { id }); /// `IndexerJobInit` receives a `location::Data` object to be indexed #[derive(Serialize, Deserialize)] @@ -42,6 +53,11 @@ pub struct IndexerJobInit { pub location: indexer_job_location::Data, } +impl Hash for IndexerJobInit { + fn hash(&self, state: &mut H) { + self.location.id.hash(state); + } +} /// `IndexerJobData` contains the state of the indexer job, which includes a `location_path` that /// is cached and casted on `PathBuf` from `local_path` column in the `location` table. It also /// contains some metadata for logging purposes. @@ -94,11 +110,7 @@ impl StatefulJob for IndexerJob { } /// Creates a vector of valid path buffers from a directory, chunked into batches of `BATCH_SIZE`. - async fn init( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> Result<(), JobError> { + async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { let location_path = state .init .location @@ -107,32 +119,17 @@ impl StatefulJob for IndexerJob { .map(PathBuf::from) .unwrap(); - // query db to highers id, so we can increment it for the new files indexed - #[derive(Deserialize, Serialize, Debug)] - struct QueryRes { - id: Option, - } - - // TODO: use a select to fetch only the id instead of entire record when prisma supports it // grab the next id so we can increment in memory for batch inserting - let first_file_id = ctx - .library_ctx() - .db - .file_path() - .find_first(vec![]) - .order_by(file_path::id::order(Direction::Desc)) - .exec() - .await? - .map(|r| r.id) - .unwrap_or(0); + let first_file_id = get_max_file_path_id(&ctx.library_ctx).await?; - let mut indexer_rules_by_kind = HashMap::new(); + let mut indexer_rules_by_kind: HashMap> = + HashMap::with_capacity(state.init.location.indexer_rules.len()); for location_rule in &state.init.location.indexer_rules { let indexer_rule = IndexerRule::try_from(&location_rule.indexer_rule)?; indexer_rules_by_kind .entry(indexer_rule.kind) - .or_insert(vec![]) + .or_default() .push(indexer_rule); } @@ -154,10 +151,15 @@ impl StatefulJob for IndexerJob { .await?; let total_paths = paths.len(); + let last_file_id = first_file_id + total_paths as i32; + + // Setting our global state for file_path ids + set_max_file_path_id(last_file_id); + let mut dirs_ids = HashMap::new(); let paths_entries = paths .into_iter() - .zip(first_file_id..(first_file_id + total_paths as i32)) + .zip(first_file_id..last_file_id) .map( |( WalkEntry { @@ -205,7 +207,7 @@ impl StatefulJob for IndexerJob { IndexerJobData::on_scan_progress( ctx.clone(), vec![ - ScanProgress::SavedChunks(i as usize), + ScanProgress::SavedChunks(i), ScanProgress::Message(format!( "Writing {} of {} to db", i * chunk_steps.len(), @@ -224,7 +226,7 @@ impl StatefulJob for IndexerJob { async fn execute_step( &self, ctx: WorkerContext, - state: &mut JobState, + state: &mut JobState, ) -> Result<(), JobError> { let data = &state .data @@ -234,51 +236,44 @@ impl StatefulJob for IndexerJob { let location_path = &data.location_path; let location_id = state.init.location.id; - let count = ctx - .library_ctx() - .db - .file_path() - .create_many( - state.steps[0] - .iter() - .map(|entry| { - let name; - let extension; + let entries = state.steps[0] + .iter() + .map(|entry| { + let name; + let extension; - // if 'entry.path' is a directory, set extension to an empty string to - // avoid periods in folder names being interpreted as file extensions - if entry.is_dir { - extension = "".to_string(); - name = extract_name(entry.path.file_name()); - } else { - // if the 'entry.path' is not a directory, then get the extension and name. - extension = extract_name(entry.path.extension()); - name = extract_name(entry.path.file_stem()); - } - let materialized_path = entry - .path - .strip_prefix(location_path) - .unwrap() - .to_string_lossy() - .to_string(); + // if 'entry.path' is a directory, set extension to an empty string to + // avoid periods in folder names being interpreted as file extensions + if entry.is_dir { + extension = None; + name = extract_name(entry.path.file_name()); + } else { + // if the 'entry.path' is not a directory, then get the extension and name. + extension = Some(extract_name(entry.path.extension()).to_lowercase()); + name = extract_name(entry.path.file_stem()); + } + let materialized_path = entry + .path + .strip_prefix(location_path) + .unwrap() + .to_str() + .expect("Found non-UTF-8 path") + .to_string(); - file_path::create_unchecked( - entry.file_id, - location_id, - materialized_path, - name, - vec![ - file_path::is_dir::set(entry.is_dir), - file_path::extension::set(Some(extension)), - file_path::parent_id::set(entry.parent_id), - file_path::date_created::set(entry.created_at.into()), - ], - ) - }) - .collect(), - ) - .exec() - .await?; + FilePathBatchCreateEntry { + id: entry.file_id, + location_id, + materialized_path, + name, + extension, + parent_id: entry.parent_id, + is_dir: entry.is_dir, + created_at: entry.created_at, + } + }) + .collect(); + + let count = create_many_file_paths(&ctx.library_ctx, entries).await?; info!("Inserted {count} records"); @@ -286,11 +281,7 @@ impl StatefulJob for IndexerJob { } /// Logs some metadata about the indexer job - async fn finalize( - &self, - _ctx: WorkerContext, - state: &mut JobState, - ) -> JobResult { + async fn finalize(&self, _ctx: WorkerContext, state: &mut JobState) -> JobResult { let data = state .data .as_ref() diff --git a/core/src/location/indexer/rules.rs b/core/src/location/indexer/rules.rs index fc146c8b7..2ac10e923 100644 --- a/core/src/location/indexer/rules.rs +++ b/core/src/location/indexer/rules.rs @@ -223,7 +223,7 @@ async fn accept_dir_for_its_children( let mut read_dir = fs::read_dir(source).await?; while let Some(entry) = read_dir.next_entry().await? { if entry.metadata().await?.is_dir() - && children.contains(entry.file_name().to_string_lossy().as_ref()) + && children.contains(entry.file_name().to_str().expect("Found non-UTF-8 path")) { return Ok(true); } @@ -240,7 +240,7 @@ async fn reject_dir_for_its_children( let mut read_dir = fs::read_dir(source).await?; while let Some(entry) = read_dir.next_entry().await? { if entry.metadata().await?.is_dir() - && children.contains(entry.file_name().to_string_lossy().as_ref()) + && children.contains(entry.file_name().to_str().expect("Found non-UTF-8 path")) { return Ok(false); } diff --git a/core/src/location/manager/helpers.rs b/core/src/location/manager/helpers.rs new file mode 100644 index 000000000..867490ec9 --- /dev/null +++ b/core/src/location/manager/helpers.rs @@ -0,0 +1,165 @@ +use crate::{library::LibraryContext, prisma::location}; + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + time::Duration, +}; + +use tokio::{fs, io::ErrorKind, time::sleep}; +use tracing::{error, warn}; +use uuid::Uuid; + +use super::{watcher::LocationWatcher, LocationId}; + +type LibraryId = Uuid; +type LocationAndLibraryKey = (LocationId, LibraryId); + +const LOCATION_CHECK_INTERVAL: Duration = Duration::from_secs(5); + +pub(super) async fn check_online(location: &location::Data, library_ctx: &LibraryContext) -> bool { + if let Some(ref local_path) = location.local_path { + match fs::metadata(local_path).await { + Ok(_) => { + if !location.is_online { + set_location_online(location.id, library_ctx, true).await; + } + true + } + Err(e) if e.kind() == ErrorKind::NotFound => { + if location.is_online { + set_location_online(location.id, library_ctx, false).await; + } + false + } + Err(e) => { + error!("Failed to check if location is online: {:#?}", e); + false + } + } + } else { + // In this case, we don't have a `local_path`, but this location was marked as online + if location.is_online { + set_location_online(location.id, library_ctx, false).await; + } + false + } +} + +pub(super) async fn set_location_online( + location_id: LocationId, + library_ctx: &LibraryContext, + online: bool, +) { + if let Err(e) = library_ctx + .db + .location() + .update( + location::id::equals(location_id), + vec![location::is_online::set(online)], + ) + .exec() + .await + { + error!( + "Failed to update location to online: (id: {}, error: {:#?})", + location_id, e + ); + } +} + +pub(super) async fn location_check_sleep( + location_id: LocationId, + library_ctx: LibraryContext, +) -> (LocationId, LibraryContext) { + sleep(LOCATION_CHECK_INTERVAL).await; + (location_id, library_ctx) +} + +pub(super) fn watch_location( + location: location::Data, + library_id: LibraryId, + location_path: impl AsRef, + locations_watched: &mut HashMap, + locations_unwatched: &mut HashMap, +) { + let location_id = location.id; + if let Some(mut watcher) = locations_unwatched.remove(&(location_id, library_id)) { + if watcher.check_path(location_path) { + watcher.watch(); + } else { + watcher.update_data(location, true); + } + + locations_watched.insert((location_id, library_id), watcher); + } +} + +pub(super) fn unwatch_location( + location: location::Data, + library_id: LibraryId, + location_path: impl AsRef, + locations_watched: &mut HashMap, + locations_unwatched: &mut HashMap, +) { + let location_id = location.id; + if let Some(mut watcher) = locations_watched.remove(&(location_id, library_id)) { + if watcher.check_path(location_path) { + watcher.unwatch(); + } else { + watcher.update_data(location, false) + } + + locations_unwatched.insert((location_id, library_id), watcher); + } +} + +pub(super) fn drop_location( + location_id: LocationId, + library_id: LibraryId, + message: &str, + locations_watched: &mut HashMap, + locations_unwatched: &mut HashMap, +) { + warn!("{message}: ",); + if let Some(mut watcher) = locations_watched.remove(&(location_id, library_id)) { + watcher.unwatch(); + } else { + locations_unwatched.remove(&(location_id, library_id)); + } +} + +pub(super) async fn get_location( + location_id: i32, + library_ctx: &LibraryContext, +) -> Option { + library_ctx + .db + .location() + .find_unique(location::id::equals(location_id)) + .exec() + .await + .unwrap_or_else(|err| { + error!("Failed to get location data from location_id: {:#?}", err); + None + }) +} + +pub(super) fn subtract_location_path( + location_path: impl AsRef, + current_path: impl AsRef, +) -> Option { + let location_path = location_path.as_ref(); + let current_path = current_path.as_ref(); + + if let Ok(stripped) = current_path.strip_prefix(location_path) { + Some(stripped.to_path_buf()) + } else { + error!( + "Failed to strip location root path ({}) from current path ({})", + location_path.display(), + current_path.display() + ); + None + } +} diff --git a/core/src/location/manager/mod.rs b/core/src/location/manager/mod.rs new file mode 100644 index 000000000..cc72c66e8 --- /dev/null +++ b/core/src/location/manager/mod.rs @@ -0,0 +1,279 @@ +use crate::library::LibraryContext; + +use std::{path::PathBuf, sync::Arc}; + +use thiserror::Error; +use tokio::{ + io, + sync::{mpsc, oneshot}, +}; +use tracing::{debug, error}; + +#[cfg(feature = "location-watcher")] +mod watcher; + +#[cfg(feature = "location-watcher")] +mod helpers; + +pub type LocationId = i32; + +type ManagerMessage = ( + LocationId, + LibraryContext, + oneshot::Sender>, +); + +#[derive(Error, Debug)] +pub enum LocationManagerError { + #[error("Unable to send location id to be checked by actor: (error: {0})")] + ActorSendLocationError(#[from] mpsc::error::SendError), + #[error("Unable to receive actor response: (error: {0})")] + ActorResponseError(#[from] oneshot::error::RecvError), + + #[cfg(feature = "location-watcher")] + #[error("Watcher error: (error: {0})")] + WatcherError(#[from] notify::Error), + + #[error("Location missing local path: ")] + LocationMissingLocalPath(LocationId), + #[error("Tried to update a non-existing file: ")] + UpdateNonExistingFile(PathBuf), + #[error("Unable to extract materialized path from location: ")] + UnableToExtractMaterializedPath(LocationId, PathBuf), + #[error("Database error: {0}")] + DatabaseError(#[from] prisma_client_rust::QueryError), + #[error("I/O error: {0}")] + IOError(#[from] io::Error), +} + +#[derive(Debug)] +pub struct LocationManager { + add_locations_tx: mpsc::Sender, + remove_locations_tx: mpsc::Sender, + stop_tx: Option>, +} + +impl LocationManager { + #[allow(unused)] + pub fn new() -> Arc { + let (add_locations_tx, add_locations_rx) = mpsc::channel(128); + let (remove_locations_tx, remove_locations_rx) = mpsc::channel(128); + let (stop_tx, stop_rx) = oneshot::channel(); + + #[cfg(feature = "location-watcher")] + tokio::spawn(Self::run_locations_checker( + add_locations_rx, + remove_locations_rx, + stop_rx, + )); + + #[cfg(not(feature = "location-watcher"))] + tracing::warn!("Location watcher is disabled, locations will not be checked"); + + debug!("Location manager initialized"); + + Arc::new(Self { + add_locations_tx, + remove_locations_tx, + stop_tx: Some(stop_tx), + }) + } + + pub async fn add( + &self, + location_id: LocationId, + library_ctx: LibraryContext, + ) -> Result<(), LocationManagerError> { + if cfg!(feature = "location-watcher") { + let (tx, rx) = oneshot::channel(); + + self.add_locations_tx + .send((location_id, library_ctx, tx)) + .await?; + + rx.await? + } else { + Ok(()) + } + } + + pub async fn remove( + &self, + location_id: LocationId, + library_ctx: LibraryContext, + ) -> Result<(), LocationManagerError> { + if cfg!(feature = "location-watcher") { + let (tx, rx) = oneshot::channel(); + + self.remove_locations_tx + .send((location_id, library_ctx, tx)) + .await?; + + rx.await? + } else { + Ok(()) + } + } + + #[cfg(feature = "location-watcher")] + async fn run_locations_checker( + mut add_locations_rx: mpsc::Receiver, + mut remove_locations_rx: mpsc::Receiver, + mut stop_rx: oneshot::Receiver<()>, + ) -> Result<(), LocationManagerError> { + use std::collections::{HashMap, HashSet}; + + use futures::stream::{FuturesUnordered, StreamExt}; + use tokio::select; + use tracing::{info, warn}; + + use helpers::{ + check_online, drop_location, get_location, location_check_sleep, unwatch_location, + watch_location, + }; + use watcher::LocationWatcher; + + let mut to_check_futures = FuturesUnordered::new(); + let mut to_remove = HashSet::new(); + let mut locations_watched = HashMap::new(); + let mut locations_unwatched = HashMap::new(); + + loop { + select! { + // To add a new location + Some((location_id, library_ctx, response_tx)) = add_locations_rx.recv() => { + if let Some(location) = get_location(location_id, &library_ctx).await { + let is_online = check_online(&location, &library_ctx).await; + let _ = response_tx.send( + LocationWatcher::new(location, library_ctx.clone()) + .await + .map(|mut watcher| { + if is_online { + watcher.watch(); + locations_watched.insert( + (location_id, library_ctx.id), + watcher + ); + } else { + locations_unwatched.insert( + (location_id, library_ctx.id), + watcher + ); + } + + to_check_futures.push( + location_check_sleep(location_id, library_ctx) + ); + } + ) + ); // ignore errors, we handle errors on receiver + } else { + warn!( + "Location not found in database to be watched: {}", + location_id + ); + } + } + + // To remove an location + Some((location_id, library_ctx, response_tx)) = remove_locations_rx.recv() => { + if let Some(location) = get_location(location_id, &library_ctx).await { + if let Some(ref local_path_str) = location.local_path.clone() { + unwatch_location( + location, + library_ctx.id, + local_path_str, + &mut locations_watched, + &mut locations_unwatched, + ); + locations_unwatched.remove(&(location_id, library_ctx.id)); + } else { + drop_location( + location_id, + library_ctx.id, + "Dropping location from location manager, because we don't have a `local_path` anymore", + &mut locations_watched, + &mut locations_unwatched + ); + } + } else { + drop_location( + location_id, + library_ctx.id, + "Removing location from manager, as we failed to fetch from db", + &mut locations_watched, + &mut locations_unwatched + ); + } + + // Marking location as removed, so we don't try to check it when the time comes + to_remove.insert((location_id, library_ctx.id)); + + let _ = response_tx.send(Ok(())); // ignore errors, we handle errors on receiver + } + + // Periodically checking locations + Some((location_id, library_ctx)) = to_check_futures.next() => { + if to_remove.contains(&(location_id, library_ctx.id)) { + // The time to check came for an already removed library, so we just ignore it + to_remove.remove(&(location_id, library_ctx.id)); + } else if let Some(location) = get_location(location_id, &library_ctx).await { + if let Some(ref local_path_str) = location.local_path.clone() { + if check_online(&location, &library_ctx).await { + watch_location( + location, + library_ctx.id, + local_path_str, + &mut locations_watched, + &mut locations_unwatched + ); + } else { + unwatch_location( + location, + library_ctx.id, + local_path_str, + &mut locations_watched, + &mut locations_unwatched + ); + } + to_check_futures.push(location_check_sleep(location_id, library_ctx)); + } else { + drop_location( + location_id, + library_ctx.id, + "Dropping location from location manager, because we don't have a `local_path` anymore", + &mut locations_watched, + &mut locations_unwatched + ); + } + } else { + drop_location( + location_id, + library_ctx.id, + "Removing location from manager, as we failed to fetch from db", + &mut locations_watched, + &mut locations_unwatched + ); + } + } + + _ = &mut stop_rx => { + info!("Stopping location manager"); + break; + } + } + } + + Ok(()) + } +} + +impl Drop for LocationManager { + fn drop(&mut self) { + if let Some(stop_tx) = self.stop_tx.take() { + if stop_tx.send(()).is_err() { + error!("Failed to send stop signal to location manager"); + } + } + } +} diff --git a/core/src/location/manager/watcher/linux.rs b/core/src/location/manager/watcher/linux.rs new file mode 100644 index 000000000..f2a4a0e96 --- /dev/null +++ b/core/src/location/manager/watcher/linux.rs @@ -0,0 +1,56 @@ +use crate::{ + library::LibraryContext, + location::{indexer::indexer_job::indexer_job_location, manager::LocationManagerError}, +}; + +use async_trait::async_trait; +use notify::{ + event::{AccessKind, AccessMode, CreateKind, ModifyKind, RenameMode}, + Event, EventKind, +}; +use tracing::trace; + +use super::{ + utils::{create_dir, file_creation_or_update, remove_event, rename_both_event}, + EventHandler, +}; + +#[derive(Debug)] +pub(super) struct LinuxEventHandler {} + +#[async_trait] +impl EventHandler for LinuxEventHandler { + fn new() -> Self { + Self {} + } + + async fn handle_event( + &mut self, + location: indexer_job_location::Data, + library_ctx: &LibraryContext, + event: Event, + ) -> Result<(), LocationManagerError> { + trace!("Received Linux event: {:#?}", event); + + match event.kind { + EventKind::Access(AccessKind::Close(AccessMode::Write)) => { + // If a file was closed with write mode, then it was updated or created + file_creation_or_update(location, event, library_ctx).await?; + } + EventKind::Create(CreateKind::Folder) => { + create_dir(location, event, library_ctx.clone()).await?; + } + EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { + rename_both_event(location, event, library_ctx).await?; + } + EventKind::Remove(remove_kind) => { + remove_event(location, event, remove_kind, library_ctx).await?; + } + other_event_kind => { + trace!("Other Linux event that we don't handle for now: {other_event_kind:#?}"); + } + } + + Ok(()) + } +} diff --git a/core/src/location/manager/watcher/macos.rs b/core/src/location/manager/watcher/macos.rs new file mode 100644 index 000000000..e058cd9de --- /dev/null +++ b/core/src/location/manager/watcher/macos.rs @@ -0,0 +1,130 @@ +use crate::{ + library::LibraryContext, + location::{indexer::indexer_job::indexer_job_location, manager::LocationManagerError}, +}; + +use std::{future::Future, time::Duration}; + +use async_trait::async_trait; +use notify::{ + event::{CreateKind, DataChange, ModifyKind, RenameMode}, + Event, EventKind, +}; +use tokio::{fs, select, spawn, sync::oneshot, time::sleep}; +use tracing::{trace, warn}; + +use super::{ + utils::{create_dir, create_file, remove_event, rename, update_file}, + EventHandler, +}; + +#[derive(Debug, Default)] +pub(super) struct MacOsEventHandler { + maybe_rename_sender: Option>, +} + +#[async_trait] +impl EventHandler for MacOsEventHandler { + fn new() -> Self + where + Self: Sized, + { + Default::default() + } + + async fn handle_event( + &mut self, + location: indexer_job_location::Data, + library_ctx: &LibraryContext, + event: Event, + ) -> Result<(), LocationManagerError> { + trace!("Received MacOS event: {:#?}", event); + + match event.kind { + EventKind::Create(create_kind) => match create_kind { + CreateKind::File => { + let (maybe_rename_tx, maybe_rename_rx) = oneshot::channel(); + spawn(wait_to_create( + location, + event, + library_ctx.clone(), + create_file, + maybe_rename_rx, + )); + self.maybe_rename_sender = Some(maybe_rename_tx); + } + CreateKind::Folder => { + let (maybe_rename_tx, maybe_rename_rx) = oneshot::channel(); + spawn(wait_to_create( + location, + event, + library_ctx.clone(), + create_dir, + maybe_rename_rx, + )); + self.maybe_rename_sender = Some(maybe_rename_tx); + } + other => { + trace!("Ignoring other create event: {:#?}", other); + } + }, + EventKind::Modify(ref modify_kind) => match modify_kind { + ModifyKind::Data(DataChange::Any) => { + if fs::metadata(&event.paths[0]).await?.is_file() { + update_file(location, event, library_ctx).await?; + } else { + trace!("Unexpected MacOS modify event on a directory"); + } + + // We ignore EventKind::Modify(ModifyKind::Data(DataChange::Any)) for directories + // as they're also used for removing files and directories, being emitted + // on the parent directory in this case + } + ModifyKind::Name(RenameMode::Any) => { + if let Some(rename_sender) = self.maybe_rename_sender.take() { + if !rename_sender.is_closed() && rename_sender.send(event).is_err() { + warn!("Failed to send rename event"); + } + } + } + other => { + trace!("Ignoring other modify event: {:#?}", other); + } + }, + EventKind::Remove(remove_kind) => { + remove_event(location, event, remove_kind, library_ctx).await?; + // An EventKind::Modify(ModifyKind::Data(DataChange::Any)) - On parent directory + // is also emitted, but we can ignore it. + } + other_event_kind => { + trace!("Other MacOS event that we don't handle for now: {other_event_kind:#?}"); + } + } + + Ok(()) + } +} + +// FIX-ME: Had some troubles with borrowck, to receive a +// impl FnOnce(indexer_job_location::Data, Event, &LibraryContext) -> Fut +// as a parameter, had to move LibraryContext into the functions +async fn wait_to_create( + location: indexer_job_location::Data, + event: Event, + library_ctx: LibraryContext, + create_fn: impl FnOnce(indexer_job_location::Data, Event, LibraryContext) -> Fut, + maybe_rename_rx: oneshot::Receiver, +) -> Result<(), LocationManagerError> +where + Fut: for<'r> Future>, +{ + select! { + () = sleep(Duration::from_secs(1)) => { + create_fn(location, event, library_ctx).await + }, + Ok(rename_event) = maybe_rename_rx => { + trace!("Renaming file or directory instead of creating a new one"); + rename(&event.paths[0], &rename_event.paths[0], location, &library_ctx).await + } + } +} diff --git a/core/src/location/manager/watcher/mod.rs b/core/src/location/manager/watcher/mod.rs new file mode 100644 index 000000000..9cbd4921f --- /dev/null +++ b/core/src/location/manager/watcher/mod.rs @@ -0,0 +1,697 @@ +use crate::{ + library::LibraryContext, + prisma::{file_path, location}, +}; + +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use notify::{Config, Event, RecommendedWatcher, RecursiveMode, Watcher}; +use tokio::{ + runtime::Handle, + select, + sync::{mpsc, oneshot}, + task::{block_in_place, JoinHandle}, +}; +use tracing::{debug, error, warn}; + +use super::{ + super::{fetch_location, indexer::indexer_job::indexer_job_location}, + LocationId, LocationManagerError, +}; + +mod linux; +mod macos; +mod windows; + +mod utils; + +use utils::{check_event, check_location_online}; + +#[cfg(target_os = "linux")] +type Handler = linux::LinuxEventHandler; + +#[cfg(target_os = "macos")] +type Handler = macos::MacOsEventHandler; + +#[cfg(target_os = "windows")] +type Handler = windows::WindowsEventHandler; + +file_path::include!(file_path_with_object { object }); + +#[async_trait] +trait EventHandler { + fn new() -> Self + where + Self: Sized; + + async fn handle_event( + &mut self, + location: indexer_job_location::Data, + library_ctx: &LibraryContext, + event: Event, + ) -> Result<(), LocationManagerError>; +} + +#[derive(Debug)] +pub(super) struct LocationWatcher { + location: location::Data, + path: PathBuf, + watcher: RecommendedWatcher, + handle: Option>, + stop_tx: Option>, +} + +impl LocationWatcher { + pub(super) async fn new( + location: location::Data, + library_ctx: LibraryContext, + ) -> Result { + let (events_tx, events_rx) = mpsc::unbounded_channel(); + let (stop_tx, stop_rx) = oneshot::channel(); + + let watcher = RecommendedWatcher::new( + move |result| { + if !events_tx.is_closed() { + if events_tx.send(result).is_err() { + error!( + "Unable to send watcher event to location manager for location: ", + location.id + ); + } + } else { + error!( + "Tried to send location file system events to a closed channel: >, + mut stop_rx: oneshot::Receiver<()>, + ) { + let mut event_handler = Handler::new(); + + loop { + select! { + Some(event) = events_rx.recv() => { + match event { + Ok(event) => { + if let Err(e) = Self::handle_single_event( + location_id, + event, + &mut event_handler, + &library_ctx + ).await { + error!("Failed to handle location file system event: \ + ", + ); + } + } + Err(e) => { + error!("watch error: {:#?}", e); + } + } + } + _ = &mut stop_rx => { + debug!("Stop Location Manager event handler for location: ", location_id); + break + } + } + } + } + + async fn handle_single_event( + location_id: LocationId, + event: Event, + event_handler: &mut impl EventHandler, + library_ctx: &LibraryContext, + ) -> Result<(), LocationManagerError> { + if check_event(&event) { + if let Some(location) = fetch_location(library_ctx, location_id) + .include(indexer_job_location::include()) + .exec() + .await? + { + if check_location_online(&location) { + return event_handler + .handle_event(location, library_ctx, event) + .await; + } else { + warn!("Tried to handle event for offline location: "); + } + } else { + warn!("Tried to handle event for unknown location: "); + } + } + + Ok(()) + } + + pub(super) fn check_path(&self, path: impl AsRef) -> bool { + self.path == path.as_ref() + } + + pub(super) fn watch(&mut self) { + if let Err(e) = self.watcher.watch(&self.path, RecursiveMode::Recursive) { + error!( + "Unable to watch location: (path: {}, error: {e:#?})", + self.path.display() + ); + } else { + debug!("Now watching location: (path: {})", self.path.display()); + } + } + + pub(super) fn unwatch(&mut self) { + if let Err(e) = self.watcher.unwatch(&self.path) { + /**************************************** TODO: **************************************** + * According to an unit test, this error may occur when a subdirectory is removed * + * and we try to unwatch the parent directory then we have to check the implications * + * of unwatch error for this case. * + **************************************************************************************/ + error!( + "Unable to unwatch location: (path: {}, error: {e:#?})", + self.path.display() + ); + } else { + debug!("Stop watching location: (path: {})", self.path.display()); + } + } + + pub(super) fn update_data(&mut self, location: location::Data, to_watch: bool) { + assert_eq!( + self.location.id, location.id, + "Updated location data must have the same id" + ); + let path = PathBuf::from(location.local_path.as_ref().unwrap_or_else(|| { + panic!( + "Tried to watch a location without local_path: ", + location.id + ) + })); + + if self.path != path { + self.unwatch(); + self.path = path; + if to_watch { + self.watch(); + } + } + self.location = location; + } +} + +impl Drop for LocationWatcher { + fn drop(&mut self) { + if let Some(stop_tx) = self.stop_tx.take() { + if stop_tx.send(()).is_err() { + error!( + "Failed to send stop signal to location watcher: ", + self.location.id + ); + } + + // FIXME: change this Drop to async drop in the future + if let Some(handle) = self.handle.take() { + if let Err(e) = + block_in_place(move || Handle::current().block_on(async move { handle.await })) + { + error!("Failed to join watcher task: {e:#?}") + } + } + } + } +} + +/*************************************************************************************************** + * Some tests to validate our assumptions of events through different file systems * + *************************************************************************************************** + * Events dispatched on Linux: * + * Create File: * + * 1) EventKind::Create(CreateKind::File) * + * 2) EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any)) * + * or EventKind::Modify(ModifyKind::Data(DataChange::Any)) * + * 3) EventKind::Access(AccessKind::Close(AccessMode::Write))) * + * Create Directory: * + * 1) EventKind::Create(CreateKind::Folder) * + * Update File: * + * 1) EventKind::Modify(ModifyKind::Data(DataChange::Any)) * + * 2) EventKind::Access(AccessKind::Close(AccessMode::Write))) * + * Update File (rename): * + * 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * + * 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * + * 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) * + * Update Directory (rename): * + * 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * + * 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * + * 3) EventKind::Modify(ModifyKind::Name(RenameMode::Both)) * + * Delete File: * + * 1) EventKind::Remove(RemoveKind::File) * + * Delete Directory: * + * 1) EventKind::Remove(RemoveKind::Folder) * + * * + * Events dispatched on MacOS: * + * Create File: * + * 1) EventKind::Create(CreateKind::File) * + * Create Directory: * + * 1) EventKind::Create(CreateKind::Folder) * + * Update File: * + * 1) EventKind::Modify(ModifyKind::Data(DataChange::Any)) * + * Update File (rename): * + * 1) EventKind::Create(CreateKind::File) * + * 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) * + * Update Directory (rename): * + * 1) EventKind::Create(CreateKind::Folder) * + * 2) EventKind::Modify(ModifyKind::Name(RenameMode::Any)) * + * Delete File: * + * 1) EventKind::Remove(RemoveKind::Any) * + * 2) EventKind::Modify(ModifyKind::Data(DataChange::Any)) - On parent directory * + * Delete Directory: * + * 1) EventKind::Remove(RemoveKind::Any) * + * 2) EventKind::Modify(ModifyKind::Data(DataChange::Any)) - On parent directory * + * * + * Events dispatched on Windows: * + * Create File: * + * 1) EventKind::Create(CreateKind::Any) * + * 2) EventKind::Modify(ModifyKind::Any) * + * Create Directory: * + * 1) EventKind::Create(CreateKind::Any) * + * Update File: * + * 1) EventKind::Modify(ModifyKind::Any) * + * Update File (rename): * + * 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * + * 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * + * Update Directory (rename): * + * 1) EventKind::Modify(ModifyKind::Name(RenameMode::From)) * + * 2) EventKind::Modify(ModifyKind::Name(RenameMode::To)) * + * Delete File: * + * 1) EventKind::Remove(RemoveKind::Any) * + * Delete Directory: * + * 1) EventKind::Remove(RemoveKind::Any) * + * * + * Events dispatched on Android: * + * TODO * + * * + * Events dispatched on iOS: * + * TODO * + * * + **************************************************************************************************/ +#[cfg(test)] +#[allow(unused)] +mod tests { + #[cfg(target_os = "macos")] + use notify::event::DataChange; + use notify::{ + event::{AccessKind, AccessMode, CreateKind, ModifyKind, RemoveKind, RenameMode}, + Config, Event, EventKind, RecommendedWatcher, Watcher, + }; + use std::io::ErrorKind; + use std::{path::Path, time::Duration}; + use tempfile::{tempdir, TempDir}; + use tokio::{fs, io::AsyncWriteExt, sync::mpsc, time::sleep}; + use tracing::{debug, error}; + use tracing_test::traced_test; + + async fn setup_watcher() -> ( + TempDir, + RecommendedWatcher, + mpsc::UnboundedReceiver>, + ) { + let (events_tx, events_rx) = mpsc::unbounded_channel(); + + let watcher = RecommendedWatcher::new( + move |result| { + events_tx + .send(result) + .expect("Unable to send watcher event"); + }, + Config::default(), + ) + .expect("Failed to create watcher"); + + (tempdir().unwrap(), watcher, events_rx) + } + + async fn expect_event( + mut events_rx: mpsc::UnboundedReceiver>, + path: impl AsRef, + expected_event: EventKind, + ) { + debug!("Expecting event: {expected_event:#?}"); + let path = path.as_ref(); + let mut tries = 0; + loop { + match events_rx.try_recv() { + Ok(maybe_event) => { + let event = maybe_event.expect("Failed to receive event"); + debug!("Received event: {event:#?}"); + // In case of file creation, we expect to see an close event on write mode + if event.paths[0] == path && event.kind == expected_event { + debug!("Received expected event: {expected_event:#?}"); + break; + } + } + Err(e) => { + debug!("No event yet: {e:#?}"); + tries += 1; + sleep(Duration::from_millis(100)).await; + } + } + + if tries == 10 { + panic!("No expected event received after 10 tries"); + } + } + } + + #[tokio::test] + #[traced_test] + async fn create_file_event() { + let (root_dir, mut watcher, events_rx) = setup_watcher().await; + + watcher + .watch(root_dir.path(), notify::RecursiveMode::Recursive) + .expect("Failed to watch root directory"); + debug!("Now watching {}", root_dir.path().display()); + + let file_path = root_dir.path().join("test.txt"); + fs::write(&file_path, "test").await.unwrap(); + + #[cfg(target_os = "windows")] + expect_event(events_rx, &file_path, EventKind::Modify(ModifyKind::Any)).await; + + #[cfg(target_os = "macos")] + expect_event(events_rx, &file_path, EventKind::Create(CreateKind::File)).await; + + #[cfg(target_os = "linux")] + expect_event( + events_rx, + &file_path, + EventKind::Access(AccessKind::Close(AccessMode::Write)), + ) + .await; + + debug!("Unwatching root directory: {}", root_dir.path().display()); + if let Err(e) = watcher.unwatch(root_dir.path()) { + error!("Failed to unwatch root directory: {e:#?}"); + } + } + + #[tokio::test] + #[traced_test] + async fn create_dir_event() { + let (root_dir, mut watcher, events_rx) = setup_watcher().await; + + watcher + .watch(root_dir.path(), notify::RecursiveMode::Recursive) + .expect("Failed to watch root directory"); + debug!("Now watching {}", root_dir.path().display()); + + let dir_path = root_dir.path().join("inner"); + fs::create_dir(&dir_path) + .await + .expect("Failed to create directory"); + + #[cfg(target_os = "windows")] + expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Any)).await; + + #[cfg(target_os = "macos")] + expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Folder)).await; + + #[cfg(target_os = "linux")] + expect_event(events_rx, &dir_path, EventKind::Create(CreateKind::Folder)).await; + + debug!("Unwatching root directory: {}", root_dir.path().display()); + if let Err(e) = watcher.unwatch(root_dir.path()) { + error!("Failed to unwatch root directory: {e:#?}"); + } + } + + #[tokio::test] + #[traced_test] + async fn update_file_event() { + let (root_dir, mut watcher, events_rx) = setup_watcher().await; + + let file_path = root_dir.path().join("test.txt"); + fs::write(&file_path, "test").await.unwrap(); + + watcher + .watch(root_dir.path(), notify::RecursiveMode::Recursive) + .expect("Failed to watch root directory"); + debug!("Now watching {}", root_dir.path().display()); + + let mut file = fs::OpenOptions::new() + .append(true) + .open(&file_path) + .await + .expect("Failed to open file"); + + // Writing then sync data before closing the file + file.write_all(b"\nanother test") + .await + .expect("Failed to write to file"); + file.sync_all().await.expect("Failed to flush file"); + drop(file); + + #[cfg(target_os = "windows")] + expect_event(events_rx, &file_path, EventKind::Modify(ModifyKind::Any)).await; + + #[cfg(target_os = "macos")] + expect_event( + events_rx, + &file_path, + EventKind::Modify(ModifyKind::Data(DataChange::Any)), + ) + .await; + + #[cfg(target_os = "linux")] + expect_event( + events_rx, + &file_path, + EventKind::Access(AccessKind::Close(AccessMode::Write)), + ) + .await; + + debug!("Unwatching root directory: {}", root_dir.path().display()); + if let Err(e) = watcher.unwatch(root_dir.path()) { + error!("Failed to unwatch root directory: {e:#?}"); + } + } + + #[tokio::test] + #[traced_test] + async fn update_file_rename_event() { + let (root_dir, mut watcher, events_rx) = setup_watcher().await; + + let file_path = root_dir.path().join("test.txt"); + fs::write(&file_path, "test").await.unwrap(); + + watcher + .watch(root_dir.path(), notify::RecursiveMode::Recursive) + .expect("Failed to watch root directory"); + debug!("Now watching {}", root_dir.path().display()); + + let new_file_name = root_dir.path().join("test2.txt"); + + fs::rename(&file_path, &new_file_name) + .await + .expect("Failed to rename file"); + + #[cfg(target_os = "windows")] + expect_event( + events_rx, + &new_file_name, + EventKind::Modify(ModifyKind::Name(RenameMode::To)), + ) + .await; + + #[cfg(target_os = "macos")] + expect_event( + events_rx, + &file_path, + EventKind::Modify(ModifyKind::Name(RenameMode::Any)), + ) + .await; + + #[cfg(target_os = "linux")] + expect_event( + events_rx, + &file_path, + EventKind::Modify(ModifyKind::Name(RenameMode::Both)), + ) + .await; + + debug!("Unwatching root directory: {}", root_dir.path().display()); + if let Err(e) = watcher.unwatch(root_dir.path()) { + error!("Failed to unwatch root directory: {e:#?}"); + } + } + + #[tokio::test] + #[traced_test] + async fn update_dir_event() { + let (root_dir, mut watcher, events_rx) = setup_watcher().await; + + let dir_path = root_dir.path().join("inner"); + fs::create_dir(&dir_path) + .await + .expect("Failed to create directory"); + + watcher + .watch(root_dir.path(), notify::RecursiveMode::Recursive) + .expect("Failed to watch root directory"); + debug!("Now watching {}", root_dir.path().display()); + + let new_dir_name = root_dir.path().join("inner2"); + + fs::rename(&dir_path, &new_dir_name) + .await + .expect("Failed to rename directory"); + + #[cfg(target_os = "windows")] + expect_event( + events_rx, + &new_dir_name, + EventKind::Modify(ModifyKind::Name(RenameMode::To)), + ) + .await; + + #[cfg(target_os = "macos")] + expect_event( + events_rx, + &dir_path, + EventKind::Modify(ModifyKind::Name(RenameMode::Any)), + ) + .await; + + #[cfg(target_os = "linux")] + expect_event( + events_rx, + &dir_path, + EventKind::Modify(ModifyKind::Name(RenameMode::Both)), + ) + .await; + + debug!("Unwatching root directory: {}", root_dir.path().display()); + if let Err(e) = watcher.unwatch(root_dir.path()) { + error!("Failed to unwatch root directory: {e:#?}"); + } + } + + #[tokio::test] + #[traced_test] + async fn delete_file_event() { + let (root_dir, mut watcher, events_rx) = setup_watcher().await; + + let file_path = root_dir.path().join("test.txt"); + fs::write(&file_path, "test").await.unwrap(); + + watcher + .watch(root_dir.path(), notify::RecursiveMode::Recursive) + .expect("Failed to watch root directory"); + debug!("Now watching {}", root_dir.path().display()); + + fs::remove_file(&file_path) + .await + .expect("Failed to remove file"); + + #[cfg(target_os = "windows")] + expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::Any)).await; + + #[cfg(target_os = "macos")] + expect_event( + events_rx, + &root_dir.path(), + EventKind::Modify(ModifyKind::Data(DataChange::Any)), + ) + .await; + + #[cfg(target_os = "linux")] + expect_event(events_rx, &file_path, EventKind::Remove(RemoveKind::File)).await; + + debug!("Unwatching root directory: {}", root_dir.path().display()); + if let Err(e) = watcher.unwatch(root_dir.path()) { + error!("Failed to unwatch root directory: {e:#?}"); + } + } + + #[tokio::test] + #[traced_test] + async fn delete_dir_event() { + let (root_dir, mut watcher, events_rx) = setup_watcher().await; + + let dir_path = root_dir.path().join("inner"); + fs::create_dir(&dir_path) + .await + .expect("Failed to create directory"); + + if let Err(e) = fs::metadata(&dir_path).await { + if e.kind() == ErrorKind::NotFound { + panic!("Directory not found"); + } else { + panic!("{e}"); + } + } + + watcher + .watch(root_dir.path(), notify::RecursiveMode::Recursive) + .expect("Failed to watch root directory"); + debug!("Now watching {}", root_dir.path().display()); + + debug!("First unwatching the inner directory before removing it"); + if let Err(e) = watcher.unwatch(&dir_path) { + error!("Failed to unwatch inner directory: {e:#?}"); + } + + fs::remove_dir(&dir_path) + .await + .expect("Failed to remove directory"); + + #[cfg(target_os = "windows")] + expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Any)).await; + + #[cfg(target_os = "macos")] + expect_event( + events_rx, + &root_dir.path(), + EventKind::Modify(ModifyKind::Data(DataChange::Any)), + ) + .await; + + #[cfg(target_os = "linux")] + expect_event(events_rx, &dir_path, EventKind::Remove(RemoveKind::Folder)).await; + + debug!("Unwatching root directory: {}", root_dir.path().display()); + if let Err(e) = watcher.unwatch(root_dir.path()) { + error!("Failed to unwatch root directory: {e:#?}"); + } + } +} diff --git a/core/src/location/manager/watcher/utils.rs b/core/src/location/manager/watcher/utils.rs new file mode 100644 index 000000000..bd31375a2 --- /dev/null +++ b/core/src/location/manager/watcher/utils.rs @@ -0,0 +1,596 @@ +use crate::{ + invalidate_query, + library::LibraryContext, + location::{ + delete_directory, + file_path_helper::create_file_path, + indexer::indexer_job::indexer_job_location, + manager::{helpers::subtract_location_path, LocationId, LocationManagerError}, + }, + object::{ + identifier_job::{assemble_object_metadata, ObjectCreationMetadata}, + preview::{ + can_generate_thumbnail_for_image, generate_image_thumbnail, THUMBNAIL_CACHE_DIR_NAME, + }, + validation::hash::file_checksum, + }, + prisma::{file_path, object}, +}; + +use std::{ + path::{Path, PathBuf}, + str::FromStr, +}; + +use chrono::{FixedOffset, Utc}; +use int_enum::IntEnum; +use notify::{event::RemoveKind, Event}; +use prisma_client_rust::{raw, PrismaValue}; +use sd_file_ext::extensions::ImageExtension; +use tokio::{fs, io::ErrorKind}; +use tracing::{error, info, trace, warn}; + +use super::file_path_with_object; + +pub(super) fn check_location_online(location: &indexer_job_location::Data) -> bool { + // if location is offline return early + // this prevents .... + if !location.is_online { + info!( + "Location is offline, skipping event: ", + location.id + ); + false + } else { + true + } +} + +pub(super) fn check_event(event: &Event) -> bool { + // if first path includes .DS_Store, ignore + if event.paths.iter().any(|p| { + p.to_str() + .expect("Found non-UTF-8 path") + .contains(".DS_Store") + }) { + return false; + } + + true +} + +pub(super) async fn create_dir( + location: indexer_job_location::Data, + event: Event, + library_ctx: LibraryContext, +) -> Result<(), LocationManagerError> { + if let Some(ref location_local_path) = location.local_path { + trace!( + "Location: creating directory: {}", + event.paths[0].display() + ); + + if let Some(subpath) = subtract_location_path(location_local_path, &event.paths[0]) { + let parent_directory = get_parent_dir(location.id, &subpath, &library_ctx).await?; + + trace!("parent_directory: {:?}", parent_directory); + + if let Some(parent_directory) = parent_directory { + let created_path = create_file_path( + &library_ctx, + location.id, + subpath.to_str().expect("Found non-UTF-8 path").to_string(), + subpath + .file_stem() + .unwrap() + .to_str() + .expect("Found non-UTF-8 path") + .to_string(), + None, + Some(parent_directory.id), + true, + ) + .await?; + + info!("Created path: {}", created_path.materialized_path); + + invalidate_query!(library_ctx, "locations.getExplorerData"); + } else { + warn!("Watcher found a path without parent"); + } + } + } + + Ok(()) +} + +pub(super) async fn create_file( + location: indexer_job_location::Data, + event: Event, + library_ctx: LibraryContext, +) -> Result<(), LocationManagerError> { + if let Some(ref location_local_path) = location.local_path { + inner_create_file(location.id, location_local_path, event, &library_ctx).await + } else { + Err(LocationManagerError::LocationMissingLocalPath(location.id)) + } +} + +async fn inner_create_file( + location_id: LocationId, + location_local_path: &str, + event: Event, + library_ctx: &LibraryContext, +) -> Result<(), LocationManagerError> { + trace!( + "Location: creating file: {}", + event.paths[0].display() + ); + if let Some(materialized_path) = subtract_location_path(location_local_path, &event.paths[0]) { + if let Some(parent_directory) = + get_parent_dir(location_id, &materialized_path, library_ctx).await? + { + let created_file = create_file_path( + library_ctx, + location_id, + materialized_path + .to_str() + .expect("Found non-UTF-8 path") + .to_string(), + materialized_path + .file_stem() + .unwrap_or_default() + .to_str() + .expect("Found non-UTF-8 path") + .to_string(), + materialized_path.extension().and_then(|ext| { + if ext.is_empty() { + None + } else { + Some(ext.to_str().expect("Found non-UTF-8 path").to_string()) + } + }), + Some(parent_directory.id), + false, + ) + .await?; + + info!("Created path: {}", created_file.materialized_path); + + // generate provisional object + let ObjectCreationMetadata { + cas_id, + size_str, + kind, + date_created, + } = assemble_object_metadata(location_local_path, &created_file).await?; + + // upsert object because in can be from a file that previously existed and was moved + let object = library_ctx + .db + .object() + .upsert( + object::cas_id::equals(cas_id.clone()), + ( + cas_id.clone(), + size_str.clone(), + vec![ + object::date_created::set(date_created), + object::kind::set(kind.int_value()), + ], + ), + vec![ + object::size_in_bytes::set(size_str), + object::date_indexed::set( + Utc::now().with_timezone(&FixedOffset::east_opt(0).unwrap()), + ), + ], + ) + .exec() + .await?; + + library_ctx + .db + .file_path() + .update( + file_path::location_id_id(location_id, created_file.id), + vec![file_path::object_id::set(Some(object.id))], + ) + .exec() + .await?; + + trace!("object: {:#?}", object); + if !object.has_thumbnail { + if let Some(ref extension) = created_file.extension { + generate_thumbnail(extension, &cas_id, &event.paths[0], library_ctx).await; + } + } + + invalidate_query!(library_ctx, "locations.getExplorerData"); + } else { + warn!("Watcher found a path without parent"); + } + } + + Ok(()) +} + +pub(super) async fn file_creation_or_update( + location: indexer_job_location::Data, + event: Event, + library_ctx: &LibraryContext, +) -> Result<(), LocationManagerError> { + if let Some(ref location_local_path) = location.local_path { + if let Some(file_path) = + get_existing_file_path(&location, &event.paths[0], false, library_ctx).await? + { + inner_update_file(location_local_path, file_path, event, library_ctx).await + } else { + // We received None because it is a new file + inner_create_file(location.id, location_local_path, event, library_ctx).await + } + } else { + Err(LocationManagerError::LocationMissingLocalPath(location.id)) + } +} + +pub(super) async fn update_file( + location: indexer_job_location::Data, + event: Event, + library_ctx: &LibraryContext, +) -> Result<(), LocationManagerError> { + if let Some(ref location_local_path) = location.local_path { + if let Some(file_path) = + get_existing_file_path(&location, &event.paths[0], false, library_ctx).await? + { + let ret = inner_update_file(location_local_path, file_path, event, library_ctx).await; + invalidate_query!(library_ctx, "locations.getExplorerData"); + ret + } else { + Err(LocationManagerError::UpdateNonExistingFile( + event.paths[0].clone(), + )) + } + } else { + Err(LocationManagerError::LocationMissingLocalPath(location.id)) + } +} + +async fn inner_update_file( + location_local_path: &str, + file_path: file_path_with_object::Data, + event: Event, + library_ctx: &LibraryContext, +) -> Result<(), LocationManagerError> { + trace!( + "Location: updating file: {}", + event.paths[0].display() + ); + // We have to separate this object, as the `assemble_object_metadata` doesn't + // accept `file_path_with_object::Data` + let file_path_only = file_path::Data { + id: file_path.id, + is_dir: file_path.is_dir, + location_id: file_path.location_id, + location: None, + materialized_path: file_path.materialized_path, + name: file_path.name, + extension: file_path.extension, + object_id: file_path.object_id, + object: None, + parent_id: file_path.parent_id, + key_id: file_path.key_id, + date_created: file_path.date_created, + date_modified: file_path.date_modified, + date_indexed: file_path.date_indexed, + key: None, + }; + let ObjectCreationMetadata { + cas_id, + size_str, + kind, + date_created, + } = assemble_object_metadata(location_local_path, &file_path_only).await?; + + if let Some(ref object) = file_path.object { + if object.cas_id != cas_id { + // file content changed + library_ctx + .db + .object() + .update( + object::id::equals(object.id), + vec![ + object::cas_id::set(cas_id.clone()), + object::size_in_bytes::set(size_str), + object::kind::set(kind.int_value()), + object::date_modified::set(date_created), + object::integrity_checksum::set(if object.integrity_checksum.is_some() { + // If a checksum was already computed, we need to recompute it + Some(file_checksum(&event.paths[0]).await?) + } else { + None + }), + ], + ) + .exec() + .await?; + + if object.has_thumbnail { + // if this file had a thumbnail previously, we update it to match the new content + if let Some(ref extension) = file_path_only.extension { + generate_thumbnail(extension, &cas_id, &event.paths[0], library_ctx).await; + } + } + } + } + + invalidate_query!(library_ctx, "locations.getExplorerData"); + + Ok(()) +} + +pub(super) async fn rename_both_event( + location: indexer_job_location::Data, + event: Event, + library_ctx: &LibraryContext, +) -> Result<(), LocationManagerError> { + rename(&event.paths[1], &event.paths[0], location, library_ctx).await +} + +pub(super) async fn rename( + new_path: impl AsRef, + old_path: impl AsRef, + location: indexer_job_location::Data, + library_ctx: &LibraryContext, +) -> Result<(), LocationManagerError> { + let mut old_path_materialized = extract_materialized_path(&location, old_path.as_ref())? + .to_str() + .expect("Found non-UTF-8 path") + .to_string(); + + let new_path_materialized = extract_materialized_path(&location, new_path.as_ref())?; + let mut new_path_materialized_str = new_path_materialized + .to_str() + .expect("Found non-UTF-8 path") + .to_string(); + + if let Some(file_path) = + get_existing_file_or_directory(&location, old_path, library_ctx).await? + { + // If the renamed path is a directory, we have to update every successor + if file_path.is_dir { + if !old_path_materialized.ends_with('/') { + old_path_materialized += "/"; + } + if !new_path_materialized_str.ends_with('/') { + new_path_materialized_str += "/"; + } + + let updated = library_ctx + .db + ._execute_raw( + raw!( + "UPDATE file_path SET materialized_path = REPLACE(materialized_path, {}, {}) WHERE location_id = {}", + PrismaValue::String(old_path_materialized), + PrismaValue::String(new_path_materialized_str.clone()), + PrismaValue::Int(location.id as i64) + ) + ) + .exec() + .await?; + trace!("Updated {updated} file_paths"); + } + + library_ctx + .db + .file_path() + .update( + file_path::location_id_id(file_path.location_id, file_path.id), + vec![ + file_path::materialized_path::set(new_path_materialized_str), + file_path::name::set( + new_path_materialized + .file_stem() + .unwrap() + .to_str() + .expect("Found non-UTF-8 path") + .to_string(), + ), + file_path::extension::set( + new_path_materialized + .extension() + .map(|s| s.to_str().expect("Found non-UTF-8 path").to_string()), + ), + ], + ) + .exec() + .await?; + invalidate_query!(library_ctx, "locations.getExplorerData"); + } + + Ok(()) +} + +pub(super) async fn remove_event( + location: indexer_job_location::Data, + event: Event, + remove_kind: RemoveKind, + library_ctx: &LibraryContext, +) -> Result<(), LocationManagerError> { + trace!("removed {remove_kind:#?}"); + + // if it doesn't either way, then we don't care + if let Some(file_path) = + get_existing_file_or_directory(&location, &event.paths[0], library_ctx).await? + { + // check file still exists on disk + match fs::metadata(&event.paths[0]).await { + Ok(_) => { + todo!("file has changed in some way, re-identify it") + } + Err(e) if e.kind() == ErrorKind::NotFound => { + // if is doesn't, we can remove it safely from our db + if file_path.is_dir { + delete_directory(library_ctx, location.id, Some(file_path.materialized_path)) + .await?; + } else { + library_ctx + .db + .file_path() + .delete(file_path::location_id_id(location.id, file_path.id)) + .exec() + .await?; + + if let Some(object_id) = file_path.object_id { + library_ctx + .db + .object() + .delete_many(vec![ + object::id::equals(object_id), + // https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#none + object::file_paths::none(vec![]), + ]) + .exec() + .await?; + } + } + } + Err(e) => return Err(e.into()), + } + + invalidate_query!(library_ctx, "locations.getExplorerData"); + } + + Ok(()) +} + +fn extract_materialized_path( + location: &indexer_job_location::Data, + path: impl AsRef, +) -> Result { + subtract_location_path( + location + .local_path + .as_ref() + .ok_or(LocationManagerError::LocationMissingLocalPath(location.id))?, + &path, + ) + .ok_or_else(|| { + LocationManagerError::UnableToExtractMaterializedPath( + location.id, + path.as_ref().to_path_buf(), + ) + }) +} + +async fn get_existing_file_path( + location: &indexer_job_location::Data, + path: impl AsRef, + is_dir: bool, + library_ctx: &LibraryContext, +) -> Result, LocationManagerError> { + let mut materialized_path = extract_materialized_path(location, path)? + .to_str() + .expect("Found non-UTF-8 path") + .to_string(); + if is_dir && !materialized_path.ends_with('/') { + materialized_path += "/"; + } + + library_ctx + .db + .file_path() + .find_first(vec![file_path::materialized_path::equals( + materialized_path, + )]) + // include object for orphan check + .include(file_path_with_object::include()) + .exec() + .await + .map_err(Into::into) +} + +async fn get_existing_file_or_directory( + location: &indexer_job_location::Data, + path: impl AsRef, + library_ctx: &LibraryContext, +) -> Result, LocationManagerError> { + let mut maybe_file_path = + get_existing_file_path(location, path.as_ref(), false, library_ctx).await?; + // First we just check if this path was a file in our db, if it isn't then we check for a directory + if maybe_file_path.is_none() { + maybe_file_path = + get_existing_file_path(location, path.as_ref(), true, library_ctx).await?; + } + + Ok(maybe_file_path) +} + +async fn get_parent_dir( + location_id: LocationId, + path: impl AsRef, + library_ctx: &LibraryContext, +) -> Result, LocationManagerError> { + let mut parent_path_str = path + .as_ref() + .parent() + // We have an "/" `materialized_path` for each location_id + .unwrap_or_else(|| Path::new("/")) + .to_str() + .expect("Found non-UTF-8 path") + .to_string(); + + // As we're looking specifically for a parent directory, it must end with '/' + if !parent_path_str.ends_with('/') { + parent_path_str += "/"; + } + + library_ctx + .db + .file_path() + .find_first(vec![ + file_path::location_id::equals(location_id), + file_path::materialized_path::equals(parent_path_str), + ]) + .exec() + .await + .map_err(Into::into) +} + +async fn generate_thumbnail( + extension: &str, + cas_id: &str, + file_path: impl AsRef, + library_ctx: &LibraryContext, +) { + let file_path = file_path.as_ref(); + let output_path = library_ctx + .config() + .data_directory() + .join(THUMBNAIL_CACHE_DIR_NAME) + .join(cas_id) + .with_extension("webp"); + + if let Ok(extension) = ImageExtension::from_str(extension) { + if can_generate_thumbnail_for_image(&extension) { + if let Err(e) = generate_image_thumbnail(file_path, &output_path).await { + error!("Failed to image thumbnail on location manager: {e:#?}"); + } + } + } + + #[cfg(feature = "ffmpeg")] + { + use crate::object::preview::{can_generate_thumbnail_for_video, generate_video_thumbnail}; + use sd_file_ext::extensions::VideoExtension; + + if let Ok(extension) = VideoExtension::from_str(extension) { + if can_generate_thumbnail_for_video(&extension) { + if let Err(e) = generate_video_thumbnail(file_path, &output_path).await { + error!("Failed to video thumbnail on location manager: {e:#?}"); + } + } + } + } +} diff --git a/core/src/location/manager/watcher/windows.rs b/core/src/location/manager/watcher/windows.rs new file mode 100644 index 000000000..dc6b26209 --- /dev/null +++ b/core/src/location/manager/watcher/windows.rs @@ -0,0 +1,84 @@ +use crate::{ + library::LibraryContext, + location::{indexer::indexer_job::indexer_job_location, manager::LocationManagerError}, +}; + +use async_trait::async_trait; +use notify::{ + event::{CreateKind, ModifyKind, RenameMode}, + Event, EventKind, +}; +use tokio::fs; +use tracing::{trace, warn}; + +use super::{ + utils::{create_dir, create_file, remove_event, rename, update_file}, + EventHandler, +}; + +#[derive(Debug, Default)] +pub(super) struct WindowsEventHandler { + rename_stack: Option, + create_file_stack: Option, +} + +#[async_trait] +impl EventHandler for WindowsEventHandler { + fn new() -> Self + where + Self: Sized, + { + Default::default() + } + + async fn handle_event( + &mut self, + location: indexer_job_location::Data, + library_ctx: &LibraryContext, + event: Event, + ) -> Result<(), LocationManagerError> { + trace!("Received Windows event: {:#?}", event); + + match event.kind { + EventKind::Create(CreateKind::Any) => { + let metadata = fs::metadata(&event.paths[0]).await?; + if metadata.is_file() { + self.create_file_stack = Some(event); + } else { + create_dir(location, event, library_ctx.clone()).await?; + } + } + EventKind::Modify(ModifyKind::Any) => { + let metadata = fs::metadata(&event.paths[0]).await?; + if metadata.is_file() { + if let Some(create_file_event) = self.create_file_stack.take() { + create_file(location, create_file_event, library_ctx.clone()).await?; + } else { + update_file(location, event, library_ctx).await?; + } + } else { + warn!("Unexpected Windows modify event on a directory"); + } + } + EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { + self.rename_stack = Some(event); + } + EventKind::Modify(ModifyKind::Name(RenameMode::To)) => { + let from_event = self + .rename_stack + .take() + .expect("Unexpectedly missing rename from windows event"); + rename(&event.paths[0], &from_event.paths[0], location, library_ctx).await?; + } + EventKind::Remove(remove_kind) => { + remove_event(location, event, remove_kind, library_ctx).await?; + } + + other_event_kind => { + trace!("Other Windows event that we don't handle for now: {other_event_kind:#?}"); + } + } + + Ok(()) + } +} diff --git a/core/src/location/metadata.rs b/core/src/location/metadata.rs new file mode 100644 index 000000000..63790adb3 --- /dev/null +++ b/core/src/location/metadata.rs @@ -0,0 +1,229 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{fs, io}; +use uuid::Uuid; + +static SPACEDRIVE_LOCATION_METADATA_FILE: &str = ".spacedrive"; + +pub(super) type LibraryId = Uuid; +pub(super) type LocationPubId = Uuid; + +#[derive(Serialize, Deserialize, Default, Debug)] +struct LocationMetadata { + pub_id: LocationPubId, + name: String, + path: PathBuf, + created_at: DateTime, + updated_at: DateTime, +} + +#[derive(Serialize, Deserialize, Default, Debug)] +struct SpacedriveLocationMetadata { + libraries: HashMap, + created_at: DateTime, + updated_at: DateTime, +} + +pub(super) struct SpacedriveLocationMetadataFile { + path: PathBuf, + metadata: SpacedriveLocationMetadata, +} + +impl SpacedriveLocationMetadataFile { + pub(super) async fn try_load( + location_path: impl AsRef, + ) -> Result, LocationMetadataError> { + let metadata_file_name = location_path + .as_ref() + .join(SPACEDRIVE_LOCATION_METADATA_FILE); + + match fs::read(&metadata_file_name).await { + Ok(data) => Ok(Some(Self { + path: metadata_file_name, + metadata: serde_json::from_slice(&data).map_err(|e| { + LocationMetadataError::Deserialize(e, location_path.as_ref().to_path_buf()) + })?, + })), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(LocationMetadataError::Read( + e, + location_path.as_ref().to_path_buf(), + )), + } + } + + pub(super) async fn create_and_save( + library_id: LibraryId, + location_pub_id: Uuid, + location_path: impl AsRef, + location_name: String, + ) -> Result<(), LocationMetadataError> { + Self { + path: location_path + .as_ref() + .join(SPACEDRIVE_LOCATION_METADATA_FILE), + metadata: SpacedriveLocationMetadata { + libraries: [( + library_id, + LocationMetadata { + pub_id: location_pub_id, + name: location_name, + path: location_path.as_ref().to_path_buf(), + created_at: Utc::now(), + updated_at: Utc::now(), + }, + )] + .into_iter() + .collect(), + created_at: Utc::now(), + updated_at: Utc::now(), + }, + } + .write_metadata() + .await + } + + pub(super) async fn relink( + &mut self, + library_id: LibraryId, + location_path: impl AsRef, + ) -> Result<(), LocationMetadataError> { + let location_metadata = self + .metadata + .libraries + .get_mut(&library_id) + .ok_or(LocationMetadataError::LibraryNotFound(library_id))?; + + let new_path = location_path.as_ref().to_path_buf(); + if location_metadata.path == new_path { + return Err(LocationMetadataError::RelinkSamePath(new_path)); + } + + location_metadata.path = new_path; + location_metadata.updated_at = Utc::now(); + self.path = location_path + .as_ref() + .join(SPACEDRIVE_LOCATION_METADATA_FILE); + + self.write_metadata().await + } + + pub(super) async fn update( + &mut self, + library_id: LibraryId, + location_name: String, + ) -> Result<(), LocationMetadataError> { + let location_metadata = self + .metadata + .libraries + .get_mut(&library_id) + .ok_or(LocationMetadataError::LibraryNotFound(library_id))?; + + location_metadata.name = location_name; + location_metadata.updated_at = Utc::now(); + + self.write_metadata().await + } + + pub(super) async fn add_library( + &mut self, + library_id: LibraryId, + location_pub_id: Uuid, + location_path: impl AsRef, + location_name: String, + ) -> Result<(), LocationMetadataError> { + self.metadata.libraries.insert( + library_id, + LocationMetadata { + pub_id: location_pub_id, + name: location_name, + path: location_path.as_ref().to_path_buf(), + created_at: Utc::now(), + updated_at: Utc::now(), + }, + ); + + self.metadata.updated_at = Utc::now(); + self.write_metadata().await + } + + pub(super) fn has_library(&self, library_id: LibraryId) -> bool { + self.metadata.libraries.contains_key(&library_id) + } + + pub(super) fn location_path( + &self, + library_id: LibraryId, + ) -> Result<&Path, LocationMetadataError> { + self.metadata + .libraries + .get(&library_id) + .map(|l| l.path.as_path()) + .ok_or(LocationMetadataError::LibraryNotFound(library_id)) + } + + pub(super) async fn remove_library( + &mut self, + library_id: LibraryId, + ) -> Result<(), LocationMetadataError> { + self.metadata + .libraries + .remove(&library_id) + .ok_or(LocationMetadataError::LibraryNotFound(library_id))?; + + self.metadata.updated_at = Utc::now(); + + if !self.metadata.libraries.is_empty() { + self.write_metadata().await + } else { + fs::remove_file(&self.path) + .await + .map_err(|e| LocationMetadataError::Delete(e, self.path.clone())) + } + } + + pub(super) fn location_pub_id( + &self, + library_id: LibraryId, + ) -> Result { + self.metadata + .libraries + .get(&library_id) + .ok_or(LocationMetadataError::LibraryNotFound(library_id)) + .map(|m| m.pub_id) + } + + async fn write_metadata(&self) -> Result<(), LocationMetadataError> { + fs::write( + &self.path, + serde_json::to_vec(&self.metadata) + .map_err(|e| LocationMetadataError::Serialize(e, self.path.clone()))?, + ) + .await + .map_err(|e| LocationMetadataError::Write(e, self.path.clone())) + } +} + +#[derive(Error, Debug)] +pub enum LocationMetadataError { + #[error("Library not found: {0}")] + LibraryNotFound(LibraryId), + #[error("Failed to read location metadata file (path: {1:?}); (error: {0:?})")] + Read(io::Error, PathBuf), + #[error("Failed to delete location metadata file (path: {1:?}); (error: {0:?})")] + Delete(io::Error, PathBuf), + #[error("Failed to serialize metadata file for location (at path: {1:?}); (error: {0:?})")] + Serialize(serde_json::Error, PathBuf), + #[error("Failed to write location metadata file (path: {1:?}); (error: {0:?})")] + Write(io::Error, PathBuf), + #[error("Failed to deserialize metadata file for location (at path: {1:?}); (error: {0:?})")] + Deserialize(serde_json::Error, PathBuf), + #[error("Failed to relink, as the new location path is the same as the old path")] + RelinkSamePath(PathBuf), +} diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index 3fb090961..892daf7c4 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -3,32 +3,34 @@ use crate::{ job::Job, library::LibraryContext, object::{ - identifier_job::{FileIdentifierJob, FileIdentifierJobInit}, + identifier_job::full_identifier_job::{FullFileIdentifierJob, FullFileIdentifierJobInit}, preview::{ThumbnailJob, ThumbnailJobInit}, - validation::validator_job::{ObjectValidatorJob, ObjectValidatorJobInit}, }, - prisma::{indexer_rules_in_location, location, node}, + prisma::{file_path, indexer_rules_in_location, location, node, object}, }; -use rspc::Type; -use serde::{Deserialize, Serialize}; -use std::{collections::HashSet, path::PathBuf}; -use tokio::{ - fs::{metadata, File}, - io::AsyncWriteExt, +use std::{ + collections::HashSet, + path::{Path, PathBuf}, }; + +use prisma_client_rust::QueryError; +use rspc::Type; +use serde::Deserialize; +use tokio::{fs, io}; use tracing::{debug, info}; use uuid::Uuid; mod error; +pub mod file_path_helper; pub mod indexer; +mod manager; +mod metadata; pub use error::LocationError; -use indexer::indexer_job::{IndexerJob, IndexerJobInit}; - -use self::indexer::indexer_job::indexer_job_location; - -static DOTFILE_NAME: &str = ".spacedrive"; +use indexer::indexer_job::{indexer_job_location, IndexerJob, IndexerJobInit}; +pub use manager::{LocationManager, LocationManagerError}; +use metadata::SpacedriveLocationMetadataFile; /// `LocationCreateArgs` is the argument received from the client using `rspc` to create a new location. /// It has the actual path and a vector of indexer rules ids, to create many-to-many relationships @@ -44,36 +46,36 @@ impl LocationCreateArgs { self, ctx: &LibraryContext, ) -> Result { - // check if we have access to this location - if !self.path.try_exists().unwrap() { - return Err(LocationError::PathNotFound(self.path)); - } - - let path_metadata = metadata(&self.path) - .await - .map_err(|e| LocationError::DotfileReadFailure(e, self.path.clone()))?; + let path_metadata = match fs::metadata(&self.path).await { + Ok(metadata) => metadata, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(LocationError::PathNotFound(self.path)) + } + Err(e) => { + return Err(LocationError::LocationPathFilesystemMetadataAccess( + e, self.path, + )); + } + }; if path_metadata.permissions().readonly() { - return Err(LocationError::ReadonlyDotFileLocationFailure(self.path)); + return Err(LocationError::ReadonlyLocationFailure(self.path)); } if !path_metadata.is_dir() { return Err(LocationError::NotDirectory(self.path)); } - // check if the location already exists - let _location_exists = ctx - .db - .location() - .find_first(vec![location::local_path::equals(Some( - self.path.to_string_lossy().to_string(), - ))]) - .exec() - .await? - .is_some(); - - if _location_exists { - return Err(LocationError::LocationAlreadyExists(self.path)); + if let Some(metadata) = SpacedriveLocationMetadataFile::try_load(&self.path).await? { + return if metadata.has_library(ctx.id) { + Err(LocationError::NeedRelink { + // SAFETY: This unwrap is ok as we checked that we have this library_id + old_path: metadata.location_path(ctx.id).unwrap().to_path_buf(), + new_path: self.path, + }) + } else { + Err(LocationError::AddLibraryToMetadata(self.path)) + }; } debug!( @@ -82,54 +84,61 @@ impl LocationCreateArgs { ); let uuid = Uuid::new_v4(); - let mut location = ctx - .db - .location() - .create( - uuid.as_bytes().to_vec(), - node::id::equals(ctx.node_local_id), - vec![ - location::name::set(Some( - self.path.file_name().unwrap().to_str().unwrap().to_string(), - )), - location::is_online::set(true), - location::local_path::set(Some(self.path.to_string_lossy().to_string())), - ], - ) - .include(indexer_job_location::include()) - .exec() - .await?; + let location = create_location(ctx, uuid, &self.path, &self.indexer_rules_ids).await?; - info!("Created location: {:?}", location); + // Write a location metadata on a .spacedrive file + SpacedriveLocationMetadataFile::create_and_save( + ctx.id, + uuid, + &self.path, + location.name.as_ref().unwrap().clone(), + ) + .await?; - if !self.indexer_rules_ids.is_empty() { - link_location_and_indexer_rules(ctx, location.id, &self.indexer_rules_ids).await?; + info!("Created location: {location:?}"); + + Ok(location) + } + + pub async fn add_library( + self, + ctx: &LibraryContext, + ) -> Result { + let mut metadata = SpacedriveLocationMetadataFile::try_load(&self.path) + .await? + .ok_or_else(|| LocationError::MetadataNotFound(self.path.clone()))?; + + if metadata.has_library(ctx.id) { + return Err(LocationError::NeedRelink { + // SAFETY: This unwrap is ok as we checked that we have this library_id + old_path: metadata.location_path(ctx.id).unwrap().to_path_buf(), + new_path: self.path, + }); } - // Updating our location variable to include information about the indexer rules - location = fetch_location(ctx, location.id) - .include(indexer_job_location::include()) - .exec() - .await? - .ok_or(LocationError::IdNotFound(location.id))?; + debug!( + "Trying to add a new library (library_id = {}) to an already existing location '{}'", + ctx.id, + self.path.display() + ); - // write a file called .spacedrive to path containing the location id in JSON format - let mut dotfile = File::create(self.path.join(DOTFILE_NAME)) - .await - .map_err(|e| LocationError::DotfileWriteFailure(e, self.path.clone()))?; + let uuid = Uuid::new_v4(); - let json_bytes = serde_json::to_vec(&DotSpacedrive { - location_uuid: uuid, - library_uuid: ctx.id, - }) - .map_err(|e| LocationError::DotfileSerializeFailure(e, self.path.clone()))?; + let location = create_location(ctx, uuid, &self.path, &self.indexer_rules_ids).await?; - dotfile - .write_all(&json_bytes) - .await - .map_err(|e| LocationError::DotfileWriteFailure(e, self.path))?; + metadata + .add_library( + ctx.id, + uuid, + &self.path, + location.name.as_ref().unwrap().clone(), + ) + .await?; - invalidate_query!(ctx, "locations.list"); + info!( + "Added library (library_id = {}) to location: {location:?}", + ctx.id + ); Ok(location) } @@ -156,15 +165,23 @@ impl LocationUpdateArgs { .await? .ok_or(LocationError::IdNotFound(self.id))?; - if location.name != self.name { + if self.name.is_some() && location.name != self.name { ctx.db .location() .update( location::id::equals(self.id), - vec![location::name::set(self.name)], + vec![location::name::set(self.name.clone())], ) .exec() .await?; + + if let Some(ref local_path) = location.local_path { + if let Some(mut metadata) = + SpacedriveLocationMetadataFile::try_load(local_path).await? + { + metadata.update(ctx.id, self.name.unwrap().clone()).await?; + } + } } let current_rules_ids = location @@ -205,25 +222,6 @@ impl LocationUpdateArgs { } } -#[derive(Serialize, Deserialize, Default)] -pub struct DotSpacedrive { - pub location_uuid: Uuid, - pub library_uuid: Uuid, -} - -// checks to see if a location is: -// - accessible on from the local filesystem -// - already exists in the database -// pub async fn check_location(path: &str) -> Result { -// let dotfile: DotSpacedrive = match fs::File::open(format!("{}/{}", path.clone(), DOTFILE_NAME)) -// { -// Ok(file) => serde_json::from_reader(file).unwrap_or(DotSpacedrive::default()), -// Err(e) => return Err(LocationError::DotfileReadFailure(e)), -// }; - -// Ok(dotfile) -// } - pub fn fetch_location(ctx: &LibraryContext, location_id: i32) -> location::FindUnique { ctx.db .location() @@ -257,38 +255,201 @@ pub async fn scan_location( return Err(LocationError::MissingLocalPath(location.id)); }; - let location_id = location.id; ctx.queue_job(Job::new( - FileIdentifierJobInit { + FullFileIdentifierJobInit { location_id: location.id, sub_path: None, }, - Box::new(FileIdentifierJob {}), - )) - .await; - ctx.spawn_job(Job::new( - IndexerJobInit { location }, - Box::new(IndexerJob {}), - )) - .await; - ctx.queue_job(Job::new( - ThumbnailJobInit { - location_id, - path: PathBuf::new(), - background: true, - }, - Box::new(ThumbnailJob {}), - )) - .await; - ctx.queue_job(Job::new( - ObjectValidatorJobInit { - location_id, - path: PathBuf::new(), - background: true, - }, - Box::new(ObjectValidatorJob {}), + FullFileIdentifierJob {}, )) .await; + ctx.queue_job(Job::new( + ThumbnailJobInit { + location_id: location.id, + root_path: PathBuf::new(), + background: true, + }, + ThumbnailJob {}, + )) + .await; + + ctx.spawn_job(Job::new(IndexerJobInit { location }, IndexerJob {})) + .await; + + Ok(()) +} + +pub async fn relink_location( + ctx: &LibraryContext, + location_path: impl AsRef, +) -> Result<(), LocationError> { + let mut metadata = SpacedriveLocationMetadataFile::try_load(&location_path) + .await? + .ok_or_else(|| LocationError::MissingMetadataFile(location_path.as_ref().to_path_buf()))?; + + metadata.relink(ctx.id, &location_path).await?; + + ctx.db + .location() + .update( + location::pub_id::equals(metadata.location_pub_id(ctx.id)?.as_ref().to_vec()), + vec![ + location::local_path::set(Some( + location_path + .as_ref() + .to_str() + .expect("Found non-UTF-8 path") + .to_string(), + )), + location::is_online::set(true), + ], + ) + .exec() + .await?; + + Ok(()) +} + +async fn create_location( + ctx: &LibraryContext, + location_pub_id: Uuid, + location_path: impl AsRef, + indexer_rules_ids: &[i32], +) -> Result { + let location_name = location_path + .as_ref() + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + + let mut location = ctx + .db + .location() + .create( + location_pub_id.as_bytes().to_vec(), + node::id::equals(ctx.node_local_id), + vec![ + location::name::set(Some(location_name.clone())), + location::is_online::set(true), + location::local_path::set(Some( + location_path + .as_ref() + .to_str() + .expect("Found non-UTF-8 path") + .to_string(), + )), + ], + ) + .include(indexer_job_location::include()) + .exec() + .await?; + + if !indexer_rules_ids.is_empty() { + link_location_and_indexer_rules(ctx, location.id, indexer_rules_ids).await?; + } + + // Updating our location variable to include information about the indexer rules + location = fetch_location(ctx, location.id) + .include(indexer_job_location::include()) + .exec() + .await? + .ok_or(LocationError::IdNotFound(location.id))?; + + invalidate_query!(ctx, "locations.list"); + + ctx.location_manager().add(location.id, ctx.clone()).await?; + + Ok(location) +} + +pub async fn delete_location(ctx: &LibraryContext, location_id: i32) -> Result<(), LocationError> { + ctx.location_manager() + .remove(location_id, ctx.clone()) + .await?; + + delete_directory(ctx, location_id, None).await?; + + ctx.db + .indexer_rules_in_location() + .delete_many(vec![indexer_rules_in_location::location_id::equals( + location_id, + )]) + .exec() + .await?; + + let location = ctx + .db + .location() + .delete(location::id::equals(location_id)) + .exec() + .await?; + + if let Some(local_path) = location.local_path { + if let Ok(Some(mut metadata)) = SpacedriveLocationMetadataFile::try_load(&local_path).await + { + metadata.remove_library(ctx.id).await?; + } + } + + info!("Location {} deleted", location_id); + invalidate_query!(ctx, "locations.list"); + + Ok(()) +} + +file_path::select!(file_path_object_id_only { object_id }); + +/// Will delete a directory recursively with Objects if left as orphans +/// this function is used to delete a location and when ingesting directory deletion events +pub async fn delete_directory( + ctx: &LibraryContext, + location_id: i32, + parent_materialized_path: Option, +) -> Result<(), QueryError> { + let children_params = if let Some(parent_materialized_path) = parent_materialized_path { + vec![ + file_path::location_id::equals(location_id), + file_path::materialized_path::starts_with(parent_materialized_path), + ] + } else { + vec![file_path::location_id::equals(location_id)] + }; + + // Fetching all object_ids from all children file_paths + let object_ids = ctx + .db + .file_path() + .find_many(children_params.clone()) + .select(file_path_object_id_only::select()) + .exec() + .await? + .into_iter() + .filter_map(|file_path| file_path.object_id) + .collect(); + + // WARNING: file_paths must be deleted before objects, as they reference objects through object_id + // delete all children file_paths + ctx.db + .file_path() + .delete_many(children_params) + .exec() + .await?; + + // delete all children objects + ctx.db + .object() + .delete_many(vec![ + object::id::in_vec(object_ids), + // https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#none + object::file_paths::none(vec![]), + ]) + .exec() + .await?; + + invalidate_query!(ctx, "locations.getExplorerData"); + Ok(()) } diff --git a/core/src/object/cas.rs b/core/src/object/cas.rs index d0cd6d227..77cd31c2d 100644 --- a/core/src/object/cas.rs +++ b/core/src/object/cas.rs @@ -1,5 +1,5 @@ use blake3::Hasher; -use std::path::PathBuf; +use std::path::Path; use tokio::{ fs::File, io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom}, @@ -17,7 +17,7 @@ async fn read_at(file: &mut File, offset: u64, size: u64) -> Result, io: Ok(buf) } -pub async fn generate_cas_id(path: PathBuf, size: u64) -> Result { +pub async fn generate_cas_id(path: impl AsRef, size: u64) -> Result { // open file reference let mut file = File::open(path).await?; @@ -43,6 +43,7 @@ pub async fn generate_cas_id(path: PathBuf, size: u64) -> Result, - ) -> Result<(), JobError> { + async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { // enumerate files to decrypt // populate the steps with them (local file paths) - let library = ctx.library_ctx(); - - let location = library + let location = ctx + .library_ctx .db .location() .find_unique(location::id::equals(state.init.location_id)) @@ -63,7 +58,8 @@ impl StatefulJob for FileDecryptorJob { .map(PathBuf::from) .expect("critical error: issue getting local path as pathbuf"); - let item = library + let item = ctx + .library_ctx .db .file_path() .find_first(vec![file_path::object_id::equals(Some( @@ -91,7 +87,7 @@ impl StatefulJob for FileDecryptorJob { async fn execute_step( &self, ctx: WorkerContext, - state: &mut JobState, + state: &mut JobState, ) -> Result<(), JobError> { let step = &state.steps[0]; // handle overwriting checks, and making sure there's enough available space @@ -133,7 +129,7 @@ impl StatefulJob for FileDecryptorJob { let index = header.find_key_index(password.clone())?; // inherit the encryption algorithm from the keyslot - ctx.library_ctx().key_manager.add_to_keystore( + ctx.library_ctx.key_manager.add_to_keystore( password.clone(), header.algorithm, header.keyslots[index].hashing_algorithm, @@ -150,7 +146,7 @@ impl StatefulJob for FileDecryptorJob { ))); } } else { - let keys = ctx.library_ctx().key_manager.enumerate_hashed_keys(); + let keys = ctx.library_ctx.key_manager.enumerate_hashed_keys(); header.decrypt_master_key_from_prehashed(keys)? }; @@ -169,11 +165,7 @@ impl StatefulJob for FileDecryptorJob { Ok(()) } - async fn finalize( - &self, - _ctx: WorkerContext, - state: &mut JobState, - ) -> JobResult { + async fn finalize(&self, _ctx: WorkerContext, state: &mut JobState) -> JobResult { // mark job as successful Ok(Some(serde_json::to_value(&state.init)?)) } diff --git a/core/src/object/fs/encrypt.rs b/core/src/object/fs/encrypt.rs index 348c7bddc..e3b40de90 100644 --- a/core/src/object/fs/encrypt.rs +++ b/core/src/object/fs/encrypt.rs @@ -26,7 +26,7 @@ enum ObjectType { #[derive(Serialize, Deserialize, Debug)] pub struct FileEncryptorJobState {} -#[derive(Serialize, Deserialize, Type)] +#[derive(Serialize, Deserialize, Type, Hash)] pub struct FileEncryptorJobInit { pub location_id: i32, pub object_id: i32, @@ -68,16 +68,11 @@ impl StatefulJob for FileEncryptorJob { JOB_NAME } - async fn init( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> Result<(), JobError> { + async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { // enumerate files to encrypt // populate the steps with them (local file paths) - let library = ctx.library_ctx(); - - let location = library + let location = ctx + .library_ctx .db .location() .find_unique(location::id::equals(state.init.location_id)) @@ -91,7 +86,8 @@ impl StatefulJob for FileEncryptorJob { .map(PathBuf::from) .expect("critical error: issue getting local path as pathbuf"); - let item = library + let item = ctx + .library_ctx .db .file_path() .find_first(vec![file_path::object_id::equals(Some( @@ -128,7 +124,7 @@ impl StatefulJob for FileEncryptorJob { async fn execute_step( &self, ctx: WorkerContext, - state: &mut JobState, + state: &mut JobState, ) -> Result<(), JobError> { let step = &state.steps[0]; @@ -137,13 +133,13 @@ impl StatefulJob for FileEncryptorJob { // handle overwriting checks, and making sure there's enough available space let user_key = ctx - .library_ctx() + .library_ctx .key_manager .access_keymount(state.init.key_uuid)? .hashed_key; let user_key_details = ctx - .library_ctx() + .library_ctx .key_manager .access_keystore(state.init.key_uuid)?; @@ -184,7 +180,7 @@ impl StatefulJob for FileEncryptorJob { if state.init.metadata || state.init.preview_media { // if any are requested, we can make the query as it'll be used at least once let object = ctx - .library_ctx() + .library_ctx .db .object() .find_unique(object::id::equals(state.init.object_id)) @@ -240,11 +236,7 @@ impl StatefulJob for FileEncryptorJob { Ok(()) } - async fn finalize( - &self, - _ctx: WorkerContext, - state: &mut JobState, - ) -> JobResult { + async fn finalize(&self, _ctx: WorkerContext, state: &mut JobState) -> JobResult { // mark job as successful Ok(Some(serde_json::to_value(&state.init)?)) } diff --git a/core/src/object/identifier_job.rs b/core/src/object/identifier_job.rs deleted file mode 100644 index 03e8c2a6a..000000000 --- a/core/src/object/identifier_job.rs +++ /dev/null @@ -1,422 +0,0 @@ -use crate::{ - job::{JobError, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext}, - library::LibraryContext, - prisma::{file_path, location, object}, -}; -use chrono::{DateTime, FixedOffset}; -use int_enum::IntEnum; -use prisma_client_rust::{prisma_models::PrismaValue, raw::Raw, Direction}; -use sd_file_ext::{extensions::Extension, kind::ObjectKind}; -use serde::{Deserialize, Serialize}; -use std::{ - collections::{HashMap, HashSet}, - path::{Path, PathBuf}, -}; -use tokio::{fs, io}; -use tracing::{error, info}; - -use super::cas::generate_cas_id; - -// we break this job into chunks of 100 to improve performance -static CHUNK_SIZE: usize = 100; -pub const IDENTIFIER_JOB_NAME: &str = "file_identifier"; - -pub struct FileIdentifierJob {} - -// FileIdentifierJobInit takes file_paths without a file_id and uniquely identifies them -// first: generating the cas_id and extracting metadata -// finally: creating unique file records, and linking them to their file_paths -#[derive(Serialize, Deserialize, Clone)] -pub struct FileIdentifierJobInit { - pub location_id: i32, - pub sub_path: Option, // subpath to start from -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct FilePathIdAndLocationIdCursor { - file_path_id: i32, - location_id: i32, -} - -impl From<&FilePathIdAndLocationIdCursor> for file_path::UniqueWhereParam { - fn from(cursor: &FilePathIdAndLocationIdCursor) -> Self { - file_path::location_id_id(cursor.location_id, cursor.file_path_id) - } -} - -#[derive(Serialize, Deserialize)] -pub struct FileIdentifierJobState { - total_count: usize, - task_count: usize, - location: location::Data, - location_path: PathBuf, - cursor: FilePathIdAndLocationIdCursor, -} - -#[async_trait::async_trait] -impl StatefulJob for FileIdentifierJob { - type Init = FileIdentifierJobInit; - type Data = FileIdentifierJobState; - type Step = (); - - fn name(&self) -> &'static str { - IDENTIFIER_JOB_NAME - } - - async fn init( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> Result<(), JobError> { - info!("Identifying orphan Paths..."); - - let library = ctx.library_ctx(); - - let location_id = state.init.location_id; - - let location = library - .db - .location() - .find_unique(location::id::equals(location_id)) - .exec() - .await? - .unwrap(); - - let location_path = location - .local_path - .as_ref() - .map(PathBuf::from) - .unwrap_or_default(); - - let total_count = count_orphan_file_paths(&library, state.init.location_id).await?; - info!("Found {} orphan file paths", total_count); - - let task_count = (total_count as f64 / CHUNK_SIZE as f64).ceil() as usize; - info!( - "Found {} orphan Paths. Will execute {} tasks...", - total_count, task_count - ); - - // update job with total task count based on orphan file_paths count - ctx.progress(vec![JobReportUpdate::TaskCount(task_count)]); - - let first_path_id = library - .db - .file_path() - .find_first(orphan_path_filters(location_id, None)) - .exec() - .await? - .map(|d| d.id) - .unwrap_or(1); - - state.data = Some(FileIdentifierJobState { - total_count, - task_count, - location, - location_path, - cursor: FilePathIdAndLocationIdCursor { - file_path_id: first_path_id, - location_id: state.init.location_id, - }, - }); - - state.steps = (0..task_count).map(|_| ()).collect(); - - Ok(()) - } - - async fn execute_step( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> Result<(), JobError> { - let db = ctx.library_ctx().db; - - // link file_path ids to a CreateObject struct containing unique file data - let mut chunk: HashMap = HashMap::new(); - let mut cas_lookup: HashMap = HashMap::new(); - - let data = state - .data - .as_mut() - .expect("Critical error: missing data on job state"); - - // get chunk of orphans to process - let file_paths = - get_orphan_file_paths(&ctx.library_ctx(), &data.cursor, data.location.id).await?; - - // if no file paths found, abort entire job early - if file_paths.is_empty() { - return Err(JobError::JobDataNotFound( - "Expected orphan Paths not returned from database query for this chunk".to_string(), - )); - } - - info!( - "Processing {:?} orphan Paths. ({} completed of {})", - file_paths.len(), - state.step_number, - data.task_count - ); - - // analyze each file_path - for file_path in &file_paths { - // get the cas_id and extract metadata - match assemble_object_metadata(&data.location_path, file_path).await { - Ok(object) => { - let cas_id = object.cas_id.clone(); - // create entry into chunks for created file data - chunk.insert(file_path.id, object); - cas_lookup.insert(cas_id, file_path.id); - } - Err(e) => { - error!("Error assembling Object metadata: {:#?}", e); - continue; - } - }; - } - - // find all existing files by cas id - let generated_cas_ids = chunk.values().map(|c| c.cas_id.clone()).collect(); - let existing_objects = db - .object() - .find_many(vec![object::cas_id::in_vec(generated_cas_ids)]) - .exec() - .await?; - - info!("Found {} existing files", existing_objects.len()); - - for existing_object in &existing_objects { - if let Err(e) = db - .file_path() - .update( - file_path::location_id_id( - state.init.location_id, - *cas_lookup.get(&existing_object.cas_id).unwrap(), - ), - vec![file_path::object_id::set(Some(existing_object.id))], - ) - .exec() - .await - { - error!("Error updating file_id: {:#?}", e); - } - } - - let existing_object_cas_ids = existing_objects - .iter() - .map(|object| object.cas_id.clone()) - .collect::>(); - - // extract objects that don't already exist in the database - let new_objects = chunk - .iter() - .map(|(_id, create_file)| create_file) - .filter(|create_file| !existing_object_cas_ids.contains(&create_file.cas_id)) - .collect::>(); - - if !new_objects.is_empty() { - // assemble prisma values for new unique files - let mut values = Vec::with_capacity(new_objects.len() * 3); - for object in &new_objects { - values.extend([ - PrismaValue::String(object.cas_id.clone()), - PrismaValue::Int(object.size_in_bytes), - PrismaValue::DateTime(object.date_created), - PrismaValue::Int(object.kind.int_value() as i64), - ]); - } - - // create new file records with assembled values - // TODO: Use create_many with skip_duplicates. Waiting on https://github.com/Brendonovich/prisma-client-rust/issues/143 - let created_files: Vec = db - ._query_raw(Raw::new( - &format!( - "INSERT INTO object (cas_id, size_in_bytes, date_created, kind) VALUES {} - ON CONFLICT (cas_id) DO NOTHING RETURNING id, cas_id", - vec!["({}, {}, {}, {})"; new_objects.len()].join(",") - ), - values, - )) - .exec() - .await - .unwrap_or_else(|e| { - error!("Error inserting files: {:#?}", e); - Vec::new() - }); - - for created_file in created_files { - // associate newly created files with their respective file_paths - // TODO: this is potentially bottle necking the chunk system, individually linking file_path to file, 100 queries per chunk - // - insert many could work, but I couldn't find a good way to do this in a single SQL query - if let Err(e) = ctx - .library_ctx() - .db - .file_path() - .update( - file_path::location_id_id( - state.init.location_id, - *cas_lookup.get(&created_file.cas_id).unwrap(), - ), - vec![file_path::object_id::set(Some(created_file.id))], - ) - .exec() - .await - { - info!("Error updating file_id: {:#?}", e); - } - } - } - - // set the step data cursor to the last row of this chunk - if let Some(last_row) = file_paths.last() { - data.cursor.file_path_id = last_row.id; - } - - ctx.progress(vec![ - JobReportUpdate::CompletedTaskCount(state.step_number), - JobReportUpdate::Message(format!( - "Processed {} of {} orphan Paths", - state.step_number * CHUNK_SIZE, - data.total_count - )), - ]); - - // let _remaining = count_orphan_file_paths(&ctx.core_ctx, location_id.into()).await?; - Ok(()) - } - - async fn finalize( - &self, - _ctx: WorkerContext, - state: &mut JobState, - ) -> JobResult { - let data = state - .data - .as_ref() - .expect("critical error: missing data on job state"); - info!( - "Finalizing identifier job at {}, total of {} tasks", - data.location_path.display(), - data.task_count - ); - - Ok(Some(serde_json::to_value(&state.init)?)) - } -} - -fn orphan_path_filters(location_id: i32, file_path_id: Option) -> Vec { - let mut params = vec![ - file_path::object_id::equals(None), - file_path::is_dir::equals(false), - file_path::location_id::equals(location_id), - ]; - // this is a workaround for the cursor not working properly - if let Some(file_path_id) = file_path_id { - params.push(file_path::id::gte(file_path_id)) - } - params -} - -#[derive(Deserialize, Serialize, Debug)] -struct CountRes { - count: Option, -} - -async fn count_orphan_file_paths( - ctx: &LibraryContext, - location_id: i32, -) -> Result { - let files_count = ctx - .db - .file_path() - .count(vec![ - file_path::object_id::equals(None), - file_path::is_dir::equals(false), - file_path::location_id::equals(location_id), - ]) - .exec() - .await?; - // Is this - Ok(files_count as usize) -} - -async fn get_orphan_file_paths( - ctx: &LibraryContext, - cursor: &FilePathIdAndLocationIdCursor, - location_id: i32, -) -> Result, prisma_client_rust::QueryError> { - info!( - "Querying {} orphan Paths at cursor: {:?}", - CHUNK_SIZE, cursor - ); - ctx.db - .file_path() - .find_many(orphan_path_filters(location_id, Some(cursor.file_path_id))) - .order_by(file_path::id::order(Direction::Asc)) - // .cursor(cursor.into()) - .take(CHUNK_SIZE as i64) - .skip(1) - .exec() - .await -} - -#[derive(Deserialize, Serialize, Debug)] -struct CreateObject { - pub cas_id: String, - pub size_in_bytes: i64, - pub date_created: DateTime, - pub kind: ObjectKind, -} - -#[derive(Deserialize, Serialize, Debug)] -struct FileCreated { - pub id: i32, - pub cas_id: String, -} - -async fn assemble_object_metadata( - location_path: impl AsRef, - file_path: &file_path::Data, -) -> Result { - let path = location_path - .as_ref() - .join(file_path.materialized_path.as_str()); - - info!("Reading path: {:?}", path); - - let metadata = fs::metadata(&path).await?; - - // derive Object kind - let object_kind: ObjectKind = match path.extension() { - Some(ext) => match ext.to_str() { - Some(ext) => { - let mut file = std::fs::File::open(&path).unwrap(); - let resolved_ext = Extension::resolve_conflicting(ext, &mut file, true); - - resolved_ext.map(Into::into).unwrap_or(ObjectKind::Unknown) - } - None => ObjectKind::Unknown, - }, - None => ObjectKind::Unknown, - }; - - let size = metadata.len(); - - let cas_id = { - if !file_path.is_dir { - let mut ret = generate_cas_id(path, size).await?; - ret.truncate(16); - ret - } else { - "".to_string() - } - }; - - Ok(CreateObject { - cas_id, - size_in_bytes: size as i64, - date_created: file_path.date_created, - kind: object_kind, - }) -} diff --git a/core/src/object/identifier_job/full_identifier_job.rs b/core/src/object/identifier_job/full_identifier_job.rs new file mode 100644 index 000000000..c4f896a9c --- /dev/null +++ b/core/src/object/identifier_job/full_identifier_job.rs @@ -0,0 +1,249 @@ +use crate::{ + invalidate_query, + job::{JobError, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext}, + library::LibraryContext, + prisma::{file_path, location}, +}; + +use std::path::PathBuf; + +use prisma_client_rust::Direction; +use serde::{Deserialize, Serialize}; +use tracing::info; + +use super::{identifier_job_step, IdentifierJobError, CHUNK_SIZE}; + +pub const FULL_IDENTIFIER_JOB_NAME: &str = "file_identifier"; + +pub struct FullFileIdentifierJob {} + +// FileIdentifierJobInit takes file_paths without a file_id and uniquely identifies them +// first: generating the cas_id and extracting metadata +// finally: creating unique file records, and linking them to their file_paths +#[derive(Serialize, Deserialize, Clone, Hash)] +pub struct FullFileIdentifierJobInit { + pub location_id: i32, + pub sub_path: Option, // subpath to start from +} + +#[derive(Serialize, Deserialize, Debug)] +struct FilePathIdAndLocationIdCursor { + file_path_id: i32, + location_id: i32, +} + +impl From<&FilePathIdAndLocationIdCursor> for file_path::UniqueWhereParam { + fn from(cursor: &FilePathIdAndLocationIdCursor) -> Self { + file_path::location_id_id(cursor.location_id, cursor.file_path_id) + } +} + +#[derive(Serialize, Deserialize)] +pub struct FullFileIdentifierJobState { + location: location::Data, + location_path: PathBuf, + cursor: FilePathIdAndLocationIdCursor, + report: FileIdentifierReport, +} + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct FileIdentifierReport { + location_path: String, + total_orphan_paths: usize, + total_objects_created: usize, + total_objects_linked: usize, + total_objects_ignored: usize, +} + +#[async_trait::async_trait] +impl StatefulJob for FullFileIdentifierJob { + type Init = FullFileIdentifierJobInit; + type Data = FullFileIdentifierJobState; + type Step = (); + + fn name(&self) -> &'static str { + FULL_IDENTIFIER_JOB_NAME + } + + async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { + info!("Identifying orphan File Paths..."); + + let location_id = state.init.location_id; + + let location = ctx + .library_ctx + .db + .location() + .find_unique(location::id::equals(location_id)) + .exec() + .await? + .ok_or(IdentifierJobError::MissingLocation(state.init.location_id))?; + + let location_path = location + .local_path + .as_ref() + .map(PathBuf::from) + .ok_or(IdentifierJobError::LocationLocalPath(location_id))?; + + let orphan_count = count_orphan_file_paths(&ctx.library_ctx, location_id).await?; + info!("Found {} orphan file paths", orphan_count); + + let task_count = (orphan_count as f64 / CHUNK_SIZE as f64).ceil() as usize; + info!( + "Found {} orphan Paths. Will execute {} tasks...", + orphan_count, task_count + ); + + // update job with total task count based on orphan file_paths count + ctx.progress(vec![JobReportUpdate::TaskCount(task_count)]); + + let first_path_id = ctx + .library_ctx + .db + .file_path() + .find_first(orphan_path_filters(location_id, None)) + .exec() + .await? + .map(|d| d.id) + .unwrap_or(1); + + state.data = Some(FullFileIdentifierJobState { + report: FileIdentifierReport { + location_path: location_path.to_str().unwrap_or("").to_string(), + total_orphan_paths: orphan_count, + ..Default::default() + }, + location, + location_path, + cursor: FilePathIdAndLocationIdCursor { + file_path_id: first_path_id, + location_id: state.init.location_id, + }, + }); + + state.steps = (0..task_count).map(|_| ()).collect(); + + Ok(()) + } + + async fn execute_step( + &self, + ctx: WorkerContext, + state: &mut JobState, + ) -> Result<(), JobError> { + let data = state + .data + .as_mut() + .expect("Critical error: missing data on job state"); + + // get chunk of orphans to process + let file_paths = + get_orphan_file_paths(&ctx.library_ctx, &data.cursor, data.location.id).await?; + + // if no file paths found, abort entire job early, there is nothing to do + // if we hit this error, there is something wrong with the data/query + if file_paths.is_empty() { + return Err(JobError::EarlyFinish { + name: self.name().to_string(), + reason: "Expected orphan Paths not returned from database query for this chunk" + .to_string(), + }); + } + + info!( + "Processing {:?} orphan Paths. ({} completed of {})", + file_paths.len(), + state.step_number, + data.report.total_orphan_paths + ); + + let (total_objects_created, total_objects_linked) = identifier_job_step( + &ctx.library_ctx, + state.init.location_id, + &data.location_path, + &file_paths, + ) + .await?; + data.report.total_objects_created += total_objects_created; + data.report.total_objects_linked += total_objects_linked; + + // set the step data cursor to the last row of this chunk + if let Some(last_row) = file_paths.last() { + data.cursor.file_path_id = last_row.id; + } + + ctx.progress(vec![ + JobReportUpdate::CompletedTaskCount(state.step_number), + JobReportUpdate::Message(format!( + "Processed {} of {} orphan Paths", + state.step_number * CHUNK_SIZE, + data.report.total_orphan_paths + )), + ]); + + invalidate_query!(ctx.library_ctx, "locations.getExplorerData"); + + // let _remaining = count_orphan_file_paths(&ctx.core_ctx, location_id.into()).await?; + Ok(()) + } + + async fn finalize(&self, _ctx: WorkerContext, state: &mut JobState) -> JobResult { + let data = state + .data + .as_ref() + .expect("critical error: missing data on job state"); + + info!("Finalizing identifier job: {:#?}", data.report); + + Ok(Some(serde_json::to_value(&data.report)?)) + } +} + +fn orphan_path_filters(location_id: i32, file_path_id: Option) -> Vec { + let mut params = vec![ + file_path::object_id::equals(None), + file_path::is_dir::equals(false), + file_path::location_id::equals(location_id), + ]; + // this is a workaround for the cursor not working properly + if let Some(file_path_id) = file_path_id { + params.push(file_path::id::gte(file_path_id)); + } + params +} + +async fn count_orphan_file_paths( + ctx: &LibraryContext, + location_id: i32, +) -> Result { + Ok(ctx + .db + .file_path() + .count(vec![ + file_path::object_id::equals(None), + file_path::is_dir::equals(false), + file_path::location_id::equals(location_id), + ]) + .exec() + .await? as usize) +} + +async fn get_orphan_file_paths( + ctx: &LibraryContext, + cursor: &FilePathIdAndLocationIdCursor, + location_id: i32, +) -> Result, prisma_client_rust::QueryError> { + info!( + "Querying {} orphan Paths at cursor: {:?}", + CHUNK_SIZE, cursor + ); + ctx.db + .file_path() + .find_many(orphan_path_filters(location_id, Some(cursor.file_path_id))) + .order_by(file_path::id::order(Direction::Asc)) + // .cursor(cursor.into()) + .take(CHUNK_SIZE as i64) + // .skip(1) + .exec() + .await +} diff --git a/core/src/object/identifier_job/mod.rs b/core/src/object/identifier_job/mod.rs new file mode 100644 index 000000000..a4ec5bb53 --- /dev/null +++ b/core/src/object/identifier_job/mod.rs @@ -0,0 +1,287 @@ +use crate::{ + job::JobError, + library::LibraryContext, + object::cas::generate_cas_id, + prisma::{file_path, object}, +}; +use chrono::{DateTime, FixedOffset}; +use std::{ + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, +}; + +use futures::future::join_all; +use int_enum::IntEnum; +use prisma_client_rust::QueryError; +use sd_file_ext::{extensions::Extension, kind::ObjectKind}; +use thiserror::Error; +use tokio::{fs, io}; +use tracing::{error, info}; + +pub mod full_identifier_job; + +// we break these jobs into chunks of 100 to improve performance +static CHUNK_SIZE: usize = 100; + +#[derive(Error, Debug)] +pub enum IdentifierJobError { + #[error("Location not found: ")] + MissingLocation(i32), + #[error("Root file path not found: ")] + MissingRootFilePath(PathBuf), + #[error("Location without local path: ")] + LocationLocalPath(i32), +} + +#[derive(Debug, Clone)] +pub struct ObjectCreationMetadata { + pub cas_id: String, + pub size_str: String, + pub kind: ObjectKind, + pub date_created: DateTime, +} + +pub async fn assemble_object_metadata( + location_path: impl AsRef, + file_path: &file_path::Data, +) -> Result { + assert!( + !file_path.is_dir, + "We can't generate cas_id for directories" + ); + + let path = location_path.as_ref().join(&file_path.materialized_path); + + let metadata = fs::metadata(&path).await?; + + // derive Object kind + let object_kind = match path.extension() { + Some(ext) => match ext.to_str() { + Some(ext) => { + let mut file = fs::File::open(&path).await?; + + Extension::resolve_conflicting(&ext.to_lowercase(), &mut file, false) + .await + .map(Into::into) + .unwrap_or(ObjectKind::Unknown) + } + None => ObjectKind::Unknown, + }, + None => ObjectKind::Unknown, + }; + + let size = metadata.len(); + + let cas_id = generate_cas_id(&path, size).await?; + + info!("Analyzed file: {:?} {:?} {:?}", path, cas_id, object_kind); + + Ok(ObjectCreationMetadata { + cas_id, + size_str: size.to_string(), + kind: object_kind, + date_created: file_path.date_created, + }) +} + +async fn batch_update_file_paths( + library: &LibraryContext, + location_id: i32, + objects: &[object::Data], + cas_id_lookup: &HashMap>, +) -> Result, QueryError> { + let mut file_path_updates = Vec::new(); + + objects.iter().for_each(|object| { + let file_path_ids = cas_id_lookup.get(&object.cas_id).unwrap(); + + file_path_updates.extend(file_path_ids.iter().map(|file_path_id| { + info!( + "Linking: ", + file_path_id, object.id + ); + library.db.file_path().update( + file_path::location_id_id(location_id, *file_path_id), + vec![file_path::object_id::set(Some(object.id))], + ) + })); + }); + + info!( + "Updating {} file paths for {} objects", + file_path_updates.len(), + objects.len() + ); + + library.db._batch(file_path_updates).await +} + +async fn generate_provisional_objects( + location_path: impl AsRef, + file_paths: &[file_path::Data], +) -> HashMap)> { + let mut provisional_objects = HashMap::with_capacity(file_paths.len()); + + // analyze each file_path + let location_path = location_path.as_ref(); + for (file_path_id, objects_result) in join_all(file_paths.iter().map(|file_path| async move { + ( + file_path.id, + assemble_object_metadata(location_path, file_path).await, + ) + })) + .await + { + // get the cas_id and extract metadata + match objects_result { + Ok(ObjectCreationMetadata { + cas_id, + size_str, + kind, + date_created, + }) => { + // create entry into chunks for created file data + provisional_objects.insert( + file_path_id, + object::create_unchecked( + cas_id, + size_str, + vec![ + object::date_created::set(date_created), + object::kind::set(kind.int_value()), + ], + ), + ); + } + Err(e) => { + error!("Error assembling Object metadata: {:#?}", e); + continue; + } + }; + } + provisional_objects +} + +async fn identifier_job_step( + library: &LibraryContext, + location_id: i32, + location_path: impl AsRef, + file_paths: &[file_path::Data], +) -> Result<(usize, usize), JobError> { + let location_path = location_path.as_ref(); + + // generate objects for all file paths + let provisional_objects = generate_provisional_objects(location_path, file_paths).await; + + let unique_cas_ids = provisional_objects + .values() + .map(|(cas_id, _, _)| cas_id.clone()) + .collect::>() + .into_iter() + .collect::>(); + + // allow easy lookup of cas_id to many file_path_ids + let mut cas_id_lookup: HashMap> = HashMap::with_capacity(unique_cas_ids.len()); + + // populate cas_id_lookup with file_path_ids + for (file_path_id, (cas_id, _, _)) in provisional_objects.iter() { + cas_id_lookup + .entry(cas_id.clone()) + .or_insert_with(Vec::new) + .push(*file_path_id); + } + + // info!("{:#?}", cas_id_lookup); + + // get all objects that already exist in the database + let existing_objects = library + .db + .object() + .find_many(vec![object::cas_id::in_vec(unique_cas_ids)]) + .exec() + .await?; + + info!( + "Found {} existing Objects in Library, linking file paths...", + existing_objects.len() + ); + + let existing_objects_linked = if !existing_objects.is_empty() { + // link file_path.object_id to existing objects + batch_update_file_paths(library, location_id, &existing_objects, &cas_id_lookup) + .await? + .len() + } else { + 0 + }; + + let existing_object_cas_ids = existing_objects + .iter() + .map(|object| object.cas_id.clone()) + .collect::>(); + + // extract objects that don't already exist in the database + let new_objects = provisional_objects + .into_iter() + .filter(|(_, (cas_id, _, _))| !existing_object_cas_ids.contains(cas_id)) + .collect::>(); + + let new_objects_cas_ids = new_objects + .iter() + .map(|(_, (cas_id, _, _))| cas_id.clone()) + .collect::>(); + + info!( + "Creating {} new Objects in Library... {:#?}", + new_objects.len(), + new_objects_cas_ids + ); + + let mut total_created: usize = 0; + if !new_objects.is_empty() { + // create new object records with assembled values + let total_created_files = library + .db + .object() + .create_many( + new_objects + .into_iter() + .map(|(_, (cas_id, size, params))| (cas_id, size, params)) + .collect(), + ) + .skip_duplicates() + .exec() + .await + .unwrap_or_else(|e| { + error!("Error inserting files: {:#?}", e); + 0 + }); + + total_created = total_created_files as usize; + + info!("Created {} new Objects in Library", total_created); + + // fetch newly created objects so we can link them to file_paths by their id + let created_files = library + .db + .object() + .find_many(vec![object::cas_id::in_vec(new_objects_cas_ids)]) + .exec() + .await + .unwrap_or_else(|e| { + error!("Error finding created files: {:#?}", e); + vec![] + }); + + info!( + "Retrieved {} newly created Objects in Library", + created_files.len() + ); + + if !created_files.is_empty() { + batch_update_file_paths(library, location_id, &created_files, &cas_id_lookup).await?; + } + } + + Ok((total_created, existing_objects_linked)) +} diff --git a/core/src/object/preview/media_data.rs b/core/src/object/preview/media_data.rs new file mode 100644 index 000000000..99eff7757 --- /dev/null +++ b/core/src/object/preview/media_data.rs @@ -0,0 +1,140 @@ +// #[cfg(feature = "ffmpeg")] +// use std::{ffi::OsStr, path::PathBuf}; +// +// #[cfg(feature = "ffmpeg")] +// use ffmpeg_next::{codec::context::Context, format, media::Type}; +// +// #[derive(Default, Debug)] +// pub struct MediaItem { +// pub created_at: Option, +// pub brand: Option, +// pub model: Option, +// pub duration_seconds: i32, +// pub best_video_stream_index: usize, +// pub best_audio_stream_index: usize, +// pub best_subtitle_stream_index: usize, +// pub steams: Vec, +// } +// +// #[derive(Debug)] +// pub struct Stream { +// pub codec: String, +// pub frames: f64, +// pub duration_seconds: f64, +// #[cfg(feature = "ffmpeg")] +// pub kind: Option, +// } +// +// #[cfg(feature = "ffmpeg")] +// #[derive(Debug, PartialEq, Eq)] +// pub enum StreamKind { +// Video(VideoStream), +// Audio(AudioStream), +// } +// +// #[derive(Debug, PartialEq, Eq)] +// pub struct VideoStream { +// pub width: u32, +// pub height: u32, +// pub aspect_ratio: String, +// #[cfg(feature = "ffmpeg")] +// pub format: format::Pixel, +// pub bitrate: usize, +// } +// +// #[derive(Debug, PartialEq, Eq)] +// pub struct AudioStream { +// pub channels: u16, +// #[cfg(feature = "ffmpeg")] +// pub format: format::Sample, +// pub bitrate: usize, +// pub rate: u32, +// } +// +// #[cfg(feature = "ffmpeg")] +// fn extract(iter: &mut ffmpeg_next::dictionary::Iter, key: &str) -> Option { +// iter.find(|k| k.0.contains(key)).map(|k| k.1.to_string()) +// } + +// #[cfg(feature = "ffmpeg")] +// pub fn extract_media_data(path: &PathBuf) -> Result { +// use chrono::NaiveDateTime; +// +// ffmpeg_next::init().unwrap(); +// +// let mut name = path +// .file_name() +// .and_then(OsStr::to_str) +// .map(ToString::to_string) +// .unwrap_or_default(); +// +// // strip to exact potential date length and attempt to parse +// name = name.chars().take(19).collect(); +// // specifically OBS uses this format for time, other checks could be added +// let potential_date = NaiveDateTime::parse_from_str(&name, "%Y-%m-%d %H-%M-%S"); +// +// let context = format::input(&path)?; +// +// let mut media_item = MediaItem::default(); +// let metadata = context.metadata(); +// let mut iter = metadata.iter(); +// +// // creation_time is usually the creation date of the file +// media_item.created_at = extract(&mut iter, "creation_time"); +// // apple photos use "com.apple.quicktime.creationdate", which we care more about than the creation_time +// media_item.created_at = extract(&mut iter, "creationdate"); +// // fallback to potential time if exists +// if media_item.created_at.is_none() { +// media_item.created_at = potential_date.map(|d| d.to_string()).ok(); +// } +// // origin metadata +// media_item.brand = extract(&mut iter, "major_brand"); +// media_item.brand = extract(&mut iter, "make"); +// media_item.model = extract(&mut iter, "model"); +// +// if let Some(stream) = context.streams().best(Type::Video) { +// media_item.best_video_stream_index = stream.index(); +// } +// if let Some(stream) = context.streams().best(Type::Audio) { +// media_item.best_audio_stream_index = stream.index(); +// } +// if let Some(stream) = context.streams().best(Type::Subtitle) { +// media_item.best_subtitle_stream_index = stream.index(); +// } +// media_item.duration_seconds = context.duration() as i32 / ffmpeg_next::ffi::AV_TIME_BASE; +// +// for stream in context.streams() { +// let codec = Context::from_parameters(stream.parameters())?; +// +// let mut stream_item = Stream { +// codec: codec.id().name().to_string(), +// frames: stream.frames() as f64, +// duration_seconds: stream.duration() as f64 * f64::from(stream.time_base()), +// kind: None, +// }; +// +// if codec.medium() == Type::Video { +// if let Ok(video) = codec.decoder().video() { +// stream_item.kind = Some(StreamKind::Video(VideoStream { +// bitrate: video.bit_rate(), +// format: video.format(), +// width: video.width(), +// height: video.height(), +// aspect_ratio: video.aspect_ratio().to_string(), +// })); +// } +// } else if codec.medium() == Type::Audio { +// if let Ok(audio) = codec.decoder().audio() { +// stream_item.kind = Some(StreamKind::Audio(AudioStream { +// channels: audio.channels(), +// bitrate: audio.bit_rate(), +// rate: audio.rate(), +// format: audio.format(), +// })); +// } +// } +// media_item.steams.push(stream_item); +// } +// +// Ok(media_item) +// } diff --git a/core/src/object/reader.rs b/core/src/object/preview/media_data_job.rs similarity index 100% rename from core/src/object/reader.rs rename to core/src/object/preview/media_data_job.rs diff --git a/core/src/object/preview/metadata.rs b/core/src/object/preview/metadata.rs deleted file mode 100644 index d22fd5317..000000000 --- a/core/src/object/preview/metadata.rs +++ /dev/null @@ -1,137 +0,0 @@ -#[cfg(feature = "ffmpeg")] -use ffmpeg_next::format; - -#[derive(Default, Debug)] -pub struct MediaItem { - pub created_at: Option, - pub brand: Option, - pub model: Option, - pub duration_seconds: f64, - pub best_video_stream_index: usize, - pub best_audio_stream_index: usize, - pub best_subtitle_stream_index: usize, - pub steams: Vec, -} - -#[derive(Debug)] -pub struct Stream { - pub codec: String, - pub frames: f64, - pub duration_seconds: f64, - pub kind: Option, -} - -#[derive(Debug)] -#[allow(dead_code)] // TODO: Remove this when we start using ffmpeg -pub enum StreamKind { - Video(VideoStream), - Audio(AudioStream), -} - -#[derive(Debug)] -pub struct VideoStream { - pub width: u32, - pub height: u32, - pub aspect_ratio: String, - #[cfg(feature = "ffmpeg")] - pub format: format::Pixel, - pub bitrate: usize, -} - -#[derive(Debug)] -pub struct AudioStream { - pub channels: u16, - #[cfg(feature = "ffmpeg")] - pub format: format::Sample, - pub bitrate: usize, - pub rate: u32, -} - -// fn extract(iter: &mut Iter, key: &str) -> Option { -// iter.find(|k| k.0.contains(key)).map(|k| k.1.to_string()) -// } - -// pub fn get_video_metadata(path: &str) -> Result<(), ffmpeg::Error> { -// ffmpeg::init().unwrap(); - -// let mut name = Path::new(path) -// .file_name() -// .and_then(OsStr::to_str) -// .map(ToString::to_string) -// .unwrap_or(String::new()); - -// // strip to exact potential date length and attempt to parse -// name = name.chars().take(19).collect(); -// // specifically OBS uses this format for time, other checks could be added -// let potential_date = NaiveDateTime::parse_from_str(&name, "%Y-%m-%d %H-%M-%S"); - -// match ffmpeg::format::input(&path) { -// Ok(context) => { -// let mut media_item = MediaItem::default(); -// let metadata = context.metadata(); -// let mut iter = metadata.iter(); - -// // creation_time is usually the creation date of the file -// media_item.created_at = extract(&mut iter, "creation_time"); -// // apple photos use "com.apple.quicktime.creationdate", which we care more about than the creation_time -// media_item.created_at = extract(&mut iter, "creationdate"); -// // fallback to potential time if exists -// if media_item.created_at.is_none() { -// media_item.created_at = potential_date.map(|d| d.to_string()).ok(); -// } -// // origin metadata -// media_item.brand = extract(&mut iter, "major_brand"); -// media_item.brand = extract(&mut iter, "make"); -// media_item.model = extract(&mut iter, "model"); - -// if let Some(stream) = context.streams().best(ffmpeg::media::Type::Video) { -// media_item.best_video_stream_index = stream.index(); -// } -// if let Some(stream) = context.streams().best(ffmpeg::media::Type::Audio) { -// media_item.best_audio_stream_index = stream.index(); -// } -// if let Some(stream) = context.streams().best(ffmpeg::media::Type::Subtitle) { -// media_item.best_subtitle_stream_index = stream.index(); -// } -// media_item.duration_seconds = -// context.duration() as f64 / f64::from(ffmpeg::ffi::AV_TIME_BASE); - -// for stream in context.streams() { -// let codec = ffmpeg::codec::context::Context::from_parameters(stream.parameters())?; - -// let mut stream_item = Stream { -// codec: codec.id().name().to_string(), -// frames: stream.frames() as f64, -// duration_seconds: stream.duration() as f64 * f64::from(stream.time_base()), -// kind: None, -// }; - -// if codec.medium() == ffmpeg::media::Type::Video { -// if let Ok(video) = codec.decoder().video() { -// stream_item.kind = Some(StreamKind::Video(VideoStream { -// bitrate: video.bit_rate(), -// format: video.format(), -// width: video.width(), -// height: video.height(), -// aspect_ratio: video.aspect_ratio().to_string(), -// })); -// } -// } else if codec.medium() == ffmpeg::media::Type::Audio { -// if let Ok(audio) = codec.decoder().audio() { -// stream_item.kind = Some(StreamKind::Audio(AudioStream { -// channels: audio.channels(), -// bitrate: audio.bit_rate(), -// rate: audio.rate(), -// format: audio.format(), -// })); -// } -// } -// media_item.steams.push(stream_item); -// } -// info!("{:#?}", media_item); -// } - -// Err(error) => error!("error: {}", error), -// } -// Ok(()) -// } diff --git a/core/src/object/preview/mod.rs b/core/src/object/preview/mod.rs index 3d2858482..98b94a7e8 100644 --- a/core/src/object/preview/mod.rs +++ b/core/src/object/preview/mod.rs @@ -1,5 +1,5 @@ -mod metadata; +mod media_data; mod thumb; -pub use metadata::*; +pub use media_data::*; pub use thumb::*; diff --git a/core/src/object/preview/thumb.rs b/core/src/object/preview/thumb.rs index f13b43acf..e59a96b3b 100644 --- a/core/src/object/preview/thumb.rs +++ b/core/src/object/preview/thumb.rs @@ -5,16 +5,18 @@ use crate::{ library::LibraryContext, prisma::{file_path, location}, }; -use sd_file_ext::extensions::{Extension, ImageExtension, VideoExtension}; -use image::{self, imageops, DynamicImage, GenericImageView}; -use serde::{Deserialize, Serialize}; -use std::collections::VecDeque; use std::{ + collections::VecDeque, error::Error, ops::Deref, path::{Path, PathBuf}, }; + +use image::{self, imageops, DynamicImage, GenericImageView}; +use sd_file_ext::extensions::{Extension, ImageExtension, VideoExtension}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; use tokio::{fs, task::block_in_place}; use tracing::{error, info, trace, warn}; use webp::Encoder; @@ -26,10 +28,10 @@ pub const THUMBNAIL_JOB_NAME: &str = "thumbnailer"; pub struct ThumbnailJob {} -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Hash)] pub struct ThumbnailJobInit { pub location_id: i32, - pub path: PathBuf, + pub root_path: PathBuf, pub background: bool, } @@ -39,7 +41,18 @@ pub struct ThumbnailJobState { root_path: PathBuf, } +#[derive(Error, Debug)] +pub enum ThumbnailError { + #[error("Location not found: ")] + MissingLocation(i32), + #[error("Root file path not found: ")] + MissingRootFilePath(PathBuf), + #[error("Location without local path: ")] + LocationLocalPath(i32), +} + file_path::include!(file_path_with_object { object }); +file_path::select!(file_path_id_only { id }); #[derive(Debug, Serialize, Deserialize, Clone, Copy)] enum ThumbnailJobStepKind { @@ -51,6 +64,7 @@ enum ThumbnailJobStepKind { #[derive(Debug, Serialize, Deserialize)] pub struct ThumbnailJobStep { file_path: file_path_with_object::Data, + object_id: i32, kind: ThumbnailJobStepKind, } @@ -64,50 +78,71 @@ impl StatefulJob for ThumbnailJob { THUMBNAIL_JOB_NAME } - async fn init( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> Result<(), JobError> { - let library_ctx = ctx.library_ctx(); - let thumbnail_dir = library_ctx + async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { + let thumbnail_dir = ctx + .library_ctx .config() .data_directory() .join(THUMBNAIL_CACHE_DIR_NAME); - let location = library_ctx + let location = ctx + .library_ctx .db .location() .find_unique(location::id::equals(state.init.location_id)) .exec() .await? - .unwrap(); + .ok_or(ThumbnailError::MissingLocation(state.init.location_id))?; + + let root_path_str = state + .init + .root_path + .to_str() + .expect("Found non-UTF-8 path") + .to_string(); + + let parent_directory_id = ctx + .library_ctx + .db + .file_path() + .find_first(vec![ + file_path::location_id::equals(state.init.location_id), + file_path::materialized_path::equals(if !root_path_str.is_empty() { + root_path_str + } else { + "/".to_string() + }), + file_path::is_dir::equals(true), + ]) + .select(file_path_id_only::select()) + .exec() + .await? + .ok_or_else(|| ThumbnailError::MissingRootFilePath(state.init.root_path.clone()))? + .id; info!( - "Searching for images in location {} at path {}", - location.id, - state.init.path.display() + "Searching for images in location {} at directory {}", + location.id, parent_directory_id ); // create all necessary directories if they don't exist fs::create_dir_all(&thumbnail_dir).await?; - let root_path = location.local_path.map(PathBuf::from).unwrap(); + let root_path = location + .local_path + .map(PathBuf::from) + .ok_or(ThumbnailError::LocationLocalPath(location.id))?; // query database for all image files in this location that need thumbnails let image_files = get_files_by_extensions( - &library_ctx, + &ctx.library_ctx, state.init.location_id, - &state.init.path, - [ - ImageExtension::Png, - ImageExtension::Jpeg, - ImageExtension::Jpg, - ImageExtension::Gif, - ImageExtension::Webp, - ] - .into_iter() - .map(Extension::Image) - .collect(), + parent_directory_id, + &sd_file_ext::extensions::ALL_IMAGE_EXTENSIONS + .iter() + .map(Clone::clone) + .filter(can_generate_thumbnail_for_image) + .map(Extension::Image) + .collect::>(), ThumbnailJobStepKind::Image, ) .await?; @@ -117,15 +152,15 @@ impl StatefulJob for ThumbnailJob { let all_files = { // query database for all video files in this location that need thumbnails let video_files = get_files_by_extensions( - &library_ctx, + &ctx.library_ctx, state.init.location_id, - &state.init.path, - sd_file_ext::extensions::ALL_VIDEO_EXTENSIONS + parent_directory_id, + &sd_file_ext::extensions::ALL_VIDEO_EXTENSIONS .iter() .map(Clone::clone) .filter(can_generate_thumbnail_for_video) .map(Extension::Video) - .collect(), + .collect::>(), ThumbnailJobStepKind::Video, ) .await?; @@ -156,7 +191,7 @@ impl StatefulJob for ThumbnailJob { async fn execute_step( &self, ctx: WorkerContext, - state: &mut JobState, + state: &mut JobState, ) -> Result<(), JobError> { let step = &state.steps[0]; ctx.progress(vec![JobReportUpdate::Message(format!( @@ -174,15 +209,14 @@ impl StatefulJob for ThumbnailJob { trace!("image_file {:?}", step); // get cas_id, if none found skip - let cas_id = match &step.file_path.object { - Some(f) => f.cas_id.clone(), - _ => { - warn!( - "skipping thumbnail generation for {}", - step.file_path.materialized_path - ); - return Ok(()); - } + let cas_id = if let Some(ref object) = step.file_path.object { + object.cas_id.clone() + } else { + warn!( + "skipping thumbnail generation for {}", + step.file_path.materialized_path + ); + return Ok(()); }; // Define and write the WebP-encoded file to a given path @@ -200,23 +234,58 @@ impl StatefulJob for ThumbnailJob { } #[cfg(feature = "ffmpeg")] ThumbnailJobStepKind::Video => { + // use crate::{ + // object::preview::{extract_media_data, StreamKind}, + // prisma::media_data, + // }; + + // use if let Err(e) = generate_video_thumbnail(&path, &output_path).await { error!("Error generating thumb for video: {:?} {:#?}", &path, e); } + // extract MediaData from video and put in the database + // TODO: this is bad here, maybe give it its own job? + // if let Ok(media_data) = extract_media_data(&path) { + // info!( + // "Extracted media data for object {}: {:?}", + // step.object_id, media_data + // ); + + // // let primary_video_stream = media_data + // // .steams + // // .iter() + // // .find(|s| s.kind == Some(StreamKind::Video(_))); + + // let params = vec![ + // media_data::duration_seconds::set(Some(media_data.duration_seconds)), + // // media_data::pixel_width::set(Some(media_data.width)), + // // media_data::pixel_height::set(Some(media_data.height)), + // ]; + // let _ = ctx + // .library_ctx() + // .db + // .media_data() + // .upsert( + // media_data::id::equals(step.object_id), + // params.clone(), + // params, + // ) + // .exec() + // .await?; + // } } } if !state.init.background { - ctx.library_ctx().emit(CoreEvent::NewThumbnail { cas_id }); + ctx.library_ctx.emit(CoreEvent::NewThumbnail { cas_id }); }; + + // With this invalidate query, we update the user interface to show each new thumbnail + invalidate_query!(ctx.library_ctx, "locations.getExplorerData"); } else { info!("Thumb exists, skipping... {}", output_path.display()); } - // With this invalidate query, we update the user interface to show each new thumbnail - let library_ctx = ctx.library_ctx(); - invalidate_query!(library_ctx, "locations.getExplorerData"); - ctx.progress(vec![JobReportUpdate::CompletedTaskCount( state.step_number + 1, )]); @@ -224,11 +293,7 @@ impl StatefulJob for ThumbnailJob { Ok(()) } - async fn finalize( - &self, - _ctx: WorkerContext, - state: &mut JobState, - ) -> JobResult { + async fn finalize(&self, _ctx: WorkerContext, state: &mut JobState) -> JobResult { let data = state .data .as_ref() @@ -244,7 +309,7 @@ impl StatefulJob for ThumbnailJob { } } -async fn generate_image_thumbnail>( +pub async fn generate_image_thumbnail>( file_path: P, output_path: P, ) -> Result<(), Box> { @@ -276,7 +341,7 @@ async fn generate_image_thumbnail>( } #[cfg(feature = "ffmpeg")] -async fn generate_video_thumbnail>( +pub async fn generate_video_thumbnail>( file_path: P, output_path: P, ) -> Result<(), Box> { @@ -290,35 +355,38 @@ async fn generate_video_thumbnail>( async fn get_files_by_extensions( ctx: &LibraryContext, location_id: i32, - path: impl AsRef, - extensions: Vec, + _parent_file_path_id: i32, + extensions: &[Extension], kind: ThumbnailJobStepKind, ) -> Result, JobError> { - let mut params = vec![ - file_path::location_id::equals(location_id), - file_path::extension::in_vec(extensions.iter().map(|e| e.to_string()).collect()), - ]; - - let path_str = path.as_ref().to_string_lossy().to_string(); - - if !path_str.is_empty() { - params.push(file_path::materialized_path::starts_with(path_str)); - } - Ok(ctx .db .file_path() - .find_many(params) + .find_many(vec![ + file_path::location_id::equals(location_id), + file_path::extension::in_vec(extensions.iter().map(ToString::to_string).collect()), + // file_path::parent_id::equals(Some(parent_file_path_id)), + ]) .include(file_path_with_object::include()) .exec() .await? .into_iter() - .map(|file_path| ThumbnailJobStep { file_path, kind }) + .map(|file_path| ThumbnailJobStep { + object_id: file_path.object.as_ref().unwrap().id, + file_path, + kind, + }) .collect()) } #[allow(unused)] pub fn can_generate_thumbnail_for_video(video_extension: &VideoExtension) -> bool { use VideoExtension::*; - !matches!(video_extension, Mpg | Swf | M2v) + // File extensions that are specifically not supported by the thumbnailer + !matches!(video_extension, Mpg | Swf | M2v | Hevc) +} +#[allow(unused)] +pub fn can_generate_thumbnail_for_image(image_extension: &ImageExtension) -> bool { + use ImageExtension::*; + matches!(image_extension, Jpg | Jpeg | Png | Webp | Gif) } diff --git a/core/src/object/validation/hash.rs b/core/src/object/validation/hash.rs index 8a0f3ecbc..adad95882 100644 --- a/core/src/object/validation/hash.rs +++ b/core/src/object/validation/hash.rs @@ -1,5 +1,5 @@ use blake3::Hasher; -use std::path::PathBuf; +use std::path::Path; use tokio::{ fs::File, io::{self, AsyncReadExt}, @@ -7,7 +7,7 @@ use tokio::{ const BLOCK_SIZE: usize = 1048576; -pub async fn file_checksum(path: PathBuf) -> Result { +pub async fn file_checksum(path: impl AsRef) -> Result { let mut reader = File::open(path).await?; let mut context = Hasher::new(); let mut buffer = vec![0; BLOCK_SIZE].into_boxed_slice(); diff --git a/core/src/object/validation/validator_job.rs b/core/src/object/validation/validator_job.rs index 403e85791..0dc5c21e7 100644 --- a/core/src/object/validation/validator_job.rs +++ b/core/src/object/validation/validator_job.rs @@ -4,13 +4,15 @@ use std::{collections::VecDeque, path::PathBuf}; use crate::{ job::{JobError, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext}, - prisma::{self, file_path, location, object}, + prisma::{file_path, location, object}, }; use tracing::info; use super::hash::file_checksum; +pub const VALIDATOR_JOB_NAME: &str = "object_validator"; + // The Validator is able to: // - generate a full byte checksum for Objects in a Location // - generate checksums for all Objects missing without one @@ -24,46 +26,49 @@ pub struct ObjectValidatorJobState { } // The validator can -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Hash)] pub struct ObjectValidatorJobInit { pub location_id: i32, pub path: PathBuf, pub background: bool, } -#[derive(Serialize, Deserialize, Debug)] -pub struct ObjectValidatorJobStep { - pub path: file_path::Data, -} +file_path::select!(file_path_and_object { + materialized_path + object: select { + id + integrity_checksum + } +}); #[async_trait::async_trait] impl StatefulJob for ObjectValidatorJob { - type Data = ObjectValidatorJobState; type Init = ObjectValidatorJobInit; - type Step = ObjectValidatorJobStep; + type Data = ObjectValidatorJobState; + type Step = file_path_and_object::Data; fn name(&self) -> &'static str { - "object_validator" + VALIDATOR_JOB_NAME } - async fn init( - &self, - ctx: WorkerContext, - state: &mut JobState, - ) -> Result<(), JobError> { - let library_ctx = ctx.library_ctx(); - - state.steps = library_ctx + async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { + state.steps = ctx + .library_ctx .db .file_path() - .find_many(vec![file_path::location_id::equals(state.init.location_id)]) + .find_many(vec![ + file_path::location_id::equals(state.init.location_id), + file_path::is_dir::equals(false), + file_path::object::is(vec![object::integrity_checksum::equals(None)]), + ]) + .select(file_path_and_object::select()) .exec() .await? .into_iter() - .map(|path| ObjectValidatorJobStep { path }) .collect::>(); - let location = library_ctx + let location = ctx + .library_ctx .db .location() .find_unique(location::id::equals(state.init.location_id)) @@ -84,46 +89,29 @@ impl StatefulJob for ObjectValidatorJob { async fn execute_step( &self, ctx: WorkerContext, - state: &mut JobState, + state: &mut JobState, ) -> Result<(), JobError> { let step = &state.steps[0]; - let library_ctx = ctx.library_ctx(); - let data = state.data.as_ref().expect("fatal: missing job state"); - let path = data.root_path.join(&step.path.materialized_path); - - // skip directories - if path.is_dir() { - return Ok(()); - } - - if let Some(object_id) = step.path.object_id { - // this is to skip files that already have checksums - // i'm unsure what the desired behaviour is in this case - // we can also compare old and new checksums here - let object = library_ctx - .db - .object() - .find_unique(object::id::equals(object_id)) - .exec() - .await? - .unwrap(); - if object.integrity_checksum.is_some() { - return Ok(()); + // this is to skip files that already have checksums + // i'm unsure what the desired behaviour is in this case + // we can also compare old and new checksums here + if let Some(ref object) = step.object { + // This if is just to make sure, we already queried objects where integrity_checksum is null + if object.integrity_checksum.is_none() { + ctx.library_ctx + .db + .object() + .update( + object::id::equals(object.id), + vec![object::SetParam::SetIntegrityChecksum(Some( + file_checksum(data.root_path.join(&step.materialized_path)).await?, + ))], + ) + .exec() + .await?; } - - let hash = file_checksum(path).await?; - - library_ctx - .db - .object() - .update( - object::id::equals(object_id), - vec![prisma::object::SetParam::SetIntegrityChecksum(Some(hash))], - ) - .exec() - .await?; } ctx.progress(vec![JobReportUpdate::CompletedTaskCount( @@ -133,11 +121,7 @@ impl StatefulJob for ObjectValidatorJob { Ok(()) } - async fn finalize( - &self, - _ctx: WorkerContext, - state: &mut JobState, - ) -> JobResult { + async fn finalize(&self, _ctx: WorkerContext, state: &mut JobState) -> JobResult { let data = state .data .as_ref() diff --git a/crates/crypto/src/crypto/stream.rs b/crates/crypto/src/crypto/stream.rs index 10588f822..6d8257c66 100644 --- a/crates/crypto/src/crypto/stream.rs +++ b/crates/crypto/src/crypto/stream.rs @@ -1,4 +1,6 @@ //! This module contains the crate's STREAM implementation, and wrappers that allow us to support multiple AEADs. +#![allow(clippy::use_self)] // I think: https://github.com/rust-lang/rust-clippy/issues/3909 + use std::io::{Cursor, Read, Write}; use crate::{primitives::BLOCK_SIZE, Error, Protected, Result}; @@ -10,14 +12,13 @@ use aes_gcm::Aes256Gcm; use chacha20poly1305::XChaCha20Poly1305; /// These are all possible algorithms that can be used for encryption and decryption -#[derive(Clone, Copy, Eq, PartialEq)] +#[derive(Clone, Copy, Eq, PartialEq, Hash)] #[cfg_attr( feature = "serde", derive(serde::Serialize), derive(serde::Deserialize) )] #[cfg_attr(feature = "rspc", derive(specta::Type))] -#[allow(clippy::use_self)] pub enum Algorithm { XChaCha20Poly1305, Aes256Gcm, diff --git a/crates/crypto/src/keys/hashing.rs b/crates/crypto/src/keys/hashing.rs index 91ed36c4e..b4c376bd3 100644 --- a/crates/crypto/src/keys/hashing.rs +++ b/crates/crypto/src/keys/hashing.rs @@ -10,6 +10,8 @@ //! let salt = generate_salt(); //! let hashed_password = hashing_algorithm.hash(password, salt).unwrap(); //! ``` +#![allow(clippy::use_self)] // I think: https://github.com/rust-lang/rust-clippy/issues/3909 + use crate::Protected; use crate::{primitives::SALT_LEN, Error, Result}; use argon2::Argon2; @@ -24,7 +26,6 @@ use argon2::Argon2; derive(serde::Deserialize) )] #[cfg_attr(feature = "rspc", derive(specta::Type))] -#[allow(clippy::use_self)] pub enum Params { Standard, Hardened, diff --git a/crates/ffmpeg/src/movie_decoder.rs b/crates/ffmpeg/src/movie_decoder.rs index 795eabe4c..cf48669c3 100644 --- a/crates/ffmpeg/src/movie_decoder.rs +++ b/crates/ffmpeg/src/movie_decoder.rs @@ -134,9 +134,7 @@ impl MovieDecoder { return Err(ThumbnailerError::SeekNotAllowed); } - let timestamp = (AV_TIME_BASE as i64) - .checked_mul(seconds as i64) - .unwrap_or(0); + let timestamp = (AV_TIME_BASE as i64).checked_mul(seconds).unwrap_or(0); check_error( unsafe { av_seek_frame(self.format_context, -1, timestamp, 0) }, @@ -334,9 +332,12 @@ impl MovieDecoder { break; } if unsafe { - CString::from_raw((*tag).key).to_string_lossy() == "filename" + CString::from_raw((*tag).key) + .to_str() + .expect("Found non-UTF-8 path") == "filename" && CString::from_raw((*tag).value) - .to_string_lossy() + .to_str() + .expect("Found non-UTF-8 path") .starts_with("cover.") } { if embedded_data_streams.is_empty() { @@ -436,8 +437,8 @@ impl MovieDecoder { (*self.video_codec_context).width, (*self.video_codec_context).height, (*self.video_codec_context).pix_fmt as i32, - (*timebase).num, - (*timebase).den, + timebase.num, + timebase.den, (*self.video_codec_context).sample_aspect_ratio.num, i32::max((*self.video_codec_context).sample_aspect_ratio.den, 1) ) diff --git a/crates/file-ext/Cargo.toml b/crates/file-ext/Cargo.toml index 7d5c81db3..66c927cc3 100644 --- a/crates/file-ext/Cargo.toml +++ b/crates/file-ext/Cargo.toml @@ -9,3 +9,9 @@ authors = ["Brendan Allen ", "Jamie Pine Option { + #[tokio::test] + async fn magic_bytes() { + async fn test_path(subpath: &str) -> Option { println!("testing {}...", subpath); let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() @@ -295,175 +301,176 @@ mod test { .join("packages/test-files/files") .join(subpath); - let mut file = File::open(path).unwrap(); + let mut file = File::open(path).await.unwrap(); Extension::resolve_conflicting(&subpath.split(".").last().unwrap(), &mut file, true) + .await } // Video extension tests assert_eq!( - dbg!(test_path("video/video.ts")), + dbg!(test_path("video/video.ts").await), Some(Extension::Video(VideoExtension::Ts)) ); assert_eq!( - dbg!(test_path("code/typescript.ts")), + dbg!(test_path("code/typescript.ts").await), Some(Extension::Code(CodeExtension::Ts)) ); assert_eq!( - dbg!(test_path("video/video.3gp")), + dbg!(test_path("video/video.3gp").await), Some(Extension::Video(VideoExtension::_3gp)) ); assert_eq!( - dbg!(test_path("video/video.mov")), + dbg!(test_path("video/video.mov").await), Some(Extension::Video(VideoExtension::Mov)) ); assert_eq!( - dbg!(test_path("video/video.asf")), + dbg!(test_path("video/video.asf").await), Some(Extension::Video(VideoExtension::Asf)) ); assert_eq!( - dbg!(test_path("video/video.avi")), + dbg!(test_path("video/video.avi").await), Some(Extension::Video(VideoExtension::Avi)) ); assert_eq!( - dbg!(test_path("video/video.flv")), + dbg!(test_path("video/video.flv").await), Some(Extension::Video(VideoExtension::Flv)) ); assert_eq!( - dbg!(test_path("video/video.m4v")), + dbg!(test_path("video/video.m4v").await), Some(Extension::Video(VideoExtension::M4v)) ); assert_eq!( - dbg!(test_path("video/video.mkv")), + dbg!(test_path("video/video.mkv").await), Some(Extension::Video(VideoExtension::Mkv)) ); assert_eq!( - dbg!(test_path("video/video.mpg")), + dbg!(test_path("video/video.mpg").await), Some(Extension::Video(VideoExtension::Mpg)) ); assert_eq!( - dbg!(test_path("video/video.mpeg")), + dbg!(test_path("video/video.mpeg").await), Some(Extension::Video(VideoExtension::Mpeg)) ); assert_eq!( - dbg!(test_path("video/video.mts")), + dbg!(test_path("video/video.mts").await), Some(Extension::Video(VideoExtension::Mts)) ); assert_eq!( - dbg!(test_path("video/video.mxf")), + dbg!(test_path("video/video.mxf").await), Some(Extension::Video(VideoExtension::Mxf)) ); assert_eq!( - dbg!(test_path("video/video.ogv")), + dbg!(test_path("video/video.ogv").await), Some(Extension::Video(VideoExtension::Ogv)) ); assert_eq!( - dbg!(test_path("video/video.swf")), + dbg!(test_path("video/video.swf").await), Some(Extension::Video(VideoExtension::Swf)) ); assert_eq!( - dbg!(test_path("video/video.ts")), + dbg!(test_path("video/video.ts").await), Some(Extension::Video(VideoExtension::Ts)) ); assert_eq!( - dbg!(test_path("video/video.vob")), + dbg!(test_path("video/video.vob").await), Some(Extension::Video(VideoExtension::Vob)) ); assert_eq!( - dbg!(test_path("video/video.ogv")), + dbg!(test_path("video/video.ogv").await), Some(Extension::Video(VideoExtension::Ogv)) ); assert_eq!( - dbg!(test_path("video/video.wmv")), + dbg!(test_path("video/video.wmv").await), Some(Extension::Video(VideoExtension::Wmv)) ); assert_eq!( - dbg!(test_path("video/video.wtv")), + dbg!(test_path("video/video.wtv").await), Some(Extension::Video(VideoExtension::Wtv)) ); // Audio extension tests assert_eq!( - dbg!(test_path("audio/audio.aac")), + dbg!(test_path("audio/audio.aac").await), Some(Extension::Audio(AudioExtension::Aac)) ); assert_eq!( - dbg!(test_path("audio/audio.adts")), + dbg!(test_path("audio/audio.adts").await), Some(Extension::Audio(AudioExtension::Adts)) ); assert_eq!( - dbg!(test_path("audio/audio.aif")), + dbg!(test_path("audio/audio.aif").await), Some(Extension::Audio(AudioExtension::Aif)) ); assert_eq!( - dbg!(test_path("audio/audio.aiff")), + dbg!(test_path("audio/audio.aiff").await), Some(Extension::Audio(AudioExtension::Aiff)) ); assert_eq!( - dbg!(test_path("audio/audio.aptx")), + dbg!(test_path("audio/audio.aptx").await), Some(Extension::Audio(AudioExtension::Aptx)) ); assert_eq!( - dbg!(test_path("audio/audio.ast")), + dbg!(test_path("audio/audio.ast").await), Some(Extension::Audio(AudioExtension::Ast)) ); assert_eq!( - dbg!(test_path("audio/audio.caf")), + dbg!(test_path("audio/audio.caf").await), Some(Extension::Audio(AudioExtension::Caf)) ); assert_eq!( - dbg!(test_path("audio/audio.flac")), + dbg!(test_path("audio/audio.flac").await), Some(Extension::Audio(AudioExtension::Flac)) ); assert_eq!( - dbg!(test_path("audio/audio.loas")), + dbg!(test_path("audio/audio.loas").await), Some(Extension::Audio(AudioExtension::Loas)) ); assert_eq!( - dbg!(test_path("audio/audio.m4a")), + dbg!(test_path("audio/audio.m4a").await), Some(Extension::Audio(AudioExtension::M4a)) ); // assert_eq!( - // dbg!(test_path("audio/audio.m4b")), + // dbg!(test_path("audio/audio.m4b").await), // Some(Extension::Audio(AudioExtension::M4b)) // ); assert_eq!( - dbg!(test_path("audio/audio.mp2")), + dbg!(test_path("audio/audio.mp2").await), Some(Extension::Audio(AudioExtension::Mp2)) ); assert_eq!( - dbg!(test_path("audio/audio.mp3")), + dbg!(test_path("audio/audio.mp3").await), Some(Extension::Audio(AudioExtension::Mp3)) ); assert_eq!( - dbg!(test_path("audio/audio.oga")), + dbg!(test_path("audio/audio.oga").await), Some(Extension::Audio(AudioExtension::Oga)) ); assert_eq!( - dbg!(test_path("audio/audio.ogg")), + dbg!(test_path("audio/audio.ogg").await), Some(Extension::Audio(AudioExtension::Ogg)) ); assert_eq!( - dbg!(test_path("audio/audio.opus")), + dbg!(test_path("audio/audio.opus").await), Some(Extension::Audio(AudioExtension::Opus)) ); assert_eq!( - dbg!(test_path("audio/audio.tta")), + dbg!(test_path("audio/audio.tta").await), Some(Extension::Audio(AudioExtension::Tta)) ); assert_eq!( - dbg!(test_path("audio/audio.voc")), + dbg!(test_path("audio/audio.voc").await), Some(Extension::Audio(AudioExtension::Voc)) ); assert_eq!( - dbg!(test_path("audio/audio.wav")), + dbg!(test_path("audio/audio.wav").await), Some(Extension::Audio(AudioExtension::Wav)) ); assert_eq!( - dbg!(test_path("audio/audio.wma")), + dbg!(test_path("audio/audio.wma").await), Some(Extension::Audio(AudioExtension::Wma)) ); assert_eq!( - dbg!(test_path("audio/audio.wv")), + dbg!(test_path("audio/audio.wv").await), Some(Extension::Audio(AudioExtension::Wv)) ); } diff --git a/crates/file-ext/src/magic.rs b/crates/file-ext/src/magic.rs index f541d1b01..74c500bf4 100644 --- a/crates/file-ext/src/magic.rs +++ b/crates/file-ext/src/magic.rs @@ -1,5 +1,12 @@ #![allow(dead_code)] + use crate::extensions::{CodeExtension, Extension, VideoExtension}; +use std::io::SeekFrom; + +use tokio::{ + fs::File, + io::{AsyncReadExt, AsyncSeekExt}, +}; #[derive(Debug, PartialEq, Eq)] pub enum ExtensionPossibility { @@ -98,8 +105,9 @@ macro_rules! extension_category_enum { $($(#[$variant_attr:meta])* $variant:ident $(= $( [$($magic_bytes:tt),*] $(+ $offset:literal)? )|+ )? ,)* } ) => { - #[derive(Debug, ::serde::Serialize, ::serde::Deserialize, Clone, Copy, PartialEq, Eq)] + #[derive(Debug, ::serde::Serialize, ::serde::Deserialize, ::strum::Display, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] + #[strum(serialize_all = "snake_case")] $(#[$enum_attr])* // construct enum @@ -120,12 +128,6 @@ macro_rules! extension_category_enum { serde_json::from_value(serde_json::Value::String(s.to_string())) } } - // convert to string - impl std::fmt::Display for $enum_name { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) // SAFETY: This is safe - } - } }; (@magic_bytes; $enum_name:ident ($($(#[$variant_attr:meta])* $variant:ident = $( [$($magic_bytes:tt),*] $(+ $offset:literal)? )|+ ),*)) => { @@ -153,14 +155,12 @@ macro_rules! extension_category_enum { } pub(crate) use extension_category_enum; -pub fn verify_magic_bytes(ext: T, file: &mut std::fs::File) -> Option { - use std::io::{Read, Seek, SeekFrom}; - +pub async fn verify_magic_bytes(ext: T, file: &mut File) -> Option { for magic in ext.magic_bytes_meta() { let mut buf = vec![0; magic.length]; - file.seek(SeekFrom::Start(magic.offset as u64)).ok()?; - file.read_exact(&mut buf).ok()?; + file.seek(SeekFrom::Start(magic.offset as u64)).await.ok()?; + file.read_exact(&mut buf).await.ok()?; if ext.has_magic_bytes(&buf) { return Some(ext); @@ -171,9 +171,9 @@ pub fn verify_magic_bytes(ext: T, file: &mut std::fs::File) -> Op } impl Extension { - pub fn resolve_conflicting( + pub async fn resolve_conflicting( ext_str: &str, - file: &mut std::fs::File, + file: &mut File, always_check_magic_bytes: bool, ) -> Option { let ext = match Extension::from_str(ext_str) { @@ -187,11 +187,20 @@ impl Extension { ExtensionPossibility::Known(e) => { if always_check_magic_bytes { match e { - Self::Image(x) => verify_magic_bytes(x, file).map(Self::Image), - Self::Audio(x) => verify_magic_bytes(x, file).map(Self::Audio), - Self::Video(x) => verify_magic_bytes(x, file).map(Self::Video), - Self::Executable(x) => verify_magic_bytes(x, file).map(Self::Executable), - _ => None, + Self::Image(x) => verify_magic_bytes(x, file).await.map(Self::Image), + Self::Audio(x) => verify_magic_bytes(x, file).await.map(Self::Audio), + Self::Video(x) => verify_magic_bytes(x, file).await.map(Self::Video), + Self::Archive(x) => verify_magic_bytes(x, file).await.map(Self::Archive), + Self::Executable(x) => { + verify_magic_bytes(x, file).await.map(Self::Executable) + } + Self::Font(x) => verify_magic_bytes(x, file).await.map(Self::Font), + Self::Encrypted(x) => { + verify_magic_bytes(x, file).await.map(Self::Encrypted) + } + Self::Mesh(x) => verify_magic_bytes(x, file).await.map(Self::Mesh), + Self::Database(x) => verify_magic_bytes(x, file).await.map(Self::Database), + _ => Some(e), } } else { Some(e) @@ -200,7 +209,9 @@ impl Extension { ExtensionPossibility::Conflicts(ext) => match ext_str { "ts" => { let maybe_video_ext = if ext.iter().any(|e| matches!(e, Extension::Video(_))) { - verify_magic_bytes(VideoExtension::Ts, file).map(Extension::Video) + verify_magic_bytes(VideoExtension::Ts, file) + .await + .map(Extension::Video) } else { None }; diff --git a/crates/sync/example/api/src/api/mod.rs b/crates/sync/example/api/src/api/mod.rs index e56fdf74d..008b23d88 100644 --- a/crates/sync/example/api/src/api/mod.rs +++ b/crates/sync/example/api/src/api/mod.rs @@ -16,7 +16,7 @@ pub struct Ctx { type Router = rspc::Router>>; fn to_map(v: &impl serde::Serialize) -> serde_json::Map { - match to_value(&v).unwrap() { + match to_value(v).unwrap() { Value::Object(m) => m, _ => unreachable!(), } @@ -144,7 +144,7 @@ pub(crate) fn new() -> RouterBuilder>> { } } - let mut array = hashmap.into_iter().map(|(_, v)| v).collect::>(); + let mut array = hashmap.into_values().collect::>(); array.sort_by(|a, b| a.id.partial_cmp(&b.id).unwrap()); diff --git a/packages/assets/icons/vue.svg b/packages/assets/icons/vue.svg index deb017c2a..ca4527ca9 100644 --- a/packages/assets/icons/vue.svg +++ b/packages/assets/icons/vue.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/packages/assets/images/Alias.png b/packages/assets/images/Alias.png new file mode 100644 index 000000000..7161229fc Binary files /dev/null and b/packages/assets/images/Alias.png differ diff --git a/packages/assets/images/Archive.png b/packages/assets/images/Archive.png new file mode 100644 index 000000000..d3b7db1a5 Binary files /dev/null and b/packages/assets/images/Archive.png differ diff --git a/packages/assets/images/Document_doc.png b/packages/assets/images/Document_doc.png new file mode 100644 index 000000000..0aebf7b3d Binary files /dev/null and b/packages/assets/images/Document_doc.png differ diff --git a/packages/assets/images/Document_pdf.png b/packages/assets/images/Document_pdf.png new file mode 100644 index 000000000..1c26fe95d Binary files /dev/null and b/packages/assets/images/Document_pdf.png differ diff --git a/packages/assets/images/Document_xls.png b/packages/assets/images/Document_xls.png new file mode 100644 index 000000000..65e8da1ab Binary files /dev/null and b/packages/assets/images/Document_xls.png differ diff --git a/packages/assets/images/Executable.png b/packages/assets/images/Executable.png new file mode 100644 index 000000000..a1d627f49 Binary files /dev/null and b/packages/assets/images/Executable.png differ diff --git a/packages/assets/images/File.png b/packages/assets/images/File.png new file mode 100644 index 000000000..445b9f24b Binary files /dev/null and b/packages/assets/images/File.png differ diff --git a/packages/assets/images/Folder.png b/packages/assets/images/Folder.png new file mode 100644 index 000000000..68770416b Binary files /dev/null and b/packages/assets/images/Folder.png differ diff --git a/packages/assets/images/Unknown.png b/packages/assets/images/Unknown.png new file mode 100644 index 000000000..86100ae42 Binary files /dev/null and b/packages/assets/images/Unknown.png differ diff --git a/packages/assets/images/Video.png b/packages/assets/images/Video.png new file mode 100644 index 000000000..c6cd33c4b Binary files /dev/null and b/packages/assets/images/Video.png differ diff --git a/packages/assets/svgs/file.svg b/packages/assets/svgs/file.svg new file mode 100644 index 000000000..cd11e8871 --- /dev/null +++ b/packages/assets/svgs/file.svg @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index da2648028..221e06bd3 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -4,7 +4,7 @@ export type Procedures = { queries: { key: "buildInfo", input: never, result: BuildInfo } | - { key: "files.readMetadata", input: LibraryArgs, result: null } | + { key: "files.get", input: LibraryArgs, result: { id: number, cas_id: string, integrity_checksum: string | null, name: string | null, extension: string | null, kind: number, size_in_bytes: string, key_id: number | null, hidden: boolean, favorite: boolean, important: boolean, has_thumbnail: boolean, has_thumbstrip: boolean, has_video_preview: boolean, ipfs_id: string | null, note: string | null, date_created: string, date_modified: string, date_indexed: string, file_paths: Array, media_data: MediaData | null } | null } | { key: "jobs.getHistory", input: LibraryArgs, result: Array } | { key: "jobs.getRunning", input: LibraryArgs, result: Array } | { key: "jobs.isRunning", input: LibraryArgs, result: boolean } | @@ -37,6 +37,7 @@ export type Procedures = { { key: "files.encryptFiles", input: LibraryArgs, result: null } | { key: "files.setFavorite", input: LibraryArgs, result: null } | { key: "files.setNote", input: LibraryArgs, result: null } | + { key: "jobs.clearAll", input: LibraryArgs, result: null } | { key: "jobs.generateThumbsForLocation", input: LibraryArgs, result: null } | { key: "jobs.identifyUniqueFiles", input: LibraryArgs, result: null } | { key: "jobs.objectValidator", input: LibraryArgs, result: null } | @@ -58,12 +59,14 @@ export type Procedures = { { key: "library.create", input: string, result: LibraryConfigWrapped } | { key: "library.delete", input: string, result: null } | { key: "library.edit", input: EditLibraryArgs, result: null } | + { key: "locations.addLibrary", input: LibraryArgs, result: null } | { key: "locations.create", input: LibraryArgs, result: null } | { key: "locations.delete", input: LibraryArgs, result: null } | { key: "locations.fullRescan", input: LibraryArgs, result: null } | { key: "locations.indexer_rules.create", input: LibraryArgs, result: IndexerRule } | { key: "locations.indexer_rules.delete", input: LibraryArgs, result: null } | { key: "locations.quickRescan", input: LibraryArgs, result: null } | + { key: "locations.relink", input: LibraryArgs, result: null } | { key: "locations.update", input: LibraryArgs, result: null } | { key: "tags.assign", input: LibraryArgs, result: null } | { key: "tags.create", input: LibraryArgs, result: Tag } | @@ -98,6 +101,8 @@ export interface FilePath { id: number, is_dir: boolean, location_id: number, ma export interface GenerateThumbsForLocationArgs { id: number, path: string } +export interface GetArgs { id: number } + export type HashingAlgorithm = { Argon2id: Params } export interface IdentifyUniqueFilesArgs { id: number, path: string } @@ -132,6 +137,8 @@ export interface LocationUpdateArgs { id: number, name: string | null, indexer_r export interface MasterPasswordChangeArgs { password: string, algorithm: Algorithm, hashing_algorithm: HashingAlgorithm } +export interface MediaData { id: number, pixel_width: number | null, pixel_height: number | null, longitude: number | null, latitude: number | null, fps: number | null, capture_device_make: string | null, capture_device_model: string | null, capture_device_software: string | null, duration_seconds: number | null, codecs: string | null, streams: number | null } + export interface Node { id: number, pub_id: Array, name: string, platform: number, version: string | null, last_seen: string, timezone: string | null, date_created: string } export interface NodeConfig { version: string | null, id: string, name: string, p2p_port: number | null } diff --git a/packages/interface/src/AppLayout.tsx b/packages/interface/src/AppLayout.tsx index 863051e18..186a4a350 100644 --- a/packages/interface/src/AppLayout.tsx +++ b/packages/interface/src/AppLayout.tsx @@ -21,6 +21,7 @@ export function AppLayout() { className={clsx( // App level styles 'flex h-screen overflow-hidden text-ink select-none cursor-default', + os === 'browser' && 'bg-app border-t border-app-line/50', os === 'macOS' && 'rounded-[10px] has-blur-effects', os !== 'browser' && os !== 'windows' && 'border border-app-frame' )} diff --git a/packages/interface/src/components/dialog/AddLocationDialog.tsx b/packages/interface/src/components/dialog/AddLocationDialog.tsx new file mode 100644 index 000000000..4a803da2d --- /dev/null +++ b/packages/interface/src/components/dialog/AddLocationDialog.tsx @@ -0,0 +1,48 @@ +import { LocationCreateArgs, useBridgeMutation, useLibraryMutation } from '@sd/client'; +import { Input } from '@sd/ui'; +import { Dialog } from '@sd/ui'; +import { useQueryClient } from '@tanstack/react-query'; +import { PropsWithChildren, useState } from 'react'; + +export default function AddLocationDialog({ + children, + onSubmit, + open, + setOpen +}: PropsWithChildren<{ onSubmit?: () => void; open: boolean; setOpen: (state: boolean) => void }>) { + // BEFORE MERGE: Remove default value + const [locationUrl, setLocationUrl] = useState( + '/Users/jamie/Projects/spacedrive/packages/test-files/files' + ); + + const createLocation = useLibraryMutation('locations.create', { + onSuccess: () => setOpen(false) + }); + + return ( + + createLocation.mutate({ + path: locationUrl, + indexer_rules_ids: [] + } as LocationCreateArgs) + } + loading={createLocation.isLoading} + submitDisabled={!locationUrl} + ctaLabel="Add" + trigger={null} + > + setLocationUrl(e.target.value)} + required + /> + + ); +} diff --git a/packages/interface/src/components/explorer/Explorer.tsx b/packages/interface/src/components/explorer/Explorer.tsx index 96d1662f7..905db4c0a 100644 --- a/packages/interface/src/components/explorer/Explorer.tsx +++ b/packages/interface/src/components/explorer/Explorer.tsx @@ -1,12 +1,12 @@ import { ExplorerData, rspc, useCurrentLibrary } from '@sd/client'; import { useEffect, useState } from 'react'; -import { useExplorerStore } from '../../util/explorerStore'; +import { useExplorerStore } from '../../hooks/useExplorerStore'; import { AlertDialog, GenericAlertDialogState } from '../dialog/AlertDialog'; import { DecryptFileDialog } from '../dialog/DecryptFileDialog'; import { EncryptFileDialog } from '../dialog/EncryptFileDialog'; import { Inspector } from '../explorer/Inspector'; -import ExplorerContextMenu from './ExplorerContextMenu'; +import { ExplorerContextMenu } from './ExplorerContextMenu'; import { TopBar } from './ExplorerTopBar'; import { VirtualizedList } from './VirtualizedList'; @@ -47,15 +47,11 @@ export default function Explorer(props: Props) { return ( <>
- +
-
+
{props.data && ( )} {expStore.showInspector && ( diff --git a/packages/interface/src/components/explorer/ExplorerContextMenu.tsx b/packages/interface/src/components/explorer/ExplorerContextMenu.tsx index a80b414e1..da90a8c6e 100644 --- a/packages/interface/src/components/explorer/ExplorerContextMenu.tsx +++ b/packages/interface/src/components/explorer/ExplorerContextMenu.tsx @@ -1,23 +1,26 @@ -import { useLibraryMutation, useLibraryQuery } from '@sd/client'; +import { ExplorerItem, useLibraryMutation, useLibraryQuery } from '@sd/client'; import { ContextMenu as CM } from '@sd/ui'; import { ArrowBendUpRight, + Image, LockSimple, LockSimpleOpen, Package, Plus, + Repeat, Share, + ShieldCheck, TagSimple, Trash, TrashSimple } from 'phosphor-react'; import { PropsWithChildren, useMemo } from 'react'; +import { getExplorerStore } from '../../hooks/useExplorerStore'; import { useOperatingSystem } from '../../hooks/useOperatingSystem'; import { usePlatform } from '../../util/Platform'; -import { getExplorerStore } from '../../util/explorerStore'; import { GenericAlertDialogProps } from '../dialog/AlertDialog'; -import { EncryptFileDialog } from '../dialog/EncryptFileDialog'; +import { isObject } from './utils'; const AssignTagMenuItems = (props: { objectId: number }) => { const tags = useLibraryQuery(['tags.list'], { suspense: true }); @@ -59,18 +62,9 @@ const AssignTagMenuItems = (props: { objectId: number }) => { ); }; -export interface ExplorerContextMenuProps extends PropsWithChildren { - setShowEncryptDialog: (isShowing: boolean) => void; - setShowDecryptDialog: (isShowing: boolean) => void; - setAlertDialogData: (data: GenericAlertDialogProps) => void; -} - -export default function ExplorerContextMenu(props: ExplorerContextMenuProps) { - const store = getExplorerStore(); - // const { mutate: generateThumbsForLocation } = useLibraryMutation( - // 'jobs.generateThumbsForLocation' - // ); +function OpenInNativeExplorer() { const platform = usePlatform(); + const store = getExplorerStore(); const os = useOperatingSystem(); const osFileBrowserName = useMemo(() => { @@ -81,6 +75,93 @@ export default function ExplorerContextMenu(props: ExplorerContextMenuProps) { } }, [os]); + return ( + <> + {platform.openPath && ( + { + alert('TODO: Open in FS'); + // console.log('TODO', store.contextMenuActiveItem); + // platform.openPath!('/Users/oscar/Desktop'); // TODO: Work out the file path from the backend + }} + /> + )} + + ); +} + +export function ExplorerContextMenu(props: PropsWithChildren) { + const store = getExplorerStore(); + + const generateThumbsForLocation = useLibraryMutation('jobs.generateThumbsForLocation'); + const objectValidator = useLibraryMutation('jobs.objectValidator'); + const rescanLocation = useLibraryMutation('locations.fullRescan'); + + return ( +
+ + + + + + { + e.preventDefault(); + + navigator.share?.({ + title: 'Spacedrive', + text: 'Check out this cool app', + url: 'https://spacedrive.com' + }); + }} + /> + + + + store.locationId && rescanLocation.mutate(store.locationId)} + label="Re-index" + icon={Repeat} + /> + + + + store.locationId && + generateThumbsForLocation.mutate({ id: store.locationId, path: '' }) + } + label="Regen Thumbnails" + icon={Image} + /> + + store.locationId && objectValidator.mutate({ id: store.locationId, path: '' }) + } + label="Generate Checksums" + icon={ShieldCheck} + /> + + + + +
+ ); +} + +export interface FileItemContextMenuProps extends PropsWithChildren { + item: ExplorerItem; + setShowEncryptDialog: (isShowing: boolean) => void; + setShowDecryptDialog: (isShowing: boolean) => void; + setAlertDialogData: (data: GenericAlertDialogProps) => void; +} + +export function FileItemContextMenu(props: FileItemContextMenuProps) { + const objectData = props.item ? (isObject(props.item) ? props.item : props.item.object) : null; + const hasMasterPasswordQuery = useLibraryQuery(['keys.hasMasterPassword']); const hasMasterPassword = hasMasterPasswordQuery.data !== undefined && hasMasterPasswordQuery.data === true @@ -100,16 +181,7 @@ export default function ExplorerContextMenu(props: ExplorerContextMenuProps) { - {platform.openPath && ( - { - console.log('TODO', store.contextMenuActiveObject); - platform.openPath!('/Users/oscar/Desktop'); // TODO: Work out the file path from the backend - }} - /> - )} + @@ -134,11 +206,10 @@ export default function ExplorerContextMenu(props: ExplorerContextMenuProps) { - {store.contextMenuObjectId && ( - - - - )} + + + + = (props) => { const store = useExplorerStore(); - // const { mutate: generateThumbsForLocation } = useLibraryMutation( - // 'jobs.generateThumbsForLocation' - // ); - // const { mutate: identifyUniqueFiles } = useLibraryMutation('jobs.identifyUniqueFiles'); - // const { mutate: objectValidator } = useLibraryMutation('jobs.objectValidator'); - const navigate = useNavigate(); //create function to focus on search box when cmd+k is pressed @@ -193,7 +191,7 @@ export const TopBar: React.FC = (props) => {
@@ -236,6 +234,24 @@ export const TopBar: React.FC = (props) => { + + (getExplorerStore().layoutMode = 'columns')} + > + + + + {/* + (getExplorerStore().layoutMode = 'timeline')} + > + + + */} = (props) => { active={store.tagAssignMode} > - - + + + // store.locationId && + // generateThumbsForLocation.mutate({ id: store.locationId, path: '' }) + // } + > diff --git a/packages/interface/src/components/explorer/FileItem.tsx b/packages/interface/src/components/explorer/FileItem.tsx index 953a24636..8eb054436 100644 --- a/packages/interface/src/components/explorer/FileItem.tsx +++ b/packages/interface/src/components/explorer/FileItem.tsx @@ -3,7 +3,10 @@ import { cva, tw } from '@sd/ui'; import clsx from 'clsx'; import { HTMLAttributes } from 'react'; -import { getExplorerStore } from '../../util/explorerStore'; +import { getExplorerStore } from '../../hooks/useExplorerStore'; +import { ObjectKind } from '../../util/kind'; +import { GenericAlertDialogProps } from '../dialog/AlertDialog'; +import { FileItemContextMenu } from './ExplorerContextMenu'; import FileThumb from './FileThumb'; import { isObject } from './utils'; @@ -24,64 +27,81 @@ interface Props extends HTMLAttributes { data: ExplorerItem; selected: boolean; index: number; + setShowEncryptDialog: (isShowing: boolean) => void; + setShowDecryptDialog: (isShowing: boolean) => void; + setAlertDialogData: (data: GenericAlertDialogProps) => void; } -function FileItem({ data, selected, index, ...rest }: Props) { +function FileItem({ + data, + selected, + index, + setShowEncryptDialog, + setShowDecryptDialog, + setAlertDialogData, + ...rest +}: Props) { + const objectData = data ? (isObject(data) ? data : data.object) : null; const isVid = isVideoExt(data.extension || ''); return ( -
{ - const objectId = isObject(data) ? data.id : data.object?.id; - if (objectId != undefined) { - getExplorerStore().contextMenuObjectId = objectId; - if (index != undefined) { - getExplorerStore().selectedRowIndex = index; - getExplorerStore().contextMenuActiveObject = isObject(data) ? data : data.object; - } - } - }} - {...rest} - draggable - className={clsx('inline-block w-[100px] mb-3', rest.className)} +
{ + if (index != undefined) { + getExplorerStore().selectedRowIndex = index; } - )} + }} + {...rest} + draggable + className={clsx('inline-block w-[100px] mb-3', rest.className)} >
- - {data?.extension && isVid && ( -
- {data.extension} -
- )} + > + + {data?.extension && isVid && ( +
+ {data.extension} +
+ )} +
+ + + {data?.name} + {data?.extension && `.${data.extension}`} + +
- - - {data?.name} - {data?.extension && `.${data.extension}`} - - -
+ ); } diff --git a/packages/interface/src/components/explorer/FileThumb.tsx b/packages/interface/src/components/explorer/FileThumb.tsx index 30c8a5b74..d14a429fb 100644 --- a/packages/interface/src/components/explorer/FileThumb.tsx +++ b/packages/interface/src/components/explorer/FileThumb.tsx @@ -1,11 +1,14 @@ -import videoSvg from '@sd/assets/svgs/video.svg'; -import zipSvg from '@sd/assets/svgs/zip.svg'; +import archive from '@sd/assets/images/Archive.png'; +import documentPdf from '@sd/assets/images/Document_pdf.png'; +import executable from '@sd/assets/images/Executable.png'; +import file from '@sd/assets/images/File.png'; +import video from '@sd/assets/images/Video.png'; import { ExplorerItem } from '@sd/client'; import clsx from 'clsx'; import { Suspense, lazy, useMemo } from 'react'; +import { useExplorerStore } from '../../hooks/useExplorerStore'; import { usePlatform } from '../../util/Platform'; -import { useExplorerStore } from '../../util/explorerStore'; import { Folder } from '../icons/Folder'; import { isObject, isPath } from './utils'; @@ -15,7 +18,7 @@ interface Props { className?: string; style?: React.CSSProperties; iconClassNames?: string; - kind?: 'video' | 'image' | 'audio' | 'zip' | 'other'; + kind?: string; } const icons = import.meta.glob('../../../../assets/icons/*.svg'); @@ -58,55 +61,12 @@ export default function FileThumb({ data, ...props }: Props) { /> ); + let icon = file; // Hacky (and temporary) way to integrate thumbnails - if (props.kind === 'video') { - return ( - - ); - } - if (props.kind === 'zip') { - return ; - } + if (props.kind === 'Archive') icon = archive; + else if (props.kind === 'Video') icon = video; + else if (props.kind === 'Document' && data.extension === 'pdf') icon = documentPdf; + else if (props.kind === 'Executable') icon = executable; - // return default file icon - return ( -
- - - - {Icon && ( -
- }> - - - - {data.extension} - -
- )} - - - -
- ); + return ; } diff --git a/packages/interface/src/components/explorer/Inspector.tsx b/packages/interface/src/components/explorer/Inspector.tsx index 050ba83b0..18a4e6bd4 100644 --- a/packages/interface/src/components/explorer/Inspector.tsx +++ b/packages/interface/src/components/explorer/Inspector.tsx @@ -1,22 +1,36 @@ // import types from '../../constants/file-types.json'; -import { useLibraryQuery } from '@sd/client'; -import { ExplorerContext, ExplorerItem } from '@sd/client'; -import { Button } from '@sd/ui'; -import { useQuery } from '@tanstack/react-query'; +import { ExplorerContext, ExplorerItem, useLibraryQuery } from '@sd/client'; +import { Button, tw } from '@sd/ui'; import clsx from 'clsx'; import dayjs from 'dayjs'; -import { Link, Share } from 'phosphor-react'; +import { Barcode, CircleWavyCheck, Clock, Cube, Link, Lock, Snowflake } from 'phosphor-react'; import { useEffect, useState } from 'react'; +import { ObjectKind } from '../../util/kind'; import { DefaultProps } from '../primitive/types'; import { Tooltip } from '../tooltip/Tooltip'; import FileThumb from './FileThumb'; import { Divider } from './inspector/Divider'; import FavoriteButton from './inspector/FavoriteButton'; -import { MetaItem } from './inspector/MetaItem'; import Note from './inspector/Note'; import { isObject } from './utils'; +export const InfoPill = tw.span`inline border border-transparent px-1 text-[11px] font-medium shadow shadow-app-shade/5 bg-app-selected rounded-md text-ink-dull`; + +export const PlaceholderPill = tw.span`inline border px-1 text-[11px] shadow shadow-app-shade/10 rounded-md bg-transparent border-dashed border-app-active transition hover:text-ink-faint hover:border-ink-faint font-medium text-ink-faint/70`; + +export const MetaContainer = tw.div`flex flex-col px-4 py-1.5`; + +export const MetaTitle = tw.h5`text-xs font-bold`; + +export const MetaValue = tw.p`text-xs break-all text-ink truncate`; + +const MetaTextLine = tw.div`flex items-center my-0.5 text-xs text-ink-dull`; + +const InspectorIcon = ({ component: Icon, ...props }: any) => ( + +); + interface Props extends DefaultProps { context?: ExplorerContext; data?: ExplorerItem; @@ -25,14 +39,10 @@ interface Props extends DefaultProps { export const Inspector = (props: Props) => { const { context, data, ...elementProps } = props; - const { data: types } = useQuery( - ['_file-types'], - () => import('../../constants/file-types.json') - ); - const is_dir = props.data?.type === 'Path' ? props.data.is_dir : false; const objectData = props.data ? (isObject(props.data) ? props.data : props.data.object) : null; + const isDir = props.data?.type === 'Path' ? props.data.is_dir : false; // this prevents the inspector from fetching data when the user is navigating quickly const [readyToFetch, setReadyToFetch] = useState(false); @@ -48,114 +58,135 @@ export const Inspector = (props: Props) => { enabled: readyToFetch }); - const isVid = isVideo(props.data?.extension || ''); + const fullObjectData = useLibraryQuery(['files.get', { id: objectData?.id || -1 }], { + enabled: readyToFetch && objectData?.id !== undefined + }); return (
{!!props.data && ( <> -
+
-
-

+
+

{props.data?.name} {props.data?.extension && `.${props.data.extension}`}

{objectData && ( -
+
- + + - +
)} - {tags?.data && tags.data.length > 0 && ( - <> - - - {tags?.data?.map((tag) => ( -
setSelectedTag(tag.id === selectedTag ? null : tag.id)} - key={tag.id} - className={clsx( - 'flex items-center rounded px-1.5 py-0.5' - // selectedTag === tag.id && 'ring' - )} - style={{ backgroundColor: tag.color + 'CC' }} - > - {tag.name} -
- ))} -
- } - /> - - )} + {props.context?.type == 'Location' && props.data?.type === 'Path' && ( - <> - - - + + URI + {`${props.context.local_path}/${props.data.materialized_path}`} + )} - - - - {!is_dir && ( - <> - -
- {props.data?.extension && ( - - {props.data?.extension} - - )} -

- {props.data?.extension - ? //@ts-ignore - types[props.data.extension.toUpperCase()]?.descriptions.join(' / ') - : 'Unknown'} -

+ { + +
+ {isDir ? 'Folder' : ObjectKind[objectData?.kind || 0]} + {props.data.extension && {props.data.extension}} + {tags?.data?.map((tag) => ( + + {tag.name} + + ))} + Add Tag
- {objectData && ( - <> - - - {objectData.cas_id && ( - - )} - - )} + + } + + + + + Size + {formatBytes(Number(objectData?.size_in_bytes || 0))} + + {fullObjectData.data?.media_data?.duration_seconds && ( + + + Duration + {fullObjectData.data.media_data.duration_seconds} + + )} + + + + + + + Created + {dayjs(props.data?.date_created).format('MMM Do YYYY')} + + + + + + Indexed + {dayjs(props.data?.date_indexed).format('MMM Do YYYY')} + + + + + {!is_dir && objectData && ( + <> + + + + + + + Content ID + {objectData?.cas_id || ''} + + + {objectData?.integrity_checksum && ( + + + + Checksum + {objectData.integrity_checksum} + + + )} + )}
@@ -165,34 +196,14 @@ export const Inspector = (props: Props) => { ); }; -function isVideo(extension: string) { - return [ - 'avi', - 'asf', - 'mpeg', - 'mts', - 'mpe', - 'vob', - 'qt', - 'mov', - 'asf', - 'asx', - 'mjpeg', - 'ts', - 'mxf', - 'm2ts', - 'f4v', - 'wm', - '3gp', - 'm4v', - 'wmv', - 'mp4', - 'webm', - 'flv', - 'mpg', - 'hevc', - 'ogv', - 'swf', - 'wtv' - ].includes(extension); +function formatBytes(bytes: number, decimals = 2) { + if (bytes === 0) return '0 Bytes'; + + const k = 1024; + const dm = decimals < 0 ? 0 : decimals; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + + const i = Math.floor(Math.log(bytes) / Math.log(k)); + + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } diff --git a/packages/interface/src/components/explorer/VirtualizedList.tsx b/packages/interface/src/components/explorer/VirtualizedList.tsx index c7322b123..9728ec094 100644 --- a/packages/interface/src/components/explorer/VirtualizedList.tsx +++ b/packages/interface/src/components/explorer/VirtualizedList.tsx @@ -4,7 +4,12 @@ import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react import { useSearchParams } from 'react-router-dom'; import { useKey, useOnWindowResize } from 'rooks'; -import { ExplorerLayoutMode, getExplorerStore, useExplorerStore } from '../../util/explorerStore'; +import { + ExplorerLayoutMode, + getExplorerStore, + useExplorerStore +} from '../../hooks/useExplorerStore'; +import { GenericAlertDialogProps } from '../dialog/AlertDialog'; import FileItem from './FileItem'; import FileRow from './FileRow'; import { isPath } from './utils'; @@ -16,9 +21,19 @@ interface Props { context: ExplorerContext; data: ExplorerItem[]; onScroll?: (posY: number) => void; + setShowEncryptDialog: (isShowing: boolean) => void; + setShowDecryptDialog: (isShowing: boolean) => void; + setAlertDialogData: (data: GenericAlertDialogProps) => void; } -export const VirtualizedList: React.FC = ({ data, context, onScroll }) => { +export const VirtualizedList: React.FC = ({ + data, + context, + onScroll, + setShowEncryptDialog, + setShowDecryptDialog, + setAlertDialogData +}) => { const scrollRef = useRef(null); const innerRef = useRef(null); @@ -142,6 +157,9 @@ export const VirtualizedList: React.FC = ({ data, context, onScroll }) => isSelected={getExplorerStore().selectedRowIndex === virtualRow.index} index={virtualRow.index} item={data[virtualRow.index]} + setShowEncryptDialog={setShowEncryptDialog} + setShowDecryptDialog={setShowDecryptDialog} + setAlertDialogData={setAlertDialogData} /> ) : ( [...Array(amountOfColumns)].map((_, i) => { @@ -157,6 +175,9 @@ export const VirtualizedList: React.FC = ({ data, context, onScroll }) => isSelected={isSelected} index={index} item={item} + setShowEncryptDialog={setShowEncryptDialog} + setShowDecryptDialog={setShowDecryptDialog} + setAlertDialogData={setAlertDialogData} /> )}
@@ -177,10 +198,21 @@ interface WrappedItemProps { index: number; isSelected: boolean; kind: ExplorerLayoutMode; + setShowEncryptDialog: (isShowing: boolean) => void; + setShowDecryptDialog: (isShowing: boolean) => void; + setAlertDialogData: (data: GenericAlertDialogProps) => void; } // Wrap either list item or grid item with click logic as it is the same for both -const WrappedItem: React.FC = ({ item, index, isSelected, kind }) => { +const WrappedItem: React.FC = ({ + item, + index, + isSelected, + kind, + setShowEncryptDialog, + setShowDecryptDialog, + setAlertDialogData +}) => { const [_, setSearchParams] = useSearchParams(); const onDoubleClick = useCallback(() => { @@ -199,6 +231,9 @@ const WrappedItem: React.FC = ({ item, index, isSelected, kind onClick={onClick} onDoubleClick={onDoubleClick} selected={isSelected} + setShowEncryptDialog={setShowEncryptDialog} + setShowDecryptDialog={setShowDecryptDialog} + setAlertDialogData={setAlertDialogData} /> ); diff --git a/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx b/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx index 05eaf92d3..a0227545c 100644 --- a/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx +++ b/packages/interface/src/components/explorer/inspector/FavoriteButton.tsx @@ -30,7 +30,7 @@ export default function FavoriteButton(props: Props) { }; return ( - ); diff --git a/packages/interface/src/components/explorer/inspector/MetaItem.tsx b/packages/interface/src/components/explorer/inspector/MetaItem.tsx deleted file mode 100644 index 9fbaa81ff..000000000 --- a/packages/interface/src/components/explorer/inspector/MetaItem.tsx +++ /dev/null @@ -1,17 +0,0 @@ -interface MetaItemProps { - title?: string; - value: string | React.ReactNode; -} - -export const MetaItem = (props: MetaItemProps) => { - return ( -
- {!!props.title &&
{props.title}
} - {typeof props.value === 'string' ? ( -

{props.value}

- ) : ( - props.value - )} -
- ); -}; diff --git a/packages/interface/src/components/explorer/inspector/Note.tsx b/packages/interface/src/components/explorer/inspector/Note.tsx index bbc999894..339d36e45 100644 --- a/packages/interface/src/components/explorer/inspector/Note.tsx +++ b/packages/interface/src/components/explorer/inspector/Note.tsx @@ -4,8 +4,8 @@ import { TextArea } from '@sd/ui'; import { useCallback, useState } from 'react'; import { useDebouncedCallback } from 'use-debounce'; +import { MetaContainer, MetaTitle } from '../Inspector'; import { Divider } from './Divider'; -import { MetaItem } from './MetaItem'; interface Props { data: SDObject; @@ -41,16 +41,14 @@ export default function Note(props: Props) { return ( <> - - } - /> + + Note +