From de85f00efccec46355a57a366ba71b7dcb95e153 Mon Sep 17 00:00:00 2001 From: "Ericson \"Fogo\" Soares" Date: Wed, 21 Jun 2023 09:55:19 -0300 Subject: [PATCH] [ENG-768 / ENG-769] Regen thumbnails / Generate checksums (#983) * Making sub_path api for jobs harder to misuse * re-enable context menu options --------- Co-authored-by: Brendan Allan --- core/src/api/files.rs | 94 +++++----------- core/src/api/jobs.rs | 12 +- core/src/job/error.rs | 4 +- core/src/location/file_path_helper/mod.rs | 4 - core/src/location/indexer/indexer_job.rs | 37 ++++--- core/src/location/indexer/shallow.rs | 2 +- .../file_identifier/file_identifier_job.rs | 41 +++---- core/src/object/file_identifier/shallow.rs | 2 +- core/src/object/preview/thumbnail/shallow.rs | 2 +- .../preview/thumbnail/thumbnailer_job.rs | 49 +++++---- core/src/object/validation/mod.rs | 20 ++++ core/src/object/validation/validator_job.rs | 103 ++++++++++++++---- .../app/$libraryId/Explorer/ContextMenu.tsx | 9 +- .../$libraryId/Explorer/File/ContextMenu.tsx | 2 +- 14 files changed, 217 insertions(+), 164 deletions(-) diff --git a/core/src/api/files.rs b/core/src/api/files.rs index 1264b37f9..747803ea9 100644 --- a/core/src/api/files.rs +++ b/core/src/api/files.rs @@ -18,7 +18,7 @@ use crate::{ use std::path::Path; use chrono::Utc; -use futures::future::try_join_all; +use futures::future::join_all; use regex::Regex; use rspc::{alpha::AlphaRouter, ErrorCode}; use serde::Deserialize; @@ -237,9 +237,10 @@ pub(crate) fn mount() -> AlphaRouter { .map_err(LocationError::FilePath)?; let mut new_file_full_path = location_path.join(iso_file_path.parent()); - new_file_full_path.push(new_file_name); if !new_extension.is_empty() { - new_file_full_path.set_extension(new_extension); + new_file_full_path.push(format!("{}.{}", new_file_name, new_extension)); + } else { + new_file_full_path.push(new_file_name); } match fs::metadata(&new_file_full_path).await { @@ -270,19 +271,6 @@ pub(crate) fn mount() -> AlphaRouter { ) })?; - library - .db - .file_path() - .update( - file_path::id::equals(from_file_path_id), - vec![ - file_path::name::set(Some(new_file_name.to_string())), - file_path::extension::set(Some(new_extension.to_string())), - ], - ) - .exec() - .await?; - Ok(()) } @@ -304,7 +292,7 @@ pub(crate) fn mount() -> AlphaRouter { )); }; - let to_update = try_join_all( + let errors = join_all( library .db .file_path() @@ -313,12 +301,8 @@ pub(crate) fn mount() -> AlphaRouter { .exec() .await? .into_iter() - .flat_map(|file_path| { - let id = file_path.id; - - IsolatedFilePathData::try_from(file_path).map(|d| (id, d)) - }) - .map(|(file_path_id, iso_file_path)| { + .flat_map(IsolatedFilePathData::try_from) + .map(|iso_file_path| { let from = location_path.join(&iso_file_path); let mut to = location_path.join(iso_file_path.parent()); let full_name = iso_file_path.full_name(); @@ -339,57 +323,37 @@ pub(crate) fn mount() -> AlphaRouter { "Invalid file name".to_string(), )) } else { - fs::rename(&from, &to) - .await - .map_err(|e| { - error!( - "Failed to rename file from: '{}' to: '{}'", + fs::rename(&from, &to).await.map_err(|e| { + error!( + "Failed to rename file from: '{}' to: '{}'; Error: {e:#?}", from.display(), to.display() ); - rspc::Error::with_cause( - ErrorCode::Conflict, - "Failed to rename file".to_string(), - e, - ) - }) - .map(|_| { - let (name, extension) = - IsolatedFilePathData::separate_name_and_extension_from_str( - &replaced_full_name, - ) - .expect("we just built this full name and validated it"); - - ( - file_path_id, - (name.to_string(), extension.to_string()), - ) - }) + rspc::Error::with_cause( + ErrorCode::Conflict, + "Failed to rename file".to_string(), + e, + ) + }) } } }), ) - .await?; + .await + .into_iter() + .filter_map(Result::err) + .collect::>(); - // TODO: dispatch sync update events - - library - .db - ._batch( - to_update + if !errors.is_empty() { + return Err(rspc::Error::new( + rspc::ErrorCode::Conflict, + errors .into_iter() - .map(|(file_path_id, (new_name, new_extension))| { - library.db.file_path().update( - file_path::id::equals(file_path_id), - vec![ - file_path::name::set(Some(new_name)), - file_path::extension::set(Some(new_extension)), - ], - ) - }) - .collect::>(), - ) - .await?; + .map(|e| e.to_string()) + .collect::>() + .join("\n"), + )); + } Ok(()) } diff --git a/core/src/api/jobs.rs b/core/src/api/jobs.rs index 1215ea87f..f16e90c9d 100644 --- a/core/src/api/jobs.rs +++ b/core/src/api/jobs.rs @@ -224,15 +224,17 @@ pub(crate) fn mount() -> AlphaRouter { R.with2(library()) .mutation(|(_, library), args: ObjectValidatorArgs| async move { - if find_location(&library, args.id).exec().await?.is_none() { + let Some(location) = find_location(&library, args.id) + .exec() + .await? + else { return Err(LocationError::IdNotFound(args.id).into()); - } + }; library .spawn_job(ObjectValidatorJobInit { - location_id: args.id, - path: args.path, - background: true, + location, + sub_path: Some(args.path), }) .await .map_err(Into::into) diff --git a/core/src/job/error.rs b/core/src/job/error.rs index 056c4f2e5..61549e0a1 100644 --- a/core/src/job/error.rs +++ b/core/src/job/error.rs @@ -2,7 +2,7 @@ use crate::{ location::{indexer::IndexerError, LocationError}, object::{ file_identifier::FileIdentifierJobError, fs::error::FileSystemJobsError, - preview::ThumbnailerError, + preview::ThumbnailerError, validation::ValidatorError, }, util::{db::MissingFieldError, error::FileIOError}, }; @@ -63,6 +63,8 @@ pub enum JobError { #[error(transparent)] IdentifierError(#[from] FileIdentifierJobError), #[error(transparent)] + Validator(#[from] ValidatorError), + #[error(transparent)] FileSystemJobsError(#[from] FileSystemJobsError), #[error(transparent)] CryptoError(#[from] CryptoError), diff --git a/core/src/location/file_path_helper/mod.rs b/core/src/location/file_path_helper/mod.rs index 55bccea0b..9d11d4838 100644 --- a/core/src/location/file_path_helper/mod.rs +++ b/core/src/location/file_path_helper/mod.rs @@ -44,10 +44,6 @@ file_path::select!(file_path_for_object_validator { name extension integrity_checksum - location: select { - id - pub_id - } }); file_path::select!(file_path_for_thumbnailer { materialized_path diff --git a/core/src/location/indexer/indexer_job.rs b/core/src/location/indexer/indexer_job.rs index 8b9256d2e..5da88d0be 100644 --- a/core/src/location/indexer/indexer_job.rs +++ b/core/src/location/indexer/indexer_job.rs @@ -77,26 +77,27 @@ impl StatefulJob for IndexerJob { .collect::, _>>() .map_err(IndexerError::from)?; - let to_walk_path = if let Some(ref sub_path) = state.init.sub_path { - let full_path = ensure_sub_path_is_in_location(location_path, sub_path) - .await - .map_err(IndexerError::from)?; - ensure_sub_path_is_directory(location_path, sub_path) - .await - .map_err(IndexerError::from)?; + let to_walk_path = match &state.init.sub_path { + Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => { + let full_path = ensure_sub_path_is_in_location(location_path, sub_path) + .await + .map_err(IndexerError::from)?; + ensure_sub_path_is_directory(location_path, sub_path) + .await + .map_err(IndexerError::from)?; - ensure_file_path_exists( - sub_path, - &IsolatedFilePathData::new(location_id, location_path, &full_path, true) - .map_err(IndexerError::from)?, - &db, - IndexerError::SubPathNotFound, - ) - .await?; + ensure_file_path_exists( + sub_path, + &IsolatedFilePathData::new(location_id, location_path, &full_path, true) + .map_err(IndexerError::from)?, + &db, + IndexerError::SubPathNotFound, + ) + .await?; - full_path - } else { - location_path.to_path_buf() + full_path + } + _ => location_path.to_path_buf(), }; let scan_start = Instant::now(); diff --git a/core/src/location/indexer/shallow.rs b/core/src/location/indexer/shallow.rs index b81d634a1..89fc76c3b 100644 --- a/core/src/location/indexer/shallow.rs +++ b/core/src/location/indexer/shallow.rs @@ -45,7 +45,7 @@ pub async fn shallow( .collect::, _>>() .map_err(IndexerError::from)?; - let (add_root, to_walk_path) = if sub_path != Path::new("") { + let (add_root, to_walk_path) = if sub_path != Path::new("") && sub_path != Path::new("/") { let full_path = ensure_sub_path_is_in_location(&location_path, &sub_path) .await .map_err(IndexerError::from)?; diff --git a/core/src/object/file_identifier/file_identifier_job.rs b/core/src/object/file_identifier/file_identifier_job.rs index 1c55c1e75..f54195da8 100644 --- a/core/src/object/file_identifier/file_identifier_job.rs +++ b/core/src/object/file_identifier/file_identifier_job.rs @@ -83,29 +83,30 @@ impl StatefulJob for FileIdentifierJob { let location_path = maybe_missing(&state.init.location.path, "location.path").map(Path::new)?; - let maybe_sub_iso_file_path = if let Some(ref sub_path) = state.init.sub_path { - let full_path = ensure_sub_path_is_in_location(location_path, sub_path) - .await - .map_err(FileIdentifierJobError::from)?; - ensure_sub_path_is_directory(location_path, sub_path) - .await - .map_err(FileIdentifierJobError::from)?; - - let sub_iso_file_path = - IsolatedFilePathData::new(location_id, location_path, &full_path, true) + let maybe_sub_iso_file_path = match &state.init.sub_path { + Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => { + let full_path = ensure_sub_path_is_in_location(location_path, sub_path) + .await + .map_err(FileIdentifierJobError::from)?; + ensure_sub_path_is_directory(location_path, sub_path) + .await .map_err(FileIdentifierJobError::from)?; - ensure_file_path_exists( - sub_path, - &sub_iso_file_path, - db, - FileIdentifierJobError::SubPathNotFound, - ) - .await?; + let sub_iso_file_path = + IsolatedFilePathData::new(location_id, location_path, &full_path, true) + .map_err(FileIdentifierJobError::from)?; - Some(sub_iso_file_path) - } else { - None + ensure_file_path_exists( + sub_path, + &sub_iso_file_path, + db, + FileIdentifierJobError::SubPathNotFound, + ) + .await?; + + Some(sub_iso_file_path) + } + _ => None, }; let orphan_count = diff --git a/core/src/object/file_identifier/shallow.rs b/core/src/object/file_identifier/shallow.rs index 3d4547f75..531b00a35 100644 --- a/core/src/object/file_identifier/shallow.rs +++ b/core/src/object/file_identifier/shallow.rs @@ -35,7 +35,7 @@ pub async fn shallow( let location_id = location.id; let location_path = maybe_missing(&location.path, "location.path").map(Path::new)?; - let sub_iso_file_path = if sub_path != Path::new("") { + let sub_iso_file_path = if sub_path != Path::new("") && sub_path != Path::new("/") { let full_path = ensure_sub_path_is_in_location(location_path, &sub_path) .await .map_err(FileIdentifierJobError::from)?; diff --git a/core/src/object/preview/thumbnail/shallow.rs b/core/src/object/preview/thumbnail/shallow.rs index bcec742a8..c1a94d4b2 100644 --- a/core/src/object/preview/thumbnail/shallow.rs +++ b/core/src/object/preview/thumbnail/shallow.rs @@ -37,7 +37,7 @@ pub async fn shallow_thumbnailer( None => return Ok(()), }; - let (path, iso_file_path) = if sub_path != Path::new("") { + let (path, iso_file_path) = if sub_path != Path::new("") && sub_path != Path::new("/") { let full_path = ensure_sub_path_is_in_location(&location_path, &sub_path) .await .map_err(ThumbnailerError::from)?; diff --git a/core/src/object/preview/thumbnail/thumbnailer_job.rs b/core/src/object/preview/thumbnail/thumbnailer_job.rs index 1a68ec379..dd44eb8db 100644 --- a/core/src/object/preview/thumbnail/thumbnailer_job.rs +++ b/core/src/object/preview/thumbnail/thumbnailer_job.rs @@ -12,7 +12,11 @@ use crate::{ prisma::{file_path, location, PrismaClient}, }; -use std::{collections::VecDeque, hash::Hash, path::PathBuf}; +use std::{ + collections::VecDeque, + hash::Hash, + path::{Path, PathBuf}, +}; use sd_file_ext::extensions::Extension; @@ -77,33 +81,34 @@ impl StatefulJob for ThumbnailerJob { None => return Ok(()), }; - let (path, iso_file_path) = if let Some(ref sub_path) = state.init.sub_path { - let full_path = ensure_sub_path_is_in_location(&location_path, sub_path) - .await - .map_err(ThumbnailerError::from)?; - ensure_sub_path_is_directory(&location_path, sub_path) - .await - .map_err(ThumbnailerError::from)?; - - let sub_iso_file_path = - IsolatedFilePathData::new(location_id, &location_path, &full_path, true) + let (path, iso_file_path) = match &state.init.sub_path { + Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => { + let full_path = ensure_sub_path_is_in_location(&location_path, sub_path) + .await + .map_err(ThumbnailerError::from)?; + ensure_sub_path_is_directory(&location_path, sub_path) + .await .map_err(ThumbnailerError::from)?; - ensure_file_path_exists( - sub_path, - &sub_iso_file_path, - db, - ThumbnailerError::SubPathNotFound, - ) - .await?; + let sub_iso_file_path = + IsolatedFilePathData::new(location_id, &location_path, &full_path, true) + .map_err(ThumbnailerError::from)?; - (full_path, sub_iso_file_path) - } else { - ( + ensure_file_path_exists( + sub_path, + &sub_iso_file_path, + db, + ThumbnailerError::SubPathNotFound, + ) + .await?; + + (full_path, sub_iso_file_path) + } + _ => ( location_path.to_path_buf(), IsolatedFilePathData::new(location_id, &location_path, &location_path, true) .map_err(ThumbnailerError::from)?, - ) + ), }; info!("Searching for images in location {location_id} at directory {iso_file_path}"); diff --git a/core/src/object/validation/mod.rs b/core/src/object/validation/mod.rs index 372e0c428..6064c92ef 100644 --- a/core/src/object/validation/mod.rs +++ b/core/src/object/validation/mod.rs @@ -1,2 +1,22 @@ +use crate::{location::file_path_helper::FilePathError, util::error::FileIOError}; + +use std::path::Path; + +use thiserror::Error; + pub mod hash; pub mod validator_job; + +#[derive(Error, Debug)] +pub enum ValidatorError { + #[error("sub path not found: ", .0.display())] + SubPathNotFound(Box), + + // Internal errors + #[error("database error: {0}")] + Database(#[from] prisma_client_rust::QueryError), + #[error(transparent)] + FilePath(#[from] FilePathError), + #[error(transparent)] + FileIO(#[from] FileIOError), +} diff --git a/core/src/object/validation/validator_job.rs b/core/src/object/validation/validator_job.rs index cd7503fee..3be49956f 100644 --- a/core/src/object/validation/validator_job.rs +++ b/core/src/object/validation/validator_job.rs @@ -4,19 +4,28 @@ use crate::{ JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext, }, library::Library, - location::file_path_helper::{file_path_for_object_validator, IsolatedFilePathData}, + location::file_path_helper::{ + ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location, + file_path_for_object_validator, IsolatedFilePathData, + }, prisma::{file_path, location}, sync, - util::{db::maybe_missing, error::FileIOError}, + util::{ + db::{chain_optional_iter, maybe_missing}, + error::FileIOError, + }, }; -use std::path::PathBuf; +use std::{ + hash::{Hash, Hasher}, + path::{Path, PathBuf}, +}; use serde::{Deserialize, Serialize}; use serde_json::json; use tracing::info; -use super::hash::file_checksum; +use super::{hash::file_checksum, ValidatorError}; // The Validator is able to: // - generate a full byte checksum for Objects in a Location @@ -26,16 +35,24 @@ pub struct ObjectValidatorJob {} #[derive(Serialize, Deserialize, Debug)] pub struct ObjectValidatorJobState { - pub root_path: PathBuf, + pub location_path: PathBuf, pub task_count: usize, } // The validator can -#[derive(Serialize, Deserialize, Debug, Hash)] +#[derive(Serialize, Deserialize, Debug)] pub struct ObjectValidatorJobInit { - pub location_id: location::id::Type, - pub path: PathBuf, - pub background: bool, + pub location: location::Data, + pub sub_path: Option, +} + +impl Hash for ObjectValidatorJobInit { + fn hash(&self, state: &mut H) { + self.location.id.hash(state); + if let Some(ref sub_path) = self.sub_path { + sub_path.hash(state); + } + } } impl JobInitData for ObjectValidatorJobInit { @@ -61,20 +78,58 @@ impl StatefulJob for ObjectValidatorJob { ) -> Result<(), JobError> { let Library { db, .. } = &ctx.library; + let location_id = state.init.location.id; + + let location_path = + maybe_missing(&state.init.location.path, "location.path").map(PathBuf::from)?; + + let maybe_sub_iso_file_path = match &state.init.sub_path { + Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => { + let full_path = ensure_sub_path_is_in_location(&location_path, sub_path) + .await + .map_err(ValidatorError::from)?; + ensure_sub_path_is_directory(&location_path, sub_path) + .await + .map_err(ValidatorError::from)?; + + let sub_iso_file_path = + IsolatedFilePathData::new(location_id, &location_path, &full_path, true) + .map_err(ValidatorError::from)?; + + ensure_file_path_exists( + sub_path, + &sub_iso_file_path, + db, + ValidatorError::SubPathNotFound, + ) + .await?; + + Some(sub_iso_file_path) + } + _ => None, + }; + state.steps.extend( db.file_path() - .find_many(vec![ - file_path::location_id::equals(Some(state.init.location_id)), - file_path::is_dir::equals(Some(false)), - file_path::integrity_checksum::equals(None), - ]) + .find_many(chain_optional_iter( + [ + file_path::location_id::equals(Some(state.init.location.id)), + file_path::is_dir::equals(Some(false)), + file_path::integrity_checksum::equals(None), + ], + [maybe_sub_iso_file_path.and_then(|iso_sub_path| { + iso_sub_path + .materialized_path_for_children() + .map(file_path::materialized_path::starts_with) + })], + )) .select(file_path_for_object_validator::select()) .exec() .await?, ); state.data = Some(ObjectValidatorJobState { - root_path: state.init.path.clone(), + location_path, task_count: state.steps.len(), }); @@ -98,13 +153,13 @@ impl StatefulJob for ObjectValidatorJob { // we can also compare old and new checksums here // This if is just to make sure, we already queried objects where integrity_checksum is null if file_path.integrity_checksum.is_none() { - let path = data.root_path.join(IsolatedFilePathData::try_from(( - maybe_missing(&file_path.location, "file_path.location")?.id, + let full_path = data.location_path.join(IsolatedFilePathData::try_from(( + state.init.location.id, file_path, ))?); - let checksum = file_checksum(&path) + let checksum = file_checksum(&full_path) .await - .map_err(|e| FileIOError::from((path, e)))?; + .map_err(|e| ValidatorError::FileIO(FileIOError::from((full_path, e))))?; sync.write_op( db, @@ -137,8 +192,14 @@ impl StatefulJob for ObjectValidatorJob { ) -> JobResult { let data = extract_job_data!(state); info!( - "finalizing validator job at {}: {} tasks", - data.root_path.display(), + "finalizing validator job at {}{}: {} tasks", + data.location_path.display(), + state + .init + .sub_path + .as_ref() + .map(|p| format!("{}", p.display())) + .unwrap_or_default(), data.task_count ); diff --git a/interface/app/$libraryId/Explorer/ContextMenu.tsx b/interface/app/$libraryId/Explorer/ContextMenu.tsx index 6d1cb713a..319a52f7b 100644 --- a/interface/app/$libraryId/Explorer/ContextMenu.tsx +++ b/interface/app/$libraryId/Explorer/ContextMenu.tsx @@ -130,20 +130,21 @@ export default (props: PropsWithChildren) => { store.locationId && - generateThumbsForLocation.mutate({ id: store.locationId, path: '' }) + generateThumbsForLocation.mutate({ + id: store.locationId, + path: params.path ?? '' + }) } label="Regen Thumbnails" icon={Image} - disabled /> store.locationId && - objectValidator.mutate({ id: store.locationId, path: '' }) + objectValidator.mutate({ id: store.locationId, path: params.path ?? '' }) } label="Generate Checksums" icon={ShieldCheck} - disabled /> diff --git a/interface/app/$libraryId/Explorer/File/ContextMenu.tsx b/interface/app/$libraryId/Explorer/File/ContextMenu.tsx index 47de0bc3f..f75e8eeae 100644 --- a/interface/app/$libraryId/Explorer/File/ContextMenu.tsx +++ b/interface/app/$libraryId/Explorer/File/ContextMenu.tsx @@ -238,7 +238,7 @@ export default ({ data }: Props) => { onClick={() => { generateThumbnails.mutate({ id: getExplorerStore().locationId!, - path: '/' + path: params.path ?? '' }); }} label="Regen Thumbnails"