[ENG-768 / ENG-769] Regen thumbnails / Generate checksums (#983)

* Making sub_path api for jobs harder to misuse

* re-enable context menu options

---------

Co-authored-by: Brendan Allan <brendonovich@outlook.com>

core/src/api/files.rs

@@ -18,7 +18,7 @@ use crate::{
 use std::path::Path;
 
 use chrono::Utc;
-use futures::future::try_join_all;
+use futures::future::join_all;
 use regex::Regex;
 use rspc::{alpha::AlphaRouter, ErrorCode};
 use serde::Deserialize;
@@ -237,9 +237,10 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 							.map_err(LocationError::FilePath)?;
 
 					let mut new_file_full_path = location_path.join(iso_file_path.parent());
-					new_file_full_path.push(new_file_name);
 					if !new_extension.is_empty() {
-						new_file_full_path.set_extension(new_extension);
+						new_file_full_path.push(format!("{}.{}", new_file_name, new_extension));
+					} else {
+						new_file_full_path.push(new_file_name);
 					}
 
 					match fs::metadata(&new_file_full_path).await {
@@ -270,19 +271,6 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 							)
 						})?;
 
-					library
-						.db
-						.file_path()
-						.update(
-							file_path::id::equals(from_file_path_id),
-							vec![
-								file_path::name::set(Some(new_file_name.to_string())),
-								file_path::extension::set(Some(new_extension.to_string())),
-							],
-						)
-						.exec()
-						.await?;
-
 					Ok(())
 				}
 
@@ -304,7 +292,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 						));
 					};
 
-					let to_update = try_join_all(
+					let errors = join_all(
 						library
 							.db
 							.file_path()
@@ -313,12 +301,8 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 							.exec()
 							.await?
 							.into_iter()
-							.flat_map(|file_path| {
-								let id = file_path.id;
-
-								IsolatedFilePathData::try_from(file_path).map(|d| (id, d))
-							})
-							.map(|(file_path_id, iso_file_path)| {
+							.flat_map(IsolatedFilePathData::try_from)
+							.map(|iso_file_path| {
 								let from = location_path.join(&iso_file_path);
 								let mut to = location_path.join(iso_file_path.parent());
 								let full_name = iso_file_path.full_name();
@@ -339,57 +323,37 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 											"Invalid file name".to_string(),
 										))
 									} else {
-										fs::rename(&from, &to)
-											.await
-											.map_err(|e| {
-												error!(
-													"Failed to rename file from: '{}' to: '{}'",
+										fs::rename(&from, &to).await.map_err(|e| {
+											error!(
+													"Failed to rename file from: '{}' to: '{}'; Error: {e:#?}",
 													from.display(),
 													to.display()
 												);
-												rspc::Error::with_cause(
-													ErrorCode::Conflict,
-													"Failed to rename file".to_string(),
-													e,
-												)
-											})
-											.map(|_| {
-												let (name, extension) =
-												IsolatedFilePathData::separate_name_and_extension_from_str(
-												&replaced_full_name,
-												)
-												.expect("we just built this full name and validated it");
-
-												(
-													file_path_id,
-													(name.to_string(), extension.to_string()),
-												)
-											})
+											rspc::Error::with_cause(
+												ErrorCode::Conflict,
+												"Failed to rename file".to_string(),
+												e,
+											)
+										})
 									}
 								}
 							}),
 					)
-					.await?;
+					.await
+					.into_iter()
+					.filter_map(Result::err)
+					.collect::<Vec<_>>();
 
-					// TODO: dispatch sync update events
-
-					library
-						.db
-						._batch(
-							to_update
+					if !errors.is_empty() {
+						return Err(rspc::Error::new(
+							rspc::ErrorCode::Conflict,
+							errors
 								.into_iter()
-								.map(|(file_path_id, (new_name, new_extension))| {
-									library.db.file_path().update(
-										file_path::id::equals(file_path_id),
-										vec![
-											file_path::name::set(Some(new_name)),
-											file_path::extension::set(Some(new_extension)),
-										],
-									)
-								})
-								.collect::<Vec<_>>(),
-						)
-						.await?;
+								.map(|e| e.to_string())
+								.collect::<Vec<_>>()
+								.join("\n"),
+						));
+					}
 
 					Ok(())
 				}

core/src/api/jobs.rs

@@ -224,15 +224,17 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 
 			R.with2(library())
 				.mutation(|(_, library), args: ObjectValidatorArgs| async move {
-					if find_location(&library, args.id).exec().await?.is_none() {
+					let Some(location) =  find_location(&library, args.id)
+						.exec()
+						.await?
+						else {
 						return Err(LocationError::IdNotFound(args.id).into());
-					}
+					};
 
 					library
 						.spawn_job(ObjectValidatorJobInit {
-							location_id: args.id,
-							path: args.path,
-							background: true,
+							location,
+							sub_path: Some(args.path),
 						})
 						.await
 						.map_err(Into::into)

core/src/job/error.rs

@@ -2,7 +2,7 @@ use crate::{
 	location::{indexer::IndexerError, LocationError},
 	object::{
 		file_identifier::FileIdentifierJobError, fs::error::FileSystemJobsError,
-		preview::ThumbnailerError,
+		preview::ThumbnailerError, validation::ValidatorError,
 	},
 	util::{db::MissingFieldError, error::FileIOError},
 };
@@ -63,6 +63,8 @@ pub enum JobError {
 	#[error(transparent)]
 	IdentifierError(#[from] FileIdentifierJobError),
 	#[error(transparent)]
+	Validator(#[from] ValidatorError),
+	#[error(transparent)]
 	FileSystemJobsError(#[from] FileSystemJobsError),
 	#[error(transparent)]
 	CryptoError(#[from] CryptoError),

core/src/location/file_path_helper/mod.rs

@@ -44,10 +44,6 @@ file_path::select!(file_path_for_object_validator {
 	name
 	extension
 	integrity_checksum
-	location: select {
-		id
-		pub_id
-	}
 });
 file_path::select!(file_path_for_thumbnailer {
 	materialized_path

core/src/location/indexer/indexer_job.rs

@@ -77,26 +77,27 @@ impl StatefulJob for IndexerJob {
 			.collect::<Result<Vec<_>, _>>()
 			.map_err(IndexerError::from)?;
 
-		let to_walk_path = if let Some(ref sub_path) = state.init.sub_path {
-			let full_path = ensure_sub_path_is_in_location(location_path, sub_path)
-				.await
-				.map_err(IndexerError::from)?;
-			ensure_sub_path_is_directory(location_path, sub_path)
-				.await
-				.map_err(IndexerError::from)?;
+		let to_walk_path = match &state.init.sub_path {
+			Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => {
+				let full_path = ensure_sub_path_is_in_location(location_path, sub_path)
+					.await
+					.map_err(IndexerError::from)?;
+				ensure_sub_path_is_directory(location_path, sub_path)
+					.await
+					.map_err(IndexerError::from)?;
 
-			ensure_file_path_exists(
-				sub_path,
-				&IsolatedFilePathData::new(location_id, location_path, &full_path, true)
-					.map_err(IndexerError::from)?,
-				&db,
-				IndexerError::SubPathNotFound,
-			)
-			.await?;
+				ensure_file_path_exists(
+					sub_path,
+					&IsolatedFilePathData::new(location_id, location_path, &full_path, true)
+						.map_err(IndexerError::from)?,
+					&db,
+					IndexerError::SubPathNotFound,
+				)
+				.await?;
 
-			full_path
-		} else {
-			location_path.to_path_buf()
+				full_path
+			}
+			_ => location_path.to_path_buf(),
 		};
 
 		let scan_start = Instant::now();

core/src/location/indexer/shallow.rs

@@ -45,7 +45,7 @@ pub async fn shallow(
 		.collect::<Result<Vec<_>, _>>()
 		.map_err(IndexerError::from)?;
 
-	let (add_root, to_walk_path) = if sub_path != Path::new("") {
+	let (add_root, to_walk_path) = if sub_path != Path::new("") && sub_path != Path::new("/") {
 		let full_path = ensure_sub_path_is_in_location(&location_path, &sub_path)
 			.await
 			.map_err(IndexerError::from)?;

core/src/object/file_identifier/file_identifier_job.rs

@@ -83,29 +83,30 @@ impl StatefulJob for FileIdentifierJob {
 		let location_path =
 			maybe_missing(&state.init.location.path, "location.path").map(Path::new)?;
 
-		let maybe_sub_iso_file_path = if let Some(ref sub_path) = state.init.sub_path {
-			let full_path = ensure_sub_path_is_in_location(location_path, sub_path)
-				.await
-				.map_err(FileIdentifierJobError::from)?;
-			ensure_sub_path_is_directory(location_path, sub_path)
-				.await
-				.map_err(FileIdentifierJobError::from)?;
-
-			let sub_iso_file_path =
-				IsolatedFilePathData::new(location_id, location_path, &full_path, true)
+		let maybe_sub_iso_file_path = match &state.init.sub_path {
+			Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => {
+				let full_path = ensure_sub_path_is_in_location(location_path, sub_path)
+					.await
+					.map_err(FileIdentifierJobError::from)?;
+				ensure_sub_path_is_directory(location_path, sub_path)
+					.await
 					.map_err(FileIdentifierJobError::from)?;
 
-			ensure_file_path_exists(
-				sub_path,
-				&sub_iso_file_path,
-				db,
-				FileIdentifierJobError::SubPathNotFound,
-			)
-			.await?;
+				let sub_iso_file_path =
+					IsolatedFilePathData::new(location_id, location_path, &full_path, true)
+						.map_err(FileIdentifierJobError::from)?;
 
-			Some(sub_iso_file_path)
-		} else {
-			None
+				ensure_file_path_exists(
+					sub_path,
+					&sub_iso_file_path,
+					db,
+					FileIdentifierJobError::SubPathNotFound,
+				)
+				.await?;
+
+				Some(sub_iso_file_path)
+			}
+			_ => None,
 		};
 
 		let orphan_count =

core/src/object/file_identifier/shallow.rs

@@ -35,7 +35,7 @@ pub async fn shallow(
 	let location_id = location.id;
 	let location_path = maybe_missing(&location.path, "location.path").map(Path::new)?;
 
-	let sub_iso_file_path = if sub_path != Path::new("") {
+	let sub_iso_file_path = if sub_path != Path::new("") && sub_path != Path::new("/") {
 		let full_path = ensure_sub_path_is_in_location(location_path, &sub_path)
 			.await
 			.map_err(FileIdentifierJobError::from)?;

core/src/object/preview/thumbnail/shallow.rs

@@ -37,7 +37,7 @@ pub async fn shallow_thumbnailer(
 		None => return Ok(()),
 	};
 
-	let (path, iso_file_path) = if sub_path != Path::new("") {
+	let (path, iso_file_path) = if sub_path != Path::new("") && sub_path != Path::new("/") {
 		let full_path = ensure_sub_path_is_in_location(&location_path, &sub_path)
 			.await
 			.map_err(ThumbnailerError::from)?;

core/src/object/preview/thumbnail/thumbnailer_job.rs

@@ -12,7 +12,11 @@ use crate::{
 	prisma::{file_path, location, PrismaClient},
 };
 
-use std::{collections::VecDeque, hash::Hash, path::PathBuf};
+use std::{
+	collections::VecDeque,
+	hash::Hash,
+	path::{Path, PathBuf},
+};
 
 use sd_file_ext::extensions::Extension;
 
@@ -77,33 +81,34 @@ impl StatefulJob for ThumbnailerJob {
 			None => return Ok(()),
 		};
 
-		let (path, iso_file_path) = if let Some(ref sub_path) = state.init.sub_path {
-			let full_path = ensure_sub_path_is_in_location(&location_path, sub_path)
-				.await
-				.map_err(ThumbnailerError::from)?;
-			ensure_sub_path_is_directory(&location_path, sub_path)
-				.await
-				.map_err(ThumbnailerError::from)?;
-
-			let sub_iso_file_path =
-				IsolatedFilePathData::new(location_id, &location_path, &full_path, true)
+		let (path, iso_file_path) = match &state.init.sub_path {
+			Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => {
+				let full_path = ensure_sub_path_is_in_location(&location_path, sub_path)
+					.await
+					.map_err(ThumbnailerError::from)?;
+				ensure_sub_path_is_directory(&location_path, sub_path)
+					.await
 					.map_err(ThumbnailerError::from)?;
 
-			ensure_file_path_exists(
-				sub_path,
-				&sub_iso_file_path,
-				db,
-				ThumbnailerError::SubPathNotFound,
-			)
-			.await?;
+				let sub_iso_file_path =
+					IsolatedFilePathData::new(location_id, &location_path, &full_path, true)
+						.map_err(ThumbnailerError::from)?;
 
-			(full_path, sub_iso_file_path)
-		} else {
-			(
+				ensure_file_path_exists(
+					sub_path,
+					&sub_iso_file_path,
+					db,
+					ThumbnailerError::SubPathNotFound,
+				)
+				.await?;
+
+				(full_path, sub_iso_file_path)
+			}
+			_ => (
 				location_path.to_path_buf(),
 				IsolatedFilePathData::new(location_id, &location_path, &location_path, true)
 					.map_err(ThumbnailerError::from)?,
-			)
+			),
 		};
 
 		info!("Searching for images in location {location_id} at directory {iso_file_path}");

core/src/object/validation/mod.rs

@@ -1,2 +1,22 @@
+use crate::{location::file_path_helper::FilePathError, util::error::FileIOError};
+
+use std::path::Path;
+
+use thiserror::Error;
+
 pub mod hash;
 pub mod validator_job;
+
+#[derive(Error, Debug)]
+pub enum ValidatorError {
+	#[error("sub path not found: <path='{}'>", .0.display())]
+	SubPathNotFound(Box<Path>),
+
+	// Internal errors
+	#[error("database error: {0}")]
+	Database(#[from] prisma_client_rust::QueryError),
+	#[error(transparent)]
+	FilePath(#[from] FilePathError),
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
+}

core/src/object/validation/validator_job.rs

@@ -4,19 +4,28 @@ use crate::{
 		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
 	},
 	library::Library,
-	location::file_path_helper::{file_path_for_object_validator, IsolatedFilePathData},
+	location::file_path_helper::{
+		ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
+		file_path_for_object_validator, IsolatedFilePathData,
+	},
 	prisma::{file_path, location},
 	sync,
-	util::{db::maybe_missing, error::FileIOError},
+	util::{
+		db::{chain_optional_iter, maybe_missing},
+		error::FileIOError,
+	},
 };
 
-use std::path::PathBuf;
+use std::{
+	hash::{Hash, Hasher},
+	path::{Path, PathBuf},
+};
 
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use tracing::info;
 
-use super::hash::file_checksum;
+use super::{hash::file_checksum, ValidatorError};
 
 // The Validator is able to:
 // - generate a full byte checksum for Objects in a Location
@@ -26,16 +35,24 @@ pub struct ObjectValidatorJob {}
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct ObjectValidatorJobState {
-	pub root_path: PathBuf,
+	pub location_path: PathBuf,
 	pub task_count: usize,
 }
 
 // The validator can
-#[derive(Serialize, Deserialize, Debug, Hash)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct ObjectValidatorJobInit {
-	pub location_id: location::id::Type,
-	pub path: PathBuf,
-	pub background: bool,
+	pub location: location::Data,
+	pub sub_path: Option<PathBuf>,
+}
+
+impl Hash for ObjectValidatorJobInit {
+	fn hash<H: Hasher>(&self, state: &mut H) {
+		self.location.id.hash(state);
+		if let Some(ref sub_path) = self.sub_path {
+			sub_path.hash(state);
+		}
+	}
 }
 
 impl JobInitData for ObjectValidatorJobInit {
@@ -61,20 +78,58 @@ impl StatefulJob for ObjectValidatorJob {
 	) -> Result<(), JobError> {
 		let Library { db, .. } = &ctx.library;
 
+		let location_id = state.init.location.id;
+
+		let location_path =
+			maybe_missing(&state.init.location.path, "location.path").map(PathBuf::from)?;
+
+		let maybe_sub_iso_file_path = match &state.init.sub_path {
+			Some(sub_path) if sub_path != Path::new("") && sub_path != Path::new("/") => {
+				let full_path = ensure_sub_path_is_in_location(&location_path, sub_path)
+					.await
+					.map_err(ValidatorError::from)?;
+				ensure_sub_path_is_directory(&location_path, sub_path)
+					.await
+					.map_err(ValidatorError::from)?;
+
+				let sub_iso_file_path =
+					IsolatedFilePathData::new(location_id, &location_path, &full_path, true)
+						.map_err(ValidatorError::from)?;
+
+				ensure_file_path_exists(
+					sub_path,
+					&sub_iso_file_path,
+					db,
+					ValidatorError::SubPathNotFound,
+				)
+				.await?;
+
+				Some(sub_iso_file_path)
+			}
+			_ => None,
+		};
+
 		state.steps.extend(
 			db.file_path()
-				.find_many(vec![
-					file_path::location_id::equals(Some(state.init.location_id)),
-					file_path::is_dir::equals(Some(false)),
-					file_path::integrity_checksum::equals(None),
-				])
+				.find_many(chain_optional_iter(
+					[
+						file_path::location_id::equals(Some(state.init.location.id)),
+						file_path::is_dir::equals(Some(false)),
+						file_path::integrity_checksum::equals(None),
+					],
+					[maybe_sub_iso_file_path.and_then(|iso_sub_path| {
+						iso_sub_path
+							.materialized_path_for_children()
+							.map(file_path::materialized_path::starts_with)
+					})],
+				))
 				.select(file_path_for_object_validator::select())
 				.exec()
 				.await?,
 		);
 
 		state.data = Some(ObjectValidatorJobState {
-			root_path: state.init.path.clone(),
+			location_path,
 			task_count: state.steps.len(),
 		});
 
@@ -98,13 +153,13 @@ impl StatefulJob for ObjectValidatorJob {
 		// we can also compare old and new checksums here
 		// This if is just to make sure, we already queried objects where integrity_checksum is null
 		if file_path.integrity_checksum.is_none() {
-			let path = data.root_path.join(IsolatedFilePathData::try_from((
-				maybe_missing(&file_path.location, "file_path.location")?.id,
+			let full_path = data.location_path.join(IsolatedFilePathData::try_from((
+				state.init.location.id,
 				file_path,
 			))?);
-			let checksum = file_checksum(&path)
+			let checksum = file_checksum(&full_path)
 				.await
-				.map_err(|e| FileIOError::from((path, e)))?;
+				.map_err(|e| ValidatorError::FileIO(FileIOError::from((full_path, e))))?;
 
 			sync.write_op(
 				db,
@@ -137,8 +192,14 @@ impl StatefulJob for ObjectValidatorJob {
 	) -> JobResult {
 		let data = extract_job_data!(state);
 		info!(
-			"finalizing validator job at {}: {} tasks",
-			data.root_path.display(),
+			"finalizing validator job at {}{}: {} tasks",
+			data.location_path.display(),
+			state
+				.init
+				.sub_path
+				.as_ref()
+				.map(|p| format!("{}", p.display()))
+				.unwrap_or_default(),
 			data.task_count
 		);
 

interface/app/$libraryId/Explorer/ContextMenu.tsx

@@ -130,20 +130,21 @@ export default (props: PropsWithChildren) => {
 				<CM.Item
 					onClick={() =>
 						store.locationId &&
-						generateThumbsForLocation.mutate({ id: store.locationId, path: '' })
+						generateThumbsForLocation.mutate({
+							id: store.locationId,
+							path: params.path ?? ''
+						})
 					}
 					label="Regen Thumbnails"
 					icon={Image}
-					disabled
 				/>
 				<CM.Item
 					onClick={() =>
 						store.locationId &&
-						objectValidator.mutate({ id: store.locationId, path: '' })
+						objectValidator.mutate({ id: store.locationId, path: params.path ?? '' })
 					}
 					label="Generate Checksums"
 					icon={ShieldCheck}
-					disabled
 				/>
 			</CM.SubMenu>
 		</CM.Root>

interface/app/$libraryId/Explorer/File/ContextMenu.tsx

@@ -238,7 +238,7 @@ export default ({ data }: Props) => {
 					onClick={() => {
 						generateThumbnails.mutate({
 							id: getExplorerStore().locationId!,
-							path: '/'
+							path: params.path ?? ''
 						});
 					}}
 					label="Regen Thumbnails"