[ENG-536] Improve indexer for huge locations (#781)

* WIP materialized_path abstraction revamp

* Optimizing indexer rules loading

* Using a better serialize impl for indexer rules

* New interruptable and faster Walker

* WIP new indexer

* WIP first success compiling after breaking the world

* Fixing some warnings

* Handling some lifetime issues in the walker

* New job completed with errors feature

* Introducing completed with errors to indexer
Removing IOError variant from JobError and using FileIOError instead

* Rust fmt

* Adding missing job status

* Better ergonomics to IsolatedFilePathData
Conversions from db's file_path kinds
Keeping original's relative path data to better conversion to OS's path
First unit tests

* Testing and fixing parent method

* Some error handling

* Rust fmt

* Some small fixes

* Fixing indexer rules decoding

* Bunch of small fixes

* Rust fmt

* Fixing indexer rules

* Updating frontend to new materialized_path format

* Trying to fix windows CI

---------

Co-authored-by: Brendan Allan <brendonovich@outlook.com>

apps/desktop/src-tauri/src/file.rs

@@ -67,7 +67,7 @@ pub async fn get_file_path_open_with_apps(
 		sd_desktop_macos::get_open_with_applications(&path.to_str().unwrap().into())
 	}
 	.as_slice()
-	.into_iter()
+	.iter()
 	.map(|app| OpenWithApplication {
 		name: app.name.to_string(),
 		url: app.url.to_string(),

apps/mobile/src/components/explorer/Explorer.tsx

@@ -34,7 +34,7 @@ const Explorer = ({ items }: ExplorerProps) => {
 		if (isPath(data) && data.item.is_dir) {
 			navigation.push('Location', {
 				id: data.item.location_id,
-				path: data.item.materialized_path
+				path: `${data.item.materialized_path}${data.item.name}/`
 			});
 		} else {
 			setData(data);

core/prisma/migrations/20230426011629_remove_parent_id_from_file_path/migration.sql

@@ -0,0 +1,43 @@
+/*
+  Warnings:
+
+  - The primary key for the `file_path` table will be changed. If it partially fails, the table could be left without primary key constraint.
+  - You are about to drop the column `parent_id` on the `file_path` table. All the data in the column will be lost.
+  - Added the required column `pub_id` to the `file_path` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- RedefineTables
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_file_path" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "pub_id" BLOB NOT NULL,
+    "is_dir" BOOLEAN NOT NULL DEFAULT false,
+    "cas_id" TEXT,
+    "integrity_checksum" TEXT,
+    "location_id" INTEGER NOT NULL,
+    "materialized_path" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "extension" TEXT NOT NULL,
+    "size_in_bytes" TEXT NOT NULL DEFAULT '0',
+    "inode" BLOB NOT NULL,
+    "device" BLOB NOT NULL,
+    "object_id" INTEGER,
+    "key_id" INTEGER,
+    "date_created" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "date_modified" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "date_indexed" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT "file_path_location_id_fkey" FOREIGN KEY ("location_id") REFERENCES "location" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+    CONSTRAINT "file_path_object_id_fkey" FOREIGN KEY ("object_id") REFERENCES "object" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "file_path_key_id_fkey" FOREIGN KEY ("key_id") REFERENCES "key" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+INSERT INTO "new_file_path" ("cas_id", "date_created", "date_indexed", "date_modified", "device", "extension", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "size_in_bytes") SELECT "cas_id", "date_created", "date_indexed", "date_modified", "device", "extension", "id", "inode", "integrity_checksum", "is_dir", "key_id", "location_id", "materialized_path", "name", "object_id", "size_in_bytes" FROM "file_path";
+DROP TABLE "file_path";
+ALTER TABLE "new_file_path" RENAME TO "file_path";
+CREATE UNIQUE INDEX "file_path_pub_id_key" ON "file_path"("pub_id");
+CREATE UNIQUE INDEX "file_path_integrity_checksum_key" ON "file_path"("integrity_checksum");
+CREATE INDEX "file_path_location_id_idx" ON "file_path"("location_id");
+CREATE INDEX "file_path_location_id_materialized_path_idx" ON "file_path"("location_id", "materialized_path");
+CREATE UNIQUE INDEX "file_path_location_id_materialized_path_name_extension_key" ON "file_path"("location_id", "materialized_path", "name", "extension");
+CREATE UNIQUE INDEX "file_path_location_id_inode_device_key" ON "file_path"("location_id", "inode", "device");
+PRAGMA foreign_key_check;
+PRAGMA foreign_keys=ON;

core/prisma/migrations/20230504031258_errors_text_field_on_job_table/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "job" ADD COLUMN "errors_text" TEXT;

core/prisma/schema.prisma

@@ -135,10 +135,10 @@ model FilePath {
 
     // the path of the file relative to its location
     materialized_path String
+
     // the name and extension
-    // Must have 'COLLATE NOCASE' in migration
-    name              String
-    extension         String
+    name      String
+    extension String // Extension MUST have 'COLLATE NOCASE' in migration
 
     size_in_bytes String @default("0")
 
@@ -149,9 +149,7 @@ model FilePath {
     object_id Int?
     object    Object? @relation(fields: [object_id], references: [id], onDelete: Restrict)
 
-    // the parent in the file tree
-    parent_id Bytes?
-    key_id    Int? // replacement for encryption
+    key_id Int? // replacement for encryption
     // permissions       String?
 
     date_created  DateTime @default(now())
@@ -167,6 +165,7 @@ model FilePath {
     @@unique([location_id, materialized_path, name, extension])
     @@unique([location_id, inode, device])
     @@index([location_id])
+    @@index([location_id, materialized_path])
     @@map("file_path")
 }
 
@@ -378,6 +377,9 @@ model Job {
     // Enum: sd_core::job::job_manager:JobStatus
     status Int @default(0) // 0 = Queued
 
+    // List of errors, separated by "\n\n" in case of failed jobs or completed with errors
+    errors_text String?
+
     data     Bytes? // Serialized data to be used on pause/resume
     metadata Bytes? // Serialized metadata field with info about the job after completion
 

core/src/api/files.rs

@@ -1,7 +1,7 @@
 use crate::{
 	api::utils::library,
 	invalidate_query,
-	location::{file_path_helper::MaterializedPath, find_location, LocationError},
+	location::{file_path_helper::IsolatedFilePathData, find_location, LocationError},
 	object::fs::{
 		copy::FileCopierJobInit, cut::FileCutterJobInit, decrypt::FileDecryptorJobInit,
 		delete::FileDeleterJobInit, encrypt::FileEncryptorJobInit, erase::FileEraserJobInit,
@@ -250,8 +250,14 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 
 					let location_path = Path::new(&location.path);
 					fs::rename(
-						location_path.join(&MaterializedPath::from((location_id, &file_name))),
-						location_path.join(&MaterializedPath::from((location_id, &new_file_name))),
+						location_path.join(IsolatedFilePathData::from_relative_str(
+							location_id,
+							&file_name,
+						)),
+						location_path.join(IsolatedFilePathData::from_relative_str(
+							location_id,
+							&new_file_name,
+						)),
 					)
 					.await
 					.map_err(|e| {

core/src/api/jobs.rs

@@ -60,7 +60,6 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 						.spawn_job(ThumbnailerJobInit {
 							location,
 							sub_path: Some(args.path),
-							background: false,
 						})
 						.await
 						.map_err(Into::into)

core/src/api/search.rs

@@ -1,4 +1,5 @@
-use std::path::{MAIN_SEPARATOR, MAIN_SEPARATOR_STR};
+use crate::location::file_path_helper::{check_file_path_exists, IsolatedFilePathData};
+use std::collections::BTreeSet;
 
 use chrono::{DateTime, Utc};
 use prisma_client_rust::{operator::or, Direction};
@@ -77,7 +78,7 @@ pub fn mount() -> AlphaRouter<Ctx> {
 				#[specta(optional)]
 				extension: Option<String>,
 				#[serde(default)]
-				kind: Vec<i32>,
+				kind: BTreeSet<i32>,
 				#[serde(default)]
 				tags: Vec<i32>,
 				#[serde(default)]
@@ -103,39 +104,30 @@ pub fn mount() -> AlphaRouter<Ctx> {
 						None
 					};
 
-					let directory_id = if let Some(mut path) = args.path.clone() {
-						if !path.ends_with(MAIN_SEPARATOR) {
-							path += MAIN_SEPARATOR_STR;
-						}
-
-						Some(
-							db.file_path()
-								.find_first(chain_optional_iter(
-									[
-										file_path::materialized_path::equals(path),
-										file_path::is_dir::equals(true),
-									],
-									[location.map(|l| file_path::location_id::equals(l.id))],
-								))
-								.select(file_path::select!({ pub_id }))
-								.exec()
+					let directory_materialized_path_str = match (args.path, location) {
+						(Some(path), Some(location)) if !path.is_empty() && path != "/" => {
+							let parent_iso_file_path =
+								IsolatedFilePathData::from_relative_str(location.id, &path);
+							if !check_file_path_exists::<LocationError>(&parent_iso_file_path, db)
 								.await?
-								.ok_or_else(|| {
-									rspc::Error::new(
-										ErrorCode::NotFound,
-										"Directory not found".into(),
-									)
-								})?
-								.pub_id,
-						)
-					} else {
-						None
+							{
+								return Err(rspc::Error::new(
+									ErrorCode::NotFound,
+									"Directory not found".into(),
+								));
+							}
+
+							parent_iso_file_path.materialized_path_for_children()
+						}
+						(Some(_empty), _) => Some("/".into()),
+						_ => None,
 					};
 
 					let object_params = chain_optional_iter(
 						[],
 						[
-							(!args.kind.is_empty()).then(|| object::kind::in_vec(args.kind)),
+							(!args.kind.is_empty())
+								.then(|| object::kind::in_vec(args.kind.into_iter().collect())),
 							(!args.tags.is_empty()).then(|| {
 								let tags = args.tags.into_iter().map(tag::id::equals).collect();
 								let tags_on_object = tag_on_object::tag::is(vec![or(tags)]);
@@ -149,7 +141,7 @@ pub fn mount() -> AlphaRouter<Ctx> {
 						args.search
 							.split(' ')
 							.map(str::to_string)
-							.map(file_path::materialized_path::contains),
+							.map(file_path::name::contains),
 						[
 							args.location_id.map(file_path::location_id::equals),
 							args.extension.map(file_path::extension::equals),
@@ -159,8 +151,8 @@ pub fn mount() -> AlphaRouter<Ctx> {
 							args.created_at
 								.to
 								.map(|v| file_path::date_created::lte(v.into())),
-							args.path.map(file_path::materialized_path::starts_with),
-							directory_id.map(Some).map(file_path::parent_id::equals),
+							directory_materialized_path_str
+								.map(file_path::materialized_path::equals),
 							(!object_params.is_empty())
 								.then(|| file_path::object::is(object_params)),
 						],
@@ -203,7 +195,7 @@ pub fn mount() -> AlphaRouter<Ctx> {
 							library
 								.thumbnail_exists(cas_id)
 								.await
-								.map_err(LocationError::IOError)?
+								.map_err(LocationError::from)?
 						} else {
 							false
 						};

core/src/custom_uri.rs

@@ -1,4 +1,9 @@
-use crate::{location::file_path_helper::MaterializedPath, prisma::file_path, Node};
+use crate::{
+	location::file_path_helper::{file_path_to_handle_custom_uri, IsolatedFilePathData},
+	prisma::file_path,
+	util::error::FileIOError,
+	Node,
+};
 
 use std::{
 	io,
@@ -106,15 +111,19 @@ async fn handle_thumbnail(
 		.join(file_cas_id)
 		.with_extension("webp");
 
-	let file = File::open(filename).await.map_err(|err| {
+	let file = File::open(&filename).await.map_err(|err| {
 		if err.kind() == io::ErrorKind::NotFound {
 			HandleCustomUriError::NotFound("file")
 		} else {
-			err.into()
+			FileIOError::from((&filename, err)).into()
 		}
 	})?;
 
-	let content_lenght = file.metadata().await?.len();
+	let content_lenght = file
+		.metadata()
+		.await
+		.map_err(|e| FileIOError::from((&filename, e)))?
+		.len();
 
 	Ok(builder
 		.header("Content-Type", "image/webp")
@@ -123,7 +132,9 @@ async fn handle_thumbnail(
 		.body(if method == Method::HEAD {
 			vec![]
 		} else {
-			read_file(file, content_lenght, None).await?
+			read_file(file, content_lenght, None)
+				.await
+				.map_err(|e| FileIOError::from((&filename, e)))?
 		})?)
 }
 
@@ -161,7 +172,7 @@ async fn handle_file(
 
 	let lru_cache_key = (library_id, file_path_id);
 
-	let (file_path_materialized_path, extension) =
+	let (file_path_full_path, extension) =
 		if let Some(entry) = FILE_METADATA_CACHE.get(&lru_cache_key) {
 			entry
 		} else {
@@ -175,16 +186,14 @@ async fn handle_file(
 				.db
 				.file_path()
 				.find_unique(file_path::id::equals(file_path_id))
-				.include(file_path::include!({ location }))
+				.select(file_path_to_handle_custom_uri::select())
 				.exec()
 				.await?
 				.ok_or_else(|| HandleCustomUriError::NotFound("object"))?;
 
 			let lru_entry = (
-				Path::new(&file_path.location.path).join(&MaterializedPath::from((
-					location_id,
-					&file_path.materialized_path,
-				))),
+				Path::new(&file_path.location.path)
+					.join(IsolatedFilePathData::from((location_id, &file_path))),
 				file_path.extension,
 			);
 			FILE_METADATA_CACHE.insert(lru_cache_key, lru_entry.clone());
@@ -192,15 +201,13 @@ async fn handle_file(
 			lru_entry
 		};
 
-	let file = File::open(file_path_materialized_path)
-		.await
-		.map_err(|err| {
-			if err.kind() == io::ErrorKind::NotFound {
-				HandleCustomUriError::NotFound("file")
-			} else {
-				err.into()
-			}
-		})?;
+	let file = File::open(&file_path_full_path).await.map_err(|err| {
+		if err.kind() == io::ErrorKind::NotFound {
+			HandleCustomUriError::NotFound("file")
+		} else {
+			FileIOError::from((&file_path_full_path, err)).into()
+		}
+	})?;
 
 	// TODO: This should be determined from magic bytes when the file is indexed and stored it in the DB on the file path
 	// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
@@ -266,7 +273,12 @@ async fn handle_file(
 		}
 	};
 
-	let mut content_lenght = file.metadata().await?.len();
+	let mut content_lenght = file
+		.metadata()
+		.await
+		.map_err(|e| FileIOError::from((&file_path_full_path, e)))?
+		.len();
+
 	// GET is the only method for which range handling is defined, according to the spec
 	// https://httpwg.org/specs/rfc9110.html#field.range
 	let range = if method == Method::GET {
@@ -330,10 +342,14 @@ async fn handle_file(
 
 			// FIXME: Add ETag support (caching on the webview)
 
-			read_file(file, content_lenght, Some(range.start)).await?
+			read_file(file, content_lenght, Some(range.start))
+				.await
+				.map_err(|e| FileIOError::from((&file_path_full_path, e)))?
 		}
 		_ if method == Method::HEAD => vec![],
-		_ => read_file(file, content_lenght, None).await?,
+		_ => read_file(file, content_lenght, None)
+			.await
+			.map_err(|e| FileIOError::from((&file_path_full_path, e)))?,
 	};
 
 	Ok(builder
@@ -360,7 +376,7 @@ pub enum HandleCustomUriError {
 	#[error("error creating http request/response: {0}")]
 	Http(#[from] httpz::http::Error),
 	#[error("io error: {0}")]
-	Io(#[from] io::Error),
+	FileIO(#[from] FileIOError),
 	#[error("query error: {0}")]
 	QueryError(#[from] QueryError),
 	#[error("{0}")]
@@ -377,19 +393,19 @@ impl From<HandleCustomUriError> for Response<Vec<u8>> {
 
 		(match value {
 			HandleCustomUriError::Http(err) => {
-				error!("Error creating http request/response: {}", err);
+				error!("Error creating http request/response: {:#?}", err);
 				builder
 					.status(StatusCode::INTERNAL_SERVER_ERROR)
 					.body(b"Internal Server Error".to_vec())
 			}
-			HandleCustomUriError::Io(err) => {
-				error!("IO error: {}", err);
+			HandleCustomUriError::FileIO(err) => {
+				error!("IO error: {:#?}", err);
 				builder
 					.status(StatusCode::INTERNAL_SERVER_ERROR)
 					.body(b"Internal Server Error".to_vec())
 			}
 			HandleCustomUriError::QueryError(err) => {
-				error!("Query error: {}", err);
+				error!("Query error: {:#?}", err);
 				builder
 					.status(StatusCode::INTERNAL_SERVER_ERROR)
 					.body(b"Internal Server Error".to_vec())

core/src/job/job_manager.rs

@@ -321,6 +321,7 @@ pub struct JobReport {
 	pub data: Option<Vec<u8>>,
 	pub metadata: Option<serde_json::Value>,
 	pub is_background: bool,
+	pub errors_text: Vec<String>,
 
 	pub created_at: Option<DateTime<Utc>>,
 	pub started_at: Option<DateTime<Utc>>,
@@ -361,6 +362,10 @@ impl From<job::Data> for JobReport {
 					None
 				})
 			}),
+			errors_text: data
+				.errors_text
+				.map(|errors_str| errors_str.split("\n\n").map(str::to_string).collect())
+				.unwrap_or_default(),
 			created_at: Some(data.date_created.into()),
 			started_at: data.date_started.map(|d| d.into()),
 			completed_at: data.date_completed.map(|d| d.into()),
@@ -386,6 +391,7 @@ impl JobReport {
 			started_at: None,
 			completed_at: None,
 			status: JobStatus::Queued,
+			errors_text: vec![],
 			task_count: 0,
 			data: None,
 			metadata: None,
@@ -449,6 +455,9 @@ impl JobReport {
 				job::id::equals(self.id.as_bytes().to_vec()),
 				vec![
 					job::status::set(self.status as i32),
+					job::errors_text::set(
+						(!self.errors_text.is_empty()).then(|| self.errors_text.join("\n\n")),
+					),
 					job::data::set(self.data.clone()),
 					job::metadata::set(serde_json::to_vec(&self.metadata).ok()),
 					job::task_count::set(self.task_count),
@@ -472,6 +481,7 @@ pub enum JobStatus {
 	Canceled = 3,
 	Failed = 4,
 	Paused = 5,
+	CompletedWithErrors = 6,
 }
 
 impl TryFrom<i32> for JobStatus {
@@ -485,6 +495,7 @@ impl TryFrom<i32> for JobStatus {
 			3 => Self::Canceled,
 			4 => Self::Failed,
 			5 => Self::Paused,
+			6 => Self::CompletedWithErrors,
 			_ => return Err(JobError::InvalidJobStatusInt(value)),
 		};
 

core/src/job/mod.rs

@@ -2,6 +2,7 @@ use crate::{
 	library::Library,
 	location::indexer::IndexerError,
 	object::{file_identifier::FileIdentifierJobError, preview::ThumbnailerError},
+	util::error::FileIOError,
 };
 
 use std::{
@@ -17,7 +18,7 @@ use rmp_serde::{decode::Error as DecodeError, encode::Error as EncodeError};
 use sd_crypto::Error as CryptoError;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use thiserror::Error;
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, warn};
 use uuid::Uuid;
 
 mod job_manager;
@@ -29,10 +30,8 @@ pub use worker::*;
 #[derive(Error, Debug)]
 pub enum JobError {
 	// General errors
-	#[error("Database error: {0}")]
+	#[error("database error")]
 	DatabaseError(#[from] prisma_client_rust::QueryError),
-	#[error("I/O error: {0}")]
-	IOError(#[from] std::io::Error),
 	#[error("Failed to join Tokio spawn blocking: {0}")]
 	JoinTaskError(#[from] tokio::task::JoinError),
 	#[error("Job state encode error: {0}")]
@@ -57,6 +56,8 @@ pub enum JobError {
 	Path,
 	#[error("invalid job status integer")]
 	InvalidJobStatusInt(i32),
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
 
 	// Specific job errors
 	#[error("Indexer error: {0}")]
@@ -73,16 +74,19 @@ pub enum JobError {
 	WouldOverwrite(PathBuf),
 
 	// Not errors
-	#[error("Job had a early finish: <name='{name}', reason='{reason}'>")]
+	#[error("step completed with errors")]
+	StepCompletedWithErrors(JobRunErrors),
+	#[error("job had a early finish: <name='{name}', reason='{reason}'>")]
 	EarlyFinish { name: String, reason: String },
-	#[error("Data needed for job execution not found: job <name='{0}'>")]
+	#[error("data needed for job execution not found: job <name='{0}'>")]
 	JobDataNotFound(String),
-	#[error("Job paused")]
+	#[error("job paused")]
 	Paused(Vec<u8>),
 }
 
 pub type JobResult = Result<JobMetadata, JobError>;
 pub type JobMetadata = Option<serde_json::Value>;
+pub type JobRunErrors = Vec<String>;
 
 /// `JobInitData` is a trait to represent the data being passed to initialize a `Job`
 pub trait JobInitData: Serialize + DeserializeOwned + Send + Sync + Hash {
@@ -131,7 +135,11 @@ pub trait DynJob: Send + Sync {
 	fn report(&self) -> &Option<JobReport>;
 	fn report_mut(&mut self) -> &mut Option<JobReport>;
 	fn name(&self) -> &'static str;
-	async fn run(&mut self, job_manager: Arc<JobManager>, ctx: WorkerContext) -> JobResult;
+	async fn run(
+		&mut self,
+		job_manager: Arc<JobManager>,
+		ctx: WorkerContext,
+	) -> Result<(JobMetadata, JobRunErrors), JobError>;
 	fn hash(&self) -> u64;
 	fn set_next_jobs(&mut self, next_jobs: VecDeque<Box<dyn DynJob>>);
 	fn serialize_state(&self) -> Result<Vec<u8>, JobError>;
@@ -306,17 +314,25 @@ impl<SJob: StatefulJob> DynJob for Job<SJob> {
 		<SJob as StatefulJob>::NAME
 	}
 
-	async fn run(&mut self, job_manager: Arc<JobManager>, ctx: WorkerContext) -> JobResult {
+	async fn run(
+		&mut self,
+		job_manager: Arc<JobManager>,
+		ctx: WorkerContext,
+	) -> Result<(JobMetadata, JobRunErrors), JobError> {
 		let mut job_should_run = true;
 
+		let mut errors = vec![];
+
 		// Checking if we have a brand new job, or if we are resuming an old one.
 		if self.state.data.is_none() {
 			if let Err(e) = self.stateful_job.init(ctx.clone(), &mut self.state).await {
-				if matches!(e, JobError::EarlyFinish { .. }) {
-					info!("{e}");
-					job_should_run = false;
-				} else {
-					return Err(e);
+				match e {
+					JobError::EarlyFinish { .. } => {
+						info!("{e}");
+						job_should_run = false;
+					}
+					JobError::StepCompletedWithErrors(errors_text) => errors.extend(errors_text),
+					other => return Err(other),
 				}
 			}
 		}
@@ -329,12 +345,18 @@ impl<SJob: StatefulJob> DynJob for Job<SJob> {
 					ctx.clone(),
 					&mut self.state,
 				) => {
-					if matches!(step_result, Err(JobError::EarlyFinish { .. })) {
-						info!("{}", step_result.unwrap_err());
-						break;
-					} else {
-						step_result?;
-					};
+					match step_result {
+						Err(JobError::EarlyFinish { .. }) => {
+							info!("{}", step_result.unwrap_err());
+							break;
+						},
+						Err(JobError::StepCompletedWithErrors(errors_text)) => {
+							warn!("Job<id='{}'> had a step with errors", self.id);
+							errors.extend(errors_text);
+						},
+						maybe_err => maybe_err?
+					}
+
 					self.state.steps.pop_front();
 				}
 				_ = shutdown_rx.recv() => {
@@ -368,7 +390,7 @@ impl<SJob: StatefulJob> DynJob for Job<SJob> {
 			}
 		}
 
-		Ok(metadata)
+		Ok((metadata, errors))
 	}
 
 	fn hash(&self) -> u64 {

core/src/job/worker.rs

@@ -14,7 +14,7 @@ use tokio::{
 };
 use tracing::{error, info, warn};
 
-use super::{JobMetadata, JobReport};
+use super::{JobMetadata, JobReport, JobRunErrors};
 
 const JOB_REPORT_UPDATE_INTERVAL: Duration = Duration::from_millis(1000 / 60);
 
@@ -23,6 +23,7 @@ const JOB_REPORT_UPDATE_INTERVAL: Duration = Duration::from_millis(1000 / 60);
 pub enum WorkerEvent {
 	Progressed(Vec<JobReportUpdate>),
 	Completed(oneshot::Sender<()>, JobMetadata),
+	CompletedWithErrors(oneshot::Sender<()>, JobMetadata, JobRunErrors),
 	Failed(oneshot::Sender<()>),
 	Paused(Vec<u8>, oneshot::Sender<()>),
 }
@@ -143,15 +144,21 @@ impl Worker {
 			let (done_tx, done_rx) = oneshot::channel();
 
 			match job.run(job_manager.clone(), worker_ctx.clone()).await {
-				Ok(metadata) => {
-					// handle completion
+				Ok((metadata, errors)) if errors.is_empty() => {
 					worker_ctx
 						.events_tx
 						.send(WorkerEvent::Completed(done_tx, metadata))
 						.expect("critical error: failed to send worker complete event");
 				}
+				Ok((metadata, errors)) => {
+					warn!("Job<id'{job_id}'> completed with errors");
+					worker_ctx
+						.events_tx
+						.send(WorkerEvent::CompletedWithErrors(done_tx, metadata, errors))
+						.expect("critical error: failed to send worker complete event");
+				}
 				Err(JobError::Paused(state)) => {
-					info!("Job <id='{job_id}'> paused, we will pause all children jobs");
+					info!("Job<id='{job_id}'> paused, we will pause all children jobs");
 					if let Err(e) = job.pause_children(&library).await {
 						error!("Failed to pause children jobs: {e:#?}");
 					}
@@ -162,7 +169,7 @@ impl Worker {
 						.expect("critical error: failed to send worker pause event");
 				}
 				Err(e) => {
-					error!("Job <id='{job_id}'> failed with error: {e:#?}; We will cancel all children jobs");
+					error!("Job<id='{job_id}'> failed with error: {e:#?}; We will cancel all children jobs");
 					if let Err(e) = job.cancel_children(&library).await {
 						error!("Failed to cancel children jobs: {e:#?}");
 					}
@@ -233,6 +240,27 @@ impl Worker {
 
 					break;
 				}
+				WorkerEvent::CompletedWithErrors(done_tx, metadata, errors) => {
+					worker.report.status = JobStatus::CompletedWithErrors;
+					worker.report.errors_text = errors;
+					worker.report.data = None;
+					worker.report.metadata = metadata;
+					worker.report.completed_at = Some(Utc::now());
+					if let Err(e) = worker.report.update(&library).await {
+						error!("failed to update job report: {:#?}", e);
+					}
+
+					invalidate_query!(library, "jobs.getRunning");
+					invalidate_query!(library, "jobs.getHistory");
+
+					info!("{}", worker.report);
+
+					done_tx
+						.send(())
+						.expect("critical error: failed to send worker completion");
+
+					break;
+				}
 				WorkerEvent::Failed(done_tx) => {
 					worker.report.status = JobStatus::Failed;
 					worker.report.data = None;

core/src/lib.rs

@@ -130,7 +130,9 @@ impl Node {
 			.init();
 
 		let event_bus = broadcast::channel(1024);
-		let config = NodeConfigManager::new(data_dir.to_path_buf()).await?;
+		let config = NodeConfigManager::new(data_dir.to_path_buf())
+			.await
+			.map_err(NodeError::FailedToInitializeConfig)?;
 
 		let jobs = JobManager::new();
 		let location_manager = LocationManager::new();
@@ -216,13 +218,11 @@ impl Node {
 /// Error type for Node related errors.
 #[derive(Error, Debug)]
 pub enum NodeError {
-	#[error("Failed to create data directory: {0}")]
-	FailedToCreateDataDirectory(#[from] std::io::Error),
-	#[error("Failed to initialize config: {0}")]
-	FailedToInitializeConfig(#[from] util::migrator::MigratorError),
-	#[error("Failed to initialize library manager: {0}")]
+	#[error("failed to initialize config")]
+	FailedToInitializeConfig(util::migrator::MigratorError),
+	#[error("failed to initialize library manager")]
 	FailedToInitializeLibraryManager(#[from] library::LibraryManagerError),
-	#[error("Location manager error: {0}")]
+	#[error(transparent)]
 	LocationManager(#[from] LocationManagerError),
 	#[error("invalid platform integer")]
 	InvalidPlatformInt(i32),

core/src/library/library.rs

@@ -1,11 +1,15 @@
 use crate::{
 	api::CoreEvent,
 	job::{IntoJob, JobInitData, JobManagerError, StatefulJob},
-	location::{file_path_helper::MaterializedPath, LocationManager},
+	location::{
+		file_path_helper::{file_path_to_full_path, IsolatedFilePathData},
+		LocationManager,
+	},
 	node::NodeConfigManager,
 	object::{orphan_remover::OrphanRemoverActor, preview::get_thumbnail_path},
 	prisma::{file_path, location, PrismaClient},
 	sync::SyncManager,
+	util::error::FileIOError,
 	NodeContext,
 };
 
@@ -16,6 +20,7 @@ use std::{
 };
 
 use sd_crypto::keys::keymanager::KeyManager;
+use tokio::{fs, io};
 use tracing::warn;
 use uuid::Uuid;
 
@@ -85,13 +90,13 @@ impl Library {
 		&self.node_context.location_manager
 	}
 
-	pub async fn thumbnail_exists(&self, cas_id: &str) -> tokio::io::Result<bool> {
+	pub async fn thumbnail_exists(&self, cas_id: &str) -> Result<bool, FileIOError> {
 		let thumb_path = get_thumbnail_path(self, cas_id);
 
-		match tokio::fs::metadata(thumb_path).await {
+		match fs::metadata(&thumb_path).await {
 			Ok(_) => Ok(true),
-			Err(e) if e.kind() == tokio::io::ErrorKind::NotFound => Ok(false),
-			Err(e) => Err(e),
+			Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false),
+			Err(e) => Err(FileIOError::from((thumb_path, e))),
 		}
 	}
 
@@ -104,20 +109,12 @@ impl Library {
 				file_path::location::is(vec![location::node_id::equals(self.node_local_id)]),
 				file_path::id::equals(id),
 			])
-			.select(file_path::select!({
-				materialized_path
-				location: select {
-					id
-					path
-				}
-			}))
+			.select(file_path_to_full_path::select())
 			.exec()
 			.await?
 			.map(|record| {
-				Path::new(&record.location.path).join(&MaterializedPath::from((
-					record.location.id,
-					&record.materialized_path,
-				)))
+				Path::new(&record.location.path)
+					.join(IsolatedFilePathData::from((record.location.id, &record)))
 			}))
 	}
 }

core/src/library/manager.rs

@@ -6,6 +6,8 @@ use crate::{
 	sync::{SyncManager, SyncMessage},
 	util::{
 		db::load_and_migrate,
+		error::{FileIOError, NonUtf8PathError},
+		migrator::MigratorError,
 		seeder::{indexer_rules_seeder, SeederError},
 	},
 	NodeContext,
@@ -15,15 +17,17 @@ use sd_crypto::{
 	keys::keymanager::{KeyManager, StoredKey},
 	types::{EncryptedKey, Nonce, Salt},
 };
+
 use std::{
-	env, fs, io,
+	env,
 	path::{Path, PathBuf},
 	str::FromStr,
 	sync::Arc,
 };
+
 use thiserror::Error;
-use tokio::sync::RwLock;
-use tracing::{debug, error};
+use tokio::{fs, io, sync::RwLock, try_join};
+use tracing::{debug, error, warn};
 use uuid::Uuid;
 
 use super::{Library, LibraryConfig, LibraryConfigWrapped};
@@ -40,28 +44,28 @@ pub struct LibraryManager {
 
 #[derive(Error, Debug)]
 pub enum LibraryManagerError {
-	#[error("error saving or loading the config from the filesystem")]
-	IO(#[from] io::Error),
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
 	#[error("error serializing or deserializing the JSON in the config file")]
 	Json(#[from] serde_json::Error),
-	#[error("Database error: {0}")]
+	#[error("database error")]
 	Database(#[from] prisma_client_rust::QueryError),
-	#[error("Library not found error")]
+	#[error("library not found error")]
 	LibraryNotFound,
 	#[error("error migrating the config file")]
 	Migration(String),
 	#[error("failed to parse uuid")]
 	Uuid(#[from] uuid::Error),
-	#[error("error opening database as the path contains non-UTF-8 characters")]
-	InvalidDatabasePath(PathBuf),
-	#[error("Failed to run seeder: {0}")]
+	#[error("failed to run seeder")]
 	Seeder(#[from] SeederError),
 	#[error("failed to initialise the key manager")]
 	KeyManager(#[from] sd_crypto::Error),
-	#[error("failed to run library migrations: {0}")]
-	MigratorError(#[from] crate::util::migrator::MigratorError),
+	#[error("failed to run library migrations")]
+	MigratorError(#[from] MigratorError),
 	#[error("invalid library configuration: {0}")]
 	InvalidConfig(String),
+	#[error(transparent)]
+	NonUtf8Path(#[from] NonUtf8PathError),
 }
 
 impl From<LibraryManagerError> for rspc::Error {
@@ -132,42 +136,56 @@ impl LibraryManager {
 		libraries_dir: PathBuf,
 		node_context: NodeContext,
 	) -> Result<Arc<Self>, LibraryManagerError> {
-		fs::create_dir_all(&libraries_dir)?;
+		fs::create_dir_all(&libraries_dir)
+			.await
+			.map_err(|e| FileIOError::from((&libraries_dir, e)))?;
 
 		let mut libraries = Vec::new();
-		for entry in fs::read_dir(&libraries_dir)?
-			.filter_map(|entry| entry.ok())
-			.filter(|entry| {
-				entry.path().is_file()
-					&& entry
-						.path()
-						.extension()
-						.map(|v| v == "sdlibrary")
-						.unwrap_or(false)
-			}) {
-			let config_path = entry.path();
-			let library_id = match Path::new(&config_path)
-				.file_stem()
-				.map(|v| v.to_str().map(Uuid::from_str))
+		let mut read_dir = fs::read_dir(&libraries_dir)
+			.await
+			.map_err(|e| FileIOError::from((&libraries_dir, e)))?;
+
+		while let Some(entry) = read_dir
+			.next_entry()
+			.await
+			.map_err(|e| FileIOError::from((&libraries_dir, e)))?
+		{
+			let entry_path = entry.path();
+			let metadata = entry
+				.metadata()
+				.await
+				.map_err(|e| FileIOError::from((&entry_path, e)))?;
+			if metadata.is_file()
+				&& entry_path
+					.extension()
+					.map(|ext| ext == "sdlibrary")
+					.unwrap_or(false)
 			{
-				Some(Some(Ok(id))) => id,
-				_ => {
-					println!("Attempted to load library from path '{}' but it has an invalid filename. Skipping...", config_path.display());
+				let Some(Ok(library_id)) = entry_path
+				.file_stem()
+				.and_then(|v| v.to_str().map(Uuid::from_str))
+			else {
+				warn!("Attempted to load library from path '{}' but it has an invalid filename. Skipping...", entry_path.display());
 					continue;
-				}
 			};
 
-			let db_path = config_path.clone().with_extension("db");
-			if !db_path.try_exists().unwrap() {
-				println!(
+				let db_path = entry_path.with_extension("db");
+				match fs::metadata(&db_path).await {
+					Ok(_) => {}
+					Err(e) if e.kind() == io::ErrorKind::NotFound => {
+						warn!(
 					"Found library '{}' but no matching database file was found. Skipping...",
-					config_path.display()
-				);
-				continue;
-			}
+						entry_path.display()
+					);
+						continue;
+					}
+					Err(e) => return Err(FileIOError::from((db_path, e)).into()),
+				}
 
-			let config = LibraryConfig::read(config_path)?;
-			libraries.push(Self::load(library_id, &db_path, config, node_context.clone()).await?);
+				let config = LibraryConfig::read(entry_path)?;
+				libraries
+					.push(Self::load(library_id, &db_path, config, node_context.clone()).await?);
+			}
 		}
 
 		let this = Arc::new(Self {
@@ -302,8 +320,21 @@ impl LibraryManager {
 			.find(|l| l.id == id)
 			.ok_or(LibraryManagerError::LibraryNotFound)?;
 
-		fs::remove_file(Path::new(&self.libraries_dir).join(format!("{}.db", library.id)))?;
-		fs::remove_file(Path::new(&self.libraries_dir).join(format!("{}.sdlibrary", library.id)))?;
+		let db_path = self.libraries_dir.join(format!("{}.db", library.id));
+		let sd_lib_path = self.libraries_dir.join(format!("{}.sdlibrary", library.id));
+
+		try_join!(
+			async {
+				fs::remove_file(&db_path)
+					.await
+					.map_err(|e| LibraryManagerError::FileIO(FileIOError::from((db_path, e))))
+			},
+			async {
+				fs::remove_file(&sd_lib_path)
+					.await
+					.map_err(|e| LibraryManagerError::FileIO(FileIOError::from((sd_lib_path, e))))
+			},
+		)?;
 
 		invalidate_query!(library, "library.list");
 
@@ -334,7 +365,7 @@ impl LibraryManager {
 			load_and_migrate(&format!(
 				"file:{}",
 				db_path.as_os_str().to_str().ok_or_else(|| {
-					LibraryManagerError::InvalidDatabasePath(db_path.to_path_buf())
+					LibraryManagerError::NonUtf8Path(NonUtf8PathError(db_path.into()))
 				})?
 			))
 			.await

core/src/location/error.rs

@@ -1,8 +1,9 @@
+use crate::util::error::FileIOError;
+
 use std::path::PathBuf;
 
 use rspc::{self, ErrorCode};
 use thiserror::Error;
-use tokio::io;
 use uuid::Uuid;
 
 use super::{
@@ -13,20 +14,20 @@ use super::{
 #[derive(Error, Debug)]
 pub enum LocationError {
 	// Not Found errors
-	#[error("Location not found (path: {})", .0.display())]
+	#[error("location not found <path='{}'>", .0.display())]
 	PathNotFound(PathBuf),
-	#[error("Location not found (uuid: {0})")]
+	#[error("location not found <uuid='{0}'>")]
 	UuidNotFound(Uuid),
-	#[error("Location not found (id: {0})")]
+	#[error("location not found <id='{0}'>")]
 	IdNotFound(i32),
 
 	// User errors
-	#[error("Location not a directory (path: {})", .0.display())]
+	#[error("location not a directory <path='{}'>", .0.display())]
 	NotDirectory(PathBuf),
-	#[error("Could not find directory in Location (path: {})", .0.display())]
+	#[error("could not find directory in location <path='{}'>", .0.display())]
 	DirectoryNotFound(PathBuf),
 	#[error(
-		"Library exists in the location metadata file, must relink: (old_path: {}, new_path: {})",
+		"library exists in the location metadata file, must relink <old_path='{}', new_path='{}'>",
 		.old_path.display(),
 		.new_path.display(),
 	)]
@@ -35,36 +36,36 @@ pub enum LocationError {
 		new_path: PathBuf,
 	},
 	#[error(
-		"This location belongs to another library, must update .spacedrive file: (path: {})",
+		"this location belongs to another library, must update .spacedrive file <path='{}'>",
 		.0.display()
 	)]
 	AddLibraryToMetadata(PathBuf),
-	#[error("Location metadata file not found: (path: {})", .0.display())]
+	#[error("location metadata file not found <path='{}'>", .0.display())]
 	MetadataNotFound(PathBuf),
-	#[error("Location already exists in database (path: {})", .0.display())]
+	#[error("location already exists in database <path='{}'>", .0.display())]
 	LocationAlreadyExists(PathBuf),
-	#[error("Nested location currently not supported (path: {})", .0.display())]
+	#[error("nested location currently not supported <path='{}'>", .0.display())]
 	NestedLocation(PathBuf),
 
 	// Internal Errors
-	#[error("Location metadata error (error: {0:?})")]
+	#[error(transparent)]
 	LocationMetadataError(#[from] LocationMetadataError),
-	#[error("Failed to read location path metadata info (path: {}); (error: {0:?})", .1.display())]
-	LocationPathFilesystemMetadataAccess(io::Error, PathBuf),
-	#[error("Missing metadata file for location (path: {})", .0.display())]
+	#[error("failed to read location path metadata info")]
+	LocationPathFilesystemMetadataAccess(FileIOError),
+	#[error("missing metadata file for location <path='{}'>", .0.display())]
 	MissingMetadataFile(PathBuf),
-	#[error("Failed to open file from local os (error: {0:?})")]
-	FileReadError(io::Error),
-	#[error("Failed to read mounted volumes from local os (error: {0:?})")]
+	#[error("failed to open file from local OS")]
+	FileReadError(FileIOError),
+	#[error("failed to read mounted volumes from local OS")]
 	VolumeReadError(String),
-	#[error("Failed to connect to database (error: {0:?})")]
-	IOError(io::Error),
-	#[error("Database error (error: {0:?})")]
+	#[error("database error")]
 	DatabaseError(#[from] prisma_client_rust::QueryError),
-	#[error("Location manager error (error: {0:?})")]
+	#[error(transparent)]
 	LocationManagerError(#[from] LocationManagerError),
-	#[error("File path related error (error: {0})")]
+	#[error(transparent)]
 	FilePathError(#[from] FilePathError),
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
 }
 
 impl From<LocationError> for rspc::Error {

core/src/location/file_path_helper.rs

@@ -1,680 +0,0 @@
-use crate::{
-	prisma::{file_path, location, PrismaClient},
-	util::db::uuid_to_bytes,
-};
-
-use std::{
-	borrow::Cow,
-	fmt::{Display, Formatter},
-	fs::Metadata,
-	path::{Path, PathBuf, MAIN_SEPARATOR, MAIN_SEPARATOR_STR},
-	time::SystemTime,
-};
-
-use chrono::{DateTime, Utc};
-use futures::future::try_join_all;
-use prisma_client_rust::QueryError;
-use serde::{Deserialize, Serialize};
-use thiserror::Error;
-use tokio::{fs, io};
-use tracing::error;
-use uuid::Uuid;
-
-use super::LocationId;
-
-// File Path selectables!
-file_path::select!(file_path_just_id_materialized_path {
-	pub_id
-	materialized_path
-});
-file_path::select!(file_path_for_file_identifier {
-	id
-	pub_id
-	materialized_path
-	date_created
-});
-file_path::select!(file_path_for_object_validator {
-	pub_id
-	materialized_path
-	integrity_checksum
-	location: select {
-		id
-		pub_id
-	}
-});
-file_path::select!(file_path_just_materialized_path_cas_id {
-	materialized_path
-	cas_id
-});
-
-// File Path includes!
-file_path::include!(file_path_with_object { object });
-
-#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct FilePathMetadata {
-	pub inode: u64,
-	pub device: u64,
-	pub size_in_bytes: u64,
-	pub created_at: DateTime<Utc>,
-	pub modified_at: DateTime<Utc>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct MaterializedPath<'a> {
-	pub(super) materialized_path: Cow<'a, str>,
-	pub(super) is_dir: bool,
-	pub(super) location_id: LocationId,
-	pub(super) name: Cow<'a, str>,
-	pub(super) extension: Cow<'a, str>,
-}
-
-impl MaterializedPath<'static> {
-	pub fn new(
-		location_id: LocationId,
-		location_path: impl AsRef<Path>,
-		full_path: impl AsRef<Path>,
-		is_dir: bool,
-	) -> Result<Self, FilePathError> {
-		let full_path = full_path.as_ref();
-		let mut materialized_path = format!(
-			"{MAIN_SEPARATOR_STR}{}",
-			extract_materialized_path(location_id, location_path, full_path)?
-				.to_str()
-				.expect("Found non-UTF-8 path")
-		);
-
-		if is_dir && !materialized_path.ends_with(MAIN_SEPARATOR) {
-			materialized_path += MAIN_SEPARATOR_STR;
-		}
-
-		let extension = if !is_dir {
-			let extension = full_path
-				.extension()
-				.unwrap_or_default()
-				.to_str()
-				.unwrap_or_default();
-
-			#[cfg(debug_assertions)]
-			{
-				// In dev mode, we lowercase the extension as we don't use the SQL migration,
-				// and using prisma.schema directly we can't set `COLLATE NOCASE` in the
-				// `extension` column at `file_path` table
-				extension.to_lowercase()
-			}
-			#[cfg(not(debug_assertions))]
-			{
-				extension.to_string()
-			}
-		} else {
-			String::new()
-		};
-
-		Ok(Self {
-			materialized_path: Cow::Owned(materialized_path),
-			is_dir,
-			location_id,
-			name: Cow::Owned(Self::prepare_name(full_path).to_string()),
-			extension: Cow::Owned(extension),
-		})
-	}
-}
-
-impl<'a> MaterializedPath<'a> {
-	pub fn location_id(&self) -> LocationId {
-		self.location_id
-	}
-
-	fn prepare_name(path: &Path) -> &str {
-		// Not using `impl AsRef<Path>` here because it's an private method
-		path.file_stem()
-			.unwrap_or_default()
-			.to_str()
-			.unwrap_or_default()
-	}
-
-	pub fn parent(&self) -> Self {
-		let parent_path = Path::new(self.materialized_path.as_ref())
-			.parent()
-			.unwrap_or_else(|| Path::new(MAIN_SEPARATOR_STR));
-
-		let mut parent_path_str = parent_path
-			.to_str()
-			.unwrap() // SAFETY: This unwrap is ok because this path was a valid UTF-8 String before
-			.to_string();
-
-		if !parent_path_str.ends_with(MAIN_SEPARATOR) {
-			parent_path_str += MAIN_SEPARATOR_STR;
-		}
-
-		Self {
-			materialized_path: Cow::Owned(parent_path_str),
-			is_dir: true,
-			location_id: self.location_id,
-			// NOTE: This way we don't use the same name for "/" `file_path`, that uses the location
-			// name in the database, check later if this is a problem
-			name: Cow::Owned(Self::prepare_name(parent_path).to_string()),
-			extension: Cow::Owned(String::new()),
-		}
-	}
-}
-
-impl<'a, S: AsRef<str> + 'a> From<(LocationId, &'a S)> for MaterializedPath<'a> {
-	fn from((location_id, materialized_path): (LocationId, &'a S)) -> Self {
-		let materialized_path = materialized_path.as_ref();
-		let is_dir = materialized_path.ends_with(MAIN_SEPARATOR);
-		let length = materialized_path.len();
-
-		let (name, extension) = if length == 1 {
-			// The case for the root path
-			(materialized_path, "")
-		} else if is_dir {
-			let first_name_char = materialized_path[..(length - 1)]
-				.rfind(MAIN_SEPARATOR)
-				.unwrap_or(0) + 1;
-			(&materialized_path[first_name_char..(length - 1)], "")
-		} else {
-			let first_name_char = materialized_path.rfind(MAIN_SEPARATOR).unwrap_or(0) + 1;
-			if let Some(last_dot_relative_idx) = materialized_path[first_name_char..].rfind('.') {
-				let last_dot_idx = first_name_char + last_dot_relative_idx;
-				(
-					&materialized_path[first_name_char..last_dot_idx],
-					&materialized_path[last_dot_idx + 1..],
-				)
-			} else {
-				(&materialized_path[first_name_char..], "")
-			}
-		};
-
-		Self {
-			materialized_path: Cow::Borrowed(materialized_path),
-			location_id,
-			is_dir,
-			name: Cow::Borrowed(name),
-			extension: Cow::Borrowed(extension),
-		}
-	}
-}
-
-impl From<MaterializedPath<'_>> for String {
-	fn from(path: MaterializedPath) -> Self {
-		path.materialized_path.into_owned()
-	}
-}
-
-impl From<&MaterializedPath<'_>> for String {
-	fn from(path: &MaterializedPath) -> Self {
-		path.materialized_path.to_string()
-	}
-}
-
-impl AsRef<str> for MaterializedPath<'_> {
-	fn as_ref(&self) -> &str {
-		self.materialized_path.as_ref()
-	}
-}
-
-impl AsRef<Path> for &MaterializedPath<'_> {
-	fn as_ref(&self) -> &Path {
-		// Skipping / because it's not a valid path to be joined
-		Path::new(&self.materialized_path[1..])
-	}
-}
-
-impl Display for MaterializedPath<'_> {
-	fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{}", self.materialized_path)
-	}
-}
-
-#[derive(Error, Debug)]
-pub enum FilePathError {
-	#[error("File Path not found: <path={0}>")]
-	NotFound(PathBuf),
-	#[error("Received an invalid sub path: <location_path={location_path}, sub_path={sub_path}>")]
-	InvalidSubPath {
-		location_path: PathBuf,
-		sub_path: PathBuf,
-	},
-	#[error("Sub path is not a directory: {0}")]
-	SubPathNotDirectory(PathBuf),
-	#[error("The parent directory of the received sub path isn't indexed in the location: <id={location_id}, sub_path={sub_path}>")]
-	SubPathParentNotInLocation {
-		location_id: LocationId,
-		sub_path: PathBuf,
-	},
-	#[error("Unable to extract materialized path from location: <id='{0}', path='{1:?}'>")]
-	UnableToExtractMaterializedPath(LocationId, PathBuf),
-	#[error("Database error (error: {0:?})")]
-	DatabaseError(#[from] QueryError),
-	#[error("Database error (error: {0:?})")]
-	IOError(#[from] io::Error),
-}
-
-#[cfg(feature = "location-watcher")]
-pub async fn create_file_path(
-	crate::location::Library { db, sync, .. }: &crate::location::Library,
-	MaterializedPath {
-		materialized_path,
-		is_dir,
-		location_id,
-		name,
-		extension,
-	}: MaterializedPath<'_>,
-	parent_id: Option<Uuid>,
-	cas_id: Option<String>,
-	metadata: FilePathMetadata,
-) -> Result<file_path::Data, FilePathError> {
-	// Keeping a reference in that map for the entire duration of the function, so we keep it locked
-
-	use crate::{sync, util};
-	use serde_json::json;
-
-	let location = db
-		.location()
-		.find_unique(location::id::equals(location_id))
-		.select(location::select!({ id pub_id }))
-		.exec()
-		.await?
-		.unwrap();
-
-	let params = {
-		use file_path::*;
-
-		util::db::chain_optional_iter(
-			[
-				(
-					location::NAME,
-					json!(sync::location::SyncId {
-						pub_id: location.pub_id
-					}),
-				),
-				(cas_id::NAME, json!(cas_id)),
-				(materialized_path::NAME, json!(materialized_path)),
-				(name::NAME, json!(name)),
-				(extension::NAME, json!(extension)),
-				(
-					size_in_bytes::NAME,
-					json!(metadata.size_in_bytes.to_string()),
-				),
-				(inode::NAME, json!(metadata.inode.to_le_bytes())),
-				(device::NAME, json!(metadata.device.to_le_bytes())),
-				(is_dir::NAME, json!(is_dir)),
-				(date_created::NAME, json!(metadata.created_at)),
-				(date_modified::NAME, json!(metadata.modified_at)),
-			],
-			[parent_id.map(|parent_id| {
-				(
-					parent_id::NAME,
-					json!(sync::file_path::SyncId {
-						pub_id: uuid_to_bytes(parent_id)
-					}),
-				)
-			})],
-		)
-	};
-
-	let pub_id = uuid_to_bytes(Uuid::new_v4());
-
-	let created_path = sync
-		.write_op(
-			db,
-			sync.unique_shared_create(
-				sync::file_path::SyncId {
-					pub_id: pub_id.clone(),
-				},
-				params,
-			),
-			db.file_path().create(
-				pub_id,
-				location::id::equals(location.id),
-				materialized_path.into_owned(),
-				name.into_owned(),
-				extension.into_owned(),
-				metadata.inode.to_le_bytes().into(),
-				metadata.device.to_le_bytes().into(),
-				{
-					use file_path::*;
-					vec![
-						cas_id::set(cas_id),
-						parent_id::set(parent_id.map(uuid_to_bytes)),
-						is_dir::set(is_dir),
-						size_in_bytes::set(metadata.size_in_bytes.to_string()),
-						date_created::set(metadata.created_at.into()),
-						date_modified::set(metadata.modified_at.into()),
-					]
-				},
-			),
-		)
-		.await?;
-
-	Ok(created_path)
-}
-
-pub fn subtract_location_path(
-	location_path: impl AsRef<Path>,
-	current_path: impl AsRef<Path>,
-) -> Option<PathBuf> {
-	let location_path = location_path.as_ref();
-	let current_path = current_path.as_ref();
-
-	if let Ok(stripped) = current_path.strip_prefix(location_path) {
-		Some(stripped.to_path_buf())
-	} else {
-		error!(
-			"Failed to strip location root path ({}) from current path ({})",
-			location_path.display(),
-			current_path.display()
-		);
-		None
-	}
-}
-
-pub fn extract_materialized_path(
-	location_id: LocationId,
-	location_path: impl AsRef<Path>,
-	path: impl AsRef<Path>,
-) -> Result<PathBuf, FilePathError> {
-	subtract_location_path(location_path, &path).ok_or_else(|| {
-		FilePathError::UnableToExtractMaterializedPath(location_id, path.as_ref().to_path_buf())
-	})
-}
-
-pub async fn filter_file_paths_by_many_full_path_params(
-	location: &location::Data,
-	full_paths: &[impl AsRef<Path>],
-) -> Result<Vec<file_path::WhereParam>, FilePathError> {
-	let is_dirs = try_join_all(
-		full_paths
-			.iter()
-			.map(|path| async move { fs::metadata(path).await.map(|metadata| metadata.is_dir()) }),
-	)
-	.await?;
-
-	let materialized_paths = full_paths
-		.iter()
-		.zip(is_dirs.into_iter())
-		.map(|(path, is_dir)| {
-			MaterializedPath::new(location.id, &location.path, path, is_dir).map(Into::into)
-		})
-		// Collecting in a Result, so we stop on the first error
-		.collect::<Result<Vec<_>, _>>()?;
-
-	Ok(vec![
-		file_path::location_id::equals(location.id),
-		file_path::materialized_path::in_vec(materialized_paths),
-	])
-}
-
-#[cfg(feature = "location-watcher")]
-pub async fn check_existing_file_path(
-	materialized_path: &MaterializedPath<'_>,
-	db: &PrismaClient,
-) -> Result<bool, FilePathError> {
-	db.file_path()
-		.count(filter_existing_file_path_params(materialized_path))
-		.exec()
-		.await
-		.map_or_else(|e| Err(e.into()), |count| Ok(count > 0))
-}
-
-pub fn filter_existing_file_path_params(
-	MaterializedPath {
-		materialized_path,
-		is_dir,
-		location_id,
-		name,
-		extension,
-	}: &MaterializedPath,
-) -> Vec<file_path::WhereParam> {
-	let mut params = vec![
-		file_path::location_id::equals(*location_id),
-		file_path::materialized_path::equals(materialized_path.to_string()),
-		file_path::is_dir::equals(*is_dir),
-		file_path::extension::equals(extension.to_string()),
-	];
-
-	// This is due to a limitation of MaterializedPath, where we don't know the location name to use
-	// as the file_path name at the root of the location "/" or "\" on Windows
-	if materialized_path != MAIN_SEPARATOR_STR {
-		params.push(file_path::name::equals(name.to_string()));
-	}
-
-	params
-}
-
-/// With this function we try to do a loose filtering of file paths, to avoid having to do check
-/// twice for directories and for files. This is because directories have a trailing `/` or `\` in
-/// the materialized path
-#[allow(unused)]
-pub fn loose_find_existing_file_path_params(
-	MaterializedPath {
-		materialized_path,
-		is_dir,
-		location_id,
-		name,
-		..
-	}: &MaterializedPath,
-) -> Vec<file_path::WhereParam> {
-	let mut materialized_path_str = materialized_path.to_string();
-	if *is_dir {
-		materialized_path_str.pop();
-	}
-
-	let mut params = vec![
-		file_path::location_id::equals(*location_id),
-		file_path::materialized_path::starts_with(materialized_path_str),
-	];
-
-	// This is due to a limitation of MaterializedPath, where we don't know the location name to use
-	// as the file_path name at the root of the location "/" or "\" on Windows
-	if materialized_path != MAIN_SEPARATOR_STR {
-		params.push(file_path::name::equals(name.to_string()));
-	}
-
-	params
-}
-
-pub async fn get_existing_file_path_id(
-	materialized_path: &MaterializedPath<'_>,
-	db: &PrismaClient,
-) -> Result<Option<Uuid>, FilePathError> {
-	db.file_path()
-		.find_first(filter_existing_file_path_params(materialized_path))
-		.select(file_path::select!({ pub_id }))
-		.exec()
-		.await
-		.map_or_else(
-			|e| Err(e.into()),
-			|r| Ok(r.map(|r| Uuid::from_slice(&r.pub_id).unwrap())),
-		)
-}
-
-#[cfg(feature = "location-watcher")]
-pub async fn get_parent_dir(
-	materialized_path: &MaterializedPath<'_>,
-	db: &PrismaClient,
-) -> Result<Option<file_path::Data>, FilePathError> {
-	db.file_path()
-		.find_first(filter_existing_file_path_params(
-			&materialized_path.parent(),
-		))
-		.exec()
-		.await
-		.map_err(Into::into)
-}
-
-#[cfg(feature = "location-watcher")]
-pub async fn get_parent_dir_id(
-	materialized_path: &MaterializedPath<'_>,
-	db: &PrismaClient,
-) -> Result<Option<Uuid>, FilePathError> {
-	get_existing_file_path_id(&materialized_path.parent(), db).await
-}
-
-pub async fn ensure_sub_path_is_in_location(
-	location_path: impl AsRef<Path>,
-	sub_path: impl AsRef<Path>,
-) -> Result<PathBuf, FilePathError> {
-	let mut sub_path = sub_path.as_ref();
-	if sub_path.starts_with(MAIN_SEPARATOR_STR) {
-		// SAFETY: we just checked that it starts with the separator
-		sub_path = sub_path.strip_prefix(MAIN_SEPARATOR_STR).unwrap();
-	}
-	let location_path = location_path.as_ref();
-
-	if !sub_path.starts_with(location_path) {
-		// If the sub_path doesn't start with the location_path, we have to check if it's a
-		// materialized path received from the frontend, then we check if the full path exists
-		let full_path = location_path.join(sub_path);
-
-		match fs::metadata(&full_path).await {
-			Ok(_) => Ok(full_path),
-			Err(e) if e.kind() == io::ErrorKind::NotFound => Err(FilePathError::InvalidSubPath {
-				sub_path: sub_path.to_path_buf(),
-				location_path: location_path.to_path_buf(),
-			}),
-			Err(e) => Err(e.into()),
-		}
-	} else {
-		Ok(sub_path.to_path_buf())
-	}
-}
-
-pub async fn ensure_sub_path_is_directory(
-	location_path: impl AsRef<Path>,
-	sub_path: impl AsRef<Path>,
-) -> Result<(), FilePathError> {
-	let mut sub_path = sub_path.as_ref();
-
-	match fs::metadata(sub_path).await {
-		Ok(meta) => {
-			if meta.is_file() {
-				Err(FilePathError::SubPathNotDirectory(sub_path.to_path_buf()))
-			} else {
-				Ok(())
-			}
-		}
-		Err(e) if e.kind() == io::ErrorKind::NotFound => {
-			if sub_path.starts_with(MAIN_SEPARATOR_STR) {
-				// SAFETY: we just checked that it starts with the separator
-				sub_path = sub_path.strip_prefix(MAIN_SEPARATOR_STR).unwrap();
-			}
-
-			let location_path = location_path.as_ref();
-
-			match fs::metadata(location_path.join(sub_path)).await {
-				Ok(meta) => {
-					if meta.is_file() {
-						Err(FilePathError::SubPathNotDirectory(sub_path.to_path_buf()))
-					} else {
-						Ok(())
-					}
-				}
-				Err(e) if e.kind() == io::ErrorKind::NotFound => {
-					Err(FilePathError::InvalidSubPath {
-						sub_path: sub_path.to_path_buf(),
-						location_path: location_path.to_path_buf(),
-					})
-				}
-				Err(e) => Err(e.into()),
-			}
-		}
-		Err(e) => Err(e.into()),
-	}
-}
-
-pub async fn retain_file_paths_in_location(
-	location_id: LocationId,
-	to_retain: Vec<Uuid>,
-	maybe_parent_file_path: Option<file_path_just_id_materialized_path::Data>,
-	db: &PrismaClient,
-) -> Result<i64, FilePathError> {
-	let mut to_delete_params = vec![
-		file_path::location_id::equals(location_id),
-		file_path::pub_id::not_in_vec(to_retain.into_iter().map(uuid_to_bytes).collect()),
-	];
-
-	if let Some(parent_file_path) = maybe_parent_file_path {
-		// If the parent_materialized_path is not the root path, we only delete file paths that start with the parent path
-		let param = if parent_file_path.materialized_path != MAIN_SEPARATOR_STR {
-			file_path::materialized_path::starts_with(parent_file_path.materialized_path)
-		} else {
-			// If the parent_materialized_path is the root path, we fetch children using the parent id
-			file_path::parent_id::equals(Some(parent_file_path.pub_id))
-		};
-
-		to_delete_params.push(param);
-	}
-
-	db.file_path()
-		.delete_many(to_delete_params)
-		.exec()
-		.await
-		.map_err(Into::into)
-}
-
-#[allow(unused)] // TODO remove this annotation when we can use it on windows
-pub fn get_inode_and_device(metadata: &Metadata) -> Result<(u64, u64), FilePathError> {
-	#[cfg(target_family = "unix")]
-	{
-		use std::os::unix::fs::MetadataExt;
-
-		Ok((metadata.ino(), metadata.dev()))
-	}
-
-	#[cfg(target_family = "windows")]
-	{
-		// TODO use this when it's stable and remove winapi-utils dependency
-
-		// use std::os::windows::fs::MetadataExt;
-
-		// Ok((
-		// 	metadata
-		// 		.file_index()
-		// 		.expect("This function must not be called from a `DirEntry`'s `Metadata"),
-		// 	metadata
-		// 		.volume_serial_number()
-		// 		.expect("This function must not be called from a `DirEntry`'s `Metadata") as u64,
-		// ))
-
-		todo!("Use metadata: {:#?}", metadata)
-	}
-}
-
-#[allow(unused)]
-pub async fn get_inode_and_device_from_path(
-	path: impl AsRef<Path>,
-) -> Result<(u64, u64), FilePathError> {
-	#[cfg(target_family = "unix")]
-	{
-		// TODO use this when it's stable and remove winapi-utils dependency
-		let metadata = fs::metadata(path.as_ref()).await?;
-
-		get_inode_and_device(&metadata)
-	}
-
-	#[cfg(target_family = "windows")]
-	{
-		use winapi_util::{file::information, Handle};
-
-		let info = information(&Handle::from_path_any(path.as_ref())?)?;
-
-		Ok((info.file_index(), info.volume_serial_number()))
-	}
-}
-
-pub trait MetadataExt {
-	fn created_or_now(&self) -> SystemTime;
-
-	fn modified_or_now(&self) -> SystemTime;
-}
-
-impl MetadataExt for Metadata {
-	fn created_or_now(&self) -> SystemTime {
-		self.created().unwrap_or_else(|_| SystemTime::now())
-	}
-
-	fn modified_or_now(&self) -> SystemTime {
-		self.modified().unwrap_or_else(|_| SystemTime::now())
-	}
-}

core/src/location/file_path_helper/isolated_file_path_data.rs

@@ -0,0 +1,636 @@
+use crate::{location::LocationId, prisma::file_path, util::error::NonUtf8PathError};
+
+use std::{borrow::Cow, fmt, path::Path};
+
+use serde::{Deserialize, Serialize};
+
+use super::{
+	file_path_for_file_identifier, file_path_for_object_validator, file_path_for_thumbnailer,
+	file_path_to_full_path, file_path_to_handle_custom_uri, file_path_to_isolate,
+	file_path_with_object, FilePathError,
+};
+
+#[derive(Serialize, Deserialize, Debug, Hash, Eq, PartialEq)]
+#[non_exhaustive]
+pub struct IsolatedFilePathData<'a> {
+	pub(in crate::location) location_id: LocationId,
+	pub(in crate::location) materialized_path: Cow<'a, str>,
+	pub(in crate::location) is_dir: bool,
+	pub(in crate::location) name: Cow<'a, str>,
+	pub(in crate::location) extension: Cow<'a, str>,
+	pub(in crate::location) relative_path: Cow<'a, str>,
+}
+
+impl IsolatedFilePathData<'static> {
+	pub fn new(
+		location_id: LocationId,
+		location_path: impl AsRef<Path>,
+		full_path: impl AsRef<Path>,
+		is_dir: bool,
+	) -> Result<Self, FilePathError> {
+		let full_path = full_path.as_ref();
+		let location_path = location_path.as_ref();
+
+		let extension = (!is_dir)
+			.then(|| {
+				let extension = full_path
+					.extension()
+					.unwrap_or_default()
+					.to_str()
+					.unwrap_or_default();
+
+				#[cfg(debug_assertions)]
+				{
+					// In dev mode, we lowercase the extension as we don't use the SQL migration,
+					// and using prisma.schema directly we can't set `COLLATE NOCASE` in the
+					// `extension` column at `file_path` table
+					extension.to_lowercase()
+				}
+				#[cfg(not(debug_assertions))]
+				{
+					extension.to_string()
+				}
+			})
+			.unwrap_or_default();
+
+		Ok(Self {
+			is_dir,
+			location_id,
+			materialized_path: Cow::Owned(extract_normalized_materialized_path_str(
+				location_id,
+				location_path,
+				full_path,
+			)?),
+			name: Cow::Owned(
+				(location_path != full_path)
+					.then(|| Self::prepare_name(full_path).to_string())
+					.unwrap_or_default(),
+			),
+			extension: Cow::Owned(extension),
+			relative_path: Cow::Owned(extract_relative_path(
+				location_id,
+				location_path,
+				full_path,
+			)?),
+		})
+	}
+}
+
+impl<'a> IsolatedFilePathData<'a> {
+	pub fn location_id(&self) -> LocationId {
+		self.location_id
+	}
+
+	pub fn parent(&'a self) -> Self {
+		let (parent_path_str, name, relative_path) = if self.materialized_path == "/" {
+			("/", "", "")
+		} else {
+			let trailing_slash_idx = self.materialized_path.len() - 1;
+			let last_slash_idx = self.materialized_path[..trailing_slash_idx]
+				.rfind('/')
+				.expect("malformed materialized path at `parent` method");
+
+			(
+				&self.materialized_path[..last_slash_idx + 1],
+				&self.materialized_path[last_slash_idx + 1..trailing_slash_idx],
+				&self.materialized_path[1..trailing_slash_idx],
+			)
+		};
+
+		Self {
+			is_dir: true,
+			location_id: self.location_id,
+			relative_path: Cow::Borrowed(relative_path),
+			materialized_path: Cow::Borrowed(parent_path_str),
+			name: Cow::Borrowed(name),
+			extension: Cow::Borrowed(""),
+		}
+	}
+
+	pub fn from_relative_str(location_id: LocationId, relative_file_path_str: &'a str) -> Self {
+		let is_dir = relative_file_path_str.ends_with('/');
+
+		let (materialized_path, maybe_name, maybe_extension) =
+			Self::separate_path_name_and_extension_from_str(relative_file_path_str, is_dir);
+
+		Self {
+			location_id,
+			materialized_path: Cow::Borrowed(materialized_path),
+			is_dir,
+			name: maybe_name.map(Cow::Borrowed).unwrap_or_default(),
+			extension: maybe_extension.map(Cow::Borrowed).unwrap_or_default(),
+			relative_path: Cow::Borrowed(relative_file_path_str),
+		}
+	}
+
+	pub fn materialized_path_for_children(&self) -> Option<String> {
+		if self.materialized_path == "/" && self.name.is_empty() && self.is_dir {
+			// We're at the root file_path
+			Some("/".to_string())
+		} else {
+			self.is_dir
+				.then(|| format!("{}{}/", self.materialized_path, self.name))
+		}
+	}
+
+	pub fn separate_path_name_and_extension_from_str(
+		source: &'a str,
+		is_dir: bool,
+	) -> (
+		&'a str,         // Materialized path
+		Option<&'a str>, // Maybe a name
+		Option<&'a str>, // Maybe an extension
+	) {
+		let length = source.len();
+
+		if length == 1 {
+			// The case for the root path
+			(source, None, None)
+		} else if is_dir {
+			let last_char_idx = if source.ends_with('/') {
+				length - 1
+			} else {
+				length
+			};
+
+			let first_name_char_idx = source[..last_char_idx].rfind('/').unwrap_or(0) + 1;
+			(
+				&source[..first_name_char_idx],
+				Some(&source[first_name_char_idx..last_char_idx]),
+				None,
+			)
+		} else {
+			let first_name_char_idx = source.rfind('/').unwrap_or(0) + 1;
+			let end_idx = first_name_char_idx - 1;
+			if let Some(last_dot_relative_idx) = source[first_name_char_idx..].rfind('.') {
+				let last_dot_idx = first_name_char_idx + last_dot_relative_idx;
+				(
+					&source[..end_idx],
+					Some(&source[first_name_char_idx..last_dot_idx]),
+					Some(&source[last_dot_idx + 1..]),
+				)
+			} else {
+				(
+					&source[..end_idx],
+					Some(&source[first_name_char_idx..]),
+					None,
+				)
+			}
+		}
+	}
+
+	fn prepare_name(path: &Path) -> &str {
+		// Not using `impl AsRef<Path>` here because it's an private method
+		path.file_stem()
+			.unwrap_or_default()
+			.to_str()
+			.unwrap_or_default()
+	}
+
+	fn from_db_data(
+		location_id: LocationId,
+		db_materialized_path: &'a str,
+		db_is_dir: bool,
+		db_name: &'a str,
+		db_extension: &'a str,
+	) -> Self {
+		Self {
+			location_id,
+			materialized_path: Cow::Borrowed(db_materialized_path),
+			is_dir: db_is_dir,
+			name: Cow::Borrowed(db_name),
+			extension: Cow::Borrowed(db_extension),
+			relative_path: Cow::Owned(assemble_relative_path(
+				db_materialized_path,
+				db_name,
+				db_extension,
+				db_is_dir,
+			)),
+		}
+	}
+}
+
+impl AsRef<Path> for IsolatedFilePathData<'_> {
+	fn as_ref(&self) -> &Path {
+		Path::new(self.relative_path.as_ref())
+	}
+}
+
+impl From<IsolatedFilePathData<'static>> for file_path::UniqueWhereParam {
+	fn from(path: IsolatedFilePathData<'static>) -> Self {
+		Self::LocationIdMaterializedPathNameExtensionEquals(
+			path.location_id,
+			path.materialized_path.into_owned(),
+			path.name.into_owned(),
+			path.extension.into_owned(),
+		)
+	}
+}
+
+impl From<IsolatedFilePathData<'static>> for file_path::WhereParam {
+	fn from(path: IsolatedFilePathData<'static>) -> Self {
+		Self::And(vec![
+			file_path::location_id::equals(path.location_id),
+			file_path::materialized_path::equals(path.materialized_path.into_owned()),
+			file_path::name::equals(path.name.into_owned()),
+			file_path::extension::equals(path.extension.into_owned()),
+		])
+	}
+}
+
+impl From<&IsolatedFilePathData<'_>> for file_path::UniqueWhereParam {
+	fn from(path: &IsolatedFilePathData<'_>) -> Self {
+		Self::LocationIdMaterializedPathNameExtensionEquals(
+			path.location_id,
+			path.materialized_path.to_string(),
+			path.name.to_string(),
+			path.extension.to_string(),
+		)
+	}
+}
+
+impl From<&IsolatedFilePathData<'_>> for file_path::WhereParam {
+	fn from(path: &IsolatedFilePathData<'_>) -> Self {
+		Self::And(vec![
+			file_path::location_id::equals(path.location_id),
+			file_path::materialized_path::equals(path.materialized_path.to_string()),
+			file_path::name::equals(path.name.to_string()),
+			file_path::extension::equals(path.extension.to_string()),
+		])
+	}
+}
+
+impl fmt::Display for IsolatedFilePathData<'_> {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "{}", self.relative_path)
+	}
+}
+
+#[macro_use]
+mod macros {
+	macro_rules! impl_from_db {
+		($($file_path_kind:ident),+ $(,)?) => {
+			$(
+				impl ::std::convert::From<$file_path_kind::Data> for $crate::
+					location::
+					file_path_helper::
+					isolated_file_path_data::
+					IsolatedFilePathData<'static>
+				{
+					fn from(path: $file_path_kind::Data) -> Self {
+						Self {
+							location_id: path.location_id,
+							relative_path: ::std::borrow::Cow::Owned(
+								$crate::
+								location::
+								file_path_helper::
+								isolated_file_path_data::
+								assemble_relative_path(
+									&path.materialized_path,
+									&path.name,
+									&path.extension,
+									path.is_dir,
+								)
+							),
+							materialized_path: ::std::borrow::Cow::Owned(path.materialized_path),
+							is_dir: path.is_dir,
+							name: ::std::borrow::Cow::Owned(path.name),
+							extension: ::std::borrow::Cow::Owned(path.extension),
+						}
+					}
+				}
+
+				impl<'a> ::std::convert::From<&'a $file_path_kind::Data> for $crate::
+					location::
+					file_path_helper::
+					isolated_file_path_data::
+					IsolatedFilePathData<'a>
+				{
+					fn from(path: &'a $file_path_kind::Data) -> Self {
+						Self::from_db_data(
+							path.location_id,
+							&path.materialized_path,
+							path.is_dir,
+							&path.name,
+							&path.extension
+						)
+					}
+				}
+			)+
+		};
+	}
+
+	macro_rules! impl_from_db_without_location_id {
+		($($file_path_kind:ident),+ $(,)?) => {
+			$(
+				impl ::std::convert::From<($crate::location::LocationId, $file_path_kind::Data)> for $crate::
+					location::
+					file_path_helper::
+					isolated_file_path_data::
+					IsolatedFilePathData<'static>
+				{
+					fn from((location_id, path): ($crate::location::LocationId, $file_path_kind::Data)) -> Self {
+						Self {
+							location_id,
+							relative_path: Cow::Owned(
+								$crate::
+								location::
+								file_path_helper::
+								isolated_file_path_data::
+								assemble_relative_path(
+									&path.materialized_path,
+									&path.name,
+									&path.extension,
+									path.is_dir,
+								)
+							),
+							materialized_path: Cow::Owned(path.materialized_path),
+							is_dir: path.is_dir,
+							name: Cow::Owned(path.name),
+							extension: Cow::Owned(path.extension),
+						}
+					}
+				}
+
+				impl<'a> ::std::convert::From<($crate::location::LocationId, &'a $file_path_kind::Data)> for $crate::
+					location::
+					file_path_helper::
+					isolated_file_path_data::
+					IsolatedFilePathData<'a>
+				{
+					fn from((location_id, path): ($crate::location::LocationId, &'a $file_path_kind::Data)) -> Self {
+						Self::from_db_data(
+							location_id,
+							&path.materialized_path,
+							path.is_dir,
+							&path.name,
+							&path.extension
+						)
+					}
+				}
+			)+
+		};
+	}
+}
+
+impl_from_db!(file_path, file_path_to_isolate, file_path_with_object);
+
+impl_from_db_without_location_id!(
+	file_path_for_file_identifier,
+	file_path_to_full_path,
+	file_path_for_thumbnailer,
+	file_path_for_object_validator,
+	file_path_to_handle_custom_uri
+);
+
+fn extract_relative_path(
+	location_id: LocationId,
+	location_path: impl AsRef<Path>,
+	path: impl AsRef<Path>,
+) -> Result<String, FilePathError> {
+	let path = path.as_ref();
+
+	path.strip_prefix(location_path)
+		.map_err(|_| FilePathError::UnableToExtractMaterializedPath {
+			location_id,
+			path: path.into(),
+		})
+		.and_then(|relative| {
+			relative
+				.to_str()
+				.map(|relative_str| relative_str.replace('\\', "/"))
+				.ok_or_else(|| NonUtf8PathError(path.into()).into())
+		})
+}
+
+/// This function separates a file path from a location path, and normalizes replacing '\' with '/'
+/// to be consistent between Windows and Unix like systems
+pub fn extract_normalized_materialized_path_str(
+	location_id: LocationId,
+	location_path: impl AsRef<Path>,
+	path: impl AsRef<Path>,
+) -> Result<String, FilePathError> {
+	let path = path.as_ref();
+
+	path.strip_prefix(location_path)
+		.map_err(|_| FilePathError::UnableToExtractMaterializedPath {
+			location_id,
+			path: path.into(),
+		})?
+		.parent()
+		.map(|materialized_path| {
+			materialized_path
+				.to_str()
+				.map(|materialized_path_str| {
+					if !materialized_path_str.is_empty() {
+						format!("/{}/", materialized_path_str.replace('\\', "/"))
+					} else {
+						"/".to_string()
+					}
+				})
+				.ok_or_else(|| NonUtf8PathError(path.into()))
+		})
+		.unwrap_or_else(|| Ok("/".to_string()))
+		.map_err(Into::into)
+}
+
+fn assemble_relative_path(
+	materialized_path: &str,
+	name: &str,
+	extension: &str,
+	is_dir: bool,
+) -> String {
+	match (is_dir, extension) {
+		(false, extension) if !extension.is_empty() => {
+			format!("{}{}.{}", &materialized_path[1..], name, extension)
+		}
+		(_, _) => format!("{}{}", &materialized_path[1..], name),
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	fn expected(
+		materialized_path: &'static str,
+		is_dir: bool,
+		name: &'static str,
+		extension: &'static str,
+		relative_path: &'static str,
+	) -> IsolatedFilePathData<'static> {
+		IsolatedFilePathData {
+			location_id: 1,
+			materialized_path: materialized_path.into(),
+			is_dir,
+			name: name.into(),
+			extension: extension.into(),
+			relative_path: relative_path.into(),
+		}
+	}
+
+	#[test]
+	fn new_method() {
+		let tester = |full_path, is_dir, expected, msg| {
+			let actual =
+				IsolatedFilePathData::new(1, "/spacedrive/location", full_path, is_dir).unwrap();
+			assert_eq!(actual, expected, "{msg}");
+		};
+
+		tester(
+			"/spacedrive/location",
+			true,
+			expected("/", true, "", "", ""),
+			"the location root directory",
+		);
+
+		tester(
+			"/spacedrive/location/file.txt",
+			false,
+			expected("/", false, "file", "txt", "file.txt"),
+			"a file in the root directory",
+		);
+
+		tester(
+			"/spacedrive/location/dir",
+			true,
+			expected("/", true, "dir", "", "dir"),
+			"a directory in the root directory",
+		);
+
+		tester(
+			"/spacedrive/location/dir/file.txt",
+			false,
+			expected("/dir/", false, "file", "txt", "dir/file.txt"),
+			"a directory with a file inside",
+		);
+
+		tester(
+			"/spacedrive/location/dir/dir2",
+			true,
+			expected("/dir/", true, "dir2", "", "dir/dir2"),
+			"a directory in a directory",
+		);
+
+		tester(
+			"/spacedrive/location/dir/dir2/dir3",
+			true,
+			expected("/dir/dir2/", true, "dir3", "", "dir/dir2/dir3"),
+			"3 level of directories",
+		);
+
+		tester(
+			"/spacedrive/location/dir/dir2/dir3/file.txt",
+			false,
+			expected(
+				"/dir/dir2/dir3/",
+				false,
+				"file",
+				"txt",
+				"dir/dir2/dir3/file.txt",
+			),
+			"a file inside a third level directory",
+		);
+	}
+
+	#[test]
+	fn parent_method() {
+		let tester = |full_path, is_dir, expected, msg| {
+			let child =
+				IsolatedFilePathData::new(1, "/spacedrive/location", full_path, is_dir).unwrap();
+
+			let actual = child.parent();
+			assert_eq!(actual, expected, "{msg}");
+		};
+
+		tester(
+			"/spacedrive/location",
+			true,
+			expected("/", true, "", "", ""),
+			"the location root directory",
+		);
+
+		tester(
+			"/spacedrive/location/file.txt",
+			false,
+			expected("/", true, "", "", ""),
+			"a file in the root directory",
+		);
+
+		tester(
+			"/spacedrive/location/dir",
+			true,
+			expected("/", true, "", "", ""),
+			"a directory in the root directory",
+		);
+
+		tester(
+			"/spacedrive/location/dir/file.txt",
+			false,
+			expected("/", true, "dir", "", "dir"),
+			"a directory with a file inside",
+		);
+
+		tester(
+			"/spacedrive/location/dir/dir2",
+			true,
+			expected("/", true, "dir", "", "dir"),
+			"a directory in a directory",
+		);
+
+		tester(
+			"/spacedrive/location/dir/dir2/dir3",
+			true,
+			expected("/dir/", true, "dir2", "", "dir/dir2"),
+			"3 level of directories",
+		);
+
+		tester(
+			"/spacedrive/location/dir/dir2/dir3/file.txt",
+			false,
+			expected("/dir/dir2/", true, "dir3", "", "dir/dir2/dir3"),
+			"a file inside a third level directory",
+		);
+	}
+
+	#[test]
+	fn extract_normalized_materialized_path() {
+		let tester = |path, expected, msg| {
+			let actual =
+				extract_normalized_materialized_path_str(1, "/spacedrive/location", path).unwrap();
+			assert_eq!(actual, expected, "{msg}");
+		};
+
+		tester("/spacedrive/location", "/", "the location root directory");
+		tester(
+			"/spacedrive/location/file.txt",
+			"/",
+			"a file in the root directory",
+		);
+		tester(
+			"/spacedrive/location/dir",
+			"/",
+			"a directory in the root directory",
+		);
+		tester(
+			"/spacedrive/location/dir/file.txt",
+			"/dir/",
+			"a directory with a file inside",
+		);
+		tester(
+			"/spacedrive/location/dir/dir2",
+			"/dir/",
+			"a directory in a directory",
+		);
+		tester(
+			"/spacedrive/location/dir/dir2/dir3",
+			"/dir/dir2/",
+			"3 level of directories",
+		);
+		tester(
+			"/spacedrive/location/dir/dir2/dir3/file.txt",
+			"/dir/dir2/dir3/",
+			"a file inside a third level directory",
+		);
+	}
+}

core/src/location/file_path_helper/mod.rs

@@ -0,0 +1,458 @@
+use crate::{
+	prisma::{file_path, PrismaClient},
+	util::error::{FileIOError, NonUtf8PathError},
+};
+
+use std::{
+	fs::Metadata,
+	path::{Path, PathBuf},
+	time::SystemTime,
+};
+
+use chrono::{DateTime, Utc};
+use prisma_client_rust::QueryError;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use tokio::{fs, io};
+use tracing::error;
+
+pub mod isolated_file_path_data;
+
+pub use isolated_file_path_data::IsolatedFilePathData;
+
+use super::LocationId;
+
+// File Path selectables!
+file_path::select!(file_path_just_pub_id { pub_id });
+file_path::select!(file_path_just_pub_id_materialized_path {
+	pub_id
+	materialized_path
+});
+file_path::select!(file_path_for_file_identifier {
+	id
+	pub_id
+	materialized_path
+	date_created
+	is_dir
+	name
+	extension
+});
+file_path::select!(file_path_for_object_validator {
+	pub_id
+	materialized_path
+	is_dir
+	name
+	extension
+	integrity_checksum
+	location: select {
+		id
+		pub_id
+	}
+});
+file_path::select!(file_path_for_thumbnailer {
+	materialized_path
+	is_dir
+	name
+	extension
+	cas_id
+});
+file_path::select!(file_path_to_isolate {
+	location_id
+	materialized_path
+	is_dir
+	name
+	extension
+});
+file_path::select!(file_path_to_handle_custom_uri {
+	materialized_path
+	is_dir
+	name
+	extension
+	location: select {
+		path
+	}
+});
+file_path::select!(file_path_to_full_path {
+	materialized_path
+	is_dir
+	name
+	extension
+	location: select {
+		id
+		path
+	}
+});
+
+// File Path includes!
+file_path::include!(file_path_with_object { object });
+
+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct FilePathMetadata {
+	pub inode: u64,
+	pub device: u64,
+	pub size_in_bytes: u64,
+	pub created_at: DateTime<Utc>,
+	pub modified_at: DateTime<Utc>,
+}
+
+#[derive(Error, Debug)]
+pub enum FilePathError {
+	#[error("file Path not found: <path='{}'>", .0.display())]
+	NotFound(Box<Path>),
+	#[error("received an invalid sub path: <location_path='{}', sub_path='{}'>", .location_path.display(), .sub_path.display())]
+	InvalidSubPath {
+		location_path: Box<Path>,
+		sub_path: Box<Path>,
+	},
+	#[error("sub path is not a directory: <path='{}'>", .0.display())]
+	SubPathNotDirectory(Box<Path>),
+	#[error(
+		"the parent directory of the received sub path isn't indexed in the location: <id='{}', sub_path='{}'>",
+		.location_id,
+		.sub_path.display()
+	)]
+	SubPathParentNotInLocation {
+		location_id: LocationId,
+		sub_path: Box<Path>,
+	},
+	#[error("unable to extract materialized path from location: <id='{}', path='{}'>", .location_id, .path.display())]
+	UnableToExtractMaterializedPath {
+		location_id: LocationId,
+		path: Box<Path>,
+	},
+	#[error("database error")]
+	Database(#[from] QueryError),
+
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
+	#[error(transparent)]
+	NonUtf8Path(#[from] NonUtf8PathError),
+}
+
+#[cfg(feature = "location-watcher")]
+pub async fn create_file_path(
+	crate::location::Library { db, sync, .. }: &crate::location::Library,
+	IsolatedFilePathData {
+		materialized_path,
+		is_dir,
+		location_id,
+		name,
+		extension,
+		..
+	}: IsolatedFilePathData<'_>,
+	cas_id: Option<String>,
+	metadata: FilePathMetadata,
+) -> Result<file_path::Data, FilePathError> {
+	use crate::{prisma::location, sync, util::db::uuid_to_bytes};
+
+	use serde_json::json;
+	use uuid::Uuid;
+
+	let location = db
+		.location()
+		.find_unique(location::id::equals(location_id))
+		.select(location::select!({ id pub_id }))
+		.exec()
+		.await?
+		.unwrap();
+
+	let params = {
+		use file_path::*;
+
+		vec![
+			(
+				location::NAME,
+				json!(sync::location::SyncId {
+					pub_id: location.pub_id
+				}),
+			),
+			(cas_id::NAME, json!(cas_id)),
+			(materialized_path::NAME, json!(materialized_path)),
+			(name::NAME, json!(name)),
+			(extension::NAME, json!(extension)),
+			(
+				size_in_bytes::NAME,
+				json!(metadata.size_in_bytes.to_string()),
+			),
+			(inode::NAME, json!(metadata.inode.to_le_bytes())),
+			(device::NAME, json!(metadata.device.to_le_bytes())),
+			(is_dir::NAME, json!(is_dir)),
+			(date_created::NAME, json!(metadata.created_at)),
+			(date_modified::NAME, json!(metadata.modified_at)),
+		]
+	};
+
+	let pub_id = uuid_to_bytes(Uuid::new_v4());
+
+	let created_path = sync
+		.write_op(
+			db,
+			sync.unique_shared_create(
+				sync::file_path::SyncId {
+					pub_id: pub_id.clone(),
+				},
+				params,
+			),
+			db.file_path().create(
+				pub_id,
+				location::id::equals(location.id),
+				materialized_path.into_owned(),
+				name.into_owned(),
+				extension.into_owned(),
+				metadata.inode.to_le_bytes().into(),
+				metadata.device.to_le_bytes().into(),
+				{
+					use file_path::*;
+					vec![
+						cas_id::set(cas_id),
+						is_dir::set(is_dir),
+						size_in_bytes::set(metadata.size_in_bytes.to_string()),
+						date_created::set(metadata.created_at.into()),
+						date_modified::set(metadata.modified_at.into()),
+					]
+				},
+			),
+		)
+		.await?;
+
+	Ok(created_path)
+}
+
+#[cfg(feature = "location-watcher")]
+pub async fn check_existing_file_path(
+	materialized_path: &IsolatedFilePathData<'_>,
+	db: &PrismaClient,
+) -> Result<bool, FilePathError> {
+	Ok(db
+		.file_path()
+		.count(filter_existing_file_path_params(materialized_path))
+		.exec()
+		.await? > 0)
+}
+
+pub fn filter_existing_file_path_params(
+	IsolatedFilePathData {
+		materialized_path,
+		is_dir,
+		location_id,
+		name,
+		extension,
+		..
+	}: &IsolatedFilePathData,
+) -> Vec<file_path::WhereParam> {
+	vec![
+		file_path::location_id::equals(*location_id),
+		file_path::materialized_path::equals(materialized_path.to_string()),
+		file_path::is_dir::equals(*is_dir),
+		file_path::name::equals(name.to_string()),
+		file_path::extension::equals(extension.to_string()),
+	]
+}
+
+/// With this function we try to do a loose filtering of file paths, to avoid having to do check
+/// twice for directories and for files. This is because directories have a trailing `/` or `\` in
+/// the materialized path
+#[allow(unused)]
+pub fn loose_find_existing_file_path_params(
+	IsolatedFilePathData {
+		materialized_path,
+		location_id,
+		name,
+		extension,
+		..
+	}: &IsolatedFilePathData,
+) -> Vec<file_path::WhereParam> {
+	vec![
+		file_path::location_id::equals(*location_id),
+		file_path::materialized_path::equals(materialized_path.to_string()),
+		file_path::name::equals(name.to_string()),
+		file_path::extension::equals(extension.to_string()),
+	]
+}
+
+#[cfg(feature = "location-watcher")]
+pub async fn get_parent_dir(
+	materialized_path: &IsolatedFilePathData<'_>,
+	db: &PrismaClient,
+) -> Result<Option<file_path::Data>, FilePathError> {
+	db.file_path()
+		.find_first(filter_existing_file_path_params(
+			&materialized_path.parent(),
+		))
+		.exec()
+		.await
+		.map_err(Into::into)
+}
+
+pub async fn ensure_sub_path_is_in_location(
+	location_path: impl AsRef<Path>,
+	sub_path: impl AsRef<Path>,
+) -> Result<PathBuf, FilePathError> {
+	let mut sub_path = sub_path.as_ref();
+	if sub_path.starts_with("/") {
+		// SAFETY: we just checked that it starts with the separator
+		sub_path = sub_path.strip_prefix("/").unwrap();
+	}
+	let location_path = location_path.as_ref();
+
+	if !sub_path.starts_with(location_path) {
+		// If the sub_path doesn't start with the location_path, we have to check if it's a
+		// materialized path received from the frontend, then we check if the full path exists
+		let full_path = location_path.join(sub_path);
+
+		match fs::metadata(&full_path).await {
+			Ok(_) => Ok(full_path),
+			Err(e) if e.kind() == io::ErrorKind::NotFound => Err(FilePathError::InvalidSubPath {
+				sub_path: sub_path.into(),
+				location_path: location_path.into(),
+			}),
+			Err(e) => Err(FileIOError::from((full_path, e)).into()),
+		}
+	} else {
+		Ok(sub_path.to_path_buf())
+	}
+}
+
+pub async fn ensure_file_path_exists<E>(
+	sub_path: impl AsRef<Path>,
+	iso_file_path: &IsolatedFilePathData<'_>,
+	db: &PrismaClient,
+	error_fn: impl FnOnce(Box<Path>) -> E,
+) -> Result<(), E>
+where
+	E: From<QueryError>,
+{
+	if !check_file_path_exists(iso_file_path, db).await? {
+		Err(error_fn(sub_path.as_ref().into()))
+	} else {
+		Ok(())
+	}
+}
+
+pub async fn check_file_path_exists<E>(
+	iso_file_path: &IsolatedFilePathData<'_>,
+	db: &PrismaClient,
+) -> Result<bool, E>
+where
+	E: From<QueryError>,
+{
+	db.file_path()
+		.count(filter_existing_file_path_params(iso_file_path))
+		.exec()
+		.await
+		.map(|count| count > 0)
+		.map_err(Into::into)
+}
+
+pub async fn ensure_sub_path_is_directory(
+	location_path: impl AsRef<Path>,
+	sub_path: impl AsRef<Path>,
+) -> Result<(), FilePathError> {
+	let mut sub_path = sub_path.as_ref();
+
+	match fs::metadata(sub_path).await {
+		Ok(meta) => {
+			if meta.is_file() {
+				Err(FilePathError::SubPathNotDirectory(sub_path.into()))
+			} else {
+				Ok(())
+			}
+		}
+		Err(e) if e.kind() == io::ErrorKind::NotFound => {
+			if sub_path.starts_with("/") {
+				// SAFETY: we just checked that it starts with the separator
+				sub_path = sub_path.strip_prefix("/").unwrap();
+			}
+
+			let location_path = location_path.as_ref();
+			let full_path = location_path.join(sub_path);
+			match fs::metadata(&full_path).await {
+				Ok(meta) => {
+					if meta.is_file() {
+						Err(FilePathError::SubPathNotDirectory(sub_path.into()))
+					} else {
+						Ok(())
+					}
+				}
+				Err(e) if e.kind() == io::ErrorKind::NotFound => {
+					Err(FilePathError::InvalidSubPath {
+						sub_path: sub_path.into(),
+						location_path: location_path.into(),
+					})
+				}
+				Err(e) => Err(FileIOError::from((full_path, e)).into()),
+			}
+		}
+		Err(e) => Err(FileIOError::from((sub_path, e)).into()),
+	}
+}
+
+#[allow(unused)] // TODO remove this annotation when we can use it on windows
+pub fn get_inode_and_device(metadata: &Metadata) -> Result<(u64, u64), FilePathError> {
+	#[cfg(target_family = "unix")]
+	{
+		use std::os::unix::fs::MetadataExt;
+
+		Ok((metadata.ino(), metadata.dev()))
+	}
+
+	#[cfg(target_family = "windows")]
+	{
+		// TODO use this when it's stable and remove winapi-utils dependency
+
+		// use std::os::windows::fs::MetadataExt;
+
+		// Ok((
+		// 	metadata
+		// 		.file_index()
+		// 		.expect("This function must not be called from a `DirEntry`'s `Metadata"),
+		// 	metadata
+		// 		.volume_serial_number()
+		// 		.expect("This function must not be called from a `DirEntry`'s `Metadata") as u64,
+		// ))
+
+		todo!("Use metadata: {:#?}", metadata)
+	}
+}
+
+#[allow(unused)]
+pub async fn get_inode_and_device_from_path(
+	path: impl AsRef<Path>,
+) -> Result<(u64, u64), FilePathError> {
+	#[cfg(target_family = "unix")]
+	{
+		// TODO use this when it's stable and remove winapi-utils dependency
+		let metadata = fs::metadata(path.as_ref())
+			.await
+			.map_err(|e| FileIOError::from((path, e)))?;
+
+		get_inode_and_device(&metadata)
+	}
+
+	#[cfg(target_family = "windows")]
+	{
+		use winapi_util::{file::information, Handle};
+
+		let info = Handle::from_path_any(path.as_ref())
+			.and_then(|ref handle| information(handle))
+			.map_err(|e| FileIOError::from((path, e)))?;
+
+		Ok((info.file_index(), info.volume_serial_number()))
+	}
+}
+
+pub trait MetadataExt {
+	fn created_or_now(&self) -> SystemTime;
+
+	fn modified_or_now(&self) -> SystemTime;
+}
+
+impl MetadataExt for Metadata {
+	fn created_or_now(&self) -> SystemTime {
+		self.created().unwrap_or_else(|_| SystemTime::now())
+	}
+
+	fn modified_or_now(&self) -> SystemTime {
+		self.modified().unwrap_or_else(|_| SystemTime::now())
+	}
+}

core/src/location/indexer/indexer_job.rs

@@ -1,31 +1,26 @@
 use crate::{
+	file_paths_db_fetcher_fn,
 	job::{JobError, JobInitData, JobResult, JobState, StatefulJob, WorkerContext},
 	location::file_path_helper::{
-		ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
-		file_path_just_id_materialized_path, filter_existing_file_path_params,
-		filter_file_paths_by_many_full_path_params, retain_file_paths_in_location,
-		MaterializedPath,
+		ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
+		IsolatedFilePathData,
 	},
-	prisma::location,
+	to_remove_db_fetcher_fn,
 };
 
-use std::{
-	collections::{HashMap, VecDeque},
-	path::Path,
-};
+use std::{path::Path, sync::Arc};
 
-use chrono::Utc;
 use itertools::Itertools;
+use serde::{Deserialize, Serialize};
 use tokio::time::Instant;
-use tracing::error;
-use uuid::Uuid;
 
 use super::{
-	execute_indexer_step, finalize_indexer,
-	rules::{IndexerRule, RuleKind},
-	walk::walk,
-	IndexerError, IndexerJobData, IndexerJobInit, IndexerJobStep, IndexerJobStepEntry,
-	ScanProgress,
+	execute_indexer_save_step, finalize_indexer, iso_file_path_factory,
+	remove_non_existing_file_paths,
+	rules::aggregate_rules_by_kind,
+	update_notifier_fn,
+	walk::{keep_walking, walk, ToWalkEntry, WalkResult},
+	IndexerError, IndexerJobData, IndexerJobInit, IndexerJobSaveStep, ScanProgress,
 };
 
 /// BATCH_SIZE is the number of files to index at each step, writing the chunk of files metadata in the database.
@@ -40,11 +35,19 @@ impl JobInitData for IndexerJobInit {
 	type Job = IndexerJob;
 }
 
+/// `IndexerJobStepInput` defines the action that should be executed in the current step
+#[derive(Serialize, Deserialize, Debug)]
+pub enum IndexerJobStepInput {
+	/// `IndexerJobStepEntry`. The size of this vector is given by the [`BATCH_SIZE`] constant.
+	Save(IndexerJobSaveStep),
+	Walk(ToWalkEntry),
+}
+
 #[async_trait::async_trait]
 impl StatefulJob for IndexerJob {
 	type Init = IndexerJobInit;
 	type Data = IndexerJobData;
-	type Step = IndexerJobStep;
+	type Step = IndexerJobStepInput;
 
 	const NAME: &'static str = "indexer";
 
@@ -61,21 +64,12 @@ impl StatefulJob for IndexerJob {
 		let location_id = state.init.location.id;
 		let location_path = Path::new(&state.init.location.path);
 
-		let mut indexer_rules_by_kind: HashMap<RuleKind, Vec<IndexerRule>> =
-			HashMap::with_capacity(state.init.location.indexer_rules.len());
-		for location_rule in &state.init.location.indexer_rules {
-			let indexer_rule = IndexerRule::try_from(&location_rule.indexer_rule)?;
+		let db = Arc::clone(&ctx.library.db);
 
-			indexer_rules_by_kind
-				.entry(indexer_rule.kind)
-				.or_default()
-				.push(indexer_rule);
-		}
+		let rules_by_kind = aggregate_rules_by_kind(state.init.location.indexer_rules.iter())
+			.map_err(IndexerError::from)?;
 
-		let mut dirs_ids = HashMap::new();
-
-		let (to_walk_path, maybe_parent_file_path) = if let Some(ref sub_path) = state.init.sub_path
-		{
+		let to_walk_path = if let Some(ref sub_path) = state.init.sub_path {
 			let full_path = ensure_sub_path_is_in_location(location_path, sub_path)
 				.await
 				.map_err(IndexerError::from)?;
@@ -83,196 +77,183 @@ impl StatefulJob for IndexerJob {
 				.await
 				.map_err(IndexerError::from)?;
 
-			let sub_path_file_path = ctx
-				.library
-				.db
-				.file_path()
-				.find_first(filter_existing_file_path_params(
-					&MaterializedPath::new(location_id, location_path, &full_path, true)
-						.map_err(IndexerError::from)?,
-				))
-				.select(file_path_just_id_materialized_path::select())
-				.exec()
-				.await
-				.map_err(IndexerError::from)?
-				.expect("Sub path should already exist in the database");
+			ensure_file_path_exists(
+				sub_path,
+				&IsolatedFilePathData::new(location_id, location_path, &full_path, true)
+					.map_err(IndexerError::from)?,
+				&db,
+				IndexerError::SubPathNotFound,
+			)
+			.await?;
 
-			// If we're operating with a sub_path, then we have to put its id on `dirs_ids` map
-			dirs_ids.insert(
-				full_path.clone(),
-				Uuid::from_slice(&sub_path_file_path.pub_id).unwrap(),
-			);
-
-			(full_path, Some(sub_path_file_path))
+			full_path
 		} else {
-			(location_path.to_path_buf(), None)
+			location_path.to_path_buf()
 		};
 
 		let scan_start = Instant::now();
-		let found_paths = {
-			let ctx = &mut ctx; // Borrow outside of closure so it's not moved
+		let WalkResult {
+			walked,
+			to_walk,
+			to_remove,
+			errors,
+		} = {
 			walk(
 				&to_walk_path,
-				&indexer_rules_by_kind,
-				|path, total_entries| {
-					IndexerJobData::on_scan_progress(
-						ctx,
-						vec![
-							ScanProgress::Message(format!("Scanning {}", path.display())),
-							ScanProgress::ChunkCount(total_entries / BATCH_SIZE),
-						],
-					);
-				},
-				// if we're not using a sub_path, then its a full indexing and we must include root dir
-				state.init.sub_path.is_none(),
+				&rules_by_kind,
+				update_notifier_fn(BATCH_SIZE, &mut ctx),
+				file_paths_db_fetcher_fn!(&db),
+				to_remove_db_fetcher_fn!(location_id, location_path, &db),
+				iso_file_path_factory(location_id, location_path),
+				50_000,
 			)
 			.await?
 		};
+		let scan_read_time = scan_start.elapsed();
 
-		// NOTE:
-		// As we're passing the list of currently existing file paths to the `find_many_file_paths_by_full_path` query,
-		// it means that `dirs_ids` contains just paths that still exists on the filesystem.
-		dirs_ids.extend(
-			ctx.library
-				.db
-				.file_path()
-				.find_many(
-					filter_file_paths_by_many_full_path_params(
-						&location::Data::from(&state.init.location),
-						&found_paths
-							.iter()
-							.map(|entry| &entry.path)
-							.collect::<Vec<_>>(),
-					)
-					.await
-					.map_err(IndexerError::from)?,
-				)
-				.select(file_path_just_id_materialized_path::select())
-				.exec()
-				.await?
+		let db_delete_start = Instant::now();
+		// TODO pass these uuids to sync system
+		let removed_count = remove_non_existing_file_paths(to_remove, &db).await?;
+		let db_delete_time = db_delete_start.elapsed();
+
+		let total_paths = &mut 0;
+		let to_walk_count = to_walk.len();
+
+		state.steps.extend(
+			walked
+				.chunks(BATCH_SIZE)
 				.into_iter()
-				.map(|file_path| {
-					(
-						location_path.join(&MaterializedPath::from((
-							location_id,
-							&file_path.materialized_path,
-						))),
-						Uuid::from_slice(&file_path.pub_id).unwrap(),
-					)
-				}),
+				.enumerate()
+				.map(|(i, chunk)| {
+					let chunk_steps = chunk.collect::<Vec<_>>();
+
+					*total_paths += chunk_steps.len() as u64;
+
+					IndexerJobStepInput::Save(IndexerJobSaveStep {
+						chunk_idx: i,
+						walked: chunk_steps,
+					})
+				})
+				.chain(to_walk.into_iter().map(IndexerJobStepInput::Walk)),
 		);
 
-		// Removing all other file paths that are not in the filesystem anymore
-		let removed_paths = retain_file_paths_in_location(
-			location_id,
-			dirs_ids.values().copied().collect(),
-			maybe_parent_file_path,
-			&ctx.library.db,
-		)
-		.await
-		.map_err(IndexerError::from)?;
-
-		let mut new_paths = found_paths
-			.into_iter()
-			.filter_map(|entry| {
-				MaterializedPath::new(
-					location_id,
-					&state.init.location.path,
-					&entry.path,
-					entry.is_dir,
-				)
-				.map_or_else(
-					|e| {
-						error!("Failed to create materialized path: {e}");
-						None
-					},
-					|materialized_path| {
-						(!dirs_ids.contains_key(&entry.path)).then(|| {
-							IndexerJobStepEntry {
-								materialized_path,
-								file_id: Uuid::new_v4(), // To be set later
-								parent_id: entry.path.parent().and_then(|parent_dir| {
-									/***************************************************************
-									 * If we're dealing with a new path which its parent already   *
-									 * exist, we fetch its parent id from our `dirs_ids` map       *
-									 **************************************************************/
-									dirs_ids.get(parent_dir).copied()
-								}),
-								full_path: entry.path,
-								metadata: entry.metadata,
-							}
-						})
-					},
-				)
-			})
-			.collect::<Vec<_>>();
-
-		new_paths.iter_mut().for_each(|entry| {
-			// If the `parent_id` is still none here, is because the parent of this entry is also
-			// a new one in the DB
-			if entry.parent_id.is_none() {
-				entry.parent_id = entry
-					.full_path
-					.parent()
-					.and_then(|parent_dir| dirs_ids.get(parent_dir).copied());
-			}
-
-			dirs_ids.insert(entry.full_path.clone(), entry.file_id);
-		});
-
-		let total_paths = new_paths.len();
+		IndexerJobData::on_scan_progress(
+			&mut ctx,
+			vec![ScanProgress::Message(format!(
+				"Starting saving {total_paths} files or directories, \
+					there still {to_walk_count} directories to index",
+			))],
+		);
 
 		state.data = Some(IndexerJobData {
 			indexed_path: to_walk_path,
-			db_write_start: Utc::now(),
-			scan_read_time: scan_start.elapsed(),
-			total_paths,
-			indexed_paths: 0,
-			removed_paths,
+			rules_by_kind,
+			db_write_time: db_delete_time,
+			scan_read_time,
+			total_paths: *total_paths,
+			indexed_count: 0,
+			removed_count,
+			total_save_steps: state.steps.len() as u64 - to_walk_count as u64,
 		});
 
-		state.steps = VecDeque::with_capacity(new_paths.len() / BATCH_SIZE);
-
-		for (i, chunk) in new_paths
-			.into_iter()
-			.chunks(BATCH_SIZE)
-			.into_iter()
-			.enumerate()
-		{
-			let chunk_steps = chunk.collect::<Vec<_>>();
-			IndexerJobData::on_scan_progress(
-				&mut ctx,
-				vec![
-					ScanProgress::SavedChunks(i),
-					ScanProgress::Message(format!(
-						"Writing {} of {} to db",
-						i * chunk_steps.len(),
-						total_paths,
-					)),
-				],
-			);
-
-			state.steps.push_back(chunk_steps);
+		if !errors.is_empty() {
+			Err(JobError::StepCompletedWithErrors(
+				errors.into_iter().map(|e| format!("{e}")).collect(),
+			))
+		} else {
+			Ok(())
 		}
-
-		Ok(())
 	}
 
 	/// Process each chunk of entries in the indexer job, writing to the `file_path` table
 	async fn execute_step(
 		&self,
-		ctx: WorkerContext,
+		mut ctx: WorkerContext,
 		state: &mut JobState<Self>,
 	) -> Result<(), JobError> {
-		execute_indexer_step(&state.init.location, &state.steps[0], ctx)
-			.await
-			.map(|indexed_paths| {
-				state
-					.data
-					.as_mut()
-					.expect("critical error: missing data on job state")
-					.indexed_paths = indexed_paths;
-			})
+		let data = state
+			.data
+			.as_mut()
+			.expect("critical error: missing data on job state");
+
+		match &state.steps[0] {
+			IndexerJobStepInput::Save(step) => {
+				execute_indexer_save_step(&state.init.location, step, data, &mut ctx)
+					.await
+					.map(|(indexed_count, elapsed_time)| {
+						data.indexed_count += indexed_count;
+						data.db_write_time += elapsed_time;
+					})?
+			}
+			IndexerJobStepInput::Walk(to_walk_entry) => {
+				let location_id = state.init.location.id;
+				let location_path = Path::new(&state.init.location.path);
+				let db = Arc::clone(&ctx.library.db);
+
+				let scan_start = Instant::now();
+
+				let WalkResult {
+					walked,
+					to_walk,
+					to_remove,
+					errors,
+				} = {
+					keep_walking(
+						to_walk_entry,
+						&data.rules_by_kind,
+						update_notifier_fn(BATCH_SIZE, &mut ctx),
+						file_paths_db_fetcher_fn!(&db),
+						to_remove_db_fetcher_fn!(location_id, location_path, &db),
+						iso_file_path_factory(location_id, location_path),
+					)
+					.await?
+				};
+
+				data.scan_read_time += scan_start.elapsed();
+
+				let db_delete_time = Instant::now();
+				// TODO pass these uuids to sync system
+				data.removed_count += remove_non_existing_file_paths(to_remove, &db).await?;
+				data.db_write_time += db_delete_time.elapsed();
+
+				let old_total = data.total_paths;
+				let old_steps_count = state.steps.len() as u64;
+
+				state.steps.extend(
+					walked
+						.chunks(BATCH_SIZE)
+						.into_iter()
+						.enumerate()
+						.map(|(i, chunk)| {
+							let chunk_steps = chunk.collect::<Vec<_>>();
+							data.total_paths += chunk_steps.len() as u64;
+
+							IndexerJobStepInput::Save(IndexerJobSaveStep {
+								chunk_idx: i,
+								walked: chunk_steps,
+							})
+						})
+						.chain(to_walk.into_iter().map(IndexerJobStepInput::Walk)),
+				);
+
+				IndexerJobData::on_scan_progress(
+					&mut ctx,
+					vec![ScanProgress::Message(format!(
+						"Scanned more {} files or directories; {} more directories to scan",
+						data.total_paths - old_total,
+						state.steps.len() as u64 - old_steps_count - data.total_paths
+					))],
+				);
+
+				if !errors.is_empty() {
+					return Err(JobError::StepCompletedWithErrors(
+						errors.into_iter().map(|e| format!("{e}")).collect(),
+					));
+				}
+			}
+		}
+
+		Ok(())
 	}
 
 	async fn finalize(&mut self, ctx: WorkerContext, state: &mut JobState<Self>) -> JobResult {

core/src/location/indexer/mod.rs

@@ -1,31 +1,29 @@
 use crate::{
 	invalidate_query,
-	job::{JobError, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext},
+	job::{JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext},
 	library::Library,
-	prisma::file_path,
+	prisma::{file_path, PrismaClient},
 	sync,
-	util::db::uuid_to_bytes,
+	util::{db::uuid_to_bytes, error::FileIOError},
 };
 
 use std::{
+	collections::HashMap,
 	hash::{Hash, Hasher},
 	path::{Path, PathBuf},
 	time::Duration,
 };
 
-use chrono::{DateTime, Utc};
-use rmp_serde::{decode, encode};
 use rspc::ErrorCode;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use serde_json::json;
 use thiserror::Error;
-use tokio::io;
+use tokio::time::Instant;
 use tracing::info;
-use uuid::Uuid;
 
 use super::{
-	file_path_helper::{FilePathError, FilePathMetadata, MaterializedPath},
-	location_with_indexer_rules,
+	file_path_helper::{file_path_just_pub_id, FilePathError, IsolatedFilePathData},
+	location_with_indexer_rules, LocationId,
 };
 
 pub mod indexer_job;
@@ -33,6 +31,9 @@ pub mod rules;
 pub mod shallow_indexer_job;
 mod walk;
 
+use rules::IndexerRuleError;
+use walk::WalkedEntry;
+
 /// `IndexerJobInit` receives a `location::Data` object to be indexed
 /// and possibly a `sub_path` to be indexed. The `sub_path` is used when
 /// we want do index just a part of a location.
@@ -56,26 +57,13 @@ impl Hash for IndexerJobInit {
 #[derive(Serialize, Deserialize)]
 pub struct IndexerJobData {
 	indexed_path: PathBuf,
-	db_write_start: DateTime<Utc>,
+	rules_by_kind: HashMap<rules::RuleKind, Vec<rules::IndexerRule>>,
+	db_write_time: Duration,
 	scan_read_time: Duration,
-	total_paths: usize,
-	indexed_paths: i64,
-	removed_paths: i64,
-}
-
-/// `IndexerJobStep` is a type alias, specifying that each step of the [`IndexerJob`] is a vector of
-/// `IndexerJobStepEntry`. The size of this vector is given by the [`BATCH_SIZE`] constant.
-pub type IndexerJobStep = Vec<IndexerJobStepEntry>;
-
-/// `IndexerJobStepEntry` represents a single file to be indexed, given its metadata to be written
-/// on the `file_path` table in the database
-#[derive(Serialize, Deserialize)]
-pub struct IndexerJobStepEntry {
-	full_path: PathBuf,
-	materialized_path: MaterializedPath<'static>,
-	file_id: Uuid,
-	parent_id: Option<Uuid>,
-	metadata: FilePathMetadata,
+	total_paths: u64,
+	total_save_steps: u64,
+	indexed_count: u64,
+	removed_count: u64,
 }
 
 impl IndexerJobData {
@@ -93,6 +81,12 @@ impl IndexerJobData {
 	}
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct IndexerJobSaveStep {
+	chunk_idx: usize,
+	walked: Vec<WalkedEntry>,
+}
+
 #[derive(Clone)]
 pub enum ScanProgress {
 	ChunkCount(usize),
@@ -104,99 +98,103 @@ pub enum ScanProgress {
 #[derive(Error, Debug)]
 pub enum IndexerError {
 	// Not Found errors
-	#[error("Indexer rule not found: <id={0}>")]
+	#[error("indexer rule not found: <id='{0}'>")]
 	IndexerRuleNotFound(i32),
-
-	// User errors
-	#[error("Invalid indexer rule kind integer: {0}")]
-	InvalidRuleKindInt(i32),
-	#[error("Glob builder error: {0}")]
-	GlobBuilderError(#[from] globset::Error),
+	#[error("received sub path not in database: <path='{}'>", .0.display())]
+	SubPathNotFound(Box<Path>),
 
 	// Internal Errors
-	#[error("Database error: {0}")]
-	DatabaseError(#[from] prisma_client_rust::QueryError),
-	#[error("I/O error: {0}")]
-	IOError(#[from] io::Error),
-	#[error("Indexer rule parameters json serialization error: {0}")]
-	RuleParametersSerdeJson(#[from] serde_json::Error),
-	#[error("Indexer rule parameters encode error: {0}")]
-	RuleParametersRMPEncode(#[from] encode::Error),
-	#[error("Indexer rule parameters decode error: {0}")]
-	RuleParametersRMPDecode(#[from] decode::Error),
-	#[error("File path related error (error: {0})")]
-	FilePathError(#[from] FilePathError),
+	#[error("database error")]
+	Database(#[from] prisma_client_rust::QueryError),
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
+	#[error(transparent)]
+	FilePath(#[from] FilePathError),
+
+	// Mixed errors
+	#[error(transparent)]
+	IndexerRules(#[from] IndexerRuleError),
 }
 
 impl From<IndexerError> for rspc::Error {
 	fn from(err: IndexerError) -> Self {
 		match err {
-			IndexerError::IndexerRuleNotFound(_) => {
+			IndexerError::IndexerRuleNotFound(_) | IndexerError::SubPathNotFound(_) => {
 				rspc::Error::with_cause(ErrorCode::NotFound, err.to_string(), err)
 			}
 
-			IndexerError::InvalidRuleKindInt(_) | IndexerError::GlobBuilderError(_) => {
-				rspc::Error::with_cause(ErrorCode::BadRequest, err.to_string(), err)
-			}
+			IndexerError::IndexerRules(rule_err) => rule_err.into(),
 
 			_ => rspc::Error::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
 		}
 	}
 }
 
-async fn execute_indexer_step(
+async fn execute_indexer_save_step(
 	location: &location_with_indexer_rules::Data,
-	step: &[IndexerJobStepEntry],
-	ctx: WorkerContext,
-) -> Result<i64, JobError> {
+	save_step: &IndexerJobSaveStep,
+	data: &IndexerJobData,
+	ctx: &mut WorkerContext,
+) -> Result<(u64, Duration), IndexerError> {
+	let start_time = Instant::now();
+
+	IndexerJobData::on_scan_progress(
+		ctx,
+		vec![
+			ScanProgress::SavedChunks(save_step.chunk_idx),
+			ScanProgress::Message(format!(
+				"Writing {}/{} to db",
+				save_step.chunk_idx, data.total_save_steps
+			)),
+		],
+	);
 	let Library { sync, db, .. } = &ctx.library;
 
-	let (sync_stuff, paths): (Vec<_>, Vec<_>) = step
+	let (sync_stuff, paths): (Vec<_>, Vec<_>) = save_step
+		.walked
 		.iter()
 		.map(|entry| {
-			let MaterializedPath {
+			let IsolatedFilePathData {
 				materialized_path,
 				is_dir,
 				name,
 				extension,
 				..
-			} = entry.materialized_path.clone();
+			} = &entry.iso_file_path;
 
 			use file_path::*;
 
 			(
 				sync.unique_shared_create(
 					sync::file_path::SyncId {
-						pub_id: uuid_to_bytes(entry.file_id),
+						pub_id: uuid_to_bytes(entry.pub_id),
 					},
 					[
-						(materialized_path::NAME, json!(materialized_path.clone())),
-						(name::NAME, json!(name.clone())),
-						(is_dir::NAME, json!(is_dir)),
-						(extension::NAME, json!(extension.clone())),
+						(materialized_path::NAME, json!(materialized_path)),
+						(name::NAME, json!(name)),
+						(is_dir::NAME, json!(*is_dir)),
+						(extension::NAME, json!(extension)),
 						(
 							size_in_bytes::NAME,
 							json!(entry.metadata.size_in_bytes.to_string()),
 						),
 						(inode::NAME, json!(entry.metadata.inode.to_le_bytes())),
 						(device::NAME, json!(entry.metadata.device.to_le_bytes())),
-						(parent_id::NAME, json!(entry.parent_id)),
 						(date_created::NAME, json!(entry.metadata.created_at)),
 						(date_modified::NAME, json!(entry.metadata.modified_at)),
 					],
 				),
 				file_path::create_unchecked(
-					uuid_to_bytes(entry.file_id),
+					uuid_to_bytes(entry.pub_id),
 					location.id,
-					materialized_path.into_owned(),
-					name.into_owned(),
-					extension.into_owned(),
+					materialized_path.to_string(),
+					name.to_string(),
+					extension.to_string(),
 					entry.metadata.inode.to_le_bytes().into(),
 					entry.metadata.device.to_le_bytes().into(),
 					vec![
-						is_dir::set(is_dir),
+						is_dir::set(*is_dir),
 						size_in_bytes::set(entry.metadata.size_in_bytes.to_string()),
-						parent_id::set(entry.parent_id.map(uuid_to_bytes)),
 						date_created::set(entry.metadata.created_at.into()),
 						date_modified::set(entry.metadata.modified_at.into()),
 					],
@@ -217,17 +215,18 @@ async fn execute_indexer_step(
 
 	info!("Inserted {count} records");
 
-	Ok(count)
+	Ok((count as u64, start_time.elapsed()))
 }
 
-fn finalize_indexer<SJob, Init>(
+fn finalize_indexer<SJob, Init, Step>(
 	location_path: impl AsRef<Path>,
 	state: &JobState<SJob>,
 	ctx: WorkerContext,
 ) -> JobResult
 where
-	SJob: StatefulJob<Init = Init, Data = IndexerJobData, Step = IndexerJobStep>,
+	SJob: StatefulJob<Init = Init, Data = IndexerJobData, Step = Step>,
 	Init: Serialize + DeserializeOwned + Send + Sync + Hash,
+	Step: Serialize + DeserializeOwned + Send + Sync,
 {
 	let data = state
 		.data
@@ -240,15 +239,95 @@ where
 		location_path.as_ref().display(),
 		data.scan_read_time,
 		data.total_paths,
-		data.indexed_paths,
-		(Utc::now() - data.db_write_start)
-			.to_std()
-			.expect("critical error: non-negative duration"),
+		data.indexed_count,
+		data.db_write_time,
 	);
 
-	if data.indexed_paths > 0 || data.removed_paths > 0 {
+	if data.indexed_count > 0 || data.removed_count > 0 {
 		invalidate_query!(ctx.library, "search.paths");
 	}
 
 	Ok(Some(serde_json::to_value(state)?))
 }
+
+fn update_notifier_fn(batch_size: usize, ctx: &mut WorkerContext) -> impl FnMut(&Path, usize) + '_ {
+	move |path, total_entries| {
+		IndexerJobData::on_scan_progress(
+			ctx,
+			vec![
+				ScanProgress::Message(format!("Scanning {}", path.display())),
+				ScanProgress::ChunkCount(total_entries / batch_size),
+			],
+		);
+	}
+}
+
+fn iso_file_path_factory(
+	location_id: LocationId,
+	location_path: &Path,
+) -> impl Fn(&Path, bool) -> Result<IsolatedFilePathData<'static>, IndexerError> + '_ {
+	move |path, is_dir| {
+		IsolatedFilePathData::new(location_id, location_path, path, is_dir).map_err(Into::into)
+	}
+}
+
+async fn remove_non_existing_file_paths(
+	to_remove: impl IntoIterator<Item = file_path_just_pub_id::Data>,
+	db: &PrismaClient,
+) -> Result<u64, IndexerError> {
+	db.file_path()
+		.delete_many(vec![file_path::pub_id::in_vec(
+			to_remove.into_iter().map(|data| data.pub_id).collect(),
+		)])
+		.exec()
+		.await
+		.map(|count| count as u64)
+		.map_err(Into::into)
+}
+
+// TODO: Change this macro to a fn when we're able to return
+// `impl Fn(Vec<file_path::WhereParam>) -> impl Future<Output = Result<Vec<file_path_to_isolate::Data>, IndexerError>>`
+// Maybe when TAITs arrive
+#[macro_export]
+macro_rules! file_paths_db_fetcher_fn {
+	($db:expr) => {{
+		|found_paths| async {
+			$db.file_path()
+				.find_many(found_paths)
+				.select($crate::location::file_path_helper::file_path_to_isolate::select())
+				.exec()
+				.await
+				.map_err(Into::into)
+		}
+	}};
+}
+
+// TODO: Change this macro to a fn when we're able to return
+// `impl Fn(&Path, Vec<file_path::WhereParam>) -> impl Future<Output = Result<Vec<file_path_just_pub_id::Data>, IndexerError>>`
+// Maybe when TAITs arrive
+// FIXME: (fogodev) I was receiving this error here https://github.com/rust-lang/rust/issues/74497
+#[macro_export]
+macro_rules! to_remove_db_fetcher_fn {
+	($location_id:expr, $location_path:expr, $db:expr) => {{
+		|iso_file_path, unique_location_id_materialized_path_name_extension_params| async {
+			let iso_file_path: $crate::location::file_path_helper::IsolatedFilePathData<'static> =
+				iso_file_path;
+			$db.file_path()
+				.find_many(vec![
+					$crate::prisma::file_path::location_id::equals($location_id),
+					$crate::prisma::file_path::materialized_path::equals(
+						iso_file_path
+							.materialized_path_for_children()
+							.expect("the received isolated file path must be from a directory"),
+					),
+					::prisma_client_rust::operator::not(
+						unique_location_id_materialized_path_name_extension_params,
+					),
+				])
+				.select($crate::location::file_path_helper::file_path_just_pub_id::select())
+				.exec()
+				.await
+				.map_err(Into::into)
+		}
+	}};
+}

core/src/location/indexer/rules.rs

@@ -1,18 +1,62 @@
 use crate::{
 	library::Library,
-	location::indexer::IndexerError,
+	location::location_with_indexer_rules,
 	prisma::{indexer_rule, PrismaClient},
+	util::error::{FileIOError, NonUtf8PathError},
 };
 
 use chrono::{DateTime, Utc};
 use globset::{Glob, GlobSet, GlobSetBuilder};
-use rmp_serde;
-use serde::{Deserialize, Serialize};
+use rmp_serde::{self, decode, encode};
+use rspc::ErrorCode;
+use serde::{de, ser, Deserialize, Serialize};
 use specta::Type;
-use std::{collections::HashSet, path::Path};
+use std::{
+	collections::{HashMap, HashSet},
+	marker::PhantomData,
+	path::Path,
+};
+use thiserror::Error;
 use tokio::fs;
 use tracing::debug;
 
+#[derive(Error, Debug)]
+pub enum IndexerRuleError {
+	// User errors
+	#[error("invalid indexer rule kind integer: {0}")]
+	InvalidRuleKindInt(i32),
+	#[error("glob builder error")]
+	Glob(#[from] globset::Error),
+	#[error(transparent)]
+	NonUtf8Path(#[from] NonUtf8PathError),
+
+	// Internal Errors
+	#[error("indexer rule parameters encode error")]
+	RuleParametersRMPEncode(#[from] encode::Error),
+	#[error("indexer rule parameters decode error")]
+	RuleParametersRMPDecode(#[from] decode::Error),
+	#[error("accept by its children file I/O error")]
+	AcceptByItsChildrenFileIO(FileIOError),
+	#[error("reject by its children file I/O error")]
+	RejectByItsChildrenFileIO(FileIOError),
+	#[error("database error")]
+	Database(#[from] prisma_client_rust::QueryError),
+}
+
+impl From<IndexerRuleError> for rspc::Error {
+	fn from(err: IndexerRuleError) -> Self {
+		match err {
+			IndexerRuleError::InvalidRuleKindInt(_)
+			| IndexerRuleError::Glob(_)
+			| IndexerRuleError::NonUtf8Path(_) => {
+				rspc::Error::with_cause(ErrorCode::BadRequest, err.to_string(), err)
+			}
+
+			_ => rspc::Error::with_cause(ErrorCode::InternalServerError, err.to_string(), err),
+		}
+	}
+}
+
 /// `IndexerRuleCreateArgs` is the argument received from the client using rspc to create a new indexer rule.
 /// Note that `parameters` field **MUST** be a JSON object serialized to bytes.
 ///
@@ -33,7 +77,7 @@ impl IndexerRuleCreateArgs {
 	pub async fn create(
 		self,
 		library: &Library,
-	) -> Result<Option<indexer_rule::Data>, IndexerError> {
+	) -> Result<Option<indexer_rule::Data>, IndexerRuleError> {
 		debug!(
 			"{} a new indexer rule (name = {}, params = {:?})",
 			if self.dry_run {
@@ -83,8 +127,15 @@ pub enum RuleKind {
 	RejectIfChildrenDirectoriesArePresent = 3,
 }
 
+impl RuleKind {
+	pub const fn variant_count() -> usize {
+		// TODO: Use https://doc.rust-lang.org/std/mem/fn.variant_count.html if it ever gets stabilized
+		4
+	}
+}
+
 impl TryFrom<i32> for RuleKind {
-	type Error = IndexerError;
+	type Error = IndexerRuleError;
 
 	fn try_from(value: i32) -> Result<Self, Self::Error> {
 		let s = match value {
@@ -92,7 +143,7 @@ impl TryFrom<i32> for RuleKind {
 			1 => Self::RejectFilesByGlob,
 			2 => Self::AcceptIfChildrenDirectoriesArePresent,
 			3 => Self::RejectIfChildrenDirectoriesArePresent,
-			_ => return Err(IndexerError::InvalidRuleKindInt(value)),
+			_ => return Err(Self::Error::InvalidRuleKindInt(value)),
 		};
 
 		Ok(s)
@@ -111,14 +162,257 @@ pub enum ParametersPerKind {
 	// TODO: Add an indexer rule that filter files based on their extended attributes
 	// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants
 	// https://en.wikipedia.org/wiki/Extended_file_attributes
-	AcceptFilesByGlob(Vec<Glob>),
-	RejectFilesByGlob(Vec<Glob>),
+	AcceptFilesByGlob(Vec<Glob>, GlobSet),
+	RejectFilesByGlob(Vec<Glob>, GlobSet),
 	AcceptIfChildrenDirectoriesArePresent(HashSet<String>),
 	RejectIfChildrenDirectoriesArePresent(HashSet<String>),
 }
 
 impl ParametersPerKind {
-	async fn apply(&self, source: impl AsRef<Path>) -> Result<bool, IndexerError> {
+	fn new_files_by_globs_str_and_kind(
+		globs_str: impl IntoIterator<Item = impl AsRef<str>>,
+		kind_fn: impl Fn(Vec<Glob>, GlobSet) -> Self,
+	) -> Result<Self, IndexerRuleError> {
+		globs_str
+			.into_iter()
+			.map(|s| s.as_ref().parse::<Glob>())
+			.collect::<Result<Vec<_>, _>>()
+			.and_then(|globs| {
+				globs
+					.iter()
+					.cloned()
+					.fold(&mut GlobSetBuilder::new(), |builder, glob| {
+						builder.add(glob)
+					})
+					.build()
+					.map(move |glob_set| kind_fn(globs, glob_set))
+					.map_err(Into::into)
+			})
+			.map_err(Into::into)
+	}
+
+	pub fn new_accept_files_by_globs_str(
+		globs_str: impl IntoIterator<Item = impl AsRef<str>>,
+	) -> Result<Self, IndexerRuleError> {
+		Self::new_files_by_globs_str_and_kind(globs_str, Self::AcceptFilesByGlob)
+	}
+
+	pub fn new_reject_files_by_glob(
+		globs_str: impl IntoIterator<Item = impl AsRef<str>>,
+	) -> Result<Self, IndexerRuleError> {
+		Self::new_files_by_globs_str_and_kind(globs_str, Self::RejectFilesByGlob)
+	}
+}
+
+/// We're implementing `Serialize` by hand as `GlobSet`s aren't serializable, so we ignore them on
+/// serialization
+impl Serialize for ParametersPerKind {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+	where
+		S: ser::Serializer,
+	{
+		match *self {
+			ParametersPerKind::AcceptFilesByGlob(ref globs, ref _glob_set) => serializer
+				.serialize_newtype_variant("ParametersPerKind", 0, "AcceptFilesByGlob", globs),
+			ParametersPerKind::RejectFilesByGlob(ref globs, ref _glob_set) => serializer
+				.serialize_newtype_variant("ParametersPerKind", 1, "RejectFilesByGlob", globs),
+			ParametersPerKind::AcceptIfChildrenDirectoriesArePresent(ref children) => serializer
+				.serialize_newtype_variant(
+					"ParametersPerKind",
+					2,
+					"AcceptIfChildrenDirectoriesArePresent",
+					children,
+				),
+			ParametersPerKind::RejectIfChildrenDirectoriesArePresent(ref children) => serializer
+				.serialize_newtype_variant(
+					"ParametersPerKind",
+					3,
+					"RejectIfChildrenDirectoriesArePresent",
+					children,
+				),
+		}
+	}
+}
+
+impl<'de> Deserialize<'de> for ParametersPerKind {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+	where
+		D: de::Deserializer<'de>,
+	{
+		const VARIANTS: &[&str] = &[
+			"AcceptFilesByGlob",
+			"RejectFilesByGlob",
+			"AcceptIfChildrenDirectoriesArePresent",
+			"RejectIfChildrenDirectoriesArePresent",
+		];
+
+		enum Fields {
+			AcceptFilesByGlob,
+			RejectFilesByGlob,
+			AcceptIfChildrenDirectoriesArePresent,
+			RejectIfChildrenDirectoriesArePresent,
+		}
+
+		struct FieldsVisitor;
+
+		impl<'de> de::Visitor<'de> for FieldsVisitor {
+			type Value = Fields;
+
+			fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+				formatter.write_str(
+					"`AcceptFilesByGlob` \
+				or `RejectFilesByGlob` \
+				or `AcceptIfChildrenDirectoriesArePresent` \
+				or `RejectIfChildrenDirectoriesArePresent`",
+				)
+			}
+
+			fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
+			where
+				E: de::Error,
+			{
+				match value {
+					0 => Ok(Fields::AcceptFilesByGlob),
+					1 => Ok(Fields::RejectFilesByGlob),
+					2 => Ok(Fields::AcceptIfChildrenDirectoriesArePresent),
+					3 => Ok(Fields::RejectIfChildrenDirectoriesArePresent),
+					_ => Err(de::Error::invalid_value(
+						de::Unexpected::Unsigned(value),
+						&"variant index 0 <= i < 3",
+					)),
+				}
+			}
+			fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+			where
+				E: de::Error,
+			{
+				match value {
+					"AcceptFilesByGlob" => Ok(Fields::AcceptFilesByGlob),
+					"RejectFilesByGlob" => Ok(Fields::RejectFilesByGlob),
+					"AcceptIfChildrenDirectoriesArePresent" => {
+						Ok(Fields::AcceptIfChildrenDirectoriesArePresent)
+					}
+					"RejectIfChildrenDirectoriesArePresent" => {
+						Ok(Fields::RejectIfChildrenDirectoriesArePresent)
+					}
+					_ => Err(de::Error::unknown_variant(value, VARIANTS)),
+				}
+			}
+			fn visit_bytes<E>(self, bytes: &[u8]) -> Result<Self::Value, E>
+			where
+				E: de::Error,
+			{
+				match bytes {
+					b"AcceptFilesByGlob" => Ok(Fields::AcceptFilesByGlob),
+					b"RejectFilesByGlob" => Ok(Fields::RejectFilesByGlob),
+					b"AcceptIfChildrenDirectoriesArePresent" => {
+						Ok(Fields::AcceptIfChildrenDirectoriesArePresent)
+					}
+					b"RejectIfChildrenDirectoriesArePresent" => {
+						Ok(Fields::RejectIfChildrenDirectoriesArePresent)
+					}
+					_ => Err(de::Error::unknown_variant(
+						&String::from_utf8_lossy(bytes),
+						VARIANTS,
+					)),
+				}
+			}
+		}
+
+		impl<'de> Deserialize<'de> for Fields {
+			#[inline]
+			fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+			where
+				D: de::Deserializer<'de>,
+			{
+				deserializer.deserialize_identifier(FieldsVisitor)
+			}
+		}
+
+		struct ParametersPerKindVisitor<'de> {
+			marker: PhantomData<ParametersPerKind>,
+			lifetime: PhantomData<&'de ()>,
+		}
+
+		impl<'de> de::Visitor<'de> for ParametersPerKindVisitor<'de> {
+			type Value = ParametersPerKind;
+
+			fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+				formatter.write_str("enum ParametersPerKind")
+			}
+
+			fn visit_enum<PPK>(self, data: PPK) -> Result<Self::Value, PPK::Error>
+			where
+				PPK: de::EnumAccess<'de>,
+			{
+				use de::Error;
+
+				de::EnumAccess::variant(data).and_then(|value| match value {
+					(Fields::AcceptFilesByGlob, accept_files_by_glob) => {
+						de::VariantAccess::newtype_variant::<Vec<Glob>>(accept_files_by_glob)
+							.and_then(|globs| {
+								globs
+									.iter()
+									.fold(&mut GlobSetBuilder::new(), |builder, glob| {
+										builder.add(glob.to_owned())
+									})
+									.build()
+									.map_or_else(
+										|e| Err(PPK::Error::custom(e)),
+										|glob_set| {
+											Ok(Self::Value::AcceptFilesByGlob(globs, glob_set))
+										},
+									)
+							})
+					}
+					(Fields::RejectFilesByGlob, reject_files_by_glob) => {
+						de::VariantAccess::newtype_variant::<Vec<Glob>>(reject_files_by_glob)
+							.and_then(|globs| {
+								globs
+									.iter()
+									.fold(&mut GlobSetBuilder::new(), |builder, glob| {
+										builder.add(glob.to_owned())
+									})
+									.build()
+									.map_or_else(
+										|e| Err(PPK::Error::custom(e)),
+										|glob_set| {
+											Ok(Self::Value::RejectFilesByGlob(globs, glob_set))
+										},
+									)
+							})
+					}
+					(
+						Fields::AcceptIfChildrenDirectoriesArePresent,
+						accept_if_children_directories_are_present,
+					) => de::VariantAccess::newtype_variant::<HashSet<String>>(
+						accept_if_children_directories_are_present,
+					)
+					.map(Self::Value::AcceptIfChildrenDirectoriesArePresent),
+					(
+						Fields::RejectIfChildrenDirectoriesArePresent,
+						reject_if_children_directories_are_present,
+					) => de::VariantAccess::newtype_variant::<HashSet<String>>(
+						reject_if_children_directories_are_present,
+					)
+					.map(Self::Value::RejectIfChildrenDirectoriesArePresent),
+				})
+			}
+		}
+
+		deserializer.deserialize_enum(
+			"ParametersPerKind",
+			VARIANTS,
+			ParametersPerKindVisitor {
+				marker: PhantomData::<ParametersPerKind>,
+				lifetime: PhantomData,
+			},
+		)
+	}
+}
+
+impl ParametersPerKind {
+	async fn apply(&self, source: impl AsRef<Path>) -> Result<bool, IndexerRuleError> {
 		match self {
 			ParametersPerKind::AcceptIfChildrenDirectoriesArePresent(children) => {
 				accept_dir_for_its_children(source, children).await
@@ -127,25 +421,17 @@ impl ParametersPerKind {
 				reject_dir_for_its_children(source, children).await
 			}
 
-			ParametersPerKind::AcceptFilesByGlob(glob) => accept_by_glob(source, glob),
-			ParametersPerKind::RejectFilesByGlob(glob) => reject_by_glob(source, glob),
-		}
-	}
-
-	fn serialize(self) -> Result<Vec<u8>, IndexerError> {
-		match self {
-			Self::AcceptFilesByGlob(glob) | Self::RejectFilesByGlob(glob) => {
-				rmp_serde::to_vec_named(&glob).map_err(Into::into)
+			ParametersPerKind::AcceptFilesByGlob(_globs, accept_glob_set) => {
+				Ok(accept_by_glob(source, accept_glob_set))
 			}
-			Self::AcceptIfChildrenDirectoriesArePresent(children)
-			| Self::RejectIfChildrenDirectoriesArePresent(children) => {
-				rmp_serde::to_vec(&children.into_iter().collect::<Vec<_>>()).map_err(Into::into)
+			ParametersPerKind::RejectFilesByGlob(_globs, reject_glob_set) => {
+				Ok(reject_by_glob(source, reject_glob_set))
 			}
 		}
 	}
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct IndexerRule {
 	pub id: Option<i32>,
 	pub kind: RuleKind,
@@ -169,11 +455,11 @@ impl IndexerRule {
 		}
 	}
 
-	pub async fn apply(&self, source: impl AsRef<Path>) -> Result<bool, IndexerError> {
+	pub async fn apply(&self, source: impl AsRef<Path>) -> Result<bool, IndexerRuleError> {
 		self.parameters.apply(source).await
 	}
 
-	pub async fn save(self, client: &PrismaClient) -> Result<(), IndexerError> {
+	pub async fn save(self, client: &PrismaClient) -> Result<(), IndexerRuleError> {
 		if let Some(id) = self.id {
 			client
 				.indexer_rule()
@@ -182,7 +468,7 @@ impl IndexerRule {
 					indexer_rule::create(
 						self.kind as i32,
 						self.name,
-						self.parameters.serialize()?,
+						rmp_serde::to_vec_named(&self.parameters)?,
 						vec![indexer_rule::default::set(self.default)],
 					),
 					vec![indexer_rule::date_modified::set(Utc::now().into())],
@@ -195,7 +481,7 @@ impl IndexerRule {
 				.create(
 					self.kind as i32,
 					self.name,
-					self.parameters.serialize()?,
+					rmp_serde::to_vec_named(&self.parameters)?,
 					vec![indexer_rule::default::set(self.default)],
 				)
 				.exec()
@@ -207,7 +493,7 @@ impl IndexerRule {
 }
 
 impl TryFrom<&indexer_rule::Data> for IndexerRule {
-	type Error = IndexerError;
+	type Error = IndexerRuleError;
 
 	fn try_from(data: &indexer_rule::Data) -> Result<Self, Self::Error> {
 		let kind = RuleKind::try_from(data.kind)?;
@@ -217,27 +503,7 @@ impl TryFrom<&indexer_rule::Data> for IndexerRule {
 			kind,
 			name: data.name.clone(),
 			default: data.default,
-			parameters: match kind {
-				RuleKind::AcceptFilesByGlob | RuleKind::RejectFilesByGlob => {
-					let glob_str = rmp_serde::from_slice(&data.parameters)?;
-					if matches!(kind, RuleKind::AcceptFilesByGlob) {
-						ParametersPerKind::AcceptFilesByGlob(glob_str)
-					} else {
-						ParametersPerKind::RejectFilesByGlob(glob_str)
-					}
-				}
-				RuleKind::AcceptIfChildrenDirectoriesArePresent
-				| RuleKind::RejectIfChildrenDirectoriesArePresent => {
-					let childrens = rmp_serde::from_slice::<Vec<String>>(&data.parameters)?
-						.into_iter()
-						.collect();
-					if matches!(kind, RuleKind::AcceptIfChildrenDirectoriesArePresent) {
-						ParametersPerKind::AcceptIfChildrenDirectoriesArePresent(childrens)
-					} else {
-						ParametersPerKind::RejectIfChildrenDirectoriesArePresent(childrens)
-					}
-				}
-			},
+			parameters: rmp_serde::from_slice(&data.parameters)?,
 			date_created: data.date_created.into(),
 			date_modified: data.date_modified.into(),
 		})
@@ -245,43 +511,46 @@ impl TryFrom<&indexer_rule::Data> for IndexerRule {
 }
 
 impl TryFrom<indexer_rule::Data> for IndexerRule {
-	type Error = IndexerError;
+	type Error = IndexerRuleError;
 
 	fn try_from(data: indexer_rule::Data) -> Result<Self, Self::Error> {
 		Self::try_from(&data)
 	}
 }
 
-// TODO: memoize this
-fn globset_from_globs(globs: &[Glob]) -> Result<GlobSet, globset::Error> {
-	globs
-		.iter()
-		.fold(&mut GlobSetBuilder::new(), |builder, glob| {
-			builder.add(glob.to_owned())
-		})
-		.build()
+fn accept_by_glob(source: impl AsRef<Path>, accept_glob_set: &GlobSet) -> bool {
+	accept_glob_set.is_match(source.as_ref())
 }
 
-fn accept_by_glob(source: impl AsRef<Path>, globs: &[Glob]) -> Result<bool, IndexerError> {
-	globset_from_globs(globs)
-		.map(|glob_set| glob_set.is_match(source.as_ref()))
-		.map_err(IndexerError::GlobBuilderError)
-}
-
-fn reject_by_glob(source: impl AsRef<Path>, reject_globs: &[Glob]) -> Result<bool, IndexerError> {
-	accept_by_glob(source.as_ref(), reject_globs).map(|accept| !accept)
+fn reject_by_glob(source: impl AsRef<Path>, reject_glob_set: &GlobSet) -> bool {
+	!accept_by_glob(source.as_ref(), reject_glob_set)
 }
 
 async fn accept_dir_for_its_children(
 	source: impl AsRef<Path>,
 	children: &HashSet<String>,
-) -> Result<bool, IndexerError> {
+) -> Result<bool, IndexerRuleError> {
 	let source = source.as_ref();
-	let mut read_dir = fs::read_dir(source).await?;
-	while let Some(entry) = read_dir.next_entry().await? {
-		if entry.metadata().await?.is_dir()
-			&& children.contains(entry.file_name().to_str().expect("Found non-UTF-8 path"))
-		{
+	let mut read_dir = fs::read_dir(source)
+		.await
+		.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?;
+	while let Some(entry) = read_dir
+		.next_entry()
+		.await
+		.map_err(|e| IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e))))?
+	{
+		if entry
+			.metadata()
+			.await
+			.map_err(|e| {
+				IndexerRuleError::AcceptByItsChildrenFileIO(FileIOError::from((source, e)))
+			})?
+			.is_dir() && children.contains(
+			entry
+				.file_name()
+				.to_str()
+				.ok_or_else(|| NonUtf8PathError(entry.path().into()))?,
+		) {
 			return Ok(true);
 		}
 	}
@@ -292,13 +561,28 @@ async fn accept_dir_for_its_children(
 async fn reject_dir_for_its_children(
 	source: impl AsRef<Path>,
 	children: &HashSet<String>,
-) -> Result<bool, IndexerError> {
+) -> Result<bool, IndexerRuleError> {
 	let source = source.as_ref();
-	let mut read_dir = fs::read_dir(source).await?;
-	while let Some(entry) = read_dir.next_entry().await? {
-		if entry.metadata().await?.is_dir()
-			&& children.contains(entry.file_name().to_str().expect("Found non-UTF-8 path"))
-		{
+	let mut read_dir = fs::read_dir(source)
+		.await
+		.map_err(|e| IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?;
+	while let Some(entry) = read_dir
+		.next_entry()
+		.await
+		.map_err(|e| IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e))))?
+	{
+		if entry
+			.metadata()
+			.await
+			.map_err(|e| {
+				IndexerRuleError::RejectByItsChildrenFileIO(FileIOError::from((source, e)))
+			})?
+			.is_dir() && children.contains(
+			entry
+				.file_name()
+				.to_str()
+				.ok_or_else(|| NonUtf8PathError(entry.path().into()))?,
+		) {
 			return Ok(false);
 		}
 	}
@@ -306,6 +590,20 @@ async fn reject_dir_for_its_children(
 	Ok(true)
 }
 
+pub fn aggregate_rules_by_kind<'r>(
+	mut rules: impl Iterator<Item = &'r location_with_indexer_rules::indexer_rules::Data>,
+) -> Result<HashMap<RuleKind, Vec<IndexerRule>>, IndexerRuleError> {
+	rules.try_fold(
+		HashMap::<_, Vec<_>>::with_capacity(RuleKind::variant_count()),
+		|mut rules_by_kind, location_rule| {
+			IndexerRule::try_from(&location_rule.indexer_rule).map(|rule| {
+				rules_by_kind.entry(rule.kind).or_default().push(rule);
+				rules_by_kind
+			})
+		},
+	)
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
@@ -324,7 +622,13 @@ mod tests {
 			RuleKind::RejectFilesByGlob,
 			"ignore hidden files".to_string(),
 			false,
-			ParametersPerKind::RejectFilesByGlob(vec![Glob::new("**/.*").unwrap()]),
+			ParametersPerKind::RejectFilesByGlob(
+				vec![],
+				GlobSetBuilder::new()
+					.add(Glob::new("**/.*").unwrap())
+					.build()
+					.unwrap(),
+			),
 		);
 		assert!(!rule.apply(hidden).await.unwrap());
 		assert!(rule.apply(normal).await.unwrap());
@@ -344,9 +648,13 @@ mod tests {
 			RuleKind::RejectFilesByGlob,
 			"ignore build directory".to_string(),
 			false,
-			ParametersPerKind::RejectFilesByGlob(vec![
-				Glob::new("{**/target/*,**/target}").unwrap()
-			]),
+			ParametersPerKind::RejectFilesByGlob(
+				vec![],
+				GlobSetBuilder::new()
+					.add(Glob::new("{**/target/*,**/target}").unwrap())
+					.build()
+					.unwrap(),
+			),
 		);
 
 		assert!(rule.apply(project_file).await.unwrap());
@@ -370,7 +678,13 @@ mod tests {
 			RuleKind::AcceptFilesByGlob,
 			"only photos".to_string(),
 			false,
-			ParametersPerKind::AcceptFilesByGlob(vec![Glob::new("*.{jpg,png,jpeg}").unwrap()]),
+			ParametersPerKind::AcceptFilesByGlob(
+				vec![],
+				GlobSetBuilder::new()
+					.add(Glob::new("*.{jpg,png,jpeg}").unwrap())
+					.build()
+					.unwrap(),
+			),
 		);
 		assert!(!rule.apply(text).await.unwrap());
 		assert!(rule.apply(png).await.unwrap());
@@ -443,4 +757,65 @@ mod tests {
 		assert!(!rule.apply(project2).await.unwrap());
 		assert!(rule.apply(not_project).await.unwrap());
 	}
+
+	impl PartialEq for ParametersPerKind {
+		fn eq(&self, other: &Self) -> bool {
+			match (self, other) {
+				(
+					ParametersPerKind::AcceptFilesByGlob(self_globs, _),
+					ParametersPerKind::AcceptFilesByGlob(other_globs, _),
+				) => self_globs == other_globs,
+				(
+					ParametersPerKind::RejectFilesByGlob(self_globs, _),
+					ParametersPerKind::RejectFilesByGlob(other_globs, _),
+				) => self_globs == other_globs,
+				(
+					ParametersPerKind::AcceptIfChildrenDirectoriesArePresent(self_childrens),
+					ParametersPerKind::AcceptIfChildrenDirectoriesArePresent(other_childrens),
+				) => self_childrens == other_childrens,
+				(
+					ParametersPerKind::RejectIfChildrenDirectoriesArePresent(self_childrens),
+					ParametersPerKind::RejectIfChildrenDirectoriesArePresent(other_childrens),
+				) => self_childrens == other_childrens,
+				_ => false,
+			}
+		}
+	}
+
+	impl Eq for ParametersPerKind {}
+
+	impl PartialEq for IndexerRule {
+		fn eq(&self, other: &Self) -> bool {
+			self.id == other.id
+				&& self.kind == other.kind
+				&& self.name == other.name
+				&& self.default == other.default
+				&& self.parameters == other.parameters
+				&& self.date_created == other.date_created
+				&& self.date_modified == other.date_modified
+		}
+	}
+
+	impl Eq for IndexerRule {}
+
+	#[test]
+	fn serde_smoke_test() {
+		let actual = IndexerRule::new(
+			RuleKind::RejectFilesByGlob,
+			"No Hidden".to_string(),
+			true,
+			ParametersPerKind::RejectFilesByGlob(
+				vec![Glob::new("**/.*").unwrap()],
+				Glob::new("**/.*")
+					.and_then(|glob| GlobSetBuilder::new().add(glob).build())
+					.unwrap(),
+			),
+		);
+
+		let expected =
+			rmp_serde::from_slice::<IndexerRule>(&rmp_serde::to_vec_named(&actual).unwrap())
+				.unwrap();
+
+		assert_eq!(actual, expected);
+	}
 }

core/src/location/indexer/shallow_indexer_job.rs

@@ -1,32 +1,28 @@
 use crate::{
+	file_paths_db_fetcher_fn,
 	job::{JobError, JobInitData, JobResult, JobState, StatefulJob, WorkerContext},
 	location::file_path_helper::{
-		ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
-		file_path_just_id_materialized_path, filter_existing_file_path_params,
-		filter_file_paths_by_many_full_path_params, retain_file_paths_in_location,
-		MaterializedPath,
+		check_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
+		IsolatedFilePathData,
 	},
-	prisma::location,
+	to_remove_db_fetcher_fn,
 };
 
 use std::{
-	collections::{HashMap, HashSet},
 	hash::{Hash, Hasher},
 	path::{Path, PathBuf},
+	sync::Arc,
 };
 
-use chrono::Utc;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 use tokio::time::Instant;
-use tracing::error;
-use uuid::Uuid;
 
 use super::{
-	execute_indexer_step, finalize_indexer, location_with_indexer_rules,
-	rules::{IndexerRule, RuleKind},
-	walk::walk_single_dir,
-	IndexerError, IndexerJobData, IndexerJobStep, IndexerJobStepEntry, ScanProgress,
+	execute_indexer_save_step, finalize_indexer, iso_file_path_factory,
+	location_with_indexer_rules, remove_non_existing_file_paths, rules::aggregate_rules_by_kind,
+	update_notifier_fn, walk::walk_single_dir, IndexerError, IndexerJobData, IndexerJobSaveStep,
+	ScanProgress,
 };
 
 /// BATCH_SIZE is the number of files to index at each step, writing the chunk of files metadata in the database.
@@ -61,7 +57,7 @@ impl JobInitData for ShallowIndexerJobInit {
 impl StatefulJob for ShallowIndexerJob {
 	type Init = ShallowIndexerJobInit;
 	type Data = IndexerJobData;
-	type Step = IndexerJobStep;
+	type Step = IndexerJobSaveStep;
 
 	const NAME: &'static str = "shallow_indexer";
 	const IS_BACKGROUND: bool = true;
@@ -79,21 +75,12 @@ impl StatefulJob for ShallowIndexerJob {
 		let location_id = state.init.location.id;
 		let location_path = Path::new(&state.init.location.path);
 
-		let db = ctx.library.db.clone();
+		let db = Arc::clone(&ctx.library.db);
 
-		let mut indexer_rules_by_kind: HashMap<RuleKind, Vec<IndexerRule>> =
-			HashMap::with_capacity(state.init.location.indexer_rules.len());
+		let rules_by_kind = aggregate_rules_by_kind(state.init.location.indexer_rules.iter())
+			.map_err(IndexerError::from)?;
 
-		for location_rule in &state.init.location.indexer_rules {
-			let indexer_rule = IndexerRule::try_from(&location_rule.indexer_rule)?;
-
-			indexer_rules_by_kind
-				.entry(indexer_rule.kind)
-				.or_default()
-				.push(indexer_rule);
-		}
-
-		let (to_walk_path, parent_file_path) = if state.init.sub_path != Path::new("") {
+		let (add_root, to_walk_path) = if state.init.sub_path != Path::new("") {
 			let full_path = ensure_sub_path_is_in_location(location_path, &state.init.sub_path)
 				.await
 				.map_err(IndexerError::from)?;
@@ -101,171 +88,105 @@ impl StatefulJob for ShallowIndexerJob {
 				.await
 				.map_err(IndexerError::from)?;
 
-			let materialized_path =
-				MaterializedPath::new(location_id, location_path, &full_path, true)
-					.map_err(IndexerError::from)?;
-
 			(
+				!check_file_path_exists::<IndexerError>(
+					&IsolatedFilePathData::new(location_id, location_path, &full_path, true)
+						.map_err(IndexerError::from)?,
+					&db,
+				)
+				.await?,
 				full_path,
-				db.file_path()
-					.find_first(filter_existing_file_path_params(&materialized_path))
-					.select(file_path_just_id_materialized_path::select())
-					.exec()
-					.await
-					.map_err(IndexerError::from)?
-					.expect("Sub path should already exist in the database"),
 			)
 		} else {
-			(
-				location_path.to_path_buf(),
-				db.file_path()
-					.find_first(filter_existing_file_path_params(
-						&MaterializedPath::new(location_id, location_path, location_path, true)
-							.map_err(IndexerError::from)?,
-					))
-					.select(file_path_just_id_materialized_path::select())
-					.exec()
-					.await
-					.map_err(IndexerError::from)?
-					.expect("Location root path should already exist in the database"),
-			)
+			(false, location_path.to_path_buf())
 		};
 
 		let scan_start = Instant::now();
-		let found_paths = {
-			let ctx = &mut ctx; // Borrow outside of closure so it's not moved
+		let (walked, to_remove, errors) = {
+			let ctx = &mut ctx;
 			walk_single_dir(
 				&to_walk_path,
-				&indexer_rules_by_kind,
-				|path, total_entries| {
-					IndexerJobData::on_scan_progress(
-						ctx,
-						vec![
-							ScanProgress::Message(format!("Scanning {}", path.display())),
-							ScanProgress::ChunkCount(total_entries / BATCH_SIZE),
-						],
-					);
-				},
+				&rules_by_kind,
+				update_notifier_fn(BATCH_SIZE, ctx),
+				file_paths_db_fetcher_fn!(&db),
+				to_remove_db_fetcher_fn!(location_id, location_path, &db),
+				iso_file_path_factory(location_id, location_path),
+				add_root,
 			)
 			.await?
 		};
 
-		let (already_existing_file_paths, mut to_retain): (HashSet<_>, Vec<_>) = db
-			.file_path()
-			.find_many(
-				filter_file_paths_by_many_full_path_params(
-					&location::Data::from(&state.init.location),
-					&found_paths
-						.iter()
-						.map(|entry| &entry.path)
-						.collect::<Vec<_>>(),
-				)
-				.await
-				.map_err(IndexerError::from)?,
-			)
-			.select(file_path_just_id_materialized_path::select())
-			.exec()
-			.await?
-			.into_iter()
-			.map(|file_path| {
-				(
-					file_path.materialized_path,
-					Uuid::from_slice(&file_path.pub_id).unwrap(),
-				)
-			})
-			.unzip();
+		let db_delete_start = Instant::now();
+		// TODO pass these uuids to sync system
+		let removed_count = remove_non_existing_file_paths(to_remove, &db).await?;
+		let db_delete_time = db_delete_start.elapsed();
 
-		let parent_pub_id = Uuid::from_slice(&parent_file_path.pub_id).unwrap();
+		let total_paths = &mut 0;
 
-		// Adding our parent path id
-		to_retain.push(parent_pub_id);
+		state.steps.extend(
+			walked
+				.chunks(BATCH_SIZE)
+				.into_iter()
+				.enumerate()
+				.map(|(i, chunk)| {
+					let chunk_steps = chunk.collect::<Vec<_>>();
 
-		// Removing all other file paths that are not in the filesystem anymore
-		let removed_paths =
-			retain_file_paths_in_location(location_id, to_retain, Some(parent_file_path), &db)
-				.await
-				.map_err(IndexerError::from)?;
+					*total_paths += chunk_steps.len() as u64;
+
+					IndexerJobSaveStep {
+						chunk_idx: i,
+						walked: chunk_steps,
+					}
+				}),
+		);
 
 		ctx.library.orphan_remover.invoke().await;
 
-		// Filter out paths that are already in the databases
-		let new_paths = found_paths
-			.into_iter()
-			.filter_map(|entry| {
-				MaterializedPath::new(location_id, location_path, &entry.path, entry.is_dir)
-					.map_or_else(
-						|e| {
-							error!("Failed to create materialized path: {e}");
-							None
-						},
-						|materialized_path| {
-							(!already_existing_file_paths
-								.contains::<str>(materialized_path.as_ref()))
-							.then_some(IndexerJobStepEntry {
-								full_path: entry.path,
-								materialized_path,
-								file_id: Uuid::new_v4(),
-								parent_id: Some(parent_pub_id),
-								metadata: entry.metadata,
-							})
-						},
-					)
-			})
-			// Sadly we have to collect here to be able to check the length so we can set
-			// the max file path id later
-			.collect::<Vec<_>>();
-
-		let total_paths = new_paths.len();
+		IndexerJobData::on_scan_progress(
+			&mut ctx,
+			vec![ScanProgress::Message(format!(
+				"Saving {total_paths} files or directories"
+			))],
+		);
 
 		state.data = Some(IndexerJobData {
 			indexed_path: to_walk_path,
-			db_write_start: Utc::now(),
+			rules_by_kind,
+			db_write_time: db_delete_time,
 			scan_read_time: scan_start.elapsed(),
-			total_paths,
-			indexed_paths: 0,
-			removed_paths,
+			total_paths: *total_paths,
+			indexed_count: 0,
+			removed_count,
+			total_save_steps: state.steps.len() as u64,
 		});
 
-		state.steps = new_paths
-			.into_iter()
-			.chunks(BATCH_SIZE)
-			.into_iter()
-			.enumerate()
-			.map(|(i, chunk)| {
-				let chunk_steps = chunk.collect::<Vec<_>>();
-				IndexerJobData::on_scan_progress(
-					&mut ctx,
-					vec![
-						ScanProgress::SavedChunks(i),
-						ScanProgress::Message(format!(
-							"Writing {} of {} to db",
-							i * chunk_steps.len(),
-							total_paths,
-						)),
-					],
-				);
-				chunk_steps
-			})
-			.collect();
-
-		Ok(())
+		if !errors.is_empty() {
+			Err(JobError::StepCompletedWithErrors(
+				errors.into_iter().map(|e| format!("{e}")).collect(),
+			))
+		} else {
+			Ok(())
+		}
 	}
 
 	/// Process each chunk of entries in the indexer job, writing to the `file_path` table
 	async fn execute_step(
 		&self,
-		ctx: WorkerContext,
+		mut ctx: WorkerContext,
 		state: &mut JobState<Self>,
 	) -> Result<(), JobError> {
-		execute_indexer_step(&state.init.location, &state.steps[0], ctx)
+		let data = state
+			.data
+			.as_mut()
+			.expect("critical error: missing data on job state");
+
+		execute_indexer_save_step(&state.init.location, &state.steps[0], data, &mut ctx)
 			.await
-			.map(|indexed_paths| {
-				state
-					.data
-					.as_mut()
-					.expect("critical error: missing data on job state")
-					.indexed_paths = indexed_paths;
+			.map(|(indexed_paths, elapsed_time)| {
+				data.indexed_count += indexed_paths;
+				data.db_write_time += elapsed_time;
 			})
+			.map_err(Into::into)
 	}
 
 	/// Logs some metadata about the indexer job

core/src/location/manager/mod.rs

@@ -1,4 +1,4 @@
-use crate::{job::JobManagerError, library::Library};
+use crate::{job::JobManagerError, library::Library, util::error::FileIOError};
 
 use std::{
 	collections::BTreeSet,
@@ -8,12 +8,9 @@ use std::{
 
 use futures::executor::block_on;
 use thiserror::Error;
-use tokio::{
-	io,
-	sync::{
-		broadcast::{self, Receiver},
-		oneshot, RwLock,
-	},
+use tokio::sync::{
+	broadcast::{self, Receiver},
+	oneshot, RwLock,
 };
 use tracing::{debug, error};
 
@@ -92,18 +89,22 @@ pub enum LocationManagerError {
 	#[error("Non local location: <id='{0}'>")]
 	NonLocalLocation(LocationId),
 
+	#[error("failed to move file '{}' for reason: {reason}", .path.display())]
+	MoveError { path: Box<Path>, reason: String },
+
 	#[error("Tried to update a non-existing file: <path='{0}'>")]
 	UpdateNonExistingFile(PathBuf),
 	#[error("Database error: {0}")]
 	DatabaseError(#[from] prisma_client_rust::QueryError),
-	#[error("I/O error: {0}")]
-	IOError(#[from] io::Error),
 	#[error("File path related error (error: {0})")]
 	FilePathError(#[from] FilePathError),
 	#[error("Corrupted location pub_id on database: (error: {0})")]
 	CorruptedLocationPubId(#[from] uuid::Error),
 	#[error("Job Manager error: (error: {0})")]
 	JobManager(#[from] JobManagerError),
+
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
 }
 
 type OnlineLocations = BTreeSet<Vec<u8>>;

core/src/location/manager/watcher/linux.rs

@@ -6,7 +6,10 @@
 //! Aside from that, when a directory is moved to our watched location from the outside, we receive
 //! a Create Dir event, this one is actually ok at least.
 
-use crate::{invalidate_query, library::Library, location::manager::LocationManagerError};
+use crate::{
+	invalidate_query, library::Library, location::manager::LocationManagerError,
+	util::error::FileIOError,
+};
 
 use std::{
 	collections::{BTreeMap, HashMap},
@@ -67,7 +70,9 @@ impl<'lib> EventHandler<'lib> for LinuxEventHandler<'lib> {
 				create_dir(
 					self.location_id,
 					path,
-					&fs::metadata(path).await?,
+					&fs::metadata(path)
+						.await
+						.map_err(|e| FileIOError::from((path, e)))?,
 					self.library,
 				)
 				.await?;

core/src/location/manager/watcher/macos.rs

@@ -13,10 +13,11 @@ use crate::{
 	invalidate_query,
 	library::Library,
 	location::{
-		file_path_helper::{check_existing_file_path, get_inode_and_device, MaterializedPath},
+		file_path_helper::{check_existing_file_path, get_inode_and_device, IsolatedFilePathData},
 		manager::LocationManagerError,
 		LocationId,
 	},
+	util::error::FileIOError,
 };
 
 use std::{
@@ -81,8 +82,9 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
 
 		match kind {
 			EventKind::Create(CreateKind::Folder) => {
-				if let Some(latest_created_dir) = self.latest_created_dir.take() {
-					if paths[0] == latest_created_dir {
+				let path = &paths[0];
+				if let Some(ref latest_created_dir) = self.latest_created_dir.take() {
+					if path == latest_created_dir {
 						// NOTE: This is a MacOS specific event that happens when a folder is created
 						// trough Finder. It creates a folder but 2 events are triggered in
 						// FSEvents. So we store and check the latest created folder to avoid
@@ -93,18 +95,23 @@ impl<'lib> EventHandler<'lib> for MacOsEventHandler<'lib> {
 
 				create_dir(
 					self.location_id,
-					&paths[0],
-					&fs::metadata(&paths[0]).await?,
+					path,
+					&fs::metadata(path)
+						.await
+						.map_err(|e| FileIOError::from((path, e)))?,
 					self.library,
 				)
 				.await?;
 				self.latest_created_dir = Some(paths.remove(0));
 			}
 			EventKind::Create(CreateKind::File) => {
+				let path = &paths[0];
 				create_file(
 					self.location_id,
-					&paths[0],
-					&fs::metadata(&paths[0]).await?,
+					path,
+					&fs::metadata(path)
+						.await
+						.map_err(|e| FileIOError::from((path, e)))?,
 					self.library,
 				)
 				.await?;
@@ -209,7 +216,12 @@ impl MacOsEventHandler<'_> {
 				let location_path = extract_location_path(self.location_id, self.library).await?;
 
 				if !check_existing_file_path(
-					&MaterializedPath::new(self.location_id, &location_path, &path, meta.is_dir())?,
+					&IsolatedFilePathData::new(
+						self.location_id,
+						&location_path,
+						&path,
+						meta.is_dir(),
+					)?,
 					&self.library.db,
 				)
 				.await?
@@ -261,7 +273,7 @@ impl MacOsEventHandler<'_> {
 						.insert(inode_and_device, (Instant::now(), path));
 				}
 			}
-			Err(e) => return Err(e.into()),
+			Err(e) => return Err(FileIOError::from((path, e)).into()),
 		}
 
 		Ok(())

core/src/location/manager/watcher/utils.rs

@@ -4,10 +4,11 @@ use crate::{
 	location::{
 		delete_directory,
 		file_path_helper::{
-			create_file_path, extract_materialized_path, file_path_with_object,
-			filter_existing_file_path_params, get_parent_dir, get_parent_dir_id,
+			check_existing_file_path, create_file_path, file_path_with_object,
+			filter_existing_file_path_params, get_parent_dir,
+			isolated_file_path_data::extract_normalized_materialized_path_str,
 			loose_find_existing_file_path_params, FilePathError, FilePathMetadata,
-			MaterializedPath, MetadataExt,
+			IsolatedFilePathData, MetadataExt,
 		},
 		find_location, location_with_indexer_rules,
 		manager::LocationManagerError,
@@ -21,7 +22,7 @@ use crate::{
 	},
 	prisma::{file_path, location, object},
 	sync,
-	util::db::uuid_to_bytes,
+	util::error::FileIOError,
 };
 
 #[cfg(target_family = "unix")]
@@ -33,7 +34,7 @@ use crate::location::file_path_helper::get_inode_and_device_from_path;
 use std::{
 	collections::HashSet,
 	fs::Metadata,
-	path::{Path, PathBuf, MAIN_SEPARATOR, MAIN_SEPARATOR_STR},
+	path::{Path, PathBuf},
 	str::FromStr,
 };
 
@@ -80,7 +81,7 @@ pub(super) async fn create_dir(
 		path.display()
 	);
 
-	let materialized_path = MaterializedPath::new(location.id, &location.path, path, true)?;
+	let materialized_path = IsolatedFilePathData::new(location.id, &location.path, path, true)?;
 
 	let (inode, device) = {
 		#[cfg(target_family = "unix")]
@@ -100,15 +101,14 @@ pub(super) async fn create_dir(
 
 	trace!("parent_directory: {:?}", parent_directory);
 
-	let Some(parent_directory) = parent_directory else {
+	if parent_directory.is_none() {
 		warn!("Watcher found a directory without parent");
-        return Ok(())
+		return Ok(());
 	};
 
 	let created_path = create_file_path(
 		library,
 		materialized_path,
-		Some(Uuid::from_slice(&parent_directory.pub_id).unwrap()),
 		None,
 		FilePathMetadata {
 			inode,
@@ -147,7 +147,7 @@ pub(super) async fn create_file(
 
 	let db = &library.db;
 
-	let materialized_path = MaterializedPath::new(location_id, &location_path, path, false)?;
+	let iso_file_path = IsolatedFilePathData::new(location_id, &location_path, path, false)?;
 
 	let (inode, device) = {
 		#[cfg(target_family = "unix")]
@@ -163,24 +163,23 @@ pub(super) async fn create_file(
 		}
 	};
 
-	let Some(parent_directory) =
-		get_parent_dir(&materialized_path, db).await?
-    else {
+	if get_parent_dir(&iso_file_path, db).await?.is_none() {
 		warn!("Watcher found a file without parent");
-        return Ok(())
-    };
+		return Ok(());
+	};
 
 	// generate provisional object
 	let FileMetadata {
 		cas_id,
 		kind,
 		fs_metadata,
-	} = FileMetadata::new(&location_path, &materialized_path).await?;
+	} = FileMetadata::new(&location_path, &iso_file_path)
+		.await
+		.map_err(|e| FileIOError::from((location_path.join(&iso_file_path), e)))?;
 
 	let created_file = create_file_path(
 		library,
-		materialized_path,
-		Some(Uuid::from_slice(&parent_directory.pub_id).unwrap()),
+		iso_file_path,
 		Some(cas_id.clone()),
 		FilePathMetadata {
 			inode,
@@ -249,7 +248,11 @@ pub(super) async fn create_dir_or_file(
 	path: impl AsRef<Path>,
 	library: &Library,
 ) -> Result<Metadata, LocationManagerError> {
-	let metadata = fs::metadata(path.as_ref()).await?;
+	let path = path.as_ref();
+	let metadata = fs::metadata(path)
+		.await
+		.map_err(|e| FileIOError::from((path, e)))?;
+
 	if metadata.is_dir() {
 		create_dir(location_id, path, &metadata, library).await
 	} else {
@@ -269,12 +272,9 @@ pub(super) async fn file_creation_or_update(
 	if let Some(ref file_path) = library
 		.db
 		.file_path()
-		.find_first(filter_existing_file_path_params(&MaterializedPath::new(
-			location_id,
-			&location_path,
-			full_path,
-			false,
-		)?))
+		.find_first(filter_existing_file_path_params(
+			&IsolatedFilePathData::new(location_id, &location_path, full_path, false)?,
+		))
 		// include object for orphan check
 		.include(file_path_with_object::include())
 		.exec()
@@ -285,7 +285,9 @@ pub(super) async fn file_creation_or_update(
 		create_file(
 			location_id,
 			full_path,
-			&fs::metadata(full_path).await?,
+			&fs::metadata(full_path)
+				.await
+				.map_err(|e| FileIOError::from((full_path, e)))?,
 			library,
 		)
 		.await
@@ -303,12 +305,9 @@ pub(super) async fn update_file(
 	if let Some(ref file_path) = library
 		.db
 		.file_path()
-		.find_first(filter_existing_file_path_params(&MaterializedPath::new(
-			location_id,
-			&location_path,
-			full_path,
-			false,
-		)?))
+		.find_first(filter_existing_file_path_params(
+			&IsolatedFilePathData::new(location_id, &location_path, full_path, false)?,
+		))
 		// include object for orphan check
 		.include(file_path_with_object::include())
 		.exec()
@@ -318,6 +317,7 @@ pub(super) async fn update_file(
 		invalidate_query!(library, "search.paths");
 		ret
 	} else {
+		// FIXME(fogodev): Have to handle files excluded by indexer rules
 		Err(LocationManagerError::UpdateNonExistingFile(
 			full_path.to_path_buf(),
 		))
@@ -346,15 +346,15 @@ async fn inner_update_file(
 		full_path.display()
 	);
 
+	let iso_file_path = IsolatedFilePathData::from(file_path);
+
 	let FileMetadata {
 		cas_id,
 		fs_metadata,
 		kind,
-	} = FileMetadata::new(
-		&location_path,
-		&MaterializedPath::from((location_id, &file_path.materialized_path)),
-	)
-	.await?;
+	} = FileMetadata::new(&location_path, &iso_file_path)
+		.await
+		.map_err(|e| FileIOError::from((location_path.join(&iso_file_path), e)))?;
 
 	if let Some(old_cas_id) = &file_path.cas_id {
 		if old_cas_id != &cas_id {
@@ -379,7 +379,11 @@ async fn inner_update_file(
 						// TODO: Should this be a skip rather than a null-set?
 						let checksum = if file_path.integrity_checksum.is_some() {
 							// If a checksum was already computed, we need to recompute it
-							Some(file_checksum(full_path).await?)
+							Some(
+								file_checksum(full_path)
+									.await
+									.map_err(|e| FileIOError::from((full_path, e)))?,
+							)
 						} else {
 							None
 						};
@@ -424,7 +428,10 @@ async fn inner_update_file(
 					generate_thumbnail(&file_path.extension, &cas_id, full_path, library).await;
 
 					// remove the old thumbnail as we're generating a new one
-					fs::remove_file(get_thumbnail_path(library, old_cas_id)).await?;
+					let thumb_path = get_thumbnail_path(library, old_cas_id);
+					fs::remove_file(&thumb_path)
+						.await
+						.map_err(|e| FileIOError::from((thumb_path, e)))?;
 				}
 
 				let int_kind = kind as i32;
@@ -462,68 +469,46 @@ pub(super) async fn rename(
 	library: &Library,
 ) -> Result<(), LocationManagerError> {
 	let location_path = extract_location_path(location_id, library).await?;
+	let old_path = old_path.as_ref();
+	let new_path = new_path.as_ref();
+	let Library { db, .. } = library;
 
-	let old_path_materialized =
-		extract_materialized_path(location_id, &location_path, old_path.as_ref())?;
-	let mut old_path_materialized_str = format!(
-		"{MAIN_SEPARATOR_STR}{}",
-		old_path_materialized
-			.to_str()
-			.expect("Found non-UTF-8 path")
-	);
+	let old_path_materialized_str =
+		extract_normalized_materialized_path_str(location_id, &location_path, old_path)?;
 
-	let new_path_materialized =
-		extract_materialized_path(location_id, &location_path, new_path.as_ref())?;
-	let mut new_path_materialized_str = format!(
-		"{MAIN_SEPARATOR_STR}{}",
-		new_path_materialized
-			.to_str()
-			.expect("Found non-UTF-8 path")
-	);
-
-	let old_materialized_path_parent = old_path_materialized
-		.parent()
-		.unwrap_or_else(|| Path::new(MAIN_SEPARATOR_STR));
-	let new_materialized_path_parent = new_path_materialized
-		.parent()
-		.unwrap_or_else(|| Path::new(MAIN_SEPARATOR_STR));
+	let new_path_materialized_str =
+		extract_normalized_materialized_path_str(location_id, &location_path, new_path)?;
 
 	// Renaming a file could potentially be a move to another directory, so we check if our parent changed
-	let changed_parent_id = if old_materialized_path_parent != new_materialized_path_parent {
-		Some(
-			get_parent_dir_id(
-				&MaterializedPath::new(
-					location_id,
-					&location_path,
-					new_path,
-					true,
-				)?,
-				&library.db,
-			)
-			.await?
-			.expect("CRITICAL ERROR: If we're puting a file in a directory inside our location, then this directory must exist"),
+	if old_path_materialized_str != new_path_materialized_str
+		&& !check_existing_file_path(
+			&IsolatedFilePathData::new(location_id, &location_path, new_path, true)?.parent(),
+			db,
 		)
-	} else {
-		None
-	};
+		.await?
+	{
+		return Err(LocationManagerError::MoveError {
+			path: new_path.into(),
+			reason: "parent directory does not exist".into(),
+		});
+	}
 
-	if let Some(file_path) = library
-		.db
+	if let Some(file_path) = db
 		.file_path()
 		.find_first(loose_find_existing_file_path_params(
-			&MaterializedPath::new(location_id, &location_path, old_path, true)?,
+			&IsolatedFilePathData::new(location_id, &location_path, old_path, true)?,
 		))
 		.exec()
 		.await?
 	{
+		let new =
+			IsolatedFilePathData::new(location_id, &location_path, new_path, file_path.is_dir)?;
+
 		// If the renamed path is a directory, we have to update every successor
 		if file_path.is_dir {
-			if !old_path_materialized_str.ends_with(MAIN_SEPARATOR) {
-				old_path_materialized_str += MAIN_SEPARATOR_STR;
-			}
-			if !new_path_materialized_str.ends_with(MAIN_SEPARATOR) {
-				new_path_materialized_str += MAIN_SEPARATOR_STR;
-			}
+			let old =
+				IsolatedFilePathData::new(location_id, &location_path, old_path, file_path.is_dir)?;
+			// TODO: Fetch all file_paths that will be updated and dispatch sync events
 
 			let updated = library
 				.db
@@ -531,8 +516,8 @@ pub(super) async fn rename(
 					"UPDATE file_path \
 						SET materialized_path = REPLACE(materialized_path, {}, {}) \
 						WHERE location_id = {}",
-					PrismaValue::String(old_path_materialized_str.clone()),
-					PrismaValue::String(new_path_materialized_str.clone()),
+					PrismaValue::String(format!("{}/{}/", old.materialized_path, old.name)),
+					PrismaValue::String(format!("{}/{}/", new.materialized_path, new.name)),
 					PrismaValue::Int(location_id as i64)
 				))
 				.exec()
@@ -540,38 +525,17 @@ pub(super) async fn rename(
 			trace!("Updated {updated} file_paths");
 		}
 
-		let mut update_params = vec![
-			file_path::materialized_path::set(new_path_materialized_str),
-			file_path::name::set(
-				new_path_materialized
-					.file_stem()
-					.unwrap()
-					.to_str()
-					.expect("Found non-UTF-8 path")
-					.to_string(),
-			),
-			file_path::extension::set(
-				new_path_materialized
-					.extension()
-					.map(|s| {
-						s.to_str()
-							.expect("Found non-UTF-8 extension in path")
-							.to_string()
-					})
-					.unwrap_or_default(),
-			),
-		];
-
-		if changed_parent_id.is_some() {
-			update_params.push(file_path::parent_id::set(
-				changed_parent_id.map(uuid_to_bytes),
-			));
-		}
-
 		library
 			.db
 			.file_path()
-			.update(file_path::pub_id::equals(file_path.pub_id), update_params)
+			.update(
+				file_path::pub_id::equals(file_path.pub_id),
+				vec![
+					file_path::materialized_path::set(new_path_materialized_str),
+					file_path::name::set(new.name.to_string()),
+					file_path::extension::set(new.extension.to_string()),
+				],
+			)
 			.exec()
 			.await?;
 
@@ -593,7 +557,7 @@ pub(super) async fn remove(
 	let Some(file_path) = library.db
 		.file_path()
 		.find_first(loose_find_existing_file_path_params(
-			&MaterializedPath::new(location_id, &location_path, full_path, true)?,
+			&IsolatedFilePathData::new(location_id, &location_path, full_path, true)?,
 		))
 		.exec()
 		.await? else {
@@ -610,7 +574,7 @@ pub(super) async fn remove_by_file_path(
 	library: &Library,
 ) -> Result<(), LocationManagerError> {
 	// check file still exists on disk
-	match fs::metadata(path).await {
+	match fs::metadata(path.as_ref()).await {
 		Ok(_) => {
 			todo!("file has changed in some way, re-identify it")
 		}
@@ -645,7 +609,7 @@ pub(super) async fn remove_by_file_path(
 
 			library.orphan_remover.invoke().await;
 		}
-		Err(e) => return Err(e.into()),
+		Err(e) => return Err(FileIOError::from((path, e)).into()),
 	}
 
 	invalidate_query!(library, "search.paths");
@@ -716,7 +680,7 @@ pub(super) async fn extract_inode_and_device_from_path(
 		.db
 		.file_path()
 		.find_first(loose_find_existing_file_path_params(
-			&MaterializedPath::new(location_id, &location.path, path, true)?,
+			&IsolatedFilePathData::new(location_id, &location.path, path, true)?,
 		))
 		.select(file_path::select!( {inode device} ))
 		.exec()
@@ -727,7 +691,7 @@ pub(super) async fn extract_inode_and_device_from_path(
 				u64::from_le_bytes(file_path.device[0..8].try_into().unwrap()),
 			)
 		})
-		.ok_or_else(|| FilePathError::NotFound(path.to_path_buf()).into())
+		.ok_or_else(|| FilePathError::NotFound(path.into()).into())
 }
 
 pub(super) async fn extract_location_path(

core/src/location/manager/watcher/windows.rs

@@ -13,6 +13,7 @@ use crate::{
 	location::{
 		file_path_helper::get_inode_and_device_from_path, manager::LocationManagerError, LocationId,
 	},
+	util::error::FileIOError,
 };
 
 use std::{
@@ -100,11 +101,14 @@ impl<'lib> EventHandler<'lib> for WindowsEventHandler<'lib> {
 				}
 			}
 			EventKind::Modify(ModifyKind::Any) => {
+				let path = &paths[0];
 				// Windows emite events of update right after create events
-				if !self.recently_created_files.contains_key(&paths[0]) {
-					let metadata = fs::metadata(&paths[0]).await?;
+				if !self.recently_created_files.contains_key(path) {
+					let metadata = fs::metadata(path)
+						.await
+						.map_err(|e| FileIOError::from((path, e)))?;
 					if metadata.is_file() {
-						update_file(self.location_id, &paths[0], self.library).await?;
+						update_file(self.location_id, path, self.library).await?;
 					}
 				}
 			}

core/src/location/mod.rs

@@ -13,7 +13,7 @@ use crate::{
 	},
 	prisma::{file_path, indexer_rules_in_location, location, node, object, PrismaClient},
 	sync,
-	util::db::uuid_to_bytes,
+	util::{db::uuid_to_bytes, error::FileIOError},
 };
 
 use std::{
@@ -71,7 +71,7 @@ impl LocationCreateArgs {
 			}
 			Err(e) => {
 				return Err(LocationError::LocationPathFilesystemMetadataAccess(
-					e, self.path,
+					FileIOError::from((self.path, e)),
 				));
 			}
 		};
@@ -378,7 +378,6 @@ pub async fn scan_location(
 			.queue_next(ThumbnailerJobInit {
 				location: location_base_data,
 				sub_path: None,
-				background: true,
 			}),
 		)
 		.await
@@ -413,7 +412,6 @@ pub async fn scan_location_sub_path(
 			.queue_next(ThumbnailerJobInit {
 				location: location_base_data,
 				sub_path: Some(sub_path),
-				background: true,
 			}),
 		)
 		.await
@@ -775,24 +773,3 @@ async fn check_nested_location(
 
 	Ok(parents_count > 0 || children_count > 0)
 }
-
-// check if a path exists in our database at that location
-// pub async fn check_virtual_path_exists(
-// 	library: &Library,
-// 	location_id: i32,
-// 	subpath: impl AsRef<Path>,
-// ) -> Result<bool, LocationError> {
-// 	let path = subpath.as_ref().to_str().unwrap().to_string();
-
-// 	let file_path = library
-// 		.db
-// 		.file_path()
-// 		.find_first(vec![
-// 			file_path::location_id::equals(location_id),
-// 			file_path::materialized_path::equals(path),
-// 		])
-// 		.exec()
-// 		.await?;
-
-// 	Ok(file_path.is_some())
-// }

core/src/object/file_identifier/file_identifier_job.rs

@@ -4,8 +4,8 @@ use crate::{
 	},
 	library::Library,
 	location::file_path_helper::{
-		ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
-		file_path_for_file_identifier, MaterializedPath,
+		ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
+		file_path_for_file_identifier, IsolatedFilePathData,
 	},
 	prisma::{file_path, location, PrismaClient},
 	util::db::chain_optional_iter,
@@ -51,7 +51,7 @@ impl Hash for FileIdentifierJobInit {
 pub struct FileIdentifierJobState {
 	cursor: i32,
 	report: FileIdentifierReport,
-	maybe_sub_materialized_path: Option<MaterializedPath<'static>>,
+	maybe_sub_iso_file_path: Option<IsolatedFilePathData<'static>>,
 }
 
 impl JobInitData for FileIdentifierJobInit {
@@ -78,7 +78,7 @@ impl StatefulJob for FileIdentifierJob {
 		let location_id = state.init.location.id;
 		let location_path = Path::new(&state.init.location.path);
 
-		let maybe_sub_materialized_path = if let Some(ref sub_path) = state.init.sub_path {
+		let maybe_sub_iso_file_path = if let Some(ref sub_path) = state.init.sub_path {
 			let full_path = ensure_sub_path_is_in_location(location_path, sub_path)
 				.await
 				.map_err(FileIdentifierJobError::from)?;
@@ -86,16 +86,25 @@ impl StatefulJob for FileIdentifierJob {
 				.await
 				.map_err(FileIdentifierJobError::from)?;
 
-			Some(
-				MaterializedPath::new(location_id, location_path, &full_path, true)
-					.map_err(FileIdentifierJobError::from)?,
+			let sub_iso_file_path =
+				IsolatedFilePathData::new(location_id, location_path, &full_path, true)
+					.map_err(FileIdentifierJobError::from)?;
+
+			ensure_file_path_exists(
+				sub_path,
+				&sub_iso_file_path,
+				db,
+				FileIdentifierJobError::SubPathNotFound,
 			)
+			.await?;
+
+			Some(sub_iso_file_path)
 		} else {
 			None
 		};
 
 		let orphan_count =
-			count_orphan_file_paths(db, location_id, &maybe_sub_materialized_path).await?;
+			count_orphan_file_paths(db, location_id, &maybe_sub_iso_file_path).await?;
 
 		// Initializing `state.data` here because we need a complete state in case of early finish
 		state.data = Some(FileIdentifierJobState {
@@ -105,10 +114,13 @@ impl StatefulJob for FileIdentifierJob {
 				..Default::default()
 			},
 			cursor: 0,
-			maybe_sub_materialized_path,
+			maybe_sub_iso_file_path,
 		});
 
-		let data = state.data.as_mut().unwrap(); // SAFETY: We just initialized it
+		let data = state
+			.data
+			.as_mut()
+			.expect("critical error: missing data on job state");
 
 		if orphan_count == 0 {
 			return Err(JobError::EarlyFinish {
@@ -133,7 +145,7 @@ impl StatefulJob for FileIdentifierJob {
 			.find_first(orphan_path_filters(
 				location_id,
 				None,
-				&data.maybe_sub_materialized_path,
+				&data.maybe_sub_iso_file_path,
 			))
 			.select(file_path::select!({ id }))
 			.exec()
@@ -142,7 +154,7 @@ impl StatefulJob for FileIdentifierJob {
 
 		data.cursor = first_path.id;
 
-		state.steps = (0..task_count).map(|_| ()).collect();
+		state.steps.extend((0..task_count).map(|_| ()));
 
 		Ok(())
 	}
@@ -155,20 +167,21 @@ impl StatefulJob for FileIdentifierJob {
 		let FileIdentifierJobState {
 			ref mut cursor,
 			ref mut report,
-			ref maybe_sub_materialized_path,
+			ref maybe_sub_iso_file_path,
 		} = state
 			.data
 			.as_mut()
-			.expect("Critical error: missing data on job state");
+			.expect("critical error: missing data on job state");
 
+		let step_number = state.step_number;
 		let location = &state.init.location;
 
 		// get chunk of orphans to process
 		let file_paths = get_orphan_file_paths(
 			&ctx.library.db,
-			state.init.location.id,
+			location.id,
 			*cursor,
-			maybe_sub_materialized_path,
+			maybe_sub_iso_file_path,
 		)
 		.await?;
 
@@ -176,7 +189,7 @@ impl StatefulJob for FileIdentifierJob {
 			<Self as StatefulJob>::NAME,
 			location,
 			&file_paths,
-			state.step_number,
+			step_number,
 			cursor,
 			report,
 			ctx,
@@ -199,7 +212,7 @@ impl StatefulJob for FileIdentifierJob {
 fn orphan_path_filters(
 	location_id: i32,
 	file_path_id: Option<i32>,
-	maybe_sub_materialized_path: &Option<MaterializedPath<'_>>,
+	maybe_sub_iso_file_path: &Option<IsolatedFilePathData<'_>>,
 ) -> Vec<file_path::WhereParam> {
 	chain_optional_iter(
 		[
@@ -210,9 +223,13 @@ fn orphan_path_filters(
 		[
 			// this is a workaround for the cursor not working properly
 			file_path_id.map(file_path::id::gte),
-			maybe_sub_materialized_path
-				.as_ref()
-				.map(|p| file_path::materialized_path::starts_with(p.into())),
+			maybe_sub_iso_file_path.as_ref().map(|sub_iso_file_path| {
+				file_path::materialized_path::starts_with(
+					sub_iso_file_path
+						.materialized_path_for_children()
+						.expect("sub path iso_file_path must be a directory"),
+				)
+			}),
 		],
 	)
 }
@@ -220,7 +237,7 @@ fn orphan_path_filters(
 async fn count_orphan_file_paths(
 	db: &PrismaClient,
 	location_id: i32,
-	maybe_sub_materialized_path: &Option<MaterializedPath<'_>>,
+	maybe_sub_materialized_path: &Option<IsolatedFilePathData<'_>>,
 ) -> Result<usize, prisma_client_rust::QueryError> {
 	db.file_path()
 		.count(orphan_path_filters(
@@ -237,7 +254,7 @@ async fn get_orphan_file_paths(
 	db: &PrismaClient,
 	location_id: i32,
 	file_path_id: i32,
-	maybe_sub_materialized_path: &Option<MaterializedPath<'_>>,
+	maybe_sub_materialized_path: &Option<IsolatedFilePathData<'_>>,
 ) -> Result<Vec<file_path_for_file_identifier::Data>, prisma_client_rust::QueryError> {
 	info!(
 		"Querying {} orphan Paths at cursor: {:?}",

core/src/object/file_identifier/mod.rs

@@ -2,7 +2,9 @@ use crate::{
 	invalidate_query,
 	job::{JobError, JobReportUpdate, JobResult, WorkerContext},
 	library::Library,
-	location::file_path_helper::{file_path_for_file_identifier, FilePathError, MaterializedPath},
+	location::file_path_helper::{
+		file_path_for_file_identifier, FilePathError, IsolatedFilePathData,
+	},
 	object::{cas::generate_cas_id, object_for_file_identifier},
 	prisma::{file_path, location, object, PrismaClient},
 	sync,
@@ -33,8 +35,14 @@ const CHUNK_SIZE: usize = 100;
 
 #[derive(Error, Debug)]
 pub enum FileIdentifierJobError {
-	#[error("File path related error (error: {0})")]
+	#[error("received sub path not in database: <path='{}'>", .0.display())]
+	SubPathNotFound(Box<Path>),
+
+	// Internal Errors
+	#[error(transparent)]
 	FilePathError(#[from] FilePathError),
+	#[error("database error")]
+	Database(#[from] prisma_client_rust::QueryError),
 }
 
 #[derive(Debug, Clone)]
@@ -48,9 +56,9 @@ impl FileMetadata {
 	/// Assembles `create_unchecked` params for a given file path
 	pub async fn new(
 		location_path: impl AsRef<Path>,
-		materialized_path: &MaterializedPath<'_>, // TODO: use dedicated CreateUnchecked type
+		iso_file_path: &IsolatedFilePathData<'_>, // TODO: use dedicated CreateUnchecked type
 	) -> Result<FileMetadata, io::Error> {
-		let path = location_path.as_ref().join(materialized_path);
+		let path = location_path.as_ref().join(iso_file_path);
 
 		let fs_metadata = fs::metadata(&path).await?;
 
@@ -95,7 +103,7 @@ async fn identifier_job_step(
 		// NOTE: `file_path`'s `materialized_path` begins with a `/` character so we remove it to join it with `location.path`
 		FileMetadata::new(
 			&location.path,
-			&MaterializedPath::from((location.id, &file_path.materialized_path)),
+			&IsolatedFilePathData::from((location.id, file_path)),
 		)
 		.await
 		.map(|params| {

core/src/object/file_identifier/shallow_file_identifier_job.rs

@@ -4,11 +4,11 @@ use crate::{
 	},
 	library::Library,
 	location::file_path_helper::{
-		ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
-		file_path_for_file_identifier, get_existing_file_path_id, MaterializedPath,
+		ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
+		file_path_for_file_identifier, IsolatedFilePathData,
 	},
 	prisma::{file_path, location, PrismaClient},
-	util::db::{chain_optional_iter, uuid_to_bytes},
+	util::db::chain_optional_iter,
 };
 
 use std::{
@@ -19,7 +19,6 @@ use std::{
 use prisma_client_rust::Direction;
 use serde::{Deserialize, Serialize};
 use tracing::info;
-use uuid::Uuid;
 
 use super::{
 	finalize_file_identifier, process_identifier_file_paths, FileIdentifierJobError,
@@ -49,7 +48,7 @@ impl Hash for ShallowFileIdentifierJobInit {
 pub struct ShallowFileIdentifierJobState {
 	cursor: i32,
 	report: FileIdentifierReport,
-	sub_path_id: Uuid,
+	sub_iso_file_path: IsolatedFilePathData<'static>,
 }
 
 impl JobInitData for ShallowFileIdentifierJobInit {
@@ -77,7 +76,7 @@ impl StatefulJob for ShallowFileIdentifierJob {
 		let location_id = state.init.location.id;
 		let location_path = Path::new(&state.init.location.path);
 
-		let sub_path_id = if state.init.sub_path != Path::new("") {
+		let sub_iso_file_path = if state.init.sub_path != Path::new("") {
 			let full_path = ensure_sub_path_is_in_location(location_path, &state.init.sub_path)
 				.await
 				.map_err(FileIdentifierJobError::from)?;
@@ -85,26 +84,25 @@ impl StatefulJob for ShallowFileIdentifierJob {
 				.await
 				.map_err(FileIdentifierJobError::from)?;
 
-			get_existing_file_path_id(
-				&MaterializedPath::new(location_id, location_path, &full_path, true)
-					.map_err(FileIdentifierJobError::from)?,
+			let sub_iso_file_path =
+				IsolatedFilePathData::new(location_id, location_path, &full_path, true)
+					.map_err(FileIdentifierJobError::from)?;
+
+			ensure_file_path_exists(
+				&state.init.sub_path,
+				&sub_iso_file_path,
 				db,
+				FileIdentifierJobError::SubPathNotFound,
 			)
-			.await
-			.map_err(FileIdentifierJobError::from)?
-			.expect("Sub path should already exist in the database")
+			.await?;
+
+			sub_iso_file_path
 		} else {
-			get_existing_file_path_id(
-				&MaterializedPath::new(location_id, location_path, location_path, true)
-					.map_err(FileIdentifierJobError::from)?,
-				db,
-			)
-			.await
-			.map_err(FileIdentifierJobError::from)?
-			.expect("Location root path should already exist in the database")
+			IsolatedFilePathData::new(location_id, location_path, location_path, true)
+				.map_err(FileIdentifierJobError::from)?
 		};
 
-		let orphan_count = count_orphan_file_paths(db, location_id, sub_path_id).await?;
+		let orphan_count = count_orphan_file_paths(db, location_id, &sub_iso_file_path).await?;
 
 		// Initializing `state.data` here because we need a complete state in case of early finish
 		state.data = Some(ShallowFileIdentifierJobState {
@@ -114,7 +112,7 @@ impl StatefulJob for ShallowFileIdentifierJob {
 				..Default::default()
 			},
 			cursor: 0,
-			sub_path_id,
+			sub_iso_file_path,
 		});
 
 		if orphan_count == 0 {
@@ -135,19 +133,27 @@ impl StatefulJob for ShallowFileIdentifierJob {
 		// update job with total task count based on orphan file_paths count
 		ctx.progress(vec![JobReportUpdate::TaskCount(task_count)]);
 
+		let mut data = state
+			.data
+			.as_mut()
+			.expect("critical error: missing data on job state");
+
 		let first_path = db
 			.file_path()
-			.find_first(orphan_path_filters(location_id, None, sub_path_id))
+			.find_first(orphan_path_filters(
+				location_id,
+				None,
+				&data.sub_iso_file_path,
+			))
 			// .order_by(file_path::id::order(Direction::Asc))
 			.select(file_path::select!({ id }))
 			.exec()
 			.await?
 			.unwrap(); // SAFETY: We already validated before that there are orphans `file_path`s
 
-		// SAFETY: We just initialized `state.data` above
-		state.data.as_mut().unwrap().cursor = first_path.id;
+		data.cursor = first_path.id;
 
-		state.steps = (0..task_count).map(|_| ()).collect();
+		state.steps.extend((0..task_count).map(|_| ()));
 
 		Ok(())
 	}
@@ -160,22 +166,17 @@ impl StatefulJob for ShallowFileIdentifierJob {
 		let ShallowFileIdentifierJobState {
 			ref mut cursor,
 			ref mut report,
-			ref sub_path_id,
+			ref sub_iso_file_path,
 		} = state
 			.data
 			.as_mut()
-			.expect("Critical error: missing data on job state");
+			.expect("critical error: missing data on job state");
 
 		let location = &state.init.location;
 
 		// get chunk of orphans to process
-		let file_paths = get_orphan_file_paths(
-			&ctx.library.db,
-			state.init.location.id,
-			*cursor,
-			*sub_path_id,
-		)
-		.await?;
+		let file_paths =
+			get_orphan_file_paths(&ctx.library.db, location.id, *cursor, sub_iso_file_path).await?;
 
 		process_identifier_file_paths(
 			<Self as StatefulJob>::NAME,
@@ -204,14 +205,18 @@ impl StatefulJob for ShallowFileIdentifierJob {
 fn orphan_path_filters(
 	location_id: i32,
 	file_path_id: Option<i32>,
-	sub_path_id: Uuid,
+	sub_iso_file_path: &IsolatedFilePathData<'_>,
 ) -> Vec<file_path::WhereParam> {
 	chain_optional_iter(
 		[
 			file_path::object_id::equals(None),
 			file_path::is_dir::equals(false),
 			file_path::location_id::equals(location_id),
-			file_path::parent_id::equals(Some(uuid_to_bytes(sub_path_id))),
+			file_path::materialized_path::equals(
+				sub_iso_file_path
+					.materialized_path_for_children()
+					.expect("sub path for shallow identifier must be a directory"),
+			),
 		],
 		[file_path_id.map(file_path::id::gte)],
 	)
@@ -220,10 +225,10 @@ fn orphan_path_filters(
 async fn count_orphan_file_paths(
 	db: &PrismaClient,
 	location_id: i32,
-	sub_path_id: Uuid,
+	sub_iso_file_path: &IsolatedFilePathData<'_>,
 ) -> Result<usize, prisma_client_rust::QueryError> {
 	db.file_path()
-		.count(orphan_path_filters(location_id, None, sub_path_id))
+		.count(orphan_path_filters(location_id, None, sub_iso_file_path))
 		.exec()
 		.await
 		.map(|c| c as usize)
@@ -232,15 +237,19 @@ async fn count_orphan_file_paths(
 async fn get_orphan_file_paths(
 	db: &PrismaClient,
 	location_id: i32,
-	cursor: i32,
-	sub_path_id: Uuid,
+	file_path_id_cursor: i32,
+	sub_iso_file_path: &IsolatedFilePathData<'_>,
 ) -> Result<Vec<file_path_for_file_identifier::Data>, prisma_client_rust::QueryError> {
 	info!(
 		"Querying {} orphan Paths at cursor: {:?}",
-		CHUNK_SIZE, cursor
+		CHUNK_SIZE, file_path_id_cursor
 	);
 	db.file_path()
-		.find_many(orphan_path_filters(location_id, Some(cursor), sub_path_id))
+		.find_many(orphan_path_filters(
+			location_id,
+			Some(file_path_id_cursor),
+			sub_iso_file_path,
+		))
 		.order_by(file_path::id::order(Direction::Asc))
 		// .cursor(cursor.into())
 		.take(CHUNK_SIZE as i64)

core/src/object/fs/copy.rs

@@ -3,6 +3,7 @@ use crate::{
 	job::{
 		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
 	},
+	util::error::FileIOError,
 };
 
 use std::{hash::Hash, path::PathBuf};
@@ -12,7 +13,7 @@ use specta::Type;
 use tokio::fs;
 use tracing::{trace, warn};
 
-use super::{context_menu_fs_info, get_path_from_location_id, osstr_to_string, FsInfo};
+use super::{context_menu_fs_info, get_location_path_from_location_id, osstr_to_string, FsInfo};
 
 pub struct FileCopierJob {}
 
@@ -76,7 +77,8 @@ impl StatefulJob for FileCopierJob {
 		.await?;
 
 		let mut full_target_path =
-			get_path_from_location_id(&ctx.library.db, state.init.target_location_id).await?;
+			get_location_path_from_location_id(&ctx.library.db, state.init.target_location_id)
+				.await?;
 
 		// add the currently viewed subdirectory to the location root
 		full_target_path.push(&state.init.target_path);
@@ -109,7 +111,7 @@ impl StatefulJob for FileCopierJob {
 			source_fs_info: source_fs_info.clone(),
 		});
 
-		state.steps = [source_fs_info.into()].into_iter().collect();
+		state.steps.push_back(source_fs_info.into());
 
 		ctx.progress(vec![JobReportUpdate::TaskCount(state.steps.len())]);
 
@@ -121,26 +123,32 @@ impl StatefulJob for FileCopierJob {
 		ctx: WorkerContext,
 		state: &mut JobState<Self>,
 	) -> Result<(), JobError> {
-		let step = &state.steps[0];
+		let data = state
+			.data
+			.as_ref()
+			.expect("critical error: missing data on job state");
 
-		let job_state = state.data.as_ref().ok_or(JobError::MissingData {
-			value: String::from("job state"),
-		})?;
-
-		match step {
+		match &state.steps[0] {
 			FileCopierJobStep::File { path } => {
-				let mut target_path = job_state.target_path.clone();
+				let mut target_path = data.target_path.clone();
 
-				if job_state.source_fs_info.path_data.is_dir {
+				if data.source_fs_info.path_data.is_dir {
 					// if root type is a dir, we need to preserve structure by making paths relative
 					target_path.push(
-						path.strip_prefix(&job_state.source_fs_info.fs_path)
+						path.strip_prefix(&data.source_fs_info.fs_path)
 							.map_err(|_| JobError::Path)?,
 					);
 				}
 
-				if fs::canonicalize(path.parent().ok_or(JobError::Path)?).await?
-					== fs::canonicalize(target_path.parent().ok_or(JobError::Path)?).await?
+				let parent_path = path.parent().ok_or(JobError::Path)?;
+				let parent_target_path = target_path.parent().ok_or(JobError::Path)?;
+
+				if fs::canonicalize(parent_path)
+					.await
+					.map_err(|e| FileIOError::from((parent_path, e)))?
+					== fs::canonicalize(parent_target_path)
+						.await
+						.map_err(|e| FileIOError::from((parent_target_path, e)))?
 				{
 					return Err(JobError::MatchingSrcDest(path.clone()));
 				}
@@ -159,35 +167,51 @@ impl StatefulJob for FileCopierJob {
 						target_path.display()
 					);
 
-					fs::copy(&path, &target_path).await?;
+					fs::copy(&path, &target_path)
+						.await
+						.map_err(|e| FileIOError::from((&target_path, e)))?;
 				}
 			}
 			FileCopierJobStep::Directory { path } => {
 				// if this is the very first path, create the target dir
 				// fixes copying dirs with no child directories
-				if job_state.source_fs_info.path_data.is_dir
-					&& &job_state.source_fs_info.fs_path == path
-				{
-					fs::create_dir_all(&job_state.target_path).await?;
+				if data.source_fs_info.path_data.is_dir && &data.source_fs_info.fs_path == path {
+					fs::create_dir_all(&data.target_path)
+						.await
+						.map_err(|e| FileIOError::from((&data.target_path, e)))?;
 				}
 
-				let mut dir = fs::read_dir(&path).await?;
+				let path = path.clone(); // To appease the borrowck
 
-				while let Some(entry) = dir.next_entry().await? {
-					if entry.metadata().await?.is_dir() {
+				let mut dir = fs::read_dir(&path)
+					.await
+					.map_err(|e| FileIOError::from((&path, e)))?;
+
+				while let Some(entry) = dir
+					.next_entry()
+					.await
+					.map_err(|e| FileIOError::from((&path, e)))?
+				{
+					let entry_path = entry.path();
+					if entry
+						.metadata()
+						.await
+						.map_err(|e| FileIOError::from((&entry_path, e)))?
+						.is_dir()
+					{
 						state
 							.steps
 							.push_back(FileCopierJobStep::Directory { path: entry.path() });
 
-						fs::create_dir_all(
-							job_state.target_path.join(
-								entry
-									.path()
-									.strip_prefix(&job_state.source_fs_info.fs_path)
-									.map_err(|_| JobError::Path)?,
-							),
-						)
-						.await?;
+						let full_path = data.target_path.join(
+							entry_path
+								.strip_prefix(&data.source_fs_info.fs_path)
+								.map_err(|_| JobError::Path)?,
+						);
+
+						fs::create_dir_all(&full_path)
+							.await
+							.map_err(|e| FileIOError::from((full_path, e)))?;
 					} else {
 						state
 							.steps

core/src/object/fs/cut.rs

@@ -3,6 +3,7 @@ use crate::{
 	job::{
 		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
 	},
+	util::error::FileIOError,
 };
 
 use std::{hash::Hash, path::PathBuf};
@@ -12,13 +13,10 @@ use specta::Type;
 use tokio::fs;
 use tracing::{trace, warn};
 
-use super::{context_menu_fs_info, get_path_from_location_id, FsInfo};
+use super::{context_menu_fs_info, get_location_path_from_location_id, FsInfo};
 
 pub struct FileCutterJob {}
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct FileCutterJobState {}
-
 #[derive(Serialize, Deserialize, Hash, Type)]
 pub struct FileCutterJobInit {
 	pub source_location_id: i32,
@@ -40,7 +38,7 @@ impl JobInitData for FileCutterJobInit {
 #[async_trait::async_trait]
 impl StatefulJob for FileCutterJob {
 	type Init = FileCutterJobInit;
-	type Data = FileCutterJobState;
+	type Data = ();
 	type Step = FileCutterJobStep;
 
 	const NAME: &'static str = "file_cutter";
@@ -58,15 +56,14 @@ impl StatefulJob for FileCutterJob {
 		.await?;
 
 		let mut full_target_path =
-			get_path_from_location_id(&ctx.library.db, state.init.target_location_id).await?;
+			get_location_path_from_location_id(&ctx.library.db, state.init.target_location_id)
+				.await?;
 		full_target_path.push(&state.init.target_path);
 
-		state.steps = [FileCutterJobStep {
+		state.steps.push_back(FileCutterJobStep {
 			source_fs_info,
 			target_directory: full_target_path,
-		}]
-		.into_iter()
-		.collect();
+		});
 
 		ctx.progress(vec![JobReportUpdate::TaskCount(state.steps.len())]);
 
@@ -85,14 +82,16 @@ impl StatefulJob for FileCutterJob {
 			.target_directory
 			.join(source_info.fs_path.file_name().ok_or(JobError::OsStr)?);
 
-		if fs::canonicalize(
-			source_info
-				.fs_path
-				.parent()
-				.map_or(Err(JobError::Path), Ok)?,
-		)
-		.await? == fs::canonicalize(full_output.parent().map_or(Err(JobError::Path), Ok)?)
-			.await?
+		let parent_source = source_info.fs_path.parent().ok_or(JobError::Path)?;
+
+		let parent_output = full_output.parent().ok_or(JobError::Path)?;
+
+		if fs::canonicalize(parent_source)
+			.await
+			.map_err(|e| FileIOError::from((parent_source, e)))?
+			== fs::canonicalize(parent_output)
+				.await
+				.map_err(|e| FileIOError::from((parent_output, e)))?
 		{
 			return Err(JobError::MatchingSrcDest(source_info.fs_path.clone()));
 		}
@@ -112,7 +111,9 @@ impl StatefulJob for FileCutterJob {
 			full_output.display()
 		);
 
-		fs::rename(&source_info.fs_path, &full_output).await?;
+		fs::rename(&source_info.fs_path, &full_output)
+			.await
+			.map_err(|e| FileIOError::from((&source_info.fs_path, e)))?;
 
 		ctx.progress(vec![JobReportUpdate::CompletedTaskCount(
 			state.step_number + 1,

core/src/object/fs/decrypt.rs

@@ -1,7 +1,7 @@
 use sd_crypto::{crypto::Decryptor, header::file::FileHeader, Protected};
 use serde::{Deserialize, Serialize};
 use specta::Type;
-use std::{collections::VecDeque, path::PathBuf};
+use std::path::PathBuf;
 use tokio::fs::File;
 
 use crate::{
@@ -9,6 +9,7 @@ use crate::{
 	job::{
 		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
 	},
+	util::error::FileIOError,
 };
 
 use super::{context_menu_fs_info, FsInfo, BYTES_EXT};
@@ -55,7 +56,6 @@ impl StatefulJob for FileDecryptorJob {
 			context_menu_fs_info(&ctx.library.db, state.init.location_id, state.init.path_id)
 				.await?;
 
-		state.steps = VecDeque::new();
 		state.steps.push_back(FileDecryptorJobStep { fs_info });
 
 		ctx.progress(vec![JobReportUpdate::TaskCount(state.steps.len())]);
@@ -68,8 +68,7 @@ impl StatefulJob for FileDecryptorJob {
 		ctx: WorkerContext,
 		state: &mut JobState<Self>,
 	) -> Result<(), JobError> {
-		let step = &state.steps[0];
-		let info = &step.fs_info;
+		let info = &&state.steps[0].fs_info;
 		let key_manager = &ctx.library.key_manager;
 
 		// handle overwriting checks, and making sure there's enough available space
@@ -89,8 +88,12 @@ impl StatefulJob for FileDecryptorJob {
 			|p| p,
 		);
 
-		let mut reader = File::open(info.fs_path.clone()).await?;
-		let mut writer = File::create(output_path).await?;
+		let mut reader = File::open(info.fs_path.clone())
+			.await
+			.map_err(|e| FileIOError::from((&info.fs_path, e)))?;
+		let mut writer = File::create(&output_path)
+			.await
+			.map_err(|e| FileIOError::from((output_path, e)))?;
 
 		let (header, aad) = FileHeader::from_reader(&mut reader).await?;
 

core/src/object/fs/delete.rs

@@ -3,6 +3,7 @@ use crate::{
 	job::{
 		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
 	},
+	util::error::FileIOError,
 };
 
 use std::hash::Hash;
@@ -44,7 +45,7 @@ impl StatefulJob for FileDeleterJob {
 			context_menu_fs_info(&ctx.library.db, state.init.location_id, state.init.path_id)
 				.await?;
 
-		state.steps = [fs_info].into_iter().collect();
+		state.steps.push_back(fs_info);
 
 		ctx.progress(vec![JobReportUpdate::TaskCount(state.steps.len())]);
 
@@ -62,10 +63,11 @@ impl StatefulJob for FileDeleterJob {
 		// maybe a files.countOccurances/and or files.getPath(location_id, path_id) to show how many of these files would be deleted (and where?)
 
 		if info.path_data.is_dir {
-			tokio::fs::remove_dir_all(info.fs_path.clone()).await
+			tokio::fs::remove_dir_all(&info.fs_path).await
 		} else {
-			tokio::fs::remove_file(info.fs_path.clone()).await
-		}?;
+			tokio::fs::remove_file(&info.fs_path).await
+		}
+		.map_err(|e| FileIOError::from((&info.fs_path, e)))?;
 
 		ctx.progress(vec![JobReportUpdate::CompletedTaskCount(
 			state.step_number + 1,

core/src/object/fs/encrypt.rs

@@ -1,4 +1,4 @@
-use crate::{invalidate_query, job::*, library::Library};
+use crate::{invalidate_query, job::*, library::Library, util::error::FileIOError};
 
 use std::path::PathBuf;
 
@@ -19,9 +19,6 @@ use super::{context_menu_fs_info, FsInfo, BYTES_EXT};
 
 pub struct FileEncryptorJob;
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct FileEncryptorJobState {}
-
 #[derive(Serialize, Deserialize, Type, Hash)]
 pub struct FileEncryptorJobInit {
 	pub location_id: i32,
@@ -51,7 +48,7 @@ impl JobInitData for FileEncryptorJobInit {
 #[async_trait::async_trait]
 impl StatefulJob for FileEncryptorJob {
 	type Init = FileEncryptorJobInit;
-	type Data = FileEncryptorJobState;
+	type Data = ();
 	type Step = FsInfo;
 
 	const NAME: &'static str = "file_encryptor";
@@ -61,14 +58,13 @@ impl StatefulJob for FileEncryptorJob {
 	}
 
 	async fn init(&self, ctx: WorkerContext, state: &mut JobState<Self>) -> Result<(), JobError> {
-		let step =
+		state.steps.push_back(
 			context_menu_fs_info(&ctx.library.db, state.init.location_id, state.init.path_id)
 				.await
 				.map_err(|_| JobError::MissingData {
 					value: String::from("file_path that matches both location id and path id"),
-				})?;
-
-		state.steps = [step].into_iter().collect();
+				})?,
+		);
 
 		ctx.progress(vec![JobReportUpdate::TaskCount(state.steps.len())]);
 
@@ -139,8 +135,12 @@ impl StatefulJob for FileEncryptorJob {
 					Some,
 				);
 
-			let mut reader = File::open(&info.fs_path).await?;
-			let mut writer = File::create(output_path).await?;
+			let mut reader = File::open(&info.fs_path)
+				.await
+				.map_err(|e| FileIOError::from((&info.fs_path, e)))?;
+			let mut writer = File::create(&output_path)
+				.await
+				.map_err(|e| FileIOError::from((output_path, e)))?;
 
 			let master_key = Key::generate();
 
@@ -199,8 +199,13 @@ impl StatefulJob for FileEncryptorJob {
 
 					if tokio::fs::metadata(&pvm_path).await.is_ok() {
 						let mut pvm_bytes = Vec::new();
-						let mut pvm_file = File::open(pvm_path).await?;
-						pvm_file.read_to_end(&mut pvm_bytes).await?;
+						let mut pvm_file = File::open(&pvm_path)
+							.await
+							.map_err(|e| FileIOError::from((&pvm_path, e)))?;
+						pvm_file
+							.read_to_end(&mut pvm_bytes)
+							.await
+							.map_err(|e| FileIOError::from((pvm_path, e)))?;
 
 						header
 							.add_preview_media(

core/src/object/fs/erase.rs

@@ -3,6 +3,7 @@ use crate::{
 	job::{
 		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
 	},
+	util::error::FileIOError,
 };
 
 use std::{hash::Hash, path::PathBuf};
@@ -11,7 +12,7 @@ use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use specta::Type;
 use tokio::{fs::OpenOptions, io::AsyncWriteExt};
-use tracing::{trace, warn};
+use tracing::trace;
 
 use super::{context_menu_fs_info, FsInfo};
 
@@ -70,7 +71,7 @@ impl StatefulJob for FileEraserJob {
 
 		state.data = Some(fs_info.clone());
 
-		state.steps = [fs_info.into()].into_iter().collect();
+		state.steps.push_back(fs_info.into());
 
 		ctx.progress(vec![JobReportUpdate::TaskCount(state.steps.len())]);
 
@@ -82,39 +83,65 @@ impl StatefulJob for FileEraserJob {
 		ctx: WorkerContext,
 		state: &mut JobState<Self>,
 	) -> Result<(), JobError> {
-		let step = &state.steps[0];
-
 		// need to handle stuff such as querying prisma for all paths of a file, and deleting all of those if requested (with a checkbox in the ui)
 		// maybe a files.countOccurances/and or files.getPath(location_id, path_id) to show how many of these files would be erased (and where?)
 
-		match step {
+		match &state.steps[0] {
 			FileEraserJobStep::File { path } => {
 				let mut file = OpenOptions::new()
 					.read(true)
 					.write(true)
-					.open(&path)
-					.await?;
-				let file_len = file.metadata().await?.len();
+					.open(path)
+					.await
+					.map_err(|e| FileIOError::from((path, e)))?;
+				let file_len = file
+					.metadata()
+					.await
+					.map_err(|e| FileIOError::from((path, e)))?
+					.len();
 
 				sd_crypto::fs::erase::erase(&mut file, file_len as usize, state.init.passes)
 					.await?;
-				file.set_len(0).await?;
-				file.flush().await?;
+
+				file.set_len(0)
+					.await
+					.map_err(|e| FileIOError::from((path, e)))?;
+				file.flush()
+					.await
+					.map_err(|e| FileIOError::from((path, e)))?;
 				drop(file);
 
 				trace!("Erasing file: {:?}", path);
 
-				tokio::fs::remove_file(&path).await?;
+				tokio::fs::remove_file(path)
+					.await
+					.map_err(|e| FileIOError::from((path, e)))?;
 			}
 			FileEraserJobStep::Directory { path } => {
-				let mut dir = tokio::fs::read_dir(&path).await?;
+				let path = path.clone(); // To appease the borrowck
 
-				while let Some(entry) = dir.next_entry().await? {
-					state.steps.push_back(if entry.metadata().await?.is_dir() {
-						FileEraserJobStep::Directory { path: entry.path() }
-					} else {
-						FileEraserJobStep::File { path: entry.path() }
-					});
+				let mut dir = tokio::fs::read_dir(&path)
+					.await
+					.map_err(|e| FileIOError::from((&path, e)))?;
+
+				while let Some(entry) = dir
+					.next_entry()
+					.await
+					.map_err(|e| FileIOError::from((&path, e)))?
+				{
+					let entry_path = entry.path();
+					state.steps.push_back(
+						if entry
+							.metadata()
+							.await
+							.map_err(|e| FileIOError::from((&entry_path, e)))?
+							.is_dir()
+						{
+							FileEraserJobStep::Directory { path: entry_path }
+						} else {
+							FileEraserJobStep::File { path: entry_path }
+						},
+					);
 
 					ctx.progress(vec![JobReportUpdate::TaskCount(state.steps.len())]);
 				}
@@ -128,12 +155,14 @@ impl StatefulJob for FileEraserJob {
 	}
 
 	async fn finalize(&mut self, ctx: WorkerContext, state: &mut JobState<Self>) -> JobResult {
-		if let Some(ref info) = state.data {
-			if info.path_data.is_dir {
-				tokio::fs::remove_dir_all(&info.fs_path).await?;
-			}
-		} else {
-			warn!("missing job state, unable to fully finalise erase job");
+		let data = state
+			.data
+			.as_ref()
+			.expect("critical error: missing data on job state");
+		if data.path_data.is_dir {
+			tokio::fs::remove_dir_all(&data.fs_path)
+				.await
+				.map_err(|e| FileIOError::from((&data.fs_path, e)))?;
 		}
 
 		invalidate_query!(ctx.library, "search.paths");

core/src/object/fs/mod.rs

@@ -1,6 +1,6 @@
 use crate::{
 	job::JobError,
-	location::file_path_helper::{file_path_with_object, MaterializedPath},
+	location::file_path_helper::{file_path_with_object, IsolatedFilePathData},
 	prisma::{file_path, location, PrismaClient},
 };
 
@@ -42,7 +42,7 @@ pub fn osstr_to_string(os_str: Option<&OsStr>) -> Result<String, JobError> {
 		.ok_or(JobError::OsStr)
 }
 
-pub async fn get_path_from_location_id(
+pub async fn get_location_path_from_location_id(
 	db: &PrismaClient,
 	location_id: i32,
 ) -> Result<PathBuf, JobError> {
@@ -74,12 +74,9 @@ pub async fn context_menu_fs_info(
 		})?;
 
 	Ok(FsInfo {
-		fs_path: get_path_from_location_id(db, location_id)
+		fs_path: get_location_path_from_location_id(db, location_id)
 			.await?
-			.join(&MaterializedPath::from((
-				location_id,
-				&path_data.materialized_path,
-			))),
+			.join(IsolatedFilePathData::from(&path_data)),
 		path_data,
 	})
 }

core/src/object/preview/thumbnail/mod.rs

@@ -1,14 +1,15 @@
 use crate::{
 	api::CoreEvent,
 	invalidate_query,
-	job::{JobError, JobReportUpdate, JobResult, WorkerContext},
+	job::{
+		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
+	},
 	library::Library,
 	location::{
-		file_path_helper::{
-			file_path_just_materialized_path_cas_id, FilePathError, MaterializedPath,
-		},
+		file_path_helper::{file_path_for_thumbnailer, FilePathError, IsolatedFilePathData},
 		LocationId,
 	},
+	util::error::FileIOError,
 };
 
 use std::{
@@ -75,16 +76,22 @@ pub struct ThumbnailerJobState {
 
 #[derive(Error, Debug)]
 pub enum ThumbnailerError {
-	#[error("File path related error (error: {0})")]
-	FilePathError(#[from] FilePathError),
-	#[error("IO error (error: {0})")]
-	IOError(#[from] io::Error),
+	#[error("sub path not found: <path='{}'>", .0.display())]
+	SubPathNotFound(Box<Path>),
+
+	// Internal errors
+	#[error("database error")]
+	Database(#[from] prisma_client_rust::QueryError),
+	#[error(transparent)]
+	FilePath(#[from] FilePathError),
+	#[error(transparent)]
+	FileIO(#[from] FileIOError),
 }
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct ThumbnailerJobReport {
 	location_id: LocationId,
-	materialized_path: String,
+	path: PathBuf,
 	thumbnails_created: u32,
 }
 
@@ -97,7 +104,7 @@ enum ThumbnailerJobStepKind {
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct ThumbnailerJobStep {
-	file_path: file_path_just_materialized_path_cas_id::Data,
+	file_path: file_path_for_thumbnailer::Data,
 	kind: ThumbnailerJobStepKind,
 }
 
@@ -160,12 +167,7 @@ fn finalize_thumbnailer(data: &ThumbnailerJobState, ctx: WorkerContext) -> JobRe
 	info!(
 		"Finished thumbnail generation for location {} at {}",
 		data.report.location_id,
-		data.location_path
-			.join(&MaterializedPath::from((
-				data.report.location_id,
-				&data.report.materialized_path
-			)))
-			.display()
+		data.report.path.display()
 	);
 
 	if data.report.thumbnails_created > 0 {
@@ -175,43 +177,56 @@ fn finalize_thumbnailer(data: &ThumbnailerJobState, ctx: WorkerContext) -> JobRe
 	Ok(Some(serde_json::to_value(&data.report)?))
 }
 
-async fn process_step(
-	is_background: bool,
-	step_number: usize,
-	step: &ThumbnailerJobStep,
-	data: &mut ThumbnailerJobState,
+async fn process_step<SJob, Init>(
+	state: &mut JobState<SJob>,
 	ctx: WorkerContext,
-) -> Result<(), JobError> {
+) -> Result<(), JobError>
+where
+	SJob: StatefulJob<Init = Init, Data = ThumbnailerJobState, Step = ThumbnailerJobStep>,
+	Init: JobInitData<Job = SJob>,
+{
+	let step = &state.steps[0];
+
 	ctx.progress(vec![JobReportUpdate::Message(format!(
 		"Processing {}",
 		step.file_path.materialized_path
 	))]);
 
-	let step_result = inner_process_step(is_background, step, data, &ctx).await;
+	let step_result = inner_process_step(state, &ctx).await;
 
-	ctx.progress(vec![JobReportUpdate::CompletedTaskCount(step_number + 1)]);
+	ctx.progress(vec![JobReportUpdate::CompletedTaskCount(
+		state.step_number + 1,
+	)]);
 
 	step_result
 }
 
-async fn inner_process_step(
-	is_background: bool,
-	step: &ThumbnailerJobStep,
-	data: &mut ThumbnailerJobState,
+async fn inner_process_step<SJob, Init>(
+	state: &mut JobState<SJob>,
 	ctx: &WorkerContext,
-) -> Result<(), JobError> {
+) -> Result<(), JobError>
+where
+	SJob: StatefulJob<Init = Init, Data = ThumbnailerJobState, Step = ThumbnailerJobStep>,
+	Init: JobInitData<Job = SJob>,
+{
+	let ThumbnailerJobStep { file_path, kind } = &state.steps[0];
+	let data = state
+		.data
+		.as_ref()
+		.expect("critical error: missing data on job state");
+
 	// assemble the file path
-	let path = data.location_path.join(&MaterializedPath::from((
+	let path = data.location_path.join(IsolatedFilePathData::from((
 		data.report.location_id,
-		&step.file_path.materialized_path,
+		file_path,
 	)));
-	trace!("image_file {:?}", step);
+	trace!("image_file {:?}", file_path);
 
 	// get cas_id, if none found skip
-	let Some(cas_id) = &step.file_path.cas_id else {
+	let Some(cas_id) = &file_path.cas_id else {
 		warn!(
 			"skipping thumbnail generation for {}",
-			step.file_path.materialized_path
+			file_path.materialized_path
 		);
 
 		return Ok(());
@@ -227,7 +242,7 @@ async fn inner_process_step(
 		Err(e) if e.kind() == io::ErrorKind::NotFound => {
 			info!("Writing {:?} to {:?}", path, output_path);
 
-			match step.kind {
+			match kind {
 				ThumbnailerJobStepKind::Image => {
 					if let Err(e) = generate_image_thumbnail(&path, &output_path).await {
 						error!("Error generating thumb for image {:#?}", e);
@@ -246,9 +261,14 @@ async fn inner_process_step(
 				cas_id: cas_id.clone(),
 			});
 
-			data.report.thumbnails_created += 1;
+			state
+				.data
+				.as_mut()
+				.expect("critical error: missing data on job state")
+				.report
+				.thumbnails_created += 1;
 		}
-		Err(e) => return Err(ThumbnailerError::from(e).into()),
+		Err(e) => return Err(ThumbnailerError::from(FileIOError::from((output_path, e))).into()),
 	}
 
 	Ok(())

core/src/object/preview/thumbnail/shallow_thumbnailer_job.rs

@@ -5,19 +5,19 @@ use crate::{
 	library::Library,
 	location::{
 		file_path_helper::{
-			ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
-			file_path_just_materialized_path_cas_id, get_existing_file_path_id, MaterializedPath,
+			ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
+			file_path_for_thumbnailer, IsolatedFilePathData,
 		},
 		LocationId,
 	},
 	prisma::{file_path, location, PrismaClient},
-	util::db::uuid_to_bytes,
+	util::error::FileIOError,
 };
 
 use std::{
 	collections::VecDeque,
 	hash::Hash,
-	path::{Path, PathBuf, MAIN_SEPARATOR_STR},
+	path::{Path, PathBuf},
 };
 
 use sd_file_ext::extensions::Extension;
@@ -25,7 +25,6 @@ use sd_file_ext::extensions::Extension;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tracing::info;
-use uuid::Uuid;
 
 use super::{
 	finalize_thumbnailer, process_step, ThumbnailerError, ThumbnailerJobReport,
@@ -80,7 +79,7 @@ impl StatefulJob for ShallowThumbnailerJob {
 		let location_id = state.init.location.id;
 		let location_path = PathBuf::from(&state.init.location.path);
 
-		let sub_path_id = if state.init.sub_path != Path::new("") {
+		let (path, iso_file_path) = if state.init.sub_path != Path::new("") {
 			let full_path = ensure_sub_path_is_in_location(&location_path, &state.init.sub_path)
 				.await
 				.map_err(ThumbnailerError::from)?;
@@ -88,35 +87,42 @@ impl StatefulJob for ShallowThumbnailerJob {
 				.await
 				.map_err(ThumbnailerError::from)?;
 
-			get_existing_file_path_id(
-				&MaterializedPath::new(location_id, &location_path, &full_path, true)
-					.map_err(ThumbnailerError::from)?,
+			let sub_iso_file_path =
+				IsolatedFilePathData::new(location_id, &location_path, &full_path, true)
+					.map_err(ThumbnailerError::from)?;
+
+			ensure_file_path_exists(
+				&state.init.sub_path,
+				&sub_iso_file_path,
 				db,
+				ThumbnailerError::SubPathNotFound,
 			)
-			.await
-			.map_err(ThumbnailerError::from)?
-			.expect("Sub path should already exist in the database")
+			.await?;
+
+			(full_path, sub_iso_file_path)
 		} else {
-			get_existing_file_path_id(
-				&MaterializedPath::new(location_id, &location_path, &location_path, true)
+			(
+				location_path.to_path_buf(),
+				IsolatedFilePathData::new(location_id, &location_path, &location_path, true)
 					.map_err(ThumbnailerError::from)?,
-				db,
 			)
-			.await
-			.map_err(ThumbnailerError::from)?
-			.expect("Location root path should already exist in the database")
 		};
 
-		info!("Searching for images in location {location_id} at parent directory with id {sub_path_id}");
+		info!(
+			"Searching for images in location {location_id} at path {}",
+			path.display()
+		);
 
 		// create all necessary directories if they don't exist
-		fs::create_dir_all(&thumbnail_dir).await?;
+		fs::create_dir_all(&thumbnail_dir)
+			.await
+			.map_err(|e| FileIOError::from((&thumbnail_dir, e)))?;
 
 		// query database for all image files in this location that need thumbnails
 		let image_files = get_files_by_extensions(
 			db,
 			location_id,
-			sub_path_id,
+			&iso_file_path,
 			&FILTERED_IMAGE_EXTENSIONS,
 			ThumbnailerJobStepKind::Image,
 		)
@@ -129,7 +135,7 @@ impl StatefulJob for ShallowThumbnailerJob {
 			let video_files = get_files_by_extensions(
 				db,
 				location_id,
-				sub_path_id,
+				&iso_file_path,
 				&FILTERED_VIDEO_EXTENSIONS,
 				ThumbnailerJobStepKind::Video,
 			)
@@ -154,16 +160,11 @@ impl StatefulJob for ShallowThumbnailerJob {
 			location_path,
 			report: ThumbnailerJobReport {
 				location_id,
-				materialized_path: if state.init.sub_path != Path::new("") {
-					// SAFETY: We know that the sub_path is a valid UTF-8 string because we validated it before
-					state.init.sub_path.to_str().unwrap().to_string()
-				} else {
-					MAIN_SEPARATOR_STR.to_string()
-				},
+				path,
 				thumbnails_created: 0,
 			},
 		});
-		state.steps = all_files;
+		state.steps.extend(all_files);
 
 		Ok(())
 	}
@@ -173,17 +174,7 @@ impl StatefulJob for ShallowThumbnailerJob {
 		ctx: WorkerContext,
 		state: &mut JobState<Self>,
 	) -> Result<(), JobError> {
-		process_step(
-			false, // On shallow thumbnailer, we want to show thumbnails ASAP
-			state.step_number,
-			&state.steps[0],
-			state
-				.data
-				.as_mut()
-				.expect("critical error: missing data on job state"),
-			ctx,
-		)
-		.await
+		process_step(state, ctx).await
 	}
 
 	async fn finalize(&mut self, ctx: WorkerContext, state: &mut JobState<Self>) -> JobResult {
@@ -200,7 +191,7 @@ impl StatefulJob for ShallowThumbnailerJob {
 async fn get_files_by_extensions(
 	db: &PrismaClient,
 	location_id: LocationId,
-	parent_id: Uuid,
+	parent_isolated_file_path_data: &IsolatedFilePathData<'_>,
 	extensions: &[Extension],
 	kind: ThumbnailerJobStepKind,
 ) -> Result<Vec<ThumbnailerJobStep>, JobError> {
@@ -209,9 +200,13 @@ async fn get_files_by_extensions(
 		.find_many(vec![
 			file_path::location_id::equals(location_id),
 			file_path::extension::in_vec(extensions.iter().map(ToString::to_string).collect()),
-			file_path::parent_id::equals(Some(uuid_to_bytes(parent_id))),
+			file_path::materialized_path::equals(
+				parent_isolated_file_path_data
+					.materialized_path_for_children()
+					.expect("sub path iso_file_path must be a directory"),
+			),
 		])
-		.select(file_path_just_materialized_path_cas_id::select())
+		.select(file_path_for_thumbnailer::select())
 		.exec()
 		.await?
 		.into_iter()

core/src/object/preview/thumbnail/thumbnailer_job.rs

@@ -4,10 +4,11 @@ use crate::{
 	},
 	library::Library,
 	location::file_path_helper::{
-		ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
-		file_path_just_materialized_path_cas_id, MaterializedPath,
+		ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
+		file_path_for_thumbnailer, IsolatedFilePathData,
 	},
 	prisma::{file_path, location, PrismaClient},
+	util::error::FileIOError,
 };
 
 use std::{collections::VecDeque, hash::Hash, path::PathBuf};
@@ -33,7 +34,6 @@ pub struct ThumbnailerJob {}
 pub struct ThumbnailerJobInit {
 	pub location: location::Data,
 	pub sub_path: Option<PathBuf>,
-	pub background: bool,
 }
 
 impl Hash for ThumbnailerJobInit {
@@ -73,7 +73,7 @@ impl StatefulJob for ThumbnailerJob {
 		let location_id = state.init.location.id;
 		let location_path = PathBuf::from(&state.init.location.path);
 
-		let materialized_path = if let Some(ref sub_path) = state.init.sub_path {
+		let (path, iso_file_path) = if let Some(ref sub_path) = state.init.sub_path {
 			let full_path = ensure_sub_path_is_in_location(&location_path, sub_path)
 				.await
 				.map_err(ThumbnailerError::from)?;
@@ -81,22 +81,38 @@ impl StatefulJob for ThumbnailerJob {
 				.await
 				.map_err(ThumbnailerError::from)?;
 
-			MaterializedPath::new(location_id, &location_path, &full_path, true)
-				.map_err(ThumbnailerError::from)?
+			let sub_iso_file_path =
+				IsolatedFilePathData::new(location_id, &location_path, &full_path, true)
+					.map_err(ThumbnailerError::from)?;
+
+			ensure_file_path_exists(
+				sub_path,
+				&sub_iso_file_path,
+				db,
+				ThumbnailerError::SubPathNotFound,
+			)
+			.await?;
+
+			(full_path, sub_iso_file_path)
 		} else {
-			MaterializedPath::new(location_id, &location_path, &location_path, true)
-				.map_err(ThumbnailerError::from)?
+			(
+				location_path.to_path_buf(),
+				IsolatedFilePathData::new(location_id, &location_path, &location_path, true)
+					.map_err(ThumbnailerError::from)?,
+			)
 		};
 
-		info!("Searching for images in location {location_id} at directory {materialized_path}");
+		info!("Searching for images in location {location_id} at directory {iso_file_path}");
 
 		// create all necessary directories if they don't exist
-		fs::create_dir_all(&thumbnail_dir).await?;
+		fs::create_dir_all(&thumbnail_dir)
+			.await
+			.map_err(|e| FileIOError::from((&thumbnail_dir, e)))?;
 
 		// query database for all image files in this location that need thumbnails
 		let image_files = get_files_by_extensions(
 			db,
-			&materialized_path,
+			&iso_file_path,
 			&FILTERED_IMAGE_EXTENSIONS,
 			ThumbnailerJobStepKind::Image,
 		)
@@ -108,7 +124,7 @@ impl StatefulJob for ThumbnailerJob {
 			// query database for all video files in this location that need thumbnails
 			let video_files = get_files_by_extensions(
 				db,
-				&materialized_path,
+				&iso_file_path,
 				&FILTERED_VIDEO_EXTENSIONS,
 				ThumbnailerJobStepKind::Video,
 			)
@@ -133,11 +149,11 @@ impl StatefulJob for ThumbnailerJob {
 			location_path,
 			report: ThumbnailerJobReport {
 				location_id,
-				materialized_path: materialized_path.into(),
+				path,
 				thumbnails_created: 0,
 			},
 		});
-		state.steps = all_files;
+		state.steps.extend(all_files);
 
 		Ok(())
 	}
@@ -147,17 +163,7 @@ impl StatefulJob for ThumbnailerJob {
 		ctx: WorkerContext,
 		state: &mut JobState<Self>,
 	) -> Result<(), JobError> {
-		process_step(
-			state.init.background,
-			state.step_number,
-			&state.steps[0],
-			state
-				.data
-				.as_mut()
-				.expect("critical error: missing data on job state"),
-			ctx,
-		)
-		.await
+		process_step(state, ctx).await
 	}
 
 	async fn finalize(&mut self, ctx: WorkerContext, state: &mut JobState<Self>) -> JobResult {
@@ -173,18 +179,22 @@ impl StatefulJob for ThumbnailerJob {
 
 async fn get_files_by_extensions(
 	db: &PrismaClient,
-	materialized_path: &MaterializedPath<'_>,
+	iso_file_path: &IsolatedFilePathData<'_>,
 	extensions: &[Extension],
 	kind: ThumbnailerJobStepKind,
 ) -> Result<Vec<ThumbnailerJobStep>, JobError> {
 	Ok(db
 		.file_path()
 		.find_many(vec![
-			file_path::location_id::equals(materialized_path.location_id()),
+			file_path::location_id::equals(iso_file_path.location_id()),
 			file_path::extension::in_vec(extensions.iter().map(ToString::to_string).collect()),
-			file_path::materialized_path::starts_with(materialized_path.into()),
+			file_path::materialized_path::starts_with(
+				iso_file_path
+					.materialized_path_for_children()
+					.expect("sub path iso_file_path must be a directory"),
+			),
 		])
-		.select(file_path_just_materialized_path_cas_id::select())
+		.select(file_path_for_thumbnailer::select())
 		.exec()
 		.await?
 		.into_iter()

core/src/object/validation/validator_job.rs

@@ -3,12 +3,13 @@ use crate::{
 		JobError, JobInitData, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext,
 	},
 	library::Library,
-	location::file_path_helper::{file_path_for_object_validator, MaterializedPath},
+	location::file_path_helper::{file_path_for_object_validator, IsolatedFilePathData},
 	prisma::{file_path, location},
 	sync,
+	util::error::FileIOError,
 };
 
-use std::{collections::VecDeque, path::PathBuf};
+use std::path::PathBuf;
 
 use serde::{Deserialize, Serialize};
 use serde_json::json;
@@ -55,18 +56,17 @@ impl StatefulJob for ObjectValidatorJob {
 	async fn init(&self, ctx: WorkerContext, state: &mut JobState<Self>) -> Result<(), JobError> {
 		let Library { db, .. } = &ctx.library;
 
-		state.steps = db
-			.file_path()
-			.find_many(vec![
-				file_path::location_id::equals(state.init.location_id),
-				file_path::is_dir::equals(false),
-				file_path::integrity_checksum::equals(None),
-			])
-			.select(file_path_for_object_validator::select())
-			.exec()
-			.await?
-			.into_iter()
-			.collect::<VecDeque<_>>();
+		state.steps.extend(
+			db.file_path()
+				.find_many(vec![
+					file_path::location_id::equals(state.init.location_id),
+					file_path::is_dir::equals(false),
+					file_path::integrity_checksum::equals(None),
+				])
+				.select(file_path_for_object_validator::select())
+				.exec()
+				.await?,
+		);
 
 		let location = db
 			.location()
@@ -93,18 +93,23 @@ impl StatefulJob for ObjectValidatorJob {
 		let Library { db, sync, .. } = &ctx.library;
 
 		let file_path = &state.steps[0];
-		let data = state.data.as_ref().expect("fatal: missing job state");
+		let data = state
+			.data
+			.as_ref()
+			.expect("critical error: missing data on job state");
 
 		// this is to skip files that already have checksums
 		// i'm unsure what the desired behaviour is in this case
 		// we can also compare old and new checksums here
 		// This if is just to make sure, we already queried objects where integrity_checksum is null
 		if file_path.integrity_checksum.is_none() {
-			let checksum = file_checksum(data.root_path.join(&MaterializedPath::from((
+			let path = data.root_path.join(IsolatedFilePathData::from((
 				file_path.location.id,
-				&file_path.materialized_path,
-			))))
-			.await?;
+				file_path,
+			)));
+			let checksum = file_checksum(&path)
+				.await
+				.map_err(|e| FileIOError::from((path, e)))?;
 
 			sync.write_op(
 				db,

core/src/util/error.rs

@@ -0,0 +1,24 @@
+use std::{io, path::Path};
+
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+#[error("error accessing path: '{}'", .path.display())]
+pub struct FileIOError {
+	path: Box<Path>,
+	#[source]
+	source: io::Error,
+}
+
+impl<P: AsRef<Path>> From<(P, io::Error)> for FileIOError {
+	fn from((path, source): (P, io::Error)) -> Self {
+		Self {
+			path: path.as_ref().into(),
+			source,
+		}
+	}
+}
+
+#[derive(Debug, Error)]
+#[error("received a non UTF-8 path: <lossy_path='{}'>", .0.to_string_lossy())]
+pub struct NonUtf8PathError(pub Box<Path>);

core/src/util/mod.rs

@@ -1,5 +1,6 @@
 pub mod db;
 #[cfg(debug_assertions)]
 pub mod debug_initializer;
+pub mod error;
 pub mod migrator;
 pub mod seeder;

core/src/util/seeder.rs

@@ -1,17 +1,13 @@
 use crate::{
-	location::indexer::{
-		rules::{IndexerRule, ParametersPerKind, RuleKind},
-		IndexerError,
-	},
+	location::indexer::rules::{IndexerRule, IndexerRuleError, ParametersPerKind, RuleKind},
 	prisma::PrismaClient,
 };
-use globset::Glob;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
 pub enum SeederError {
 	#[error("Failed to run indexer rules seeder: {0}")]
-	IndexerRules(#[from] IndexerError),
+	IndexerRules(#[from] IndexerRuleError),
 	#[error("An error occurred with the database while applying migrations: {0}")]
 	DatabaseError(#[from] prisma_client_rust::QueryError),
 }
@@ -25,106 +21,105 @@ pub async fn indexer_rules_seeder(client: &PrismaClient) -> Result<(), SeederErr
 				// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#FILE_ATTRIBUTE_SYSTEM
 				"No OS protected".to_string(),
 				true,
-				ParametersPerKind::RejectFilesByGlob([
-					vec![
-						"**/.spacedrive",
-					],
-					// Globset, even on Windows, requires the use of / as a separator
-					// https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
-					// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
-					#[cfg(target_os = "windows")]
-					vec![
-						// Windows thumbnail cache files
-						"**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}",
-						// Dump file
-						"**/*.stackdump",
-						// Folder config file
-						"**/[Dd]esktop.ini",
-						// Recycle Bin used on file shares
-						"**/$RECYCLE.BIN",
-						// Chkdsk recovery directory
-						"**/FOUND.[0-9][0-9][0-9]",
-						// Reserved names
-						"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}",
-						"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*",
-						// User special files
-						"C:/Users/*/NTUSER.DAT*",
-						"C:/Users/*/ntuser.dat*",
-						"C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}",
-						// User special folders (most of these the user dont even have permission to access)
-						"C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}",
-						// System special folders
-						"C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}",
-						// NTFS internal dir, can exists on any drive
-						"[A-Z]:/System Volume Information",
-						// System special files
-						"C:/{config,pagefile,hiberfil}.sys",
-						// Windows can create a swapfile on any drive
-						"[A-Z]:/swapfile.sys",
-						"C:/DumpStack.log.tmp",
-					],
-					// https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
-					// https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14
-					#[cfg(any(target_os = "ios", target_os = "macos"))]
-					vec![
-						"**/.{DS_Store,AppleDouble,LSOverride}",
-						// Icon must end with two \r
-						"**/Icon\r\r",
-						// Thumbnails
-						"**/._*",
-					],
-					#[cfg(target_os = "macos")]
-					vec![
-						"/{System,Network,Library,Applications}",
-						"/Users/*/{Library,Applications}",
-						// Files that might appear in the root of a volume
-						"**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}",
-						// Directories potentially created on remote AFP share
-						"**/.{AppleDB,AppleDesktop,apdisk}",
-						"**/{Network Trash Folder,Temporary Items}",
-					],
-					// https://github.com/github/gitignore/blob/main/Global/Linux.gitignore
-					#[cfg(target_os = "linux")]
-					vec![
-						"**/*~",
-						// temporary files which can be created if a process still has a handle open of a deleted file
-						"**/.fuse_hidden*",
-						// KDE directory preferences
-						"**/.directory",
-						// Linux trash folder which might appear on any partition or disk
-						"**/.Trash-*",
-						// .nfs files are created when an open file is removed but is still being accessed
-						"**/.nfs*",
-					],
-					#[cfg(target_os = "android")]
-					vec![
-						"**/.nomedia",
-						"**/.thumbnails",
-					],
-					// https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout
-					// https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
-					#[cfg(target_family = "unix")]
-					vec![
-						// Directories containing unix memory/device mapped files/dirs
-						"/{dev,sys,proc}",
-						// Directories containing special files for current running programs
-						"/{run,var,boot}",
-						// ext2-4 recovery directory
-						"**/lost+found",
-					],
-				]
-				.into_iter()
-				.flatten()
-				.map(Glob::new)
-				.collect::<Result<Vec<Glob>, _>>().map_err(IndexerError::GlobBuilderError)?),
+				ParametersPerKind::new_reject_files_by_glob([
+						vec![
+							"**/.spacedrive",
+						],
+						// Globset, even on Windows, requires the use of / as a separator
+						// https://github.com/github/gitignore/blob/main/Global/Windows.gitignore
+						// https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+						#[cfg(target_os = "windows")]
+						vec![
+							// Windows thumbnail cache files
+							"**/{Thumbs.db,Thumbs.db:encryptable,ehthumbs.db,ehthumbs_vista.db}",
+							// Dump file
+							"**/*.stackdump",
+							// Folder config file
+							"**/[Dd]esktop.ini",
+							// Recycle Bin used on file shares
+							"**/$RECYCLE.BIN",
+							// Chkdsk recovery directory
+							"**/FOUND.[0-9][0-9][0-9]",
+							// Reserved names
+							"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}",
+							"**/{CON,PRN,AUX,NUL,COM0,COM1,COM2,COM3,COM4,COM5,COM6,COM7,COM8,COM9,LPT0,LPT1,LPT2,LPT3,LPT4,LPT5,LPT6,LPT7,LPT8,LPT9}.*",
+							// User special files
+							"C:/Users/*/NTUSER.DAT*",
+							"C:/Users/*/ntuser.dat*",
+							"C:/Users/*/{ntuser.ini,ntuser.dat,NTUSER.DAT}",
+							// User special folders (most of these the user dont even have permission to access)
+							"C:/Users/*/{Cookies,AppData,NetHood,Recent,PrintHood,SendTo,Templates,Start Menu,Application Data,Local Settings}",
+							// System special folders
+							"C:/{$Recycle.Bin,$WinREAgent,Documents and Settings,Program Files,Program Files (x86),ProgramData,Recovery,PerfLogs,Windows,Windows.old}",
+							// NTFS internal dir, can exists on any drive
+							"[A-Z]:/System Volume Information",
+							// System special files
+							"C:/{config,pagefile,hiberfil}.sys",
+							// Windows can create a swapfile on any drive
+							"[A-Z]:/swapfile.sys",
+							"C:/DumpStack.log.tmp",
+						],
+						// https://github.com/github/gitignore/blob/main/Global/macOS.gitignore
+						// https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html#//apple_ref/doc/uid/TP40010672-CH2-SW14
+						#[cfg(any(target_os = "ios", target_os = "macos"))]
+						vec![
+							"**/.{DS_Store,AppleDouble,LSOverride}",
+							// Icon must end with two \r
+							"**/Icon\r\r",
+							// Thumbnails
+							"**/._*",
+						],
+						#[cfg(target_os = "macos")]
+						vec![
+							"/{System,Network,Library,Applications}",
+							"/Users/*/{Library,Applications}",
+							// Files that might appear in the root of a volume
+							"**/.{DocumentRevisions-V100,fseventsd,Spotlight-V100,TemporaryItems,Trashes,VolumeIcon.icns,com.apple.timemachine.donotpresent}",
+							// Directories potentially created on remote AFP share
+							"**/.{AppleDB,AppleDesktop,apdisk}",
+							"**/{Network Trash Folder,Temporary Items}",
+						],
+						// https://github.com/github/gitignore/blob/main/Global/Linux.gitignore
+						#[cfg(target_os = "linux")]
+						vec![
+							"**/*~",
+							// temporary files which can be created if a process still has a handle open of a deleted file
+							"**/.fuse_hidden*",
+							// KDE directory preferences
+							"**/.directory",
+							// Linux trash folder which might appear on any partition or disk
+							"**/.Trash-*",
+							// .nfs files are created when an open file is removed but is still being accessed
+							"**/.nfs*",
+						],
+						#[cfg(target_os = "android")]
+						vec![
+							"**/.nomedia",
+							"**/.thumbnails",
+						],
+						// https://en.wikipedia.org/wiki/Unix_filesystem#Conventional_directory_layout
+						// https://en.wikipedia.org/wiki/Filesystem_Hierarchy_Standard
+						#[cfg(target_family = "unix")]
+						vec![
+							// Directories containing unix memory/device mapped files/dirs
+							"/{dev,sys,proc}",
+							// Directories containing special files for current running programs
+							"/{run,var,boot}",
+							// ext2-4 recovery directory
+							"**/lost+found",
+						],
+					]
+					.into_iter()
+					.flatten()
+				)?
 			),
 			IndexerRule::new(
 				RuleKind::RejectFilesByGlob,
 				"No Hidden".to_string(),
 				true,
-				ParametersPerKind::RejectFilesByGlob(vec![
-					Glob::new("**/.*").map_err(IndexerError::GlobBuilderError)?
-				]),
+				ParametersPerKind::new_reject_files_by_glob(
+					["**/.*"],
+				)?
 			),
 			IndexerRule::new(
 				RuleKind::AcceptIfChildrenDirectoriesArePresent,
@@ -138,10 +133,9 @@ pub async fn indexer_rules_seeder(client: &PrismaClient) -> Result<(), SeederErr
 				RuleKind::AcceptFilesByGlob,
 				"Only Images".to_string(),
 				false,
-				ParametersPerKind::AcceptFilesByGlob(vec![Glob::new(
-					"*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}",
-				)
-				.map_err(IndexerError::GlobBuilderError)?]),
+				ParametersPerKind::new_accept_files_by_globs_str(
+					["*.{avif,bmp,gif,ico,jpeg,jpg,png,svg,tif,tiff,webp}"],
+				)?,
 			),
 		] {
 			rule.save(client).await?;

crates/file-ext/Cargo.toml

@@ -14,7 +14,7 @@ serde_json = "1.0.85"
 strum = { version = "0.24", features = ["derive"] }
 strum_macros = "0.24"
 tokio = { workspace = true, features = ["fs", "rt", "io-util"] }
-specta.workspace = true
+specta = { workspace = true }
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["fs", "rt", "macros"] }

interface/app/$libraryId/Explorer/Inspector/index.tsx

@@ -3,7 +3,15 @@ import clsx from 'clsx';
 import dayjs from 'dayjs';
 import { Barcode, CircleWavyCheck, Clock, Cube, Hash, Link, Lock, Snowflake } from 'phosphor-react';
 import { ComponentProps, useEffect, useState } from 'react';
-import { ExplorerItem, ObjectKind, Tag, formatBytes, isPath, useLibraryQuery } from '@sd/client';
+import {
+	ExplorerItem,
+	Location,
+	ObjectKind,
+	Tag,
+	formatBytes,
+	isPath,
+	useLibraryQuery
+} from '@sd/client';
 import { Button, Divider, DropdownMenu, Tooltip, tw } from '@sd/ui';
 import { useExplorerStore } from '~/hooks/useExplorerStore';
 import { TOP_BAR_HEIGHT } from '../../TopBar';
@@ -57,7 +65,7 @@ export const Inspector = ({ data, context, ...elementProps }: Props) => {
 		enabled: readyToFetch && objectData?.id !== undefined
 	});
 
-	const item = data?.item;
+	const { item } = data;
 
 	// map array of numbers into string
 	const pub_id = fullObjectData?.data?.pub_id.map((n: number) => n.toString(16)).join('');
@@ -68,174 +76,150 @@ export const Inspector = ({ data, context, ...elementProps }: Props) => {
 			className="custom-scroll inspector-scroll h-screen w-full overflow-x-hidden pb-4 pl-1.5 pr-1"
 			style={{ paddingTop: TOP_BAR_HEIGHT + 12 }}
 		>
-			{data && (
-				<>
-					{explorerStore.layoutMode !== 'media' && (
-						<div
-							className={clsx(
-								'mb-[10px] flex h-[240] w-full items-center justify-center overflow-hidden'
-							)}
-						>
-							<FileThumb loadOriginal size={240} data={data} />
-						</div>
+			{explorerStore.layoutMode !== 'media' && (
+				<div
+					className={clsx(
+						'mb-[10px] flex h-[240] w-full items-center justify-center overflow-hidden'
 					)}
-					<div className="flex w-full select-text flex-col overflow-hidden rounded-lg border border-app-line bg-app-box py-0.5 shadow-app-shade/10">
-						<h3 className="truncate px-3 pb-1 pt-2 text-base font-bold">
-							{filePathData?.name}
-							{filePathData?.extension && `.${filePathData.extension}`}
-						</h3>
-						{objectData && (
-							<div className="mx-3 mb-0.5 mt-1 flex flex-row space-x-0.5">
-								<Tooltip label="Favorite">
-									<FavoriteButton data={objectData} />
-								</Tooltip>
+				>
+					<FileThumb loadOriginal size={240} data={data} />
+				</div>
+			)}
+			<div className="flex w-full select-text flex-col overflow-hidden rounded-lg border border-app-line bg-app-box py-0.5 shadow-app-shade/10">
+				<h3 className="truncate px-3 pb-1 pt-2 text-base font-bold">
+					{filePathData?.name}
+					{filePathData?.extension && `.${filePathData.extension}`}
+				</h3>
+				{objectData && (
+					<div className="mx-3 mb-0.5 mt-1 flex flex-row space-x-0.5">
+						<Tooltip label="Favorite">
+							<FavoriteButton data={objectData} />
+						</Tooltip>
 
-								<Tooltip label="Encrypt">
-									<Button size="icon">
-										<Lock className="h-[18px] w-[18px]" />
-									</Button>
-								</Tooltip>
-								<Tooltip label="Share">
-									<Button size="icon">
-										<Link className="h-[18px] w-[18px]" />
-									</Button>
-								</Tooltip>
-							</div>
-						)}
-						{isPath(data) && <PathDisplay data={data} />}
-						<Divider />
-						<MetaContainer>
-							<div className="flex flex-wrap gap-1 overflow-hidden">
-								<InfoPill>
-									{isDir ? 'Folder' : ObjectKind[objectData?.kind || 0]}
+						<Tooltip label="Encrypt">
+							<Button size="icon">
+								<Lock className="h-[18px] w-[18px]" />
+							</Button>
+						</Tooltip>
+						<Tooltip label="Share">
+							<Button size="icon">
+								<Link className="h-[18px] w-[18px]" />
+							</Button>
+						</Tooltip>
+					</div>
+				)}
+				{isPath(data) && context && 'path' in context && (
+					<MetaContainer>
+						<MetaTitle>URI</MetaTitle>
+						<MetaValue>
+							{`${context.path}/${data.item.materialized_path}${data.item.name}${
+								data.item.is_dir ? `.${data.item.extension}` : '/'
+							}`}
+						</MetaValue>
+					</MetaContainer>
+				)}
+				<Divider />
+				<MetaContainer>
+					<div className="flex flex-wrap gap-1 overflow-hidden">
+						<InfoPill>{isDir ? 'Folder' : ObjectKind[objectData?.kind || 0]}</InfoPill>
+						{filePathData?.extension && <InfoPill>{filePathData.extension}</InfoPill>}
+						{tags.data?.map((tag) => (
+							<Tooltip
+								key={tag.id}
+								label={tag.name || ''}
+								className="flex overflow-hidden"
+							>
+								<InfoPill
+									className="truncate !text-white"
+									style={{ backgroundColor: tag.color + 'CC' }}
+								>
+									{tag.name}
 								</InfoPill>
-								{filePathData?.extension && (
-									<InfoPill>{filePathData.extension}</InfoPill>
-								)}
-								{tags?.data?.map((tag) => (
-									<Tooltip
-										key={tag.id}
-										label={tag.name || ''}
-										className="flex overflow-hidden"
-									>
-										<InfoPill
-											className="truncate !text-white"
-											style={{ backgroundColor: tag.color + 'CC' }}
-										>
-											{tag.name}
-										</InfoPill>
-									</Tooltip>
-								))}
-								{objectData?.id && (
-									<DropdownMenu.Root
-										trigger={<PlaceholderPill>Add Tag</PlaceholderPill>}
-										side="left"
-										sideOffset={5}
-										alignOffset={-10}
-									>
-										<AssignTagMenuItems objectId={objectData.id} />
-									</DropdownMenu.Root>
-								)}
-							</div>
-						</MetaContainer>
-						<Divider />
-						<MetaContainer className="!flex-row space-x-2">
-							<MetaTextLine>
-								<InspectorIcon component={Cube} />
-								<span className="mr-1.5">Size</span>
-								<MetaValue>
-									{formatBytes(Number(filePathData?.size_in_bytes || 0))}
-								</MetaValue>
-							</MetaTextLine>
-							{fullObjectData.data?.media_data?.duration_seconds && (
-								<MetaTextLine>
-									<InspectorIcon component={Clock} />
-									<span className="mr-1.5">Duration</span>
-									<MetaValue>
-										{fullObjectData.data.media_data.duration_seconds}
-									</MetaValue>
-								</MetaTextLine>
-							)}
-						</MetaContainer>
-						<Divider />
-						<MetaContainer>
-							<Tooltip label={dayjs(item?.date_created).format('h:mm:ss a')}>
-								<MetaTextLine>
-									<InspectorIcon component={Clock} />
-									<MetaKeyName className="mr-1.5">Created</MetaKeyName>
-									<MetaValue>
-										{dayjs(item?.date_created).format('MMM Do YYYY')}
-									</MetaValue>
-								</MetaTextLine>
 							</Tooltip>
-							<Tooltip label={dayjs(item?.date_created).format('h:mm:ss a')}>
-								<MetaTextLine>
-									<InspectorIcon component={Barcode} />
-									<MetaKeyName className="mr-1.5">Indexed</MetaKeyName>
-									<MetaValue>
-										{dayjs(filePathData?.date_indexed).format('MMM Do YYYY')}
-									</MetaValue>
-								</MetaTextLine>
-							</Tooltip>
-						</MetaContainer>
-
-						{!isDir && objectData && (
-							<>
-								<Note data={objectData} />
-								<Divider />
-								<MetaContainer>
-									<Tooltip label={filePathData?.cas_id || ''}>
-										<MetaTextLine>
-											<InspectorIcon component={Snowflake} />
-											<MetaKeyName className="mr-1.5">Content ID</MetaKeyName>
-											<MetaValue>{filePathData?.cas_id || ''}</MetaValue>
-										</MetaTextLine>
-									</Tooltip>
-									{filePathData?.integrity_checksum && (
-										<Tooltip label={filePathData?.integrity_checksum || ''}>
-											<MetaTextLine>
-												<InspectorIcon component={CircleWavyCheck} />
-												<MetaKeyName className="mr-1.5">
-													Checksum
-												</MetaKeyName>
-												<MetaValue>
-													{filePathData?.integrity_checksum}
-												</MetaValue>
-											</MetaTextLine>
-										</Tooltip>
-									)}
-									{pub_id && (
-										<Tooltip label={pub_id || ''}>
-											<MetaTextLine>
-												<InspectorIcon component={Hash} />
-												<MetaKeyName className="mr-1.5">
-													Object ID
-												</MetaKeyName>
-												<MetaValue>{pub_id}</MetaValue>
-											</MetaTextLine>
-										</Tooltip>
-									)}
-								</MetaContainer>
-							</>
+						))}
+						{objectData?.id && (
+							<DropdownMenu.Root
+								trigger={<PlaceholderPill>Add Tag</PlaceholderPill>}
+								side="left"
+								sideOffset={5}
+								alignOffset={-10}
+							>
+								<AssignTagMenuItems objectId={objectData.id} />
+							</DropdownMenu.Root>
 						)}
 					</div>
-				</>
-			)}
+				</MetaContainer>
+				<Divider />
+				<MetaContainer className="!flex-row space-x-2">
+					<MetaTextLine>
+						<InspectorIcon component={Cube} />
+						<span className="mr-1.5">Size</span>
+						<MetaValue>
+							{formatBytes(Number(filePathData?.size_in_bytes || 0))}
+						</MetaValue>
+					</MetaTextLine>
+					{fullObjectData.data?.media_data?.duration_seconds && (
+						<MetaTextLine>
+							<InspectorIcon component={Clock} />
+							<span className="mr-1.5">Duration</span>
+							<MetaValue>{fullObjectData.data.media_data.duration_seconds}</MetaValue>
+						</MetaTextLine>
+					)}
+				</MetaContainer>
+				<Divider />
+				<MetaContainer>
+					<Tooltip label={dayjs(item.date_created).format('h:mm:ss a')}>
+						<MetaTextLine>
+							<InspectorIcon component={Clock} />
+							<MetaKeyName className="mr-1.5">Created</MetaKeyName>
+							<MetaValue>{dayjs(item.date_created).format('MMM Do YYYY')}</MetaValue>
+						</MetaTextLine>
+					</Tooltip>
+					<Tooltip label={dayjs(item.date_created).format('h:mm:ss a')}>
+						<MetaTextLine>
+							<InspectorIcon component={Barcode} />
+							<MetaKeyName className="mr-1.5">Indexed</MetaKeyName>
+							<MetaValue>
+								{dayjs(filePathData?.date_indexed).format('MMM Do YYYY')}
+							</MetaValue>
+						</MetaTextLine>
+					</Tooltip>
+				</MetaContainer>
+
+				{!isDir && objectData && (
+					<>
+						<Note data={objectData} />
+						<Divider />
+						<MetaContainer>
+							<Tooltip label={filePathData?.cas_id || ''}>
+								<MetaTextLine>
+									<InspectorIcon component={Snowflake} />
+									<MetaKeyName className="mr-1.5">Content ID</MetaKeyName>
+									<MetaValue>{filePathData?.cas_id || ''}</MetaValue>
+								</MetaTextLine>
+							</Tooltip>
+							{filePathData?.integrity_checksum && (
+								<Tooltip label={filePathData?.integrity_checksum || ''}>
+									<MetaTextLine>
+										<InspectorIcon component={CircleWavyCheck} />
+										<MetaKeyName className="mr-1.5">Checksum</MetaKeyName>
+										<MetaValue>{filePathData?.integrity_checksum}</MetaValue>
+									</MetaTextLine>
+								</Tooltip>
+							)}
+							{pub_id && (
+								<Tooltip label={pub_id || ''}>
+									<MetaTextLine>
+										<InspectorIcon component={Hash} />
+										<MetaKeyName className="mr-1.5">Object ID</MetaKeyName>
+										<MetaValue>{pub_id}</MetaValue>
+									</MetaTextLine>
+								</Tooltip>
+							)}
+						</MetaContainer>
+					</>
+				)}
+			</div>
 		</div>
 	);
 };
-
-const PathDisplay = ({ data }: { data: Extract<ExplorerItem, { type: 'Path' }> }) => {
-	const location = useLibraryQuery(['locations.get', data.item.location_id]);
-
-	return (
-		<>
-			{location.data && (
-				<MetaContainer>
-					<MetaTitle>URI</MetaTitle>
-					<MetaValue>{`${location.data.path}/${data.item.materialized_path}`}</MetaValue>
-				</MetaContainer>
-			)}
-		</>
-	);
-};

interface/app/$libraryId/Explorer/View.tsx

@@ -1,7 +1,7 @@
+import { ExplorerItem, isPath, useLibraryContext } from '@sd/client';
 import clsx from 'clsx';
 import { HTMLAttributes, PropsWithChildren, memo, useRef } from 'react';
 import { createSearchParams, useMatch, useNavigate } from 'react-router-dom';
-import { ExplorerItem, isPath, useLibraryContext } from '@sd/client';
 import { getExplorerStore, useExplorerStore } from '~/hooks/useExplorerStore';
 import { TOP_BAR_HEIGHT } from '../TopBar';
 import DismissibleNotice from './DismissibleNotice';
@@ -32,7 +32,7 @@ export const ViewItem = ({
 		if (isPath(data) && data.item.is_dir) {
 			navigate({
 				pathname: `/${library.uuid}/location/${getItemFilePath(data)?.location_id}`,
-				search: createSearchParams({ path: data.item.materialized_path }).toString()
+				search: createSearchParams({ path: `${data.item.materialized_path}${data.item.name}/` }).toString()
 			});
 
 			getExplorerStore().selectedRowIndex = null;

interface/app/$libraryId/Layout/Sidebar/JobManager/Job.tsx

@@ -103,6 +103,7 @@ const StatusColors: Record<JobReport['status'], string> = {
 	Running: 'text-blue-500',
 	Failed: 'text-red-500',
 	Completed: 'text-green-500',
+	CompletedWithErrors: 'text-orange-500',
 	Queued: 'text-yellow-500',
 	Canceled: 'text-gray-500',
 	Paused: 'text-gray-500'

packages/client/src/core.ts

@@ -126,8 +126,6 @@ export type LibraryConfigWrapped = { uuid: string; config: LibraryConfig }
  */
 export type Params = "Standard" | "Hardened" | "Paranoid"
 
-export type Tag = { id: number; pub_id: number[]; name: string | null; color: string | null; total_objects: number | null; redundancy_goal: number | null; date_created: string; date_modified: string }
-
 /**
  * `LocationUpdateArgs` is the argument received from the client using `rspc` to update a location.
  * It contains the id of the location to be updated, possible a name to change the current location's name
@@ -140,12 +138,16 @@ export type LocationUpdateArgs = { id: number; name: string | null; generate_pre
 
 export type SetFavoriteArgs = { id: number; favorite: boolean }
 
+export type FilePath = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string }
+
 /**
  * Represents the operating system which the remote peer is running.
  * This is not used internally and predominantly is designed to be used for display purposes by the embedding application.
  */
 export type OperatingSystem = "Windows" | "Linux" | "MacOS" | "Ios" | "Android" | { Other: string }
 
+export type RuleKind = "AcceptFilesByGlob" | "RejectFilesByGlob" | "AcceptIfChildrenDirectoriesArePresent" | "RejectIfChildrenDirectoriesArePresent"
+
 /**
  * This is a stored key, and can be freely written to the database.
  * 
@@ -155,28 +157,20 @@ export type StoredKey = { uuid: string; version: StoredKeyVersion; key_type: Sto
 
 export type OnboardingConfig = { password: Protected<string>; algorithm: Algorithm; hashing_algorithm: HashingAlgorithm }
 
-export type Object = { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; date_accessed: string | null }
+export type FileDecryptorJobInit = { location_id: number; path_id: number; mount_associated_key: boolean; output_path: string | null; password: string | null; save_to_library: boolean | null }
 
 export type Volume = { name: string; mount_point: string; total_capacity: string; available_capacity: string; is_removable: boolean; disk_type: string | null; file_system: string | null; is_root_filesystem: boolean }
 
 export type TagCreateArgs = { name: string; color: string }
 
-/**
- * `IndexerRuleCreateArgs` is the argument received from the client using rspc to create a new indexer rule.
- * Note that `parameters` field **MUST** be a JSON object serialized to bytes.
- * 
- * In case of  `RuleKind::AcceptFilesByGlob` or `RuleKind::RejectFilesByGlob`, it will be a
- * single string containing a glob pattern.
- * 
- * In case of `RuleKind::AcceptIfChildrenDirectoriesArePresent` or `RuleKind::RejectIfChildrenDirectoriesArePresent` the
- * `parameters` field must be a vector of strings containing the names of the directories.
- */
-export type IndexerRuleCreateArgs = { kind: RuleKind; name: string; dry_run: boolean; parameters: string[] }
-
 export type EditLibraryArgs = { id: string; name: string | null; description: string | null }
 
 export type LightScanArgs = { location_id: number; sub_path: string }
 
+export type JobStatus = "Queued" | "Running" | "Completed" | "Canceled" | "Failed" | "Paused" | "CompletedWithErrors"
+
+export type FileEraserJobInit = { location_id: number; path_id: number; passes: string }
+
 /**
  * This should be used for providing a nonce to encrypt/decrypt functions.
  * 
@@ -186,10 +180,6 @@ export type Nonce = { XChaCha20Poly1305: number[] } | { Aes256Gcm: number[] }
 
 export type UnlockKeyManagerArgs = { password: Protected<string>; secret_key: Protected<string> }
 
-export type OptionalRange<T> = { from: T | null; to: T | null }
-
-export type FileEncryptorJobInit = { location_id: number; path_id: number; key_uuid: string; algorithm: Algorithm; metadata: boolean; preview_media: boolean; output_path: string | null }
-
 export type InvalidateOperationEvent = { key: string; arg: any; result: any | null }
 
 export type Location = { id: number; pub_id: number[]; node_id: number; name: string; path: string; total_capacity: number | null; available_capacity: number | null; is_archived: boolean; generate_preview_media: boolean; sync_preview_media: boolean; hidden: boolean; date_created: string }
@@ -205,26 +195,22 @@ export type CRDTOperation = { node: string; timestamp: number; id: string; typ:
  */
 export type Salt = number[]
 
-export type FileCutterJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string }
+export type Ordering = { name: boolean }
 
-export type JobStatus = "Queued" | "Running" | "Completed" | "Canceled" | "Failed" | "Paused"
+export type FileCopierJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string; target_file_name_suffix: string | null }
 
-export type ObjectValidatorArgs = { id: number; path: string }
+export type IndexerRule = { id: number; kind: number; name: string; default: boolean; parameters: number[]; date_created: string; date_modified: string }
 
-export type FileEraserJobInit = { location_id: number; path_id: number; passes: string }
-
-export type MediaData = { id: number; pixel_width: number | null; pixel_height: number | null; longitude: number | null; latitude: number | null; fps: number | null; capture_device_make: string | null; capture_device_model: string | null; capture_device_software: string | null; duration_seconds: number | null; codecs: string | null; streams: number | null }
-
-export type FileDeleterJobInit = { location_id: number; path_id: number }
-
-export type FilePath = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; parent_id: number[] | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string }
+export type IdentifyUniqueFilesArgs = { id: number; path: string }
 
 /**
  * These are all possible algorithms that can be used for encryption and decryption
  */
 export type Algorithm = "XChaCha20Poly1305" | "Aes256Gcm"
 
-export type JobReport = { id: string; name: string; action: string | null; data: number[] | null; metadata: any | null; is_background: boolean; created_at: string | null; started_at: string | null; completed_at: string | null; parent_id: string | null; status: JobStatus; task_count: number; completed_task_count: number; message: string }
+export type JobReport = { id: string; name: string; action: string | null; data: number[] | null; metadata: any | null; is_background: boolean; errors_text: string[]; created_at: string | null; started_at: string | null; completed_at: string | null; parent_id: string | null; status: JobStatus; task_count: number; completed_task_count: number; message: string }
+
+export type Object = { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; date_accessed: string | null }
 
 export type OwnedOperationItem = { id: any; data: OwnedOperationData }
 
@@ -247,17 +233,33 @@ export type NodeState = ({ id: string; name: string; p2p_port: number | null; p2
 
 export type RelationOperationData = "Create" | { Update: { field: string; value: any } } | "Delete"
 
+export type FileDeleterJobInit = { location_id: number; path_id: number }
+
 export type Node = { id: number; pub_id: number[]; name: string; platform: number; version: string | null; last_seen: string; timezone: string | null; date_created: string }
 
+/**
+ * `IndexerRuleCreateArgs` is the argument received from the client using rspc to create a new indexer rule.
+ * Note that `parameters` field **MUST** be a JSON object serialized to bytes.
+ * 
+ * In case of  `RuleKind::AcceptFilesByGlob` or `RuleKind::RejectFilesByGlob`, it will be a
+ * single string containing a glob pattern.
+ * 
+ * In case of `RuleKind::AcceptIfChildrenDirectoriesArePresent` or `RuleKind::RejectIfChildrenDirectoriesArePresent` the
+ * `parameters` field must be a vector of strings containing the names of the directories.
+ */
+export type IndexerRuleCreateArgs = { kind: RuleKind; name: string; dry_run: boolean; parameters: string[] }
+
 export type SharedOperationCreateData = { u: { [key: string]: any } } | "a"
 
 export type KeyAddArgs = { algorithm: Algorithm; hashing_algorithm: HashingAlgorithm; key: Protected<string>; library_sync: boolean; automount: boolean }
 
 export type BuildInfo = { version: string; commit: string }
 
+export type MediaData = { id: number; pixel_width: number | null; pixel_height: number | null; longitude: number | null; latitude: number | null; fps: number | null; capture_device_make: string | null; capture_device_model: string | null; capture_device_software: string | null; duration_seconds: number | null; codecs: string | null; streams: number | null }
+
 export type SetNoteArgs = { id: number; note: string | null }
 
-export type RuleKind = "AcceptFilesByGlob" | "RejectFilesByGlob" | "AcceptIfChildrenDirectoriesArePresent" | "RejectIfChildrenDirectoriesArePresent"
+export type FileEncryptorJobInit = { location_id: number; path_id: number; key_uuid: string; algorithm: Algorithm; metadata: boolean; preview_media: boolean; output_path: string | null }
 
 /**
  * `LocationCreateArgs` is the argument received from the client using `rspc` to create a new location.
@@ -273,15 +275,19 @@ export type ExplorerItem = { type: "Path"; has_thumbnail: boolean; item: FilePat
  */
 export type LibraryArgs<T> = { library_id: string; arg: T }
 
-export type IdentifyUniqueFilesArgs = { id: number; path: string }
+export type FileCutterJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string }
 
 export type OwnedOperationData = { Create: { [key: string]: any } } | { CreateMany: { values: ([any, { [key: string]: any }])[]; skip_duplicates: boolean } } | { Update: { [key: string]: any } } | "Delete"
 
 export type SharedOperationData = SharedOperationCreateData | { field: string; value: any } | null
 
+export type SearchData<T> = { cursor: number[] | null; items: T[] }
+
+export type OptionalRange<T> = { from: T | null; to: T | null }
+
 export type TagUpdateArgs = { id: number; name: string | null; color: string | null }
 
-export type FileCopierJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string; target_file_name_suffix: string | null }
+export type ObjectValidatorArgs = { id: number; path: string }
 
 export type TagAssignArgs = { object_id: number; tag_id: number; unassign: boolean }
 
@@ -294,7 +300,9 @@ export type HashingAlgorithm = { name: "Argon2id"; params: Params } | { name: "B
 
 export type RenameFileArgs = { location_id: number; file_name: string; new_file_name: string }
 
-export type FilePathWithObject = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; parent_id: number[] | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string; object: Object | null }
+export type Tag = { id: number; pub_id: number[]; name: string | null; color: string | null; total_objects: number | null; redundancy_goal: number | null; date_created: string; date_modified: string }
+
+export type FilePathWithObject = { id: number; pub_id: number[]; is_dir: boolean; cas_id: string | null; integrity_checksum: string | null; location_id: number; materialized_path: string; name: string; extension: string; size_in_bytes: string; inode: number[]; device: number[]; object_id: number | null; key_id: number | null; date_created: string; date_modified: string; date_indexed: string; object: Object | null }
 
 export type LocationWithIndexerRules = { id: number; pub_id: number[]; node_id: number; name: string; path: string; total_capacity: number | null; available_capacity: number | null; is_archived: boolean; generate_preview_media: boolean; sync_preview_media: boolean; hidden: boolean; date_created: string; indexer_rules: ({ indexer_rule: IndexerRule })[] }
 
@@ -303,24 +311,16 @@ export type LocationWithIndexerRules = { id: number; pub_id: number[]; node_id:
  */
 export type LibraryConfig = { name: string; description: string }
 
-export type SearchData<T> = { cursor: number[] | null; items: T[] }
-
 export type CreateLibraryArgs = { name: string }
 
-export type FileDecryptorJobInit = { location_id: number; path_id: number; mount_associated_key: boolean; output_path: string | null; password: string | null; save_to_library: boolean | null }
-
 export type AutomountUpdateArgs = { uuid: string; status: boolean }
 
 export type Protected<T> = T
 
-export type Ordering = { name: boolean }
-
 export type Statistics = { id: number; date_captured: string; total_object_count: number; library_db_size: string; total_bytes_used: string; total_bytes_capacity: string; total_unique_bytes: string; total_bytes_free: string; preview_media_bytes: string }
 
 export type RestoreBackupArgs = { password: Protected<string>; secret_key: Protected<string>; path: string }
 
-export type IndexerRule = { id: number; kind: number; name: string; default: boolean; parameters: number[]; date_created: string; date_modified: string }
-
 export type RelationOperation = { relation_item: string; relation_group: string; relation: string; data: RelationOperationData }
 
 /**