[ENG-708] Thumbnail sharding (#925)

* first phase, basic sharding

* improved API for sharding using a "thumbnailKey"

* clean up param handling for custom_uri

* added version manager with migrations for the thumbnail directory

* remove redundant hash of a hash, silly

* fix mobile

* fix clippy

---------

Co-authored-by: Utku Bakir <74243531+utkubakir@users.noreply.github.com>

apps/desktop/src/App.tsx

@@ -58,7 +58,7 @@ if (customUriServerUrl && !customUriServerUrl?.endsWith('/')) {
 
 const platform: Platform = {
 	platform: 'tauri',
-	getThumbnailUrlById: (casId) => convertFileSrc(`thumbnail/${casId}`, 'spacedrive'),
+	getThumbnailUrlByThumbKey: (keyParts) => convertFileSrc(`thumbnail/${keyParts.map(i => encodeURIComponent(i)).join("/")}`, 'spacedrive'),
 	getFileUrl: (libraryId, locationLocalId, filePathId, _linux_workaround) => {
 		const path = `file/${libraryId}/${locationLocalId}/${filePathId}`;
 		if (_linux_workaround && customUriServerUrl) {

apps/mobile/src/components/explorer/FileThumb.tsx

@@ -15,8 +15,10 @@ type FileThumbProps = {
 	size?: number;
 };
 
-export const getThumbnailUrlById = (casId: string) =>
-	`${DocumentDirectoryPath}/thumbnails/${encodeURIComponent(casId)}.webp`;
+export const getThumbnailUrlById = (keyParts: string[]) =>
+	`${DocumentDirectoryPath}/thumbnails/${keyParts
+		.map((i) => encodeURIComponent(i))
+		.join('/')}.webp`;
 
 type KindType = keyof typeof icons | 'Unknown';
 
@@ -29,7 +31,8 @@ function getExplorerItemData(data: ExplorerItem) {
 		casId: filePath?.cas_id || null,
 		isDir: isPath(data) && data.item.is_dir,
 		kind: ObjectKind[objectData?.kind || 0] as KindType,
-		hasThumbnail: data.has_thumbnail,
+		hasLocalThumbnail: data.has_local_thumbnail, // this will be overwritten if new thumbnail is generated
+		thumbnailKey: data.thumbnail_key,
 		extension: filePath?.extension
 	};
 }
@@ -41,7 +44,8 @@ const FileThumbWrapper = ({ children, size = 1 }: PropsWithChildren<{ size: numb
 );
 
 export default function FileThumb({ data, size = 1 }: FileThumbProps) {
-	const { casId, isDir, kind, hasThumbnail, extension } = getExplorerItemData(data);
+	const { casId, isDir, kind, hasLocalThumbnail, extension, thumbnailKey } =
+		getExplorerItemData(data);
 
 	if (isPath(data) && data.item.is_dir) {
 		return (
@@ -51,12 +55,12 @@ export default function FileThumb({ data, size = 1 }: FileThumbProps) {
 		);
 	}
 
-	if (hasThumbnail && casId) {
+	if (hasLocalThumbnail && thumbnailKey) {
 		// TODO: Handle Image checkers bg?
 		return (
 			<FileThumbWrapper size={size}>
 				<Image
-					source={{ uri: getThumbnailUrlById(casId) }}
+					source={{ uri: getThumbnailUrlById(thumbnailKey) }}
 					resizeMode="contain"
 					style={tw`h-full w-full`}
 				/>

apps/web/src/App.tsx

@@ -28,8 +28,8 @@ const spacedriveProtocol = `${http}://${serverOrigin}/spacedrive`;
 
 const platform: Platform = {
 	platform: 'web',
-	getThumbnailUrlById: (casId) =>
-		`${spacedriveProtocol}/thumbnail/${encodeURIComponent(casId)}.webp`,
+	getThumbnailUrlByThumbKey: (keyParts) =>
+		`${spacedriveProtocol}/thumbnail/${keyParts.map(i => encodeURIComponent(i)).join("/")}.webp`,
 	getFileUrl: (libraryId, locationLocalId, filePathId) =>
 		`${spacedriveProtocol}/file/${encodeURIComponent(libraryId)}/${encodeURIComponent(
 			locationLocalId
@@ -42,12 +42,12 @@ const queryClient = new QueryClient({
 	defaultOptions: {
 		queries: import.meta.env.VITE_SD_DEMO_MODE
 			? {
-					refetchOnWindowFocus: false,
-					staleTime: Infinity,
-					cacheTime: Infinity,
-					networkMode: 'offlineFirst',
-					enabled: false
-			  }
+				refetchOnWindowFocus: false,
+				staleTime: Infinity,
+				cacheTime: Infinity,
+				networkMode: 'offlineFirst',
+				enabled: false
+			}
 			: undefined
 		// TODO: Mutations can't be globally disable which is annoying!
 	}

core/Cargo.toml

@@ -10,11 +10,8 @@ edition.workspace = true
 
 [features]
 default = []
-mobile = [
-] # This feature allows features to be disabled when the Core is running on mobile.
-ffmpeg = [
-	"dep:sd-ffmpeg",
-] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg.
+mobile = [] # This feature allows features to be disabled when the Core is running on mobile.
+ffmpeg = ["dep:sd-ffmpeg"] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg.
 location-watcher = ["dep:notify"]
 sync-messages = []
 heif = ["dep:sd-heif"]
@@ -91,6 +88,8 @@ normpath = { version = "1.1.1", features = ["localization"] }
 tracing-appender = { git = "https://github.com/tokio-rs/tracing", rev = "29146260fb4615d271d2e899ad95a753bb42915e" } # Unreleased changes for log deletion
 strum = { version = "0.24", features = ["derive"] }
 strum_macros = "0.24"
+hex = "0.4.3"
+int-enum = "0.4.0"
 
 
 [target.'cfg(windows)'.dependencies.winapi-util]

core/src/api/jobs.rs

@@ -120,7 +120,7 @@ pub(crate) fn mount() -> AlphaRouter<Ctx> {
 					async_stream::stream! {
 						while let Ok(event) = event_bus_rx.recv().await {
 							match event {
-								CoreEvent::NewThumbnail { cas_id } => yield cas_id,
+								CoreEvent::NewThumbnail { thumb_key } => yield thumb_key,
 								_ => {}
 							}
 						}

core/src/api/locations.rs

@@ -28,12 +28,16 @@ pub enum ExplorerContext {
 #[serde(tag = "type")]
 pub enum ExplorerItem {
 	Path {
-		// has_thumbnail is determined by the local existence of a thumbnail
-		has_thumbnail: bool,
+		// has_local_thumbnail is true only if there is local existence of a thumbnail
+		has_local_thumbnail: bool,
+		// thumbnail_key is present if there is a cas_id
+		// it includes the shard hex formatted as (["f0", "cab34a76fbf3469f"])
+		thumbnail_key: Option<Vec<String>>,
 		item: file_path_with_object::Data,
 	},
 	Object {
-		has_thumbnail: bool,
+		has_local_thumbnail: bool,
+		thumbnail_key: Option<Vec<String>>,
 		item: object_with_file_paths::Data,
 	},
 }

core/src/api/mod.rs

@@ -16,7 +16,7 @@ pub type Router = rspc::Router<Ctx>;
 /// Represents an internal core event, these are exposed to client via a rspc subscription.
 #[derive(Debug, Clone, Serialize, Type)]
 pub enum CoreEvent {
-	NewThumbnail { cas_id: String },
+	NewThumbnail { thumb_key: Vec<String> },
 	InvalidateOperation(InvalidateOperationEvent),
 }
 

core/src/api/search.rs

@@ -1,4 +1,7 @@
-use crate::location::file_path_helper::{check_file_path_exists, IsolatedFilePathData};
+use crate::{
+	location::file_path_helper::{check_file_path_exists, IsolatedFilePathData},
+	object::preview::get_thumb_key,
+};
 use std::collections::BTreeSet;
 
 use chrono::{DateTime, FixedOffset, Utc};
@@ -321,7 +324,7 @@ pub fn mount() -> AlphaRouter<Ctx> {
 					let mut items = Vec::with_capacity(file_paths.len());
 
 					for file_path in file_paths {
-						let has_thumbnail = if let Some(cas_id) = &file_path.cas_id {
+						let thumbnail_exists_locally = if let Some(cas_id) = &file_path.cas_id {
 							library
 								.thumbnail_exists(cas_id)
 								.await
@@ -331,7 +334,8 @@ pub fn mount() -> AlphaRouter<Ctx> {
 						};
 
 						items.push(ExplorerItem::Path {
-							has_thumbnail,
+							has_local_thumbnail: thumbnail_exists_locally,
+							thumbnail_key: file_path.cas_id.as_ref().map(|i| get_thumb_key(i)),
 							item: file_path,
 						})
 					}
@@ -389,7 +393,7 @@ pub fn mount() -> AlphaRouter<Ctx> {
 							.map(|fp| fp.cas_id.as_ref())
 							.find_map(|c| c);
 
-						let has_thumbnail = if let Some(cas_id) = cas_id {
+						let thumbnail_exists_locally = if let Some(cas_id) = cas_id {
 							library.thumbnail_exists(cas_id).await.map_err(|e| {
 								rspc::Error::with_cause(
 									ErrorCode::InternalServerError,
@@ -402,7 +406,8 @@ pub fn mount() -> AlphaRouter<Ctx> {
 						};
 
 						items.push(ExplorerItem::Object {
-							has_thumbnail,
+							has_local_thumbnail: thumbnail_exists_locally,
+							thumbnail_key: cas_id.map(|i| get_thumb_key(i)),
 							item: object,
 						});
 					}

core/src/custom_uri.rs

@@ -100,16 +100,18 @@ async fn handle_thumbnail(
 		return Ok(response?);
 	}
 
-	let file_cas_id = path
-		.get(1)
-		.ok_or_else(|| HandleCustomUriError::BadRequest("Invalid number of parameters!"))?;
+	if path.len() < 3 {
+		return Err(HandleCustomUriError::BadRequest(
+			"Invalid number of parameters!",
+		));
+	}
 
-	let filename = node
-		.config
-		.data_directory()
-		.join("thumbnails")
-		.join(file_cas_id)
-		.with_extension("webp");
+	let mut thumbnail_path = node.config.data_directory().join("thumbnails");
+	// if we ever wish to support multiple levels of sharding, we need only supply more params here
+	for path_part in &path[1..] {
+		thumbnail_path = thumbnail_path.join(path_part);
+	}
+	let filename = thumbnail_path.with_extension("webp");
 
 	let file = File::open(&filename).await.map_err(|err| {
 		if err.kind() == io::ErrorKind::NotFound {
@@ -119,7 +121,7 @@ async fn handle_thumbnail(
 		}
 	})?;
 
-	let content_lenght = file
+	let content_length = file
 		.metadata()
 		.await
 		.map_err(|e| FileIOError::from((&filename, e)))?
@@ -127,12 +129,12 @@ async fn handle_thumbnail(
 
 	Ok(builder
 		.header("Content-Type", "image/webp")
-		.header("Content-Length", content_lenght)
+		.header("Content-Length", content_length)
 		.status(StatusCode::OK)
 		.body(if method == Method::HEAD {
 			vec![]
 		} else {
-			read_file(file, content_lenght, None)
+			read_file(file, content_length, None)
 				.await
 				.map_err(|e| FileIOError::from((&filename, e)))?
 		})?)

core/src/object/preview/thumbnail/directory.rs

@@ -0,0 +1,102 @@
+use std::path::PathBuf;
+use tokio::fs as async_fs;
+
+use int_enum::IntEnum;
+use tracing::{error, info};
+
+use crate::util::{error::FileIOError, version_manager::VersionManager};
+
+use super::{get_shard_hex, ThumbnailerError, THUMBNAIL_CACHE_DIR_NAME};
+
+#[derive(IntEnum, Debug, Clone, Copy, Eq, PartialEq)]
+#[repr(i32)]
+pub enum ThumbnailVersion {
+	V1 = 1,
+	V2 = 2,
+	Unknown = 0,
+}
+
+pub async fn init_thumbnail_dir(data_dir: PathBuf) -> Result<PathBuf, ThumbnailerError> {
+	info!("Initializing thumbnail directory");
+	let thumbnail_dir = data_dir.join(THUMBNAIL_CACHE_DIR_NAME);
+
+	let version_file = thumbnail_dir.join("version.txt");
+	let version_manager =
+		VersionManager::<ThumbnailVersion>::new(version_file.to_str().expect("Invalid path"));
+
+	info!("Thumbnail directory: {:?}", thumbnail_dir);
+
+	// create all necessary directories if they don't exist
+	async_fs::create_dir_all(&thumbnail_dir)
+		.await
+		.map_err(|e| FileIOError::from((&thumbnail_dir, e)))?;
+
+	let mut current_version = match version_manager.get_version() {
+		Ok(version) => version,
+		Err(_) => {
+			info!("Thumbnail version file does not exist, starting fresh");
+			// Version file does not exist, start fresh
+			version_manager.set_version(ThumbnailVersion::V1)?;
+			ThumbnailVersion::V1
+		}
+	};
+
+	while current_version != ThumbnailVersion::V2 {
+		match current_version {
+			ThumbnailVersion::V1 => {
+				let thumbnail_dir_for_task = thumbnail_dir.clone();
+				// If the migration fails, it will return the error and exit the function
+				move_webp_files(&thumbnail_dir_for_task).await?;
+				version_manager.set_version(ThumbnailVersion::V2)?;
+				current_version = ThumbnailVersion::V2;
+			}
+			// If the current version is not handled explicitly, break the loop or return an error.
+			_ => {
+				error!("Thumbnail version is not handled: {:?}", current_version);
+			}
+		}
+	}
+
+	Ok(thumbnail_dir)
+}
+
+/// This function moves all webp files in the thumbnail directory to their respective shard folders.
+/// It is used to migrate from V1 to V2.
+async fn move_webp_files(dir: &PathBuf) -> Result<(), ThumbnailerError> {
+	let mut dir_entries = async_fs::read_dir(dir)
+		.await
+		.map_err(|source| FileIOError::from((dir, source)))?;
+	let mut count = 0;
+
+	while let Ok(Some(entry)) = dir_entries.next_entry().await {
+		let path = entry.path();
+		if path.is_file() {
+			if let Some(extension) = path.extension() {
+				if extension == "webp" {
+					let filename = path
+						.file_name()
+						.expect("Missing file name")
+						.to_str()
+						.expect("Failed to parse UTF8"); // we know they're cas_id's, so they're valid utf8
+					let shard_folder = get_shard_hex(filename);
+
+					let new_dir = dir.join(shard_folder);
+					async_fs::create_dir_all(&new_dir)
+						.await
+						.map_err(|source| FileIOError::from((new_dir.clone(), source)))?;
+
+					let new_path = new_dir.join(filename);
+					async_fs::rename(&path, &new_path)
+						.await
+						.map_err(|source| FileIOError::from((path.clone(), source)))?;
+					count += 1;
+				}
+			}
+		}
+	}
+	info!(
+		"Moved {} webp files to their respective shard folders.",
+		count
+	);
+	Ok(())
+}

core/src/object/preview/thumbnail/mod.rs

@@ -8,7 +8,7 @@ use crate::{
 		LocationId,
 	},
 	prisma::location,
-	util::error::FileIOError,
+	util::{error::FileIOError, version_manager::VersionManagerError},
 };
 
 use std::{
@@ -32,10 +32,14 @@ use webp::Encoder;
 
 use self::thumbnailer_job::ThumbnailerJob;
 
+mod directory;
 mod shallow;
+mod shard;
 pub mod thumbnailer_job;
 
+pub use directory::*;
 pub use shallow::*;
+pub use shard::*;
 
 const THUMBNAIL_SIZE_FACTOR: f32 = 0.2;
 const THUMBNAIL_QUALITY: f32 = 30.0;
@@ -47,10 +51,17 @@ pub fn get_thumbnail_path(library: &Library, cas_id: &str) -> PathBuf {
 		.config()
 		.data_directory()
 		.join(THUMBNAIL_CACHE_DIR_NAME)
+		.join(get_shard_hex(cas_id))
 		.join(cas_id)
 		.with_extension("webp")
 }
 
+// this is used to pass the relevant data to the frontend so it can request the thumbnail
+// it supports extending the shard hex to support deeper directory structures in the future
+pub fn get_thumb_key(cas_id: &str) -> Vec<String> {
+	vec![get_shard_hex(cas_id), cas_id.to_string()]
+}
+
 #[cfg(feature = "ffmpeg")]
 static FILTERED_VIDEO_EXTENSIONS: Lazy<Vec<Extension>> = Lazy::new(|| {
 	sd_file_ext::extensions::ALL_VIDEO_EXTENSIONS
@@ -89,6 +100,8 @@ pub enum ThumbnailerError {
 	FilePath(#[from] FilePathError),
 	#[error(transparent)]
 	FileIO(#[from] FileIOError),
+	#[error(transparent)]
+	VersionManager(#[from] VersionManagerError),
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -269,12 +282,22 @@ pub async fn inner_process_step(
 		return Ok(());
 	};
 
+	let thumb_dir = thumbnail_dir.join(get_shard_hex(cas_id));
+
+	// Create the directory if it doesn't exist
+	if let Err(e) = fs::create_dir_all(&thumb_dir).await {
+		error!("Error creating thumbnail directory {:#?}", e);
+	}
+
 	// Define and write the WebP-encoded file to a given path
-	let output_path = thumbnail_dir.join(format!("{cas_id}.webp"));
+	let output_path = thumb_dir.join(format!("{cas_id}.webp"));
 
 	match fs::metadata(&output_path).await {
 		Ok(_) => {
-			info!("Thumb exists, skipping... {}", output_path.display());
+			info!(
+				"Thumb already exists, skipping generation for {}",
+				output_path.display()
+			);
 		}
 		Err(e) if e.kind() == io::ErrorKind::NotFound => {
 			info!("Writing {:?} to {:?}", path, output_path);
@@ -293,9 +316,9 @@ pub async fn inner_process_step(
 				}
 			}
 
-			println!("emitting new thumbnail event");
+			info!("Emitting new thumbnail event");
 			library.emit(CoreEvent::NewThumbnail {
-				cas_id: cas_id.clone(),
+				thumb_key: get_thumb_key(cas_id),
 			});
 		}
 		Err(e) => return Err(ThumbnailerError::from(FileIOError::from((output_path, e))).into()),

core/src/object/preview/thumbnail/shallow.rs

@@ -1,6 +1,5 @@
 use super::{
 	ThumbnailerError, ThumbnailerJobStep, ThumbnailerJobStepKind, FILTERED_IMAGE_EXTENSIONS,
-	THUMBNAIL_CACHE_DIR_NAME,
 };
 use crate::{
 	invalidate_query,
@@ -19,6 +18,7 @@ use crate::{
 };
 use sd_file_ext::extensions::Extension;
 use std::path::{Path, PathBuf};
+use thumbnail::init_thumbnail_dir;
 use tokio::fs;
 use tracing::info;
 
@@ -32,10 +32,7 @@ pub async fn shallow_thumbnailer(
 ) -> Result<(), JobError> {
 	let Library { db, .. } = &library;
 
-	let thumbnail_dir = library
-		.config()
-		.data_directory()
-		.join(THUMBNAIL_CACHE_DIR_NAME);
+	let thumbnail_dir = init_thumbnail_dir(library.config().data_directory()).await?;
 
 	let location_id = location.id;
 	let location_path = PathBuf::from(&location.path);

core/src/object/preview/thumbnail/shard.rs

@@ -0,0 +1,8 @@
+/// The practice of dividing files into hex coded folders, often called "sharding," is mainly used to optimize file system performance. File systems can start to slow down as the number of files in a directory increases. Thus, it's often beneficial to split files into multiple directories to avoid this performance degradation.
+
+/// `get_shard_hex` takes a cas_id (a hexadecimal hash) as input and returns the first two characters of the hash as the directory name. Because we're using the first two characters of a the hash, this will give us 256 (16*16) possible directories, named 00 to ff.
+pub fn get_shard_hex(cas_id: &str) -> String {
+	// Use the first two characters of the hash as the directory name
+	let directory_name = &cas_id[0..2];
+	directory_name.to_string()
+}

core/src/object/preview/thumbnail/thumbnailer_job.rs

@@ -7,8 +7,8 @@ use crate::{
 		ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
 		file_path_for_thumbnailer, IsolatedFilePathData,
 	},
+	object::preview::thumbnail::directory::init_thumbnail_dir,
 	prisma::{file_path, location, PrismaClient},
-	util::error::FileIOError,
 };
 
 use std::{collections::VecDeque, hash::Hash, path::PathBuf};
@@ -16,13 +16,12 @@ use std::{collections::VecDeque, hash::Hash, path::PathBuf};
 use sd_file_ext::extensions::Extension;
 
 use serde::{Deserialize, Serialize};
-use tokio::fs;
+
 use tracing::info;
 
 use super::{
 	finalize_thumbnailer, process_step, ThumbnailerError, ThumbnailerJobReport,
 	ThumbnailerJobState, ThumbnailerJobStep, ThumbnailerJobStepKind, FILTERED_IMAGE_EXTENSIONS,
-	THUMBNAIL_CACHE_DIR_NAME,
 };
 
 #[cfg(feature = "ffmpeg")]
@@ -64,11 +63,8 @@ impl StatefulJob for ThumbnailerJob {
 	async fn init(&self, ctx: WorkerContext, state: &mut JobState<Self>) -> Result<(), JobError> {
 		let Library { db, .. } = &ctx.library;
 
-		let thumbnail_dir = ctx
-			.library
-			.config()
-			.data_directory()
-			.join(THUMBNAIL_CACHE_DIR_NAME);
+		let thumbnail_dir = init_thumbnail_dir(ctx.library.config().data_directory()).await?;
+		// .join(THUMBNAIL_CACHE_DIR_NAME);
 
 		let location_id = state.init.location.id;
 		let location_path = PathBuf::from(&state.init.location.path);
@@ -104,11 +100,6 @@ impl StatefulJob for ThumbnailerJob {
 
 		info!("Searching for images in location {location_id} at directory {iso_file_path}");
 
-		// create all necessary directories if they don't exist
-		fs::create_dir_all(&thumbnail_dir)
-			.await
-			.map_err(|e| FileIOError::from((&thumbnail_dir, e)))?;
-
 		// query database for all image files in this location that need thumbnails
 		let image_files = get_files_by_extensions(
 			db,

core/src/util/mod.rs

@@ -4,5 +4,6 @@ pub mod db;
 pub mod debug_initializer;
 pub mod error;
 pub mod migrator;
+pub mod version_manager;
 
 pub use abort_on_drop::*;

core/src/util/version_manager.rs

@@ -0,0 +1,76 @@
+use int_enum::IntEnum;
+use std::fs;
+use std::io::prelude::*;
+use std::path::Path;
+use std::str::FromStr;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum VersionManagerError {
+	#[error("Invalid version")]
+	InvalidVersion,
+	#[error("Version file does not exist")]
+	VersionFileDoesNotExist,
+	#[error("Error while converting integer to enum")]
+	IntConversionError,
+	#[error("Malformed version file")]
+	MalformedVersionFile,
+	#[error(transparent)]
+	IO(#[from] std::io::Error),
+	#[error(transparent)]
+	ParseIntError(#[from] std::num::ParseIntError),
+}
+
+///
+/// An abstract system for saving a text file containing a version number.
+/// The version number is an integer that can be converted to and from an enum.
+/// The enum must implement the IntEnum trait.
+///
+pub struct VersionManager<T: IntEnum<Int = i32>> {
+	version_file_path: String,
+	_marker: std::marker::PhantomData<T>,
+}
+
+impl<T: IntEnum<Int = i32>> VersionManager<T> {
+	pub fn new(version_file_path: &str) -> Self {
+		VersionManager {
+			version_file_path: version_file_path.to_string(),
+			_marker: std::marker::PhantomData,
+		}
+	}
+
+	pub fn get_version(&self) -> Result<T, VersionManagerError> {
+		if Path::new(&self.version_file_path).exists() {
+			let contents = fs::read_to_string(&self.version_file_path)?;
+			let version = i32::from_str(contents.trim())?;
+			T::from_int(version).map_err(|_| VersionManagerError::IntConversionError)
+		} else {
+			Err(VersionManagerError::VersionFileDoesNotExist)
+		}
+	}
+
+	pub fn set_version(&self, version: T) -> Result<(), VersionManagerError> {
+		let mut file = fs::File::create(&self.version_file_path)?;
+		file.write_all(version.int_value().to_string().as_bytes())?;
+		Ok(())
+	}
+
+	// pub async fn migrate<F: FnMut(T) -> Result<(), VersionManagerError>>(
+	// 	&self,
+	// 	current: T,
+	// 	latest: T,
+	// 	mut migrate_fn: F,
+	// ) -> Result<(), VersionManagerError> {
+	// 	for version_int in (current.int_value() + 1)..=latest.int_value() {
+	// 		let version = match T::from_int(version_int) {
+	// 			Ok(version) => version,
+	// 			Err(_) => return Err(VersionManagerError::IntConversionError),
+	// 		};
+	// 		migrate_fn(version)?;
+	// 	}
+
+	// 	self.set_version(latest)?;
+
+	// 	Ok(())
+	// }
+}

interface/app/$libraryId/Explorer/File/Thumb.tsx

@@ -131,7 +131,7 @@ function FileThumb({ size, cover, ...props }: ThumbProps) {
 
 		if (props.loadOriginal) {
 			setThumbType(ThumbType.Original);
-		} else if (itemData.hasThumbnail) {
+		} else if (itemData.hasLocalThumbnail) {
 			setThumbType(ThumbType.Thumbnail);
 		} else {
 			setThumbType(ThumbType.Icon);
@@ -139,7 +139,7 @@ function FileThumb({ size, cover, ...props }: ThumbProps) {
 	}, [props.loadOriginal, itemData]);
 
 	useEffect(() => {
-		const { casId, kind, isDir, extension, locationId: itemLocationId } = itemData;
+		const { casId, kind, isDir, extension, locationId: itemLocationId, thumbnailKey } = itemData;
 		const locationId = itemLocationId ?? explorerLocationId;
 		switch (thumbType) {
 			case ThumbType.Original:
@@ -158,8 +158,8 @@ function FileThumb({ size, cover, ...props }: ThumbProps) {
 				}
 				break;
 			case ThumbType.Thumbnail:
-				if (casId) {
-					setSrc(platform.getThumbnailUrlById(casId));
+				if (casId && thumbnailKey) {
+					setSrc(platform.getThumbnailUrlByThumbKey(thumbnailKey));
 				} else {
 					setThumbType(ThumbType.Icon);
 				}
@@ -183,7 +183,7 @@ function FileThumb({ size, cover, ...props }: ThumbProps) {
 	const onError = () => {
 		setLoaded(false);
 		setThumbType((prevThumbType) => {
-			return prevThumbType === ThumbType.Original && itemData.hasThumbnail
+			return prevThumbType === ThumbType.Original && itemData.hasLocalThumbnail
 				? ThumbType.Thumbnail
 				: ThumbType.Icon;
 		});

interface/app/$libraryId/Explorer/index.tsx

@@ -42,9 +42,8 @@ export default function Explorer(props: Props) {
 		onError: (err) => {
 			console.error('Error in RSPC subscription new thumbnail', err);
 		},
-		onData: (cas_id) => {
-			console.log({ cas_id });
-			explorerStore.addNewThumbnail(cas_id);
+		onData: (thumbKey) => {
+			explorerStore.addNewThumbnail(thumbKey);
 		}
 	});
 

interface/app/$libraryId/Explorer/util.ts

@@ -49,7 +49,8 @@ export function getExplorerItemData(data: ExplorerItem) {
 		isDir: isPath(data) && data.item.is_dir,
 		extension: filePath?.extension || null,
 		locationId: filePath?.location_id || null,
-		hasThumbnail: data.has_thumbnail
+		hasLocalThumbnail: data.has_local_thumbnail, // this will be overwritten if new thumbnail is generated
+		thumbnailKey: data.thumbnail_key
 	};
 }
 

interface/app/$libraryId/location/$id.tsx

@@ -42,7 +42,7 @@ export const Component = () => {
 				sub_path: path ?? ''
 			}
 		],
-		{ onData() {} }
+		{ onData() { } }
 	);
 
 	const explorerStore = getExplorerStore();
@@ -114,7 +114,8 @@ const useItems = () => {
 				}
 			]),
 		getNextPageParam: (lastPage) => lastPage.cursor ?? undefined,
-		keepPreviousData: true
+		keepPreviousData: true,
+		onSuccess: () => getExplorerStore().resetNewThumbnails()
 	});
 
 	const items = useMemo(() => query.data?.pages.flatMap((d) => d.items) || null, [query.data]);

interface/app/$libraryId/overview/data.ts

@@ -11,7 +11,7 @@ import {
 	useLibraryContext,
 	useRspcLibraryContext
 } from '@sd/client';
-import { useExplorerStore } from '~/hooks';
+import { getExplorerStore, useExplorerStore } from '~/hooks';
 import { useExplorerOrder } from '../Explorer/util';
 
 export const IconForCategory: Partial<Record<Category, string>> = {
@@ -86,7 +86,8 @@ export function useItems(selectedCategory: Category) {
 					cursor
 				}
 			]),
-		getNextPageParam: (lastPage) => lastPage.cursor ?? undefined
+		getNextPageParam: (lastPage) => lastPage.cursor ?? undefined,
+		onSuccess: () => getExplorerStore().resetNewThumbnails()
 	});
 
 	const pathsItems = useMemo(

interface/app/$libraryId/search.tsx

@@ -22,7 +22,7 @@ export const SEARCH_PARAMS = z.object({
 
 export type SearchArgs = z.infer<typeof SEARCH_PARAMS>;
 
-const ExplorerStuff = memo((props: { args: SearchArgs }) => {
+const SearchExplorer = memo((props: { args: SearchArgs }) => {
 	const explorerStore = useExplorerStore();
 	const { explorerViewOptions, explorerControlOptions, explorerToolOptions } =
 		useExplorerTopBarOptions();
@@ -41,7 +41,8 @@ const ExplorerStuff = memo((props: { args: SearchArgs }) => {
 		],
 		{
 			suspense: true,
-			enabled: !!search
+			enabled: !!search,
+			onSuccess: () => getExplorerStore().resetNewThumbnails()
 		}
 	);
 
@@ -98,7 +99,7 @@ export const Component = () => {
 
 	return (
 		<Suspense fallback="LOADING FIRST RENDER">
-			<ExplorerStuff args={search} />
+			<SearchExplorer args={search} />
 		</Suspense>
 	);
 };

interface/hooks/useExplorerItemData.ts

@@ -1,18 +1,23 @@
 import { useMemo } from 'react';
 import { ExplorerItem } from '@sd/client';
-import { getExplorerItemData, getItemFilePath } from '~/app/$libraryId/Explorer/util';
-import { useExplorerStore } from './useExplorerStore';
+import { getExplorerItemData } from '~/app/$libraryId/Explorer/util';
+import { flattenThumbnailKey, useExplorerStore } from './useExplorerStore';
 
 export function useExplorerItemData(explorerItem: ExplorerItem) {
-	const filePath = getItemFilePath(explorerItem);
-	const { newThumbnails } = useExplorerStore();
+	const explorerStore = useExplorerStore();
+
+	const newThumbnail = !!(
+		explorerItem.thumbnail_key &&
+		explorerStore.newThumbnails.has(flattenThumbnailKey(explorerItem.thumbnail_key))
+	);
 
-	const newThumbnail = newThumbnails?.[filePath?.cas_id || ''] || false;
 	return useMemo(() => {
 		const itemData = getExplorerItemData(explorerItem);
-		if (!itemData.hasThumbnail) {
-			itemData.hasThumbnail = newThumbnail;
+
+		if (!itemData.hasLocalThumbnail) {
+			itemData.hasLocalThumbnail = newThumbnail;
 		}
+
 		return itemData;
 	}, [explorerItem, newThumbnail]);
 }

interface/hooks/useExplorerStore.tsx

@@ -1,13 +1,12 @@
+import { ExplorerItem, FilePathSearchOrdering, ObjectSearchOrdering, resetStore } from '@sd/client';
 import { proxy, useSnapshot } from 'valtio';
-import { proxyMap, proxySet } from 'valtio/utils';
+import { proxySet } from 'valtio/utils';
 import { z } from 'zod';
-import { ExplorerItem, FilePathSearchOrdering, ObjectSearchOrdering } from '@sd/client';
-import { resetStore } from '@sd/client';
 
 type Join<K, P> = K extends string | number
 	? P extends string | number
-		? `${K}${'' extends P ? '' : '.'}${P}`
-		: never
+	? `${K}${'' extends P ? '' : '.'}${P}`
+	: never
 	: never;
 
 type Leaves<T> = T extends object ? { [K in keyof T]-?: Join<K, Leaves<T[K]>> }[keyof T] : '';
@@ -25,7 +24,7 @@ export enum ExplorerKind {
 export type CutCopyType = 'Cut' | 'Copy';
 
 export type FilePathSearchOrderingKeys = UnionKeys<FilePathSearchOrdering> | 'none';
-export type ObjectSearchOrderingKyes = UnionKeys<ObjectSearchOrdering> | 'none';
+export type ObjectSearchOrderingKeys = UnionKeys<ObjectSearchOrdering> | 'none';
 
 export const SortOrder = z.union([z.literal('Asc'), z.literal('Desc')]);
 
@@ -39,7 +38,7 @@ const state = {
 	tagAssignMode: false,
 	showInspector: false,
 	multiSelectIndexes: [] as number[],
-	newThumbnails: {} as Record<string, boolean | undefined>,
+	newThumbnails: proxySet() as Set<string>,
 	cutCopyState: {
 		sourcePath: '', // this is used solely for preventing copy/cutting to the same path (as that will truncate the file)
 		sourceLocationId: 0,
@@ -56,13 +55,22 @@ const state = {
 	groupBy: 'none'
 };
 
+export function flattenThumbnailKey(thumbKey: string[]) {
+	return thumbKey.join('/');
+}
+
 // Keep the private and use `useExplorerState` or `getExplorerStore` or you will get production build issues.
 const explorerStore = proxy({
 	...state,
 	reset: () => resetStore(explorerStore, state),
-	addNewThumbnail: (casId: string) => {
-		explorerStore.newThumbnails[casId] = true;
-	}
+	addNewThumbnail: (thumbKey: string[]) => {
+		explorerStore.newThumbnails.add(flattenThumbnailKey(thumbKey))
+	},
+	// this should be done when the explorer query is refreshed
+	// prevents memory leak
+	resetNewThumbnails: () => {
+		explorerStore.newThumbnails.clear();
+	},
 });
 
 export function useExplorerStore() {

interface/util/Platform.tsx

@@ -6,7 +6,7 @@ export type OperatingSystem = 'browser' | 'linux' | 'macOS' | 'windows' | 'unkno
 // This could be Tauri or web.
 export type Platform = {
 	platform: 'web' | 'tauri'; // This represents the specific platform implementation
-	getThumbnailUrlById: (casId: string) => string;
+	getThumbnailUrlByThumbKey: (thumbKey: string[]) => string;
 	getFileUrl: (
 		libraryId: string,
 		locationLocalId: number,

packages/client/src/core.ts

@@ -87,7 +87,7 @@ export type Procedures = {
         { key: "tags.update", input: LibraryArgs<TagUpdateArgs>, result: null },
     subscriptions: 
         { key: "invalidation.listen", input: never, result: InvalidateOperationEvent[] } | 
-        { key: "jobs.newThumbnail", input: LibraryArgs<null>, result: string } | 
+        { key: "jobs.newThumbnail", input: LibraryArgs<null>, result: string[] } | 
         { key: "locations.online", input: never, result: number[][] } | 
         { key: "locations.quickRescan", input: LibraryArgs<LightScanArgs>, result: null } | 
         { key: "p2p.events", input: never, result: P2PEvent } | 
@@ -127,7 +127,7 @@ export type EditLibraryArgs = { id: string; name: string | null; description: st
  */
 export type EncryptedKey = number[]
 
-export type ExplorerItem = { type: "Path"; has_thumbnail: boolean; item: FilePathWithObject } | { type: "Object"; has_thumbnail: boolean; item: ObjectWithFilePaths }
+export type ExplorerItem = { type: "Path"; has_local_thumbnail: boolean; thumbnail_key: string[] | null; item: FilePathWithObject } | { type: "Object"; has_local_thumbnail: boolean; thumbnail_key: string[] | null; item: ObjectWithFilePaths }
 
 export type FileCopierJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string; target_file_name_suffix: string | null }