From df5cd0a449f2d4366d06355c17cd938ffeac182c Mon Sep 17 00:00:00 2001 From: Jamie Pine <32987599+jamiepine@users.noreply.github.com> Date: Thu, 8 Jun 2023 00:13:45 -0700 Subject: [PATCH] [ENG-708] Thumbnail sharding (#925) * first phase, basic sharding * improved API for sharding using a "thumbnailKey" * clean up param handling for custom_uri * added version manager with migrations for the thumbnail directory * remove redundant hash of a hash, silly * fix mobile * fix clippy --------- Co-authored-by: Utku Bakir <74243531+utkubakir@users.noreply.github.com> --- Cargo.lock | Bin 238608 -> 239640 bytes apps/desktop/src/App.tsx | 2 +- .../src/components/explorer/FileThumb.tsx | 16 +-- apps/web/src/App.tsx | 16 +-- core/Cargo.toml | 9 +- core/src/api/jobs.rs | 2 +- core/src/api/locations.rs | 10 +- core/src/api/mod.rs | 2 +- core/src/api/search.rs | 15 ++- core/src/custom_uri.rs | 26 ++--- .../src/object/preview/thumbnail/directory.rs | 102 ++++++++++++++++++ core/src/object/preview/thumbnail/mod.rs | 33 +++++- core/src/object/preview/thumbnail/shallow.rs | 7 +- core/src/object/preview/thumbnail/shard.rs | 8 ++ .../preview/thumbnail/thumbnailer_job.rs | 17 +-- core/src/util/mod.rs | 1 + core/src/util/version_manager.rs | 76 +++++++++++++ .../app/$libraryId/Explorer/File/Thumb.tsx | 10 +- interface/app/$libraryId/Explorer/index.tsx | 5 +- interface/app/$libraryId/Explorer/util.ts | 3 +- interface/app/$libraryId/location/$id.tsx | 5 +- interface/app/$libraryId/overview/data.ts | 5 +- interface/app/$libraryId/search.tsx | 7 +- interface/hooks/useExplorerItemData.ts | 19 ++-- interface/hooks/useExplorerStore.tsx | 28 +++-- interface/util/Platform.tsx | 2 +- packages/client/src/core.ts | 4 +- 27 files changed, 329 insertions(+), 101 deletions(-) create mode 100644 core/src/object/preview/thumbnail/directory.rs create mode 100644 core/src/object/preview/thumbnail/shard.rs create mode 100644 core/src/util/version_manager.rs diff --git a/Cargo.lock b/Cargo.lock index de7157dc3b9bfd22aa8ed5b01fcff963e2773804..8b02b0c487129dba1a4b56d6ea8c030c81841229 100644 GIT binary patch delta 1069 zcmZuvTSyd980MVWT+=icFL{k!yFF;@Idj{Y3cR4EViA-e7}cDavm~VzR7OQq52XkEKc8F%BU8qto+`g_G-c=Z}k>}pp(r}{3rGZy;i9T*+R7rIv_zzTky zN(=d|dQj$0y>5#56x%SJ&su-}@S)oqf_P7Paxnwm*< zN*#hM9CMIGupHNkB_oP4(rsHdX<@`#GVF9>{_>Iw3v46Z>In&>tfY4a3}fJ+Nu1uxlECS;CGd z!D4p#agZMzgkMF*_DlcC@cM^fPIT51pM>S!_!vA~BA&HOz(lI4ZdA|Xf$TtB1 delta 441 zcmbPnfp5YIz6}xVlj}LfCmW>jPnYLq;-7quM`H47_I;c4IUY%E4p;svxH-u5n80Q^ zx8u^2vqeHS|45Q%gzES*IiJINazYBn=G@eIijxJ~pHCL-w3uwxad7kB4k<<;M|`tM z7aNe#F0(nN+fHEfg4rGH(+g}Fr8avkekQotWa}Bh$qSjKHY*&TExP&Oo!N?$8~=+= zF8}v)bJ>4pm+2F27-hB}bYZ!~ z+pm6P+9qu3>CEiP#H6G%-P@a4cDt|}vy$($ O1ZK7EiIbVPivR%mlct^k diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index b94f3033d..bee433395 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -58,7 +58,7 @@ if (customUriServerUrl && !customUriServerUrl?.endsWith('/')) { const platform: Platform = { platform: 'tauri', - getThumbnailUrlById: (casId) => convertFileSrc(`thumbnail/${casId}`, 'spacedrive'), + getThumbnailUrlByThumbKey: (keyParts) => convertFileSrc(`thumbnail/${keyParts.map(i => encodeURIComponent(i)).join("/")}`, 'spacedrive'), getFileUrl: (libraryId, locationLocalId, filePathId, _linux_workaround) => { const path = `file/${libraryId}/${locationLocalId}/${filePathId}`; if (_linux_workaround && customUriServerUrl) { diff --git a/apps/mobile/src/components/explorer/FileThumb.tsx b/apps/mobile/src/components/explorer/FileThumb.tsx index e7f4b230f..9a75fce06 100644 --- a/apps/mobile/src/components/explorer/FileThumb.tsx +++ b/apps/mobile/src/components/explorer/FileThumb.tsx @@ -15,8 +15,10 @@ type FileThumbProps = { size?: number; }; -export const getThumbnailUrlById = (casId: string) => - `${DocumentDirectoryPath}/thumbnails/${encodeURIComponent(casId)}.webp`; +export const getThumbnailUrlById = (keyParts: string[]) => + `${DocumentDirectoryPath}/thumbnails/${keyParts + .map((i) => encodeURIComponent(i)) + .join('/')}.webp`; type KindType = keyof typeof icons | 'Unknown'; @@ -29,7 +31,8 @@ function getExplorerItemData(data: ExplorerItem) { casId: filePath?.cas_id || null, isDir: isPath(data) && data.item.is_dir, kind: ObjectKind[objectData?.kind || 0] as KindType, - hasThumbnail: data.has_thumbnail, + hasLocalThumbnail: data.has_local_thumbnail, // this will be overwritten if new thumbnail is generated + thumbnailKey: data.thumbnail_key, extension: filePath?.extension }; } @@ -41,7 +44,8 @@ const FileThumbWrapper = ({ children, size = 1 }: PropsWithChildren<{ size: numb ); export default function FileThumb({ data, size = 1 }: FileThumbProps) { - const { casId, isDir, kind, hasThumbnail, extension } = getExplorerItemData(data); + const { casId, isDir, kind, hasLocalThumbnail, extension, thumbnailKey } = + getExplorerItemData(data); if (isPath(data) && data.item.is_dir) { return ( @@ -51,12 +55,12 @@ export default function FileThumb({ data, size = 1 }: FileThumbProps) { ); } - if (hasThumbnail && casId) { + if (hasLocalThumbnail && thumbnailKey) { // TODO: Handle Image checkers bg? return ( diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 9b7bb033e..be4635c93 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -28,8 +28,8 @@ const spacedriveProtocol = `${http}://${serverOrigin}/spacedrive`; const platform: Platform = { platform: 'web', - getThumbnailUrlById: (casId) => - `${spacedriveProtocol}/thumbnail/${encodeURIComponent(casId)}.webp`, + getThumbnailUrlByThumbKey: (keyParts) => + `${spacedriveProtocol}/thumbnail/${keyParts.map(i => encodeURIComponent(i)).join("/")}.webp`, getFileUrl: (libraryId, locationLocalId, filePathId) => `${spacedriveProtocol}/file/${encodeURIComponent(libraryId)}/${encodeURIComponent( locationLocalId @@ -42,12 +42,12 @@ const queryClient = new QueryClient({ defaultOptions: { queries: import.meta.env.VITE_SD_DEMO_MODE ? { - refetchOnWindowFocus: false, - staleTime: Infinity, - cacheTime: Infinity, - networkMode: 'offlineFirst', - enabled: false - } + refetchOnWindowFocus: false, + staleTime: Infinity, + cacheTime: Infinity, + networkMode: 'offlineFirst', + enabled: false + } : undefined // TODO: Mutations can't be globally disable which is annoying! } diff --git a/core/Cargo.toml b/core/Cargo.toml index 4e49f78a5..537fbbe12 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -10,11 +10,8 @@ edition.workspace = true [features] default = [] -mobile = [ -] # This feature allows features to be disabled when the Core is running on mobile. -ffmpeg = [ - "dep:sd-ffmpeg", -] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. +mobile = [] # This feature allows features to be disabled when the Core is running on mobile. +ffmpeg = ["dep:sd-ffmpeg"] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. location-watcher = ["dep:notify"] sync-messages = [] heif = ["dep:sd-heif"] @@ -91,6 +88,8 @@ normpath = { version = "1.1.1", features = ["localization"] } tracing-appender = { git = "https://github.com/tokio-rs/tracing", rev = "29146260fb4615d271d2e899ad95a753bb42915e" } # Unreleased changes for log deletion strum = { version = "0.24", features = ["derive"] } strum_macros = "0.24" +hex = "0.4.3" +int-enum = "0.4.0" [target.'cfg(windows)'.dependencies.winapi-util] diff --git a/core/src/api/jobs.rs b/core/src/api/jobs.rs index fa7e32f21..435ecc40f 100644 --- a/core/src/api/jobs.rs +++ b/core/src/api/jobs.rs @@ -120,7 +120,7 @@ pub(crate) fn mount() -> AlphaRouter { async_stream::stream! { while let Ok(event) = event_bus_rx.recv().await { match event { - CoreEvent::NewThumbnail { cas_id } => yield cas_id, + CoreEvent::NewThumbnail { thumb_key } => yield thumb_key, _ => {} } } diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs index de0f80352..29a80bf03 100644 --- a/core/src/api/locations.rs +++ b/core/src/api/locations.rs @@ -28,12 +28,16 @@ pub enum ExplorerContext { #[serde(tag = "type")] pub enum ExplorerItem { Path { - // has_thumbnail is determined by the local existence of a thumbnail - has_thumbnail: bool, + // has_local_thumbnail is true only if there is local existence of a thumbnail + has_local_thumbnail: bool, + // thumbnail_key is present if there is a cas_id + // it includes the shard hex formatted as (["f0", "cab34a76fbf3469f"]) + thumbnail_key: Option>, item: file_path_with_object::Data, }, Object { - has_thumbnail: bool, + has_local_thumbnail: bool, + thumbnail_key: Option>, item: object_with_file_paths::Data, }, } diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs index 9066fc973..3b09a6258 100644 --- a/core/src/api/mod.rs +++ b/core/src/api/mod.rs @@ -16,7 +16,7 @@ pub type Router = rspc::Router; /// Represents an internal core event, these are exposed to client via a rspc subscription. #[derive(Debug, Clone, Serialize, Type)] pub enum CoreEvent { - NewThumbnail { cas_id: String }, + NewThumbnail { thumb_key: Vec }, InvalidateOperation(InvalidateOperationEvent), } diff --git a/core/src/api/search.rs b/core/src/api/search.rs index 98e9efb39..2859eb709 100644 --- a/core/src/api/search.rs +++ b/core/src/api/search.rs @@ -1,4 +1,7 @@ -use crate::location::file_path_helper::{check_file_path_exists, IsolatedFilePathData}; +use crate::{ + location::file_path_helper::{check_file_path_exists, IsolatedFilePathData}, + object::preview::get_thumb_key, +}; use std::collections::BTreeSet; use chrono::{DateTime, FixedOffset, Utc}; @@ -321,7 +324,7 @@ pub fn mount() -> AlphaRouter { let mut items = Vec::with_capacity(file_paths.len()); for file_path in file_paths { - let has_thumbnail = if let Some(cas_id) = &file_path.cas_id { + let thumbnail_exists_locally = if let Some(cas_id) = &file_path.cas_id { library .thumbnail_exists(cas_id) .await @@ -331,7 +334,8 @@ pub fn mount() -> AlphaRouter { }; items.push(ExplorerItem::Path { - has_thumbnail, + has_local_thumbnail: thumbnail_exists_locally, + thumbnail_key: file_path.cas_id.as_ref().map(|i| get_thumb_key(i)), item: file_path, }) } @@ -389,7 +393,7 @@ pub fn mount() -> AlphaRouter { .map(|fp| fp.cas_id.as_ref()) .find_map(|c| c); - let has_thumbnail = if let Some(cas_id) = cas_id { + let thumbnail_exists_locally = if let Some(cas_id) = cas_id { library.thumbnail_exists(cas_id).await.map_err(|e| { rspc::Error::with_cause( ErrorCode::InternalServerError, @@ -402,7 +406,8 @@ pub fn mount() -> AlphaRouter { }; items.push(ExplorerItem::Object { - has_thumbnail, + has_local_thumbnail: thumbnail_exists_locally, + thumbnail_key: cas_id.map(|i| get_thumb_key(i)), item: object, }); } diff --git a/core/src/custom_uri.rs b/core/src/custom_uri.rs index 1bb244f1f..29b1c0478 100644 --- a/core/src/custom_uri.rs +++ b/core/src/custom_uri.rs @@ -100,16 +100,18 @@ async fn handle_thumbnail( return Ok(response?); } - let file_cas_id = path - .get(1) - .ok_or_else(|| HandleCustomUriError::BadRequest("Invalid number of parameters!"))?; + if path.len() < 3 { + return Err(HandleCustomUriError::BadRequest( + "Invalid number of parameters!", + )); + } - let filename = node - .config - .data_directory() - .join("thumbnails") - .join(file_cas_id) - .with_extension("webp"); + let mut thumbnail_path = node.config.data_directory().join("thumbnails"); + // if we ever wish to support multiple levels of sharding, we need only supply more params here + for path_part in &path[1..] { + thumbnail_path = thumbnail_path.join(path_part); + } + let filename = thumbnail_path.with_extension("webp"); let file = File::open(&filename).await.map_err(|err| { if err.kind() == io::ErrorKind::NotFound { @@ -119,7 +121,7 @@ async fn handle_thumbnail( } })?; - let content_lenght = file + let content_length = file .metadata() .await .map_err(|e| FileIOError::from((&filename, e)))? @@ -127,12 +129,12 @@ async fn handle_thumbnail( Ok(builder .header("Content-Type", "image/webp") - .header("Content-Length", content_lenght) + .header("Content-Length", content_length) .status(StatusCode::OK) .body(if method == Method::HEAD { vec![] } else { - read_file(file, content_lenght, None) + read_file(file, content_length, None) .await .map_err(|e| FileIOError::from((&filename, e)))? })?) diff --git a/core/src/object/preview/thumbnail/directory.rs b/core/src/object/preview/thumbnail/directory.rs new file mode 100644 index 000000000..ca0a33680 --- /dev/null +++ b/core/src/object/preview/thumbnail/directory.rs @@ -0,0 +1,102 @@ +use std::path::PathBuf; +use tokio::fs as async_fs; + +use int_enum::IntEnum; +use tracing::{error, info}; + +use crate::util::{error::FileIOError, version_manager::VersionManager}; + +use super::{get_shard_hex, ThumbnailerError, THUMBNAIL_CACHE_DIR_NAME}; + +#[derive(IntEnum, Debug, Clone, Copy, Eq, PartialEq)] +#[repr(i32)] +pub enum ThumbnailVersion { + V1 = 1, + V2 = 2, + Unknown = 0, +} + +pub async fn init_thumbnail_dir(data_dir: PathBuf) -> Result { + info!("Initializing thumbnail directory"); + let thumbnail_dir = data_dir.join(THUMBNAIL_CACHE_DIR_NAME); + + let version_file = thumbnail_dir.join("version.txt"); + let version_manager = + VersionManager::::new(version_file.to_str().expect("Invalid path")); + + info!("Thumbnail directory: {:?}", thumbnail_dir); + + // create all necessary directories if they don't exist + async_fs::create_dir_all(&thumbnail_dir) + .await + .map_err(|e| FileIOError::from((&thumbnail_dir, e)))?; + + let mut current_version = match version_manager.get_version() { + Ok(version) => version, + Err(_) => { + info!("Thumbnail version file does not exist, starting fresh"); + // Version file does not exist, start fresh + version_manager.set_version(ThumbnailVersion::V1)?; + ThumbnailVersion::V1 + } + }; + + while current_version != ThumbnailVersion::V2 { + match current_version { + ThumbnailVersion::V1 => { + let thumbnail_dir_for_task = thumbnail_dir.clone(); + // If the migration fails, it will return the error and exit the function + move_webp_files(&thumbnail_dir_for_task).await?; + version_manager.set_version(ThumbnailVersion::V2)?; + current_version = ThumbnailVersion::V2; + } + // If the current version is not handled explicitly, break the loop or return an error. + _ => { + error!("Thumbnail version is not handled: {:?}", current_version); + } + } + } + + Ok(thumbnail_dir) +} + +/// This function moves all webp files in the thumbnail directory to their respective shard folders. +/// It is used to migrate from V1 to V2. +async fn move_webp_files(dir: &PathBuf) -> Result<(), ThumbnailerError> { + let mut dir_entries = async_fs::read_dir(dir) + .await + .map_err(|source| FileIOError::from((dir, source)))?; + let mut count = 0; + + while let Ok(Some(entry)) = dir_entries.next_entry().await { + let path = entry.path(); + if path.is_file() { + if let Some(extension) = path.extension() { + if extension == "webp" { + let filename = path + .file_name() + .expect("Missing file name") + .to_str() + .expect("Failed to parse UTF8"); // we know they're cas_id's, so they're valid utf8 + let shard_folder = get_shard_hex(filename); + + let new_dir = dir.join(shard_folder); + async_fs::create_dir_all(&new_dir) + .await + .map_err(|source| FileIOError::from((new_dir.clone(), source)))?; + + let new_path = new_dir.join(filename); + async_fs::rename(&path, &new_path) + .await + .map_err(|source| FileIOError::from((path.clone(), source)))?; + count += 1; + } + } + } + } + info!( + "Moved {} webp files to their respective shard folders.", + count + ); + Ok(()) +} diff --git a/core/src/object/preview/thumbnail/mod.rs b/core/src/object/preview/thumbnail/mod.rs index c7f4b810e..231bcd820 100644 --- a/core/src/object/preview/thumbnail/mod.rs +++ b/core/src/object/preview/thumbnail/mod.rs @@ -8,7 +8,7 @@ use crate::{ LocationId, }, prisma::location, - util::error::FileIOError, + util::{error::FileIOError, version_manager::VersionManagerError}, }; use std::{ @@ -32,10 +32,14 @@ use webp::Encoder; use self::thumbnailer_job::ThumbnailerJob; +mod directory; mod shallow; +mod shard; pub mod thumbnailer_job; +pub use directory::*; pub use shallow::*; +pub use shard::*; const THUMBNAIL_SIZE_FACTOR: f32 = 0.2; const THUMBNAIL_QUALITY: f32 = 30.0; @@ -47,10 +51,17 @@ pub fn get_thumbnail_path(library: &Library, cas_id: &str) -> PathBuf { .config() .data_directory() .join(THUMBNAIL_CACHE_DIR_NAME) + .join(get_shard_hex(cas_id)) .join(cas_id) .with_extension("webp") } +// this is used to pass the relevant data to the frontend so it can request the thumbnail +// it supports extending the shard hex to support deeper directory structures in the future +pub fn get_thumb_key(cas_id: &str) -> Vec { + vec![get_shard_hex(cas_id), cas_id.to_string()] +} + #[cfg(feature = "ffmpeg")] static FILTERED_VIDEO_EXTENSIONS: Lazy> = Lazy::new(|| { sd_file_ext::extensions::ALL_VIDEO_EXTENSIONS @@ -89,6 +100,8 @@ pub enum ThumbnailerError { FilePath(#[from] FilePathError), #[error(transparent)] FileIO(#[from] FileIOError), + #[error(transparent)] + VersionManager(#[from] VersionManagerError), } #[derive(Debug, Serialize, Deserialize)] @@ -269,12 +282,22 @@ pub async fn inner_process_step( return Ok(()); }; + let thumb_dir = thumbnail_dir.join(get_shard_hex(cas_id)); + + // Create the directory if it doesn't exist + if let Err(e) = fs::create_dir_all(&thumb_dir).await { + error!("Error creating thumbnail directory {:#?}", e); + } + // Define and write the WebP-encoded file to a given path - let output_path = thumbnail_dir.join(format!("{cas_id}.webp")); + let output_path = thumb_dir.join(format!("{cas_id}.webp")); match fs::metadata(&output_path).await { Ok(_) => { - info!("Thumb exists, skipping... {}", output_path.display()); + info!( + "Thumb already exists, skipping generation for {}", + output_path.display() + ); } Err(e) if e.kind() == io::ErrorKind::NotFound => { info!("Writing {:?} to {:?}", path, output_path); @@ -293,9 +316,9 @@ pub async fn inner_process_step( } } - println!("emitting new thumbnail event"); + info!("Emitting new thumbnail event"); library.emit(CoreEvent::NewThumbnail { - cas_id: cas_id.clone(), + thumb_key: get_thumb_key(cas_id), }); } Err(e) => return Err(ThumbnailerError::from(FileIOError::from((output_path, e))).into()), diff --git a/core/src/object/preview/thumbnail/shallow.rs b/core/src/object/preview/thumbnail/shallow.rs index 6f9a5f18b..21978d16e 100644 --- a/core/src/object/preview/thumbnail/shallow.rs +++ b/core/src/object/preview/thumbnail/shallow.rs @@ -1,6 +1,5 @@ use super::{ ThumbnailerError, ThumbnailerJobStep, ThumbnailerJobStepKind, FILTERED_IMAGE_EXTENSIONS, - THUMBNAIL_CACHE_DIR_NAME, }; use crate::{ invalidate_query, @@ -19,6 +18,7 @@ use crate::{ }; use sd_file_ext::extensions::Extension; use std::path::{Path, PathBuf}; +use thumbnail::init_thumbnail_dir; use tokio::fs; use tracing::info; @@ -32,10 +32,7 @@ pub async fn shallow_thumbnailer( ) -> Result<(), JobError> { let Library { db, .. } = &library; - let thumbnail_dir = library - .config() - .data_directory() - .join(THUMBNAIL_CACHE_DIR_NAME); + let thumbnail_dir = init_thumbnail_dir(library.config().data_directory()).await?; let location_id = location.id; let location_path = PathBuf::from(&location.path); diff --git a/core/src/object/preview/thumbnail/shard.rs b/core/src/object/preview/thumbnail/shard.rs new file mode 100644 index 000000000..81fd417e8 --- /dev/null +++ b/core/src/object/preview/thumbnail/shard.rs @@ -0,0 +1,8 @@ +/// The practice of dividing files into hex coded folders, often called "sharding," is mainly used to optimize file system performance. File systems can start to slow down as the number of files in a directory increases. Thus, it's often beneficial to split files into multiple directories to avoid this performance degradation. + +/// `get_shard_hex` takes a cas_id (a hexadecimal hash) as input and returns the first two characters of the hash as the directory name. Because we're using the first two characters of a the hash, this will give us 256 (16*16) possible directories, named 00 to ff. +pub fn get_shard_hex(cas_id: &str) -> String { + // Use the first two characters of the hash as the directory name + let directory_name = &cas_id[0..2]; + directory_name.to_string() +} diff --git a/core/src/object/preview/thumbnail/thumbnailer_job.rs b/core/src/object/preview/thumbnail/thumbnailer_job.rs index 2edb9191f..ac09b942d 100644 --- a/core/src/object/preview/thumbnail/thumbnailer_job.rs +++ b/core/src/object/preview/thumbnail/thumbnailer_job.rs @@ -7,8 +7,8 @@ use crate::{ ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location, file_path_for_thumbnailer, IsolatedFilePathData, }, + object::preview::thumbnail::directory::init_thumbnail_dir, prisma::{file_path, location, PrismaClient}, - util::error::FileIOError, }; use std::{collections::VecDeque, hash::Hash, path::PathBuf}; @@ -16,13 +16,12 @@ use std::{collections::VecDeque, hash::Hash, path::PathBuf}; use sd_file_ext::extensions::Extension; use serde::{Deserialize, Serialize}; -use tokio::fs; + use tracing::info; use super::{ finalize_thumbnailer, process_step, ThumbnailerError, ThumbnailerJobReport, ThumbnailerJobState, ThumbnailerJobStep, ThumbnailerJobStepKind, FILTERED_IMAGE_EXTENSIONS, - THUMBNAIL_CACHE_DIR_NAME, }; #[cfg(feature = "ffmpeg")] @@ -64,11 +63,8 @@ impl StatefulJob for ThumbnailerJob { async fn init(&self, ctx: WorkerContext, state: &mut JobState) -> Result<(), JobError> { let Library { db, .. } = &ctx.library; - let thumbnail_dir = ctx - .library - .config() - .data_directory() - .join(THUMBNAIL_CACHE_DIR_NAME); + let thumbnail_dir = init_thumbnail_dir(ctx.library.config().data_directory()).await?; + // .join(THUMBNAIL_CACHE_DIR_NAME); let location_id = state.init.location.id; let location_path = PathBuf::from(&state.init.location.path); @@ -104,11 +100,6 @@ impl StatefulJob for ThumbnailerJob { info!("Searching for images in location {location_id} at directory {iso_file_path}"); - // create all necessary directories if they don't exist - fs::create_dir_all(&thumbnail_dir) - .await - .map_err(|e| FileIOError::from((&thumbnail_dir, e)))?; - // query database for all image files in this location that need thumbnails let image_files = get_files_by_extensions( db, diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs index 849552798..c61f99f09 100644 --- a/core/src/util/mod.rs +++ b/core/src/util/mod.rs @@ -4,5 +4,6 @@ pub mod db; pub mod debug_initializer; pub mod error; pub mod migrator; +pub mod version_manager; pub use abort_on_drop::*; diff --git a/core/src/util/version_manager.rs b/core/src/util/version_manager.rs new file mode 100644 index 000000000..a982fb1f2 --- /dev/null +++ b/core/src/util/version_manager.rs @@ -0,0 +1,76 @@ +use int_enum::IntEnum; +use std::fs; +use std::io::prelude::*; +use std::path::Path; +use std::str::FromStr; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum VersionManagerError { + #[error("Invalid version")] + InvalidVersion, + #[error("Version file does not exist")] + VersionFileDoesNotExist, + #[error("Error while converting integer to enum")] + IntConversionError, + #[error("Malformed version file")] + MalformedVersionFile, + #[error(transparent)] + IO(#[from] std::io::Error), + #[error(transparent)] + ParseIntError(#[from] std::num::ParseIntError), +} + +/// +/// An abstract system for saving a text file containing a version number. +/// The version number is an integer that can be converted to and from an enum. +/// The enum must implement the IntEnum trait. +/// +pub struct VersionManager> { + version_file_path: String, + _marker: std::marker::PhantomData, +} + +impl> VersionManager { + pub fn new(version_file_path: &str) -> Self { + VersionManager { + version_file_path: version_file_path.to_string(), + _marker: std::marker::PhantomData, + } + } + + pub fn get_version(&self) -> Result { + if Path::new(&self.version_file_path).exists() { + let contents = fs::read_to_string(&self.version_file_path)?; + let version = i32::from_str(contents.trim())?; + T::from_int(version).map_err(|_| VersionManagerError::IntConversionError) + } else { + Err(VersionManagerError::VersionFileDoesNotExist) + } + } + + pub fn set_version(&self, version: T) -> Result<(), VersionManagerError> { + let mut file = fs::File::create(&self.version_file_path)?; + file.write_all(version.int_value().to_string().as_bytes())?; + Ok(()) + } + + // pub async fn migrate Result<(), VersionManagerError>>( + // &self, + // current: T, + // latest: T, + // mut migrate_fn: F, + // ) -> Result<(), VersionManagerError> { + // for version_int in (current.int_value() + 1)..=latest.int_value() { + // let version = match T::from_int(version_int) { + // Ok(version) => version, + // Err(_) => return Err(VersionManagerError::IntConversionError), + // }; + // migrate_fn(version)?; + // } + + // self.set_version(latest)?; + + // Ok(()) + // } +} diff --git a/interface/app/$libraryId/Explorer/File/Thumb.tsx b/interface/app/$libraryId/Explorer/File/Thumb.tsx index bd7b04654..aa6b42df1 100644 --- a/interface/app/$libraryId/Explorer/File/Thumb.tsx +++ b/interface/app/$libraryId/Explorer/File/Thumb.tsx @@ -131,7 +131,7 @@ function FileThumb({ size, cover, ...props }: ThumbProps) { if (props.loadOriginal) { setThumbType(ThumbType.Original); - } else if (itemData.hasThumbnail) { + } else if (itemData.hasLocalThumbnail) { setThumbType(ThumbType.Thumbnail); } else { setThumbType(ThumbType.Icon); @@ -139,7 +139,7 @@ function FileThumb({ size, cover, ...props }: ThumbProps) { }, [props.loadOriginal, itemData]); useEffect(() => { - const { casId, kind, isDir, extension, locationId: itemLocationId } = itemData; + const { casId, kind, isDir, extension, locationId: itemLocationId, thumbnailKey } = itemData; const locationId = itemLocationId ?? explorerLocationId; switch (thumbType) { case ThumbType.Original: @@ -158,8 +158,8 @@ function FileThumb({ size, cover, ...props }: ThumbProps) { } break; case ThumbType.Thumbnail: - if (casId) { - setSrc(platform.getThumbnailUrlById(casId)); + if (casId && thumbnailKey) { + setSrc(platform.getThumbnailUrlByThumbKey(thumbnailKey)); } else { setThumbType(ThumbType.Icon); } @@ -183,7 +183,7 @@ function FileThumb({ size, cover, ...props }: ThumbProps) { const onError = () => { setLoaded(false); setThumbType((prevThumbType) => { - return prevThumbType === ThumbType.Original && itemData.hasThumbnail + return prevThumbType === ThumbType.Original && itemData.hasLocalThumbnail ? ThumbType.Thumbnail : ThumbType.Icon; }); diff --git a/interface/app/$libraryId/Explorer/index.tsx b/interface/app/$libraryId/Explorer/index.tsx index 6ac3582dc..2069b8630 100644 --- a/interface/app/$libraryId/Explorer/index.tsx +++ b/interface/app/$libraryId/Explorer/index.tsx @@ -42,9 +42,8 @@ export default function Explorer(props: Props) { onError: (err) => { console.error('Error in RSPC subscription new thumbnail', err); }, - onData: (cas_id) => { - console.log({ cas_id }); - explorerStore.addNewThumbnail(cas_id); + onData: (thumbKey) => { + explorerStore.addNewThumbnail(thumbKey); } }); diff --git a/interface/app/$libraryId/Explorer/util.ts b/interface/app/$libraryId/Explorer/util.ts index fafa3ca3c..945582f3b 100644 --- a/interface/app/$libraryId/Explorer/util.ts +++ b/interface/app/$libraryId/Explorer/util.ts @@ -49,7 +49,8 @@ export function getExplorerItemData(data: ExplorerItem) { isDir: isPath(data) && data.item.is_dir, extension: filePath?.extension || null, locationId: filePath?.location_id || null, - hasThumbnail: data.has_thumbnail + hasLocalThumbnail: data.has_local_thumbnail, // this will be overwritten if new thumbnail is generated + thumbnailKey: data.thumbnail_key }; } diff --git a/interface/app/$libraryId/location/$id.tsx b/interface/app/$libraryId/location/$id.tsx index 62d88bb97..6794eeaf8 100644 --- a/interface/app/$libraryId/location/$id.tsx +++ b/interface/app/$libraryId/location/$id.tsx @@ -42,7 +42,7 @@ export const Component = () => { sub_path: path ?? '' } ], - { onData() {} } + { onData() { } } ); const explorerStore = getExplorerStore(); @@ -114,7 +114,8 @@ const useItems = () => { } ]), getNextPageParam: (lastPage) => lastPage.cursor ?? undefined, - keepPreviousData: true + keepPreviousData: true, + onSuccess: () => getExplorerStore().resetNewThumbnails() }); const items = useMemo(() => query.data?.pages.flatMap((d) => d.items) || null, [query.data]); diff --git a/interface/app/$libraryId/overview/data.ts b/interface/app/$libraryId/overview/data.ts index acd4241b8..0d125b717 100644 --- a/interface/app/$libraryId/overview/data.ts +++ b/interface/app/$libraryId/overview/data.ts @@ -11,7 +11,7 @@ import { useLibraryContext, useRspcLibraryContext } from '@sd/client'; -import { useExplorerStore } from '~/hooks'; +import { getExplorerStore, useExplorerStore } from '~/hooks'; import { useExplorerOrder } from '../Explorer/util'; export const IconForCategory: Partial> = { @@ -86,7 +86,8 @@ export function useItems(selectedCategory: Category) { cursor } ]), - getNextPageParam: (lastPage) => lastPage.cursor ?? undefined + getNextPageParam: (lastPage) => lastPage.cursor ?? undefined, + onSuccess: () => getExplorerStore().resetNewThumbnails() }); const pathsItems = useMemo( diff --git a/interface/app/$libraryId/search.tsx b/interface/app/$libraryId/search.tsx index 46c62d5b2..b1b703237 100644 --- a/interface/app/$libraryId/search.tsx +++ b/interface/app/$libraryId/search.tsx @@ -22,7 +22,7 @@ export const SEARCH_PARAMS = z.object({ export type SearchArgs = z.infer; -const ExplorerStuff = memo((props: { args: SearchArgs }) => { +const SearchExplorer = memo((props: { args: SearchArgs }) => { const explorerStore = useExplorerStore(); const { explorerViewOptions, explorerControlOptions, explorerToolOptions } = useExplorerTopBarOptions(); @@ -41,7 +41,8 @@ const ExplorerStuff = memo((props: { args: SearchArgs }) => { ], { suspense: true, - enabled: !!search + enabled: !!search, + onSuccess: () => getExplorerStore().resetNewThumbnails() } ); @@ -98,7 +99,7 @@ export const Component = () => { return ( - + ); }; diff --git a/interface/hooks/useExplorerItemData.ts b/interface/hooks/useExplorerItemData.ts index bd9cb70d5..529ad61cc 100644 --- a/interface/hooks/useExplorerItemData.ts +++ b/interface/hooks/useExplorerItemData.ts @@ -1,18 +1,23 @@ import { useMemo } from 'react'; import { ExplorerItem } from '@sd/client'; -import { getExplorerItemData, getItemFilePath } from '~/app/$libraryId/Explorer/util'; -import { useExplorerStore } from './useExplorerStore'; +import { getExplorerItemData } from '~/app/$libraryId/Explorer/util'; +import { flattenThumbnailKey, useExplorerStore } from './useExplorerStore'; export function useExplorerItemData(explorerItem: ExplorerItem) { - const filePath = getItemFilePath(explorerItem); - const { newThumbnails } = useExplorerStore(); + const explorerStore = useExplorerStore(); + + const newThumbnail = !!( + explorerItem.thumbnail_key && + explorerStore.newThumbnails.has(flattenThumbnailKey(explorerItem.thumbnail_key)) + ); - const newThumbnail = newThumbnails?.[filePath?.cas_id || ''] || false; return useMemo(() => { const itemData = getExplorerItemData(explorerItem); - if (!itemData.hasThumbnail) { - itemData.hasThumbnail = newThumbnail; + + if (!itemData.hasLocalThumbnail) { + itemData.hasLocalThumbnail = newThumbnail; } + return itemData; }, [explorerItem, newThumbnail]); } diff --git a/interface/hooks/useExplorerStore.tsx b/interface/hooks/useExplorerStore.tsx index 1c8c40bb8..cab249cc7 100644 --- a/interface/hooks/useExplorerStore.tsx +++ b/interface/hooks/useExplorerStore.tsx @@ -1,13 +1,12 @@ +import { ExplorerItem, FilePathSearchOrdering, ObjectSearchOrdering, resetStore } from '@sd/client'; import { proxy, useSnapshot } from 'valtio'; -import { proxyMap, proxySet } from 'valtio/utils'; +import { proxySet } from 'valtio/utils'; import { z } from 'zod'; -import { ExplorerItem, FilePathSearchOrdering, ObjectSearchOrdering } from '@sd/client'; -import { resetStore } from '@sd/client'; type Join = K extends string | number ? P extends string | number - ? `${K}${'' extends P ? '' : '.'}${P}` - : never + ? `${K}${'' extends P ? '' : '.'}${P}` + : never : never; type Leaves = T extends object ? { [K in keyof T]-?: Join> }[keyof T] : ''; @@ -25,7 +24,7 @@ export enum ExplorerKind { export type CutCopyType = 'Cut' | 'Copy'; export type FilePathSearchOrderingKeys = UnionKeys | 'none'; -export type ObjectSearchOrderingKyes = UnionKeys | 'none'; +export type ObjectSearchOrderingKeys = UnionKeys | 'none'; export const SortOrder = z.union([z.literal('Asc'), z.literal('Desc')]); @@ -39,7 +38,7 @@ const state = { tagAssignMode: false, showInspector: false, multiSelectIndexes: [] as number[], - newThumbnails: {} as Record, + newThumbnails: proxySet() as Set, cutCopyState: { sourcePath: '', // this is used solely for preventing copy/cutting to the same path (as that will truncate the file) sourceLocationId: 0, @@ -56,13 +55,22 @@ const state = { groupBy: 'none' }; +export function flattenThumbnailKey(thumbKey: string[]) { + return thumbKey.join('/'); +} + // Keep the private and use `useExplorerState` or `getExplorerStore` or you will get production build issues. const explorerStore = proxy({ ...state, reset: () => resetStore(explorerStore, state), - addNewThumbnail: (casId: string) => { - explorerStore.newThumbnails[casId] = true; - } + addNewThumbnail: (thumbKey: string[]) => { + explorerStore.newThumbnails.add(flattenThumbnailKey(thumbKey)) + }, + // this should be done when the explorer query is refreshed + // prevents memory leak + resetNewThumbnails: () => { + explorerStore.newThumbnails.clear(); + }, }); export function useExplorerStore() { diff --git a/interface/util/Platform.tsx b/interface/util/Platform.tsx index 682017247..430d85c59 100644 --- a/interface/util/Platform.tsx +++ b/interface/util/Platform.tsx @@ -6,7 +6,7 @@ export type OperatingSystem = 'browser' | 'linux' | 'macOS' | 'windows' | 'unkno // This could be Tauri or web. export type Platform = { platform: 'web' | 'tauri'; // This represents the specific platform implementation - getThumbnailUrlById: (casId: string) => string; + getThumbnailUrlByThumbKey: (thumbKey: string[]) => string; getFileUrl: ( libraryId: string, locationLocalId: number, diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index b6978f18b..b51c39778 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -87,7 +87,7 @@ export type Procedures = { { key: "tags.update", input: LibraryArgs, result: null }, subscriptions: { key: "invalidation.listen", input: never, result: InvalidateOperationEvent[] } | - { key: "jobs.newThumbnail", input: LibraryArgs, result: string } | + { key: "jobs.newThumbnail", input: LibraryArgs, result: string[] } | { key: "locations.online", input: never, result: number[][] } | { key: "locations.quickRescan", input: LibraryArgs, result: null } | { key: "p2p.events", input: never, result: P2PEvent } | @@ -127,7 +127,7 @@ export type EditLibraryArgs = { id: string; name: string | null; description: st */ export type EncryptedKey = number[] -export type ExplorerItem = { type: "Path"; has_thumbnail: boolean; item: FilePathWithObject } | { type: "Object"; has_thumbnail: boolean; item: ObjectWithFilePaths } +export type ExplorerItem = { type: "Path"; has_local_thumbnail: boolean; thumbnail_key: string[] | null; item: FilePathWithObject } | { type: "Object"; has_local_thumbnail: boolean; thumbnail_key: string[] | null; item: ObjectWithFilePaths } export type FileCopierJobInit = { source_location_id: number; source_path_id: number; target_location_id: number; target_path: string; target_file_name_suffix: string | null }