From 8765f95e003fdfb678b71d860c176e8596cd57ce Mon Sep 17 00:00:00 2001 From: "Ericson \"Fogo\" Soares" Date: Fri, 22 Sep 2023 07:11:59 -0300 Subject: [PATCH] [ENG-1135] Locations: Total size query and return to Frontend (#1366) * Done * Fix size text to take its space --------- Co-authored-by: ameer2468 <33054370+ameer2468@users.noreply.github.com> --- .../migration.sql | 2 + core/prisma/schema.prisma | 1 + core/src/location/indexer/indexer_job.rs | 41 +++++++++------ core/src/location/indexer/shallow.rs | 22 +++++--- core/src/location/manager/watcher/utils.rs | 20 +++++--- core/src/location/mod.rs | 50 +++++++++++++++++++ .../settings/library/locations/ListItem.tsx | 25 ++++++++-- packages/client/src/core.ts | 4 +- 8 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql diff --git a/core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql b/core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql new file mode 100644 index 000000000..7af11991e --- /dev/null +++ b/core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "location" ADD COLUMN "size_in_bytes" BLOB; diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma index d0cda5328..9592bff9b 100644 --- a/core/prisma/schema.prisma +++ b/core/prisma/schema.prisma @@ -135,6 +135,7 @@ model Location { path String? total_capacity Int? available_capacity Int? + size_in_bytes Bytes? is_archived Boolean? generate_preview_media Boolean? sync_preview_media Boolean? diff --git a/core/src/location/indexer/indexer_job.rs b/core/src/location/indexer/indexer_job.rs index 5f1e8f444..fa5921620 100644 --- a/core/src/location/indexer/indexer_job.rs +++ b/core/src/location/indexer/indexer_job.rs @@ -10,7 +10,7 @@ use crate::{ ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location, IsolatedFilePathData, }, - location_with_indexer_rules, + location_with_indexer_rules, update_location_size, }, prisma::{file_path, location}, to_remove_db_fetcher_fn, @@ -496,24 +496,33 @@ impl StatefulJob for IndexerJobInit { ctx.library.orphan_remover.invoke().await; } - if let Some(data) = data { - update_directories_sizes( - &run_metadata.paths_and_sizes, - init.location.id, - &data.indexed_path, - &ctx.library, - ) - .await?; - - if data.indexed_path != data.location_path { - reverse_update_directories_sizes( - &data.indexed_path, + if run_metadata.indexed_count > 0 + || run_metadata.removed_count > 0 + || run_metadata.updated_count > 0 + { + if let Some(data) = data { + update_directories_sizes( + &run_metadata.paths_and_sizes, init.location.id, - &data.location_path, + &data.indexed_path, &ctx.library, ) - .await - .map_err(IndexerError::from)?; + .await?; + + if data.indexed_path != data.location_path { + reverse_update_directories_sizes( + &data.indexed_path, + init.location.id, + &data.location_path, + &ctx.library, + ) + .await + .map_err(IndexerError::from)?; + } + + update_location_size(init.location.id, &ctx.library) + .await + .map_err(IndexerError::from)?; } } diff --git a/core/src/location/indexer/shallow.rs b/core/src/location/indexer/shallow.rs index 96242d216..8885a2d4d 100644 --- a/core/src/location/indexer/shallow.rs +++ b/core/src/location/indexer/shallow.rs @@ -10,7 +10,7 @@ use crate::{ indexer::{ execute_indexer_update_step, reverse_update_directories_sizes, IndexerJobUpdateStep, }, - scan_location_sub_path, + scan_location_sub_path, update_location_size, }, to_remove_db_fetcher_fn, util::db::maybe_missing, @@ -88,10 +88,9 @@ pub async fn shallow( .await? }; - debug!( - "Walker at shallow indexer found {} file_paths to be removed", - to_remove.len() - ); + let to_remove_count = to_remove.len(); + + debug!("Walker at shallow indexer found {to_remove_count} file_paths to be removed"); node.thumbnail_remover .remove_cas_ids( @@ -109,12 +108,15 @@ pub async fn shallow( let mut new_directories_to_scan = HashSet::new(); + let mut to_create_count = 0; + let save_steps = walked .chunks(BATCH_SIZE) .into_iter() .enumerate() .map(|(i, chunk)| { let walked = chunk.collect::>(); + to_create_count += walked.len(); walked .iter() @@ -171,8 +173,14 @@ pub async fn shallow( execute_indexer_update_step(&step, library).await?; } - if to_walk_path != location_path { - reverse_update_directories_sizes(to_walk_path, location_id, location_path, library) + if to_create_count > 0 || to_update_count > 0 || to_remove_count > 0 { + if to_walk_path != location_path { + reverse_update_directories_sizes(to_walk_path, location_id, location_path, library) + .await + .map_err(IndexerError::from)?; + } + + update_location_size(location.id, library) .await .map_err(IndexerError::from)?; } diff --git a/core/src/location/manager/watcher/utils.rs b/core/src/location/manager/watcher/utils.rs index 287e7f357..500202de8 100644 --- a/core/src/location/manager/watcher/utils.rs +++ b/core/src/location/manager/watcher/utils.rs @@ -14,7 +14,7 @@ use crate::{ indexer::reverse_update_directories_sizes, location_with_indexer_rules, manager::LocationManagerError, - scan_location_sub_path, + scan_location_sub_path, update_location_size, }, object::{ file_identifier::FileMetadata, @@ -818,12 +818,13 @@ pub(super) async fn recalculate_directories_size( ) -> Result<(), LocationManagerError> { let mut location_path_cache = None; let mut should_invalidate = false; + let mut should_update_location_size = false; buffer.clear(); for (path, instant) in candidates.drain() { if instant.elapsed() > HUNDRED_MILLIS * 5 { if location_path_cache.is_none() { - location_path_cache = Some(maybe_missing( + location_path_cache = Some(PathBuf::from(maybe_missing( find_location(library, location_id) .select(location::select!({ path })) .exec() @@ -831,18 +832,21 @@ pub(super) async fn recalculate_directories_size( .ok_or(LocationManagerError::MissingLocation(location_id))? .path, "location.path", - )?) + )?)) } if let Some(location_path) = &location_path_cache { - if path != Path::new(location_path) { + if path != *location_path { trace!( - "Reverse calculating directory sizes starting at {} until {location_path}", - path.display() + "Reverse calculating directory sizes starting at {} until {}", + path.display(), + location_path.display(), ); reverse_update_directories_sizes(path, location_id, location_path, library) .await?; should_invalidate = true; + } else { + should_update_location_size = true; } } } else { @@ -850,6 +854,10 @@ pub(super) async fn recalculate_directories_size( } } + if should_update_location_size { + update_location_size(location_id, library).await?; + } + if should_invalidate { invalidate_query!(library, "search.paths"); } diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index 5ed8dea2f..dabfb6535 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -807,6 +807,7 @@ impl From for location::Data { total_capacity: data.total_capacity, available_capacity: data.available_capacity, is_archived: data.is_archived, + size_in_bytes: data.size_in_bytes, generate_preview_media: data.generate_preview_media, sync_preview_media: data.sync_preview_media, hidden: data.hidden, @@ -828,6 +829,7 @@ impl From<&location_with_indexer_rules::Data> for location::Data { name: data.name.clone(), total_capacity: data.total_capacity, available_capacity: data.available_capacity, + size_in_bytes: data.size_in_bytes.clone(), is_archived: data.is_archived, generate_preview_media: data.generate_preview_media, sync_preview_media: data.sync_preview_media, @@ -949,3 +951,51 @@ pub(super) async fn generate_thumbnail( thumb_key: get_thumb_key(cas_id), }); } + +pub async fn update_location_size( + location_id: location::id::Type, + library: &Library, +) -> Result<(), QueryError> { + let Library { db, .. } = library; + + let total_size = db + .file_path() + .find_many(vec![ + file_path::location_id::equals(Some(location_id)), + file_path::materialized_path::equals(Some("/".to_string())), + ]) + .select(file_path::select!({ size_in_bytes_bytes })) + .exec() + .await? + .into_iter() + .filter_map(|file_path| { + file_path.size_in_bytes_bytes.map(|size_in_bytes_bytes| { + u64::from_be_bytes([ + size_in_bytes_bytes[0], + size_in_bytes_bytes[1], + size_in_bytes_bytes[2], + size_in_bytes_bytes[3], + size_in_bytes_bytes[4], + size_in_bytes_bytes[5], + size_in_bytes_bytes[6], + size_in_bytes_bytes[7], + ]) + }) + }) + .sum::(); + + db.location() + .update( + location::id::equals(location_id), + vec![location::size_in_bytes::set(Some( + total_size.to_be_bytes().to_vec(), + ))], + ) + .exec() + .await?; + + invalidate_query!(library, "locations.list"); + invalidate_query!(library, "locations.get"); + + Ok(()) +} diff --git a/interface/app/$libraryId/settings/library/locations/ListItem.tsx b/interface/app/$libraryId/settings/library/locations/ListItem.tsx index c976a6e84..943bef3a6 100644 --- a/interface/app/$libraryId/settings/library/locations/ListItem.tsx +++ b/interface/app/$libraryId/settings/library/locations/ListItem.tsx @@ -2,10 +2,15 @@ import { Repeat, Trash } from '@phosphor-icons/react'; import clsx from 'clsx'; import { useState } from 'react'; import { useNavigate } from 'react-router'; -import { arraysEqual, Location, useLibraryMutation, useOnlineLocations } from '@sd/client'; +import { + arraysEqual, + byteSize, + Location, + useLibraryMutation, + useOnlineLocations +} from '@sd/client'; import { Button, Card, dialogManager, Tooltip } from '@sd/ui'; import { Folder } from '~/components'; -import { useIsDark } from '~/hooks'; import DeleteDialog from './DeleteDialog'; @@ -34,6 +39,7 @@ export default ({ location }: Props) => {

{location.name}

+

{/* // TODO: This is ephemeral so it should not come from the DB. Eg. a external USB can move between nodes */} {/* {location.node && ( @@ -45,9 +51,8 @@ export default ({ location }: Props) => {

-
+
{/* This is a fake button, do not add disabled prop pls */} - +