[ENG-1135] Locations: Total size query and return to Frontend (#1366)

* Done

* Fix size text to take its space

---------

Co-authored-by: ameer2468 <33054370+ameer2468@users.noreply.github.com>

core/prisma/migrations/20230921203938_add_size_bytes_in_location_table/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "location" ADD COLUMN "size_in_bytes" BLOB;

core/prisma/schema.prisma

@@ -135,6 +135,7 @@ model Location {
     path                   String?
     total_capacity         Int?
     available_capacity     Int?
+    size_in_bytes          Bytes?
     is_archived            Boolean?
     generate_preview_media Boolean?
     sync_preview_media     Boolean?

core/src/location/indexer/indexer_job.rs

@@ -10,7 +10,7 @@ use crate::{
 			ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location,
 			IsolatedFilePathData,
 		},
-		location_with_indexer_rules,
+		location_with_indexer_rules, update_location_size,
 	},
 	prisma::{file_path, location},
 	to_remove_db_fetcher_fn,
@@ -496,24 +496,33 @@ impl StatefulJob for IndexerJobInit {
 			ctx.library.orphan_remover.invoke().await;
 		}
 
-		if let Some(data) = data {
-			update_directories_sizes(
-				&run_metadata.paths_and_sizes,
-				init.location.id,
-				&data.indexed_path,
-				&ctx.library,
-			)
-			.await?;
-
-			if data.indexed_path != data.location_path {
-				reverse_update_directories_sizes(
-					&data.indexed_path,
+		if run_metadata.indexed_count > 0
+			|| run_metadata.removed_count > 0
+			|| run_metadata.updated_count > 0
+		{
+			if let Some(data) = data {
+				update_directories_sizes(
+					&run_metadata.paths_and_sizes,
 					init.location.id,
-					&data.location_path,
+					&data.indexed_path,
 					&ctx.library,
 				)
-				.await
-				.map_err(IndexerError::from)?;
+				.await?;
+
+				if data.indexed_path != data.location_path {
+					reverse_update_directories_sizes(
+						&data.indexed_path,
+						init.location.id,
+						&data.location_path,
+						&ctx.library,
+					)
+					.await
+					.map_err(IndexerError::from)?;
+				}
+
+				update_location_size(init.location.id, &ctx.library)
+					.await
+					.map_err(IndexerError::from)?;
 			}
 		}
 

core/src/location/indexer/shallow.rs

@@ -10,7 +10,7 @@ use crate::{
 		indexer::{
 			execute_indexer_update_step, reverse_update_directories_sizes, IndexerJobUpdateStep,
 		},
-		scan_location_sub_path,
+		scan_location_sub_path, update_location_size,
 	},
 	to_remove_db_fetcher_fn,
 	util::db::maybe_missing,
@@ -88,10 +88,9 @@ pub async fn shallow(
 		.await?
 	};
 
-	debug!(
-		"Walker at shallow indexer found {} file_paths to be removed",
-		to_remove.len()
-	);
+	let to_remove_count = to_remove.len();
+
+	debug!("Walker at shallow indexer found {to_remove_count} file_paths to be removed");
 
 	node.thumbnail_remover
 		.remove_cas_ids(
@@ -109,12 +108,15 @@ pub async fn shallow(
 
 	let mut new_directories_to_scan = HashSet::new();
 
+	let mut to_create_count = 0;
+
 	let save_steps = walked
 		.chunks(BATCH_SIZE)
 		.into_iter()
 		.enumerate()
 		.map(|(i, chunk)| {
 			let walked = chunk.collect::<Vec<_>>();
+			to_create_count += walked.len();
 
 			walked
 				.iter()
@@ -171,8 +173,14 @@ pub async fn shallow(
 		execute_indexer_update_step(&step, library).await?;
 	}
 
-	if to_walk_path != location_path {
-		reverse_update_directories_sizes(to_walk_path, location_id, location_path, library)
+	if to_create_count > 0 || to_update_count > 0 || to_remove_count > 0 {
+		if to_walk_path != location_path {
+			reverse_update_directories_sizes(to_walk_path, location_id, location_path, library)
+				.await
+				.map_err(IndexerError::from)?;
+		}
+
+		update_location_size(location.id, library)
 			.await
 			.map_err(IndexerError::from)?;
 	}

core/src/location/manager/watcher/utils.rs

@@ -14,7 +14,7 @@ use crate::{
 		indexer::reverse_update_directories_sizes,
 		location_with_indexer_rules,
 		manager::LocationManagerError,
-		scan_location_sub_path,
+		scan_location_sub_path, update_location_size,
 	},
 	object::{
 		file_identifier::FileMetadata,
@@ -818,12 +818,13 @@ pub(super) async fn recalculate_directories_size(
 ) -> Result<(), LocationManagerError> {
 	let mut location_path_cache = None;
 	let mut should_invalidate = false;
+	let mut should_update_location_size = false;
 	buffer.clear();
 
 	for (path, instant) in candidates.drain() {
 		if instant.elapsed() > HUNDRED_MILLIS * 5 {
 			if location_path_cache.is_none() {
-				location_path_cache = Some(maybe_missing(
+				location_path_cache = Some(PathBuf::from(maybe_missing(
 					find_location(library, location_id)
 						.select(location::select!({ path }))
 						.exec()
@@ -831,18 +832,21 @@ pub(super) async fn recalculate_directories_size(
 						.ok_or(LocationManagerError::MissingLocation(location_id))?
 						.path,
 					"location.path",
-				)?)
+				)?))
 			}
 
 			if let Some(location_path) = &location_path_cache {
-				if path != Path::new(location_path) {
+				if path != *location_path {
 					trace!(
-						"Reverse calculating directory sizes starting at {} until {location_path}",
-						path.display()
+						"Reverse calculating directory sizes starting at {} until {}",
+						path.display(),
+						location_path.display(),
 					);
 					reverse_update_directories_sizes(path, location_id, location_path, library)
 						.await?;
 					should_invalidate = true;
+				} else {
+					should_update_location_size = true;
 				}
 			}
 		} else {
@@ -850,6 +854,10 @@ pub(super) async fn recalculate_directories_size(
 		}
 	}
 
+	if should_update_location_size {
+		update_location_size(location_id, library).await?;
+	}
+
 	if should_invalidate {
 		invalidate_query!(library, "search.paths");
 	}

core/src/location/mod.rs

@@ -807,6 +807,7 @@ impl From<location_with_indexer_rules::Data> for location::Data {
 			total_capacity: data.total_capacity,
 			available_capacity: data.available_capacity,
 			is_archived: data.is_archived,
+			size_in_bytes: data.size_in_bytes,
 			generate_preview_media: data.generate_preview_media,
 			sync_preview_media: data.sync_preview_media,
 			hidden: data.hidden,
@@ -828,6 +829,7 @@ impl From<&location_with_indexer_rules::Data> for location::Data {
 			name: data.name.clone(),
 			total_capacity: data.total_capacity,
 			available_capacity: data.available_capacity,
+			size_in_bytes: data.size_in_bytes.clone(),
 			is_archived: data.is_archived,
 			generate_preview_media: data.generate_preview_media,
 			sync_preview_media: data.sync_preview_media,
@@ -949,3 +951,51 @@ pub(super) async fn generate_thumbnail(
 		thumb_key: get_thumb_key(cas_id),
 	});
 }
+
+pub async fn update_location_size(
+	location_id: location::id::Type,
+	library: &Library,
+) -> Result<(), QueryError> {
+	let Library { db, .. } = library;
+
+	let total_size = db
+		.file_path()
+		.find_many(vec![
+			file_path::location_id::equals(Some(location_id)),
+			file_path::materialized_path::equals(Some("/".to_string())),
+		])
+		.select(file_path::select!({ size_in_bytes_bytes }))
+		.exec()
+		.await?
+		.into_iter()
+		.filter_map(|file_path| {
+			file_path.size_in_bytes_bytes.map(|size_in_bytes_bytes| {
+				u64::from_be_bytes([
+					size_in_bytes_bytes[0],
+					size_in_bytes_bytes[1],
+					size_in_bytes_bytes[2],
+					size_in_bytes_bytes[3],
+					size_in_bytes_bytes[4],
+					size_in_bytes_bytes[5],
+					size_in_bytes_bytes[6],
+					size_in_bytes_bytes[7],
+				])
+			})
+		})
+		.sum::<u64>();
+
+	db.location()
+		.update(
+			location::id::equals(location_id),
+			vec![location::size_in_bytes::set(Some(
+				total_size.to_be_bytes().to_vec(),
+			))],
+		)
+		.exec()
+		.await?;
+
+	invalidate_query!(library, "locations.list");
+	invalidate_query!(library, "locations.get");
+
+	Ok(())
+}

interface/app/$libraryId/settings/library/locations/ListItem.tsx

@@ -2,10 +2,15 @@ import { Repeat, Trash } from '@phosphor-icons/react';
 import clsx from 'clsx';
 import { useState } from 'react';
 import { useNavigate } from 'react-router';
-import { arraysEqual, Location, useLibraryMutation, useOnlineLocations } from '@sd/client';
+import {
+	arraysEqual,
+	byteSize,
+	Location,
+	useLibraryMutation,
+	useOnlineLocations
+} from '@sd/client';
 import { Button, Card, dialogManager, Tooltip } from '@sd/ui';
 import { Folder } from '~/components';
-import { useIsDark } from '~/hooks';
 
 import DeleteDialog from './DeleteDialog';
 
@@ -34,6 +39,7 @@ export default ({ location }: Props) => {
 			<Folder className="mr-3 h-10 w-10 self-center" />
 			<div className="grid min-w-[110px] grid-cols-1">
 				<h1 className="truncate pt-0.5 text-sm font-semibold">{location.name}</h1>
+
 				<p className="mt-0.5 select-text truncate text-sm text-ink-dull">
 					{/* // TODO: This is ephemeral so it should not come from the DB. Eg. a external USB can move between nodes */}
 					{/* {location.node && (
@@ -45,9 +51,8 @@ export default ({ location }: Props) => {
 				</p>
 			</div>
 			<div className="flex grow" />
-			<div className="flex h-[45px] space-x-2 p-2">
+			<div className="flex h-[45px] w-full max-w-fit space-x-2 p-2">
 				{/* This is a fake button, do not add disabled prop pls */}
-
 				<Button
 					onClick={(e: { stopPropagation: () => void }) => {
 						e.stopPropagation();
@@ -65,6 +70,18 @@ export default ({ location }: Props) => {
 						{online ? 'Online' : 'Offline'}
 					</span>
 				</Button>
+				<Button
+					onClick={(e: { stopPropagation: () => void }) => {
+						e.stopPropagation();
+					}}
+					variant="gray"
+					className="pointer-events-none flex !px-2 !py-1.5"
+				>
+					<p className="text-ink-dull">Size:</p>
+					<span className="ml-1.5 text-xs text-ink-dull">{`${byteSize(
+						location.size_in_bytes
+					)}`}</span>
+				</Button>
 				<Button
 					variant="gray"
 					className="!p-1.5"

packages/client/src/core.ts

@@ -248,7 +248,7 @@ export type LibraryPreferences = { location?: { [key: string]: LocationSettings
 
 export type LightScanArgs = { location_id: number; sub_path: string }
 
-export type Location = { id: number; pub_id: number[]; name: string | null; path: string | null; total_capacity: number | null; available_capacity: number | null; is_archived: boolean | null; generate_preview_media: boolean | null; sync_preview_media: boolean | null; hidden: boolean | null; date_created: string | null; instance_id: number | null }
+export type Location = { id: number; pub_id: number[]; name: string | null; path: string | null; total_capacity: number | null; available_capacity: number | null; size_in_bytes: number[] | null; is_archived: boolean | null; generate_preview_media: boolean | null; sync_preview_media: boolean | null; hidden: boolean | null; date_created: string | null; instance_id: number | null }
 
 /**
  * `LocationCreateArgs` is the argument received from the client using `rspc` to create a new location.
@@ -269,7 +269,7 @@ export type LocationSettings = { explorer: ExplorerSettings<FilePathOrder> }
  */
 export type LocationUpdateArgs = { id: number; name: string | null; generate_preview_media: boolean | null; sync_preview_media: boolean | null; hidden: boolean | null; indexer_rules_ids: number[]; path: string | null }
 
-export type LocationWithIndexerRules = { id: number; pub_id: number[]; name: string | null; path: string | null; total_capacity: number | null; available_capacity: number | null; is_archived: boolean | null; generate_preview_media: boolean | null; sync_preview_media: boolean | null; hidden: boolean | null; date_created: string | null; instance_id: number | null; indexer_rules: { indexer_rule: IndexerRule }[] }
+export type LocationWithIndexerRules = { id: number; pub_id: number[]; name: string | null; path: string | null; total_capacity: number | null; available_capacity: number | null; size_in_bytes: number[] | null; is_archived: boolean | null; generate_preview_media: boolean | null; sync_preview_media: boolean | null; hidden: boolean | null; date_created: string | null; instance_id: number | null; indexer_rules: { indexer_rule: IndexerRule }[] }
 
 export type MaybeNot<T> = T | { not: T }