Move location online status out of database (#560)

* move location online status out of database

* remove useLibrarySubscription

* remove old code

apps/mobile/src/screens/settings/library/LocationSettings.tsx

@@ -1,19 +1,28 @@
 import { CaretRight, Repeat, Trash } from 'phosphor-react-native';
 import { Animated, FlatList, Pressable, Text, View } from 'react-native';
 import { Swipeable } from 'react-native-gesture-handler';
-import { Location, Node, useLibraryMutation, useLibraryQuery } from '@sd/client';
+import {
+	Location,
+	Node,
+	arraysEqual,
+	useLibraryMutation,
+	useLibraryQuery,
+	useOnlineLocations
+} from '@sd/client';
 import FolderIcon from '~/components/icons/FolderIcon';
 import DeleteLocationDialog from '~/containers/dialog/DeleteLocationDialog';
 import tw from '~/lib/tailwind';
 import { SettingsStackScreenProps } from '~/navigation/SettingsNavigator';
 
 function LocationItem({ location, index }: { location: Location & { node: Node }; index: number }) {
-	const { mutate: fullRescan } = useLibraryMutation('locations.fullRescan', {
+	const fullRescan = useLibraryMutation('locations.fullRescan', {
 		onMutate: () => {
 			// TODO: Show Toast
 		}
 	});
 
+	const onlineLocations = useOnlineLocations();
+
 	const renderRightActions = (progress: Animated.AnimatedInterpolation<number>) => {
 		const translate = progress.interpolate({
 			inputRange: [0, 1],
@@ -35,7 +44,7 @@ function LocationItem({ location, index }: { location: Location & { node: Node }
 				{/* Full Re-scan IS too much here */}
 				<Pressable
 					style={tw`py-1.5 px-3 bg-app-button border-app-line border rounded-md items-center justify-center shadow-sm mx-2`}
-					onPress={() => fullRescan(location.id)}
+					onPress={() => fullRescan.mutate(location.id)}
 				>
 					<Repeat size={18} color="white" />
 				</Pressable>
@@ -59,7 +68,9 @@ function LocationItem({ location, index }: { location: Location & { node: Node }
 					<View
 						style={tw.style(
 							'absolute w-2 h-2 right-0 bottom-0.5 rounded-full',
-							location.is_online ? 'bg-green-500' : 'bg-red-500'
+							onlineLocations.some((l) => arraysEqual(location.pub_id, l))
+								? 'bg-green-500'
+								: 'bg-red-500'
 						)}
 					/>
 				</View>

core/prisma/schema.prisma

@@ -102,7 +102,6 @@ model Location {
     filesystem         String?
     disk_type          Int?
     is_removable       Boolean?
-    is_online          Boolean  @default(true)
     is_archived        Boolean  @default(false)
     date_created       DateTime @default(now())
 

core/src/api/locations.rs

@@ -218,6 +218,23 @@ pub(crate) fn mount() -> rspc::RouterBuilder<
 				Ok(todo!())
 			})
 		})
+		.subscription("online", |t| {
+			t(|ctx, _: ()| {
+				let location_manager = ctx.library_manager.node_context.location_manager.clone();
+
+				let mut rx = location_manager.online_rx();
+
+				async_stream::stream! {
+					let online = location_manager.get_online().await;
+					dbg!(&online);
+					yield online;
+
+					while let Ok(locations) = rx.recv().await {
+						yield locations;
+					}
+				}
+			})
+		})
 		.merge("indexer_rules.", mount_indexer_rule_routes())
 }
 

core/src/library/library_manager.rs

@@ -33,7 +33,7 @@ pub struct LibraryManager {
 	/// libraries holds the list of libraries which are currently loaded into the node.
 	libraries: RwLock<Vec<LibraryContext>>,
 	/// node_context holds the context for the node which this library manager is running on.
-	node_context: NodeContext,
+	pub node_context: NodeContext,
 }
 
 #[derive(Error, Debug)]
@@ -332,9 +332,9 @@ impl LibraryManager {
 			.await?;
 
 		let key_manager = Arc::new(KeyManager::new(vec![]).await?);
-
 		seed_keymanager(&db, &key_manager).await?;
-		let (sync_manager, _) = SyncManager::new(db.clone(), id);
+
+		let (sync_manager, _) = SyncManager::new(&db, id);
 
 		Ok(LibraryContext {
 			id,

core/src/location/manager/helpers.rs

@@ -18,18 +18,16 @@ type LocationAndLibraryKey = (LocationId, LibraryId);
 const LOCATION_CHECK_INTERVAL: Duration = Duration::from_secs(5);
 
 pub(super) async fn check_online(location: &location::Data, library_ctx: &LibraryContext) -> bool {
+	let pub_id = &location.pub_id;
+
 	if let Some(ref local_path) = location.local_path {
 		match fs::metadata(local_path).await {
 			Ok(_) => {
-				if !location.is_online {
-					set_location_online(location.id, library_ctx, true).await;
-				}
+				library_ctx.location_manager().add_online(pub_id).await;
 				true
 			}
 			Err(e) if e.kind() == ErrorKind::NotFound => {
-				if location.is_online {
-					set_location_online(location.id, library_ctx, false).await;
-				}
+				library_ctx.location_manager().remove_online(pub_id).await;
 				false
 			}
 			Err(e) => {
@@ -39,35 +37,11 @@ pub(super) async fn check_online(location: &location::Data, library_ctx: &Librar
 		}
 	} else {
 		// In this case, we don't have a `local_path`, but this location was marked as online
-		if location.is_online {
-			set_location_online(location.id, library_ctx, false).await;
-		}
+		library_ctx.location_manager().remove_online(pub_id).await;
 		false
 	}
 }
 
-pub(super) async fn set_location_online(
-	location_id: LocationId,
-	library_ctx: &LibraryContext,
-	online: bool,
-) {
-	if let Err(e) = library_ctx
-		.db
-		.location()
-		.update(
-			location::id::equals(location_id),
-			vec![location::is_online::set(online)],
-		)
-		.exec()
-		.await
-	{
-		error!(
-			"Failed to update location to online: (id: {}, error: {:#?})",
-			location_id, e
-		);
-	}
-}
-
 pub(super) async fn location_check_sleep(
 	location_id: LocationId,
 	library_ctx: LibraryContext,

core/src/location/manager/mod.rs

@@ -1,13 +1,20 @@
 use crate::library::LibraryContext;
 
 use std::{
+	collections::BTreeSet,
 	path::{Path, PathBuf},
 	sync::Arc,
 };
 
 use futures::executor::block_on;
 use thiserror::Error;
-use tokio::{io, sync::oneshot};
+use tokio::{
+	io,
+	sync::{
+		broadcast::{self, Receiver},
+		oneshot, RwLock,
+	},
+};
 use tracing::error;
 
 #[cfg(feature = "location-watcher")]
@@ -93,8 +100,12 @@ pub enum LocationManagerError {
 	IOError(#[from] io::Error),
 }
 
+type OnlineLocations = BTreeSet<Vec<u8>>;
+
 #[derive(Debug)]
 pub struct LocationManager {
+	online_locations: RwLock<OnlineLocations>,
+	pub online_tx: broadcast::Sender<OnlineLocations>,
 	#[cfg(feature = "location-watcher")]
 	location_management_tx: mpsc::Sender<LocationManagementMessage>,
 	#[cfg(feature = "location-watcher")]
@@ -104,6 +115,8 @@ pub struct LocationManager {
 
 impl LocationManager {
 	pub fn new() -> Arc<Self> {
+		let online_tx = broadcast::channel(16).0;
+
 		#[cfg(feature = "location-watcher")]
 		{
 			let (location_management_tx, location_management_rx) = mpsc::channel(128);
@@ -120,6 +133,8 @@ impl LocationManager {
 			debug!("Location manager initialized");
 
 			Arc::new(Self {
+				online_locations: Default::default(),
+				online_tx,
 				location_management_tx,
 				watcher_management_tx,
 				stop_tx: Some(stop_tx),
@@ -129,7 +144,11 @@ impl LocationManager {
 		#[cfg(not(feature = "location-watcher"))]
 		{
 			tracing::warn!("Location watcher is disabled, locations will not be checked");
-			Arc::new(Self { stop_tx: None })
+			Arc::new(Self {
+				online_tx,
+				online_locations: Default::default(),
+				stop_tx: None,
+			})
 		}
 	}
 
@@ -466,6 +485,34 @@ impl LocationManager {
 
 		Ok(())
 	}
+
+	pub async fn is_online(&self, id: &Vec<u8>) -> bool {
+		let online_locations = self.online_locations.read().await;
+		online_locations.contains(id)
+	}
+
+	pub async fn get_online(&self) -> OnlineLocations {
+		self.online_locations.read().await.clone()
+	}
+
+	async fn broadcast_online(&self) {
+		self.online_tx.send(self.get_online().await).ok();
+	}
+
+	pub async fn add_online(&self, id: &[u8]) {
+		self.online_locations.write().await.insert(id.into());
+		self.broadcast_online().await;
+	}
+
+	pub async fn remove_online(&self, id: &Vec<u8>) {
+		let mut online_locations = self.online_locations.write().await;
+		online_locations.remove(id);
+		self.broadcast_online().await;
+	}
+
+	pub fn online_rx(&self) -> Receiver<OnlineLocations> {
+		self.online_tx.subscribe()
+	}
 }
 
 impl Drop for LocationManager {

core/src/location/manager/watcher/mod.rs

@@ -29,7 +29,7 @@ mod windows;
 
 mod utils;
 
-use utils::{check_event, check_location_online};
+use utils::check_event;
 
 #[cfg(target_os = "linux")]
 type Handler = linux::LinuxEventHandler;
@@ -178,25 +178,31 @@ impl LocationWatcher {
 		library_ctx: &LibraryContext,
 		ignore_paths: &HashSet<PathBuf>,
 	) -> Result<(), LocationManagerError> {
-		if check_event(&event, ignore_paths) {
-			if let Some(location) = fetch_location(library_ctx, location_id)
-				.include(indexer_job_location::include())
-				.exec()
-				.await?
-			{
-				if check_location_online(&location) {
-					return event_handler
-						.handle_event(location, library_ctx, event)
-						.await;
-				} else {
-					warn!("Tried to handle event for offline location: <id='{location_id}'>");
-				}
-			} else {
-				warn!("Tried to handle event for unknown location: <id='{location_id}'>");
-			}
+		if !check_event(&event, ignore_paths) {
+			return Ok(());
 		}
 
-		Ok(())
+		let Some(location) = fetch_location(library_ctx, location_id)
+			.include(indexer_job_location::include())
+			.exec()
+			.await?
+		else {
+			warn!("Tried to handle event for unknown location: <id='{location_id}'>");
+            return Ok(())
+        };
+
+		if !library_ctx
+			.location_manager()
+			.is_online(&location.pub_id)
+			.await
+		{
+			warn!("Tried to handle event for offline location: <id='{location_id}'>");
+			return Ok(());
+		}
+
+		event_handler
+			.handle_event(location, library_ctx, event)
+			.await
 	}
 
 	pub(super) fn ignore_path(

core/src/location/manager/watcher/utils.rs

@@ -34,20 +34,6 @@ use uuid::Uuid;
 
 use super::file_path_with_object;
 
-pub(super) fn check_location_online(location: &indexer_job_location::Data) -> bool {
-	// if location is offline return early
-	// this prevents ....
-	if !location.is_online {
-		info!(
-			"Location is offline, skipping event: <id='{}'>",
-			location.id
-		);
-		false
-	} else {
-		true
-	}
-}
-
 pub(super) fn check_event(event: &Event, ignore_paths: &HashSet<PathBuf>) -> bool {
 	// if first path includes .DS_Store, ignore
 	if event.paths.iter().any(|p| {

core/src/location/mod.rs

@@ -296,16 +296,13 @@ pub async fn relink_location(
 		.location()
 		.update(
 			location::pub_id::equals(metadata.location_pub_id(ctx.id)?.as_ref().to_vec()),
-			vec![
-				location::local_path::set(Some(
-					location_path
-						.as_ref()
-						.to_str()
-						.expect("Found non-UTF-8 path")
-						.to_string(),
-				)),
-				location::is_online::set(true),
-			],
+			vec![location::local_path::set(Some(
+				location_path
+					.as_ref()
+					.to_str()
+					.expect("Found non-UTF-8 path")
+					.to_string(),
+			))],
 		)
 		.exec()
 		.await?;
@@ -346,7 +343,6 @@ async fn create_location(
 				[
 					("node", json!({ "pub_id": ctx.id.as_bytes() })),
 					("name", json!(location_name)),
-					("is_online", json!(true)),
 					("local_path", json!(&local_path)),
 				],
 			),
@@ -356,7 +352,6 @@ async fn create_location(
 					node::id::equals(ctx.node_local_id),
 					vec![
 						location::name::set(Some(location_name.clone())),
-						location::is_online::set(true),
 						location::local_path::set(Some(local_path)),
 					],
 				)

core/src/sync/manager.rs

@@ -24,12 +24,12 @@ pub struct SyncManager {
 }
 
 impl SyncManager {
-	pub fn new(db: Arc<PrismaClient>, node: Uuid) -> (Self, Receiver<CRDTOperation>) {
+	pub fn new(db: &Arc<PrismaClient>, node: Uuid) -> (Self, Receiver<CRDTOperation>) {
 		let (tx, rx) = mpsc::channel(64);
 
 		(
 			Self {
-				db,
+				db: db.clone(),
 				node,
 				clock: HLCBuilder::new().with_id(node.into()).build(),
 				_clocks: Default::default(),

packages/client/src/core.ts

@@ -21,7 +21,7 @@ export type Procedures = {
         { key: "locations.getExplorerData", input: LibraryArgs<LocationExplorerArgs>, result: ExplorerData } | 
         { key: "locations.indexer_rules.get", input: LibraryArgs<number>, result: IndexerRule } | 
         { key: "locations.indexer_rules.list", input: LibraryArgs<null>, result: Array<IndexerRule> } | 
-        { key: "locations.list", input: LibraryArgs<null>, result: Array<{ id: number, pub_id: Array<number>, node_id: number, name: string | null, local_path: string | null, total_capacity: number | null, available_capacity: number | null, filesystem: string | null, disk_type: number | null, is_removable: boolean | null, is_online: boolean, is_archived: boolean, date_created: string, node: Node }> } | 
+        { key: "locations.list", input: LibraryArgs<null>, result: Array<{ id: number, pub_id: Array<number>, node_id: number, name: string | null, local_path: string | null, total_capacity: number | null, available_capacity: number | null, filesystem: string | null, disk_type: number | null, is_removable: boolean | null, is_archived: boolean, date_created: string, node: Node }> } | 
         { key: "nodeState", input: never, result: NodeState } | 
         { key: "normi.composite", input: never, result: NormalisedCompositeId } | 
         { key: "normi.org", input: never, result: NormalisedOrganisation } | 
@@ -79,7 +79,8 @@ export type Procedures = {
         { key: "tags.update", input: LibraryArgs<TagUpdateArgs>, result: null },
     subscriptions: 
         { key: "invalidateQuery", input: never, result: InvalidateOperationEvent } | 
-        { key: "jobs.newThumbnail", input: LibraryArgs<null>, result: string }
+        { key: "jobs.newThumbnail", input: LibraryArgs<null>, result: string } | 
+        { key: "locations.online", input: never, result: Array<Array<number>> }
 };
 
 export type Algorithm = "XChaCha20Poly1305" | "Aes256Gcm"
@@ -142,7 +143,7 @@ export interface LibraryConfig { version: string | null, name: string, descripti
 
 export interface LibraryConfigWrapped { uuid: string, config: LibraryConfig }
 
-export interface Location { id: number, pub_id: Array<number>, node_id: number, name: string | null, local_path: string | null, total_capacity: number | null, available_capacity: number | null, filesystem: string | null, disk_type: number | null, is_removable: boolean | null, is_online: boolean, is_archived: boolean, date_created: string }
+export interface Location { id: number, pub_id: Array<number>, node_id: number, name: string | null, local_path: string | null, total_capacity: number | null, available_capacity: number | null, filesystem: string | null, disk_type: number | null, is_removable: boolean | null, is_archived: boolean, date_created: string }
 
 export interface LocationCreateArgs { path: string, indexer_rules_ids: Array<number> }
 

packages/client/src/hooks/index.ts

@@ -1 +1,2 @@
 export * from './useCurrentLibrary';
+export * from './useOnlineLocations';

packages/client/src/hooks/useCurrentLibrary.tsx

@@ -47,6 +47,7 @@ export const useCurrentLibrary = () => {
 		keepPreviousData: true,
 		initialData: () => {
 			const cachedData = localStorage.getItem(libraryCacheLocalStorageKey);
+
 			if (cachedData) {
 				// If we fail to load cached data, it's fine
 				try {
@@ -55,13 +56,14 @@ export const useCurrentLibrary = () => {
 					console.error("Error loading cached 'sd-library-list' data", e);
 				}
 			}
+
 			return undefined;
 		},
 		onSuccess: (data) => {
 			localStorage.setItem(libraryCacheLocalStorageKey, JSON.stringify(data));
 
 			// Redirect to the onboarding flow if the user doesn't have any libraries
-			if (data?.length === 0) {
+			if (!data?.length) {
 				ctx.onNoLibrary();
 			}
 		}

packages/client/src/hooks/useOnlineLocations.ts

@@ -0,0 +1,13 @@
+import { useState } from 'react';
+import { useBridgeSubscription } from '../rspc';
+
+export function useOnlineLocations() {
+	const [state, setState] = useState<number[][] | null>(null);
+
+	// @ts-expect-error
+	useBridgeSubscription(['locations.online', null], {
+		onData: (d) => setState(d)
+	});
+
+	return state;
+}

packages/client/src/rspc.ts

@@ -44,7 +44,7 @@ const libraryHooks = hooks.createHooks<
 	// Normalized<StripLibraryArgsFromInput<LibraryProcedures<'mutations'>>>,
 	StripLibraryArgsFromInput<LibraryProcedures<'queries'>>,
 	StripLibraryArgsFromInput<LibraryProcedures<'mutations'>>,
-	never
+	StripLibraryArgsFromInput<LibraryProcedures<'subscriptions'>>
 >({
 	internal: {
 		customHooks: normiCustomHooks({ contextSharing: true }, () => {
@@ -71,6 +71,7 @@ export const rspc = hooks.createHooks<Procedures>();
 
 export const useBridgeQuery = nonLibraryHooks.useQuery;
 export const useBridgeMutation = nonLibraryHooks.useMutation;
+export const useBridgeSubscription = nonLibraryHooks.useSubscription;
 export const useLibraryQuery = libraryHooks.useQuery;
 export const useLibraryMutation = libraryHooks.useMutation;
 

packages/client/src/utils/index.ts

@@ -1 +1,9 @@
 export * from './isVideoExt';
+
+export function arraysEqual<T>(a: T[], b: T[]) {
+	if (a === b) return true;
+	if (a == null || b == null) return false;
+	if (a.length !== b.length) return false;
+
+	return a.every((n, i) => b[i] === n);
+}

packages/interface/src/components/layout/Sidebar.tsx

@@ -4,13 +4,16 @@ import { CheckCircle, CirclesFour, Gear, Lock, Planet, Plus } from 'phosphor-rea
 import React, { PropsWithChildren } from 'react';
 import { NavLink, NavLinkProps } from 'react-router-dom';
 import {
+	Location,
 	LocationCreateArgs,
+	arraysEqual,
 	getDebugState,
 	useBridgeQuery,
 	useCurrentLibrary,
 	useDebugState,
 	useLibraryMutation,
-	useLibraryQuery
+	useLibraryQuery,
+	useOnlineLocations
 } from '@sd/client';
 import {
 	Button,
@@ -119,7 +122,6 @@ export function Sidebar() {
 				{library && <LibraryScopedSection />}
 				<div className="flex-grow" />
 			</SidebarContents>
-
 			<SidebarFooter>
 				<div className="flex">
 					<ButtonLink
@@ -313,6 +315,8 @@ function LibraryScopedSection() {
 
 	const locations = useLibraryQuery(['locations.list'], { keepPreviousData: true });
 	const tags = useLibraryQuery(['tags.list'], { keepPreviousData: true });
+	const onlineLocations = useOnlineLocations();
+
 	const createLocation = useLibraryMutation('locations.create');
 
 	return (
@@ -328,21 +332,10 @@ function LibraryScopedSection() {
 					}
 				>
 					{locations.data?.map((location) => {
-						return (
-							<div key={location.id} className="flex flex-row items-center">
-								<SidebarLink
-									className="relative w-full group"
-									to={{
-										pathname: `location/${location.id}`
-									}}
-								>
-									<div className="-mt-0.5 mr-1 flex-grow-0 flex-shrink-0">
-										<Folder size={18} />
-									</div>
+						const online = onlineLocations?.some((l) => arraysEqual(location.pub_id, l));
 
-									<span className="flex-grow flex-shrink-0">{location.name}</span>
-								</SidebarLink>
-							</div>
+						return (
+							<SidebarLocation location={location} online={online ?? false} key={location.id} />
 						);
 					})}
 					{(locations.data?.length || 0) < 4 && (
@@ -398,6 +391,36 @@ function LibraryScopedSection() {
 	);
 }
 
+interface SidebarLocationProps {
+	location: Location;
+	online: boolean;
+}
+
+function SidebarLocation({ location, online }: SidebarLocationProps) {
+	return (
+		<div className="flex flex-row items-center">
+			<SidebarLink
+				className="relative w-full group"
+				to={{
+					pathname: `location/${location.id}`
+				}}
+			>
+				<div className="relative -mt-0.5 mr-1 flex-grow-0 flex-shrink-0">
+					<Folder size={18} />
+					<div
+						className={clsx(
+							'absolute w-1.5 h-1.5 right-0 bottom-0.5 rounded-full',
+							online ? 'bg-green-500' : 'bg-red-500'
+						)}
+					/>
+				</div>
+
+				<span className="flex-grow flex-shrink-0">{location.name}</span>
+			</SidebarLink>
+		</div>
+	);
+}
+
 const Icon = ({ component: Icon, ...props }: any) => (
 	<Icon weight="bold" {...props} className={clsx('w-4 h-4 mr-2', props.className)} />
 );

packages/interface/src/components/location/LocationListItem.tsx

@@ -1,7 +1,7 @@
 import clsx from 'clsx';
 import { Repeat, Trash } from 'phosphor-react';
 import { useState } from 'react';
-import { useLibraryMutation } from '@sd/client';
+import { arraysEqual, useLibraryMutation, useOnlineLocations } from '@sd/client';
 import { Location, Node } from '@sd/client';
 import { Button, Card, Dialog, UseDialogProps, dialogManager, useDialog } from '@sd/ui';
 import { useZodForm, z } from '@sd/ui/src/forms';
@@ -16,9 +16,12 @@ export default function LocationListItem({ location }: LocationListItemProps) {
 	const [hide, setHide] = useState(false);
 
 	const fullRescan = useLibraryMutation('locations.fullRescan');
+	const onlineLocations = useOnlineLocations();
 
 	if (hide) return <></>;
 
+	const online = onlineLocations?.some((l) => arraysEqual(location.pub_id, l)) || false;
+
 	return (
 		<Card>
 			<Folder size={30} className="mr-3" />
@@ -33,15 +36,8 @@ export default function LocationListItem({ location }: LocationListItemProps) {
 			<div className="flex h-[45px] p-2 space-x-2">
 				{/* This is a fake button, do not add disabled prop pls */}
 				<Button variant="gray" className="!py-1.5 !px-2 pointer-events-none flex">
-					<div
-						className={clsx(
-							'w-2 h-2  rounded-full',
-							location.is_online ? 'bg-green-500' : 'bg-red-500'
-						)}
-					/>
-					<span className="ml-1.5 text-xs text-ink-dull">
-						{location.is_online ? 'Online' : 'Offline'}
-					</span>
+					<div className={clsx('w-2 h-2  rounded-full', online ? 'bg-green-500' : 'bg-red-500')} />
+					<span className="ml-1.5 text-xs text-ink-dull">{online ? 'Online' : 'Offline'}</span>
 				</Button>
 				<Button
 					variant="gray"