From ba9ca3dd1626c8c8e548b187b402124d55e68ae3 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Wed, 31 May 2023 16:55:46 +0800 Subject: [PATCH] [ENG-674] fix invalidation (#891) * reenable the invalidate requests system * a tester for the invalidate system * watch locations in Library::load --------- Co-authored-by: Brendan Allan --- apps/server/Cargo.toml | 2 +- apps/web/src/demoData.json | 4 +- core/src/api/libraries.rs | 2 +- core/src/api/utils/invalidate.rs | 92 +++++++++++-------- core/src/library/manager.rs | 19 +++- crates/p2p/examples/basic.rs | 2 +- .../Layout/Sidebar/DebugPopover.tsx | 22 ++++- packages/client/src/core.ts | 2 + 8 files changed, 102 insertions(+), 43 deletions(-) diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index e1ba5bcc7..28e1dc5ba 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -9,7 +9,7 @@ edition.workspace = true assets = [] [dependencies] -sd-core = { path = "../../core", features = ["ffmpeg"] } +sd-core = { path = "../../core", features = ["ffmpeg", "location-watcher"] } rspc = { workspace = true, features = ["axum"] } httpz = { workspace = true, features = ["axum"] } axum = "0.6.18" diff --git a/apps/web/src/demoData.json b/apps/web/src/demoData.json index 4bf298eb5..07a474742 100644 --- a/apps/web/src/demoData.json +++ b/apps/web/src/demoData.json @@ -173,10 +173,10 @@ "fetchStatus": "fetching" }, "queryKey": [ - "library.getStatistics", + "library.statistics", { "library_id": "dc1c41b9-65c6-4a9d-a418-79e628b3ac59", "arg": null } ], - "queryHash": "[\"library.getStatistics\",{\"arg\":null,\"library_id\":\"dc1c41b9-65c6-4a9d-a418-79e628b3ac59\"}]" + "queryHash": "[\"library.statistics\",{\"arg\":null,\"library_id\":\"dc1c41b9-65c6-4a9d-a418-79e628b3ac59\"}]" } ] } diff --git a/core/src/api/libraries.rs b/core/src/api/libraries.rs index 21edbfc10..5f8ed8c50 100644 --- a/core/src/api/libraries.rs +++ b/core/src/api/libraries.rs @@ -110,7 +110,7 @@ pub(crate) fn mount() -> AlphaRouter { .get_library(new_library.uuid) .await .expect("We just created the library. Where do it be?"), - "library.getStatistics" + "library.statistics" ); Ok(new_library) diff --git a/core/src/api/utils/invalidate.rs b/core/src/api/utils/invalidate.rs index 63df249c4..f46b77ac6 100644 --- a/core/src/api/utils/invalidate.rs +++ b/core/src/api/utils/invalidate.rs @@ -119,22 +119,22 @@ macro_rules! invalidate_query { ($ctx:expr, $key:literal) => {{ let ctx: &crate::library::Library = &$ctx; // Assert the context is the correct type - // #[cfg(debug_assertions)] - // { - // #[ctor::ctor] - // fn invalidate() { - // crate::api::utils::INVALIDATION_REQUESTS - // .lock() - // .unwrap() - // .queries - // .push(crate::api::utils::InvalidationRequest { - // key: $key, - // arg_ty: None, - // result_ty: None, - // macro_src: concat!(file!(), ":", line!()), - // }) - // } - // } + #[cfg(debug_assertions)] + { + #[ctor::ctor] + fn invalidate() { + crate::api::utils::INVALIDATION_REQUESTS + .lock() + .unwrap() + .queries + .push(crate::api::utils::InvalidationRequest { + key: $key, + arg_ty: None, + result_ty: None, + macro_src: concat!(file!(), ":", line!()), + }) + } + } // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. ctx.emit(crate::api::CoreEvent::InvalidateOperation( @@ -145,25 +145,25 @@ macro_rules! invalidate_query { let _: $arg_ty = $arg; // Assert the type the user provided is correct let ctx: &crate::library::Library = &$ctx; // Assert the context is the correct type - // #[cfg(debug_assertions)] - // { - // #[ctor::ctor] - // fn invalidate() { - // crate::api::utils::INVALIDATION_REQUESTS - // .lock() - // .unwrap() - // .queries - // .push(crate::api::utils::InvalidationRequest { - // key: $key, - // arg_ty: Some(<$arg_ty as rspc::internal::specta::Type>::reference(rspc::internal::specta::DefOpts { - // parent_inline: false, - // type_map: &mut rspc::internal::specta::TypeDefs::new(), - // }, &[])), - // result_ty: None, - // macro_src: concat!(file!(), ":", line!()), - // }) - // } - // } + #[cfg(debug_assertions)] + { + #[ctor::ctor] + fn invalidate() { + crate::api::utils::INVALIDATION_REQUESTS + .lock() + .unwrap() + .queries + .push(crate::api::utils::InvalidationRequest { + key: $key, + arg_ty: Some(<$arg_ty as rspc::internal::specta::Type>::reference(rspc::internal::specta::DefOpts { + parent_inline: false, + type_map: &mut rspc::internal::specta::TypeDefs::new(), + }, &[])), + result_ty: None, + macro_src: concat!(file!(), ":", line!()), + }) + } + } // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. let _ = serde_json::to_value($arg) @@ -224,7 +224,27 @@ pub(crate) fn mount_invalidate() -> AlphaRouter { let manager_thread_active = Arc::new(AtomicBool::new(false)); // TODO: Scope the invalidate queries to a specific library (filtered server side) - R.router().procedure("listen", { + let mut r = R.router(); + + #[cfg(debug_assertions)] + { + let count = Arc::new(std::sync::atomic::AtomicU16::new(0)); + + r = r + .procedure( + "test-invalidate", + R.query(move |_, _: ()| count.fetch_add(1, Ordering::SeqCst)), + ) + .procedure( + "test-invalidate-mutation", + R.with2(super::library()).mutation(|(_, library), _: ()| { + invalidate_query!(library, "invalidation.test-invalidate"); + Ok(()) + }), + ); + } + + r.procedure("listen", { R.subscription(move |ctx, _: ()| { // This thread is used to deal with batching and deduplication. // Their is only ever one of these management threads per Node but we spawn it like this so we can steal the event bus from the rspc context. diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index 6683325ca..c2f5a9d93 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -1,8 +1,9 @@ use crate::{ invalidate_query, + location::LocationManagerError, node::Platform, object::orphan_remover::OrphanRemoverActor, - prisma::{node, PrismaClient}, + prisma::{location, node, PrismaClient}, sync::{SyncManager, SyncMessage}, util::{ db::{load_and_migrate, MigrationError}, @@ -68,6 +69,8 @@ pub enum LibraryManagerError { InvalidConfig(String), #[error(transparent)] NonUtf8Path(#[from] NonUtf8PathError), + #[error("failed to watch locations: {0}")] + LocationWatcher(#[from] LocationManagerError), } impl From for rspc::Error { @@ -425,6 +428,20 @@ impl LibraryManager { node_context, }; + for location in library + .db + .location() + .find_many(vec![location::node_id::equals(node_data.id)]) + .exec() + .await? + { + library + .node_context + .location_manager + .add(location.id, library.clone()) + .await?; + } + if let Err(e) = library .node_context .jobs diff --git a/crates/p2p/examples/basic.rs b/crates/p2p/examples/basic.rs index b1fdd79b8..d482e73d6 100644 --- a/crates/p2p/examples/basic.rs +++ b/crates/p2p/examples/basic.rs @@ -119,7 +119,7 @@ async fn main() { sleep(Duration::from_secs(3)).await; manager .broadcast( - format!("Hello World From {}", keypair.public().to_peer_id()) + format!("Hello World From {}", keypair.peer_id()) .as_bytes() .to_vec(), ) diff --git a/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx b/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx index 919904ba3..57466979e 100644 --- a/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx @@ -1,4 +1,4 @@ -import { getDebugState, useBridgeQuery, useDebugState } from '@sd/client'; +import { getDebugState, useBridgeQuery, useDebugState, useLibraryMutation } from '@sd/client'; import { Button, Popover, Select, SelectOption, Switch } from '@sd/ui'; import { usePlatform } from '~/util/Platform'; import Setting from '../../settings/Setting'; @@ -97,6 +97,7 @@ export default () => { Enabled + {/* {platform.showDevtools && ( { ); }; + +function InvalidateDebugPanel() { + const { data: count } = useBridgeQuery(['invalidation.test-invalidate']); + const { mutate } = useLibraryMutation(['invalidation.test-invalidate-mutation']); + + return ( + +
+ +
+ + ); +} diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index 95f7e170b..9d1dfaab2 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -6,6 +6,7 @@ export type Procedures = { { key: "buildInfo", input: never, result: BuildInfo } | { key: "categories.list", input: LibraryArgs, result: { [key in Category]: number } } | { key: "files.get", input: LibraryArgs, result: { id: number; pub_id: number[]; kind: number; key_id: number | null; hidden: boolean; favorite: boolean; important: boolean; has_thumbnail: boolean; has_thumbstrip: boolean; has_video_preview: boolean; ipfs_id: string | null; note: string | null; date_created: string; date_accessed: string | null; file_paths: FilePath[]; media_data: MediaData | null } | null } | + { key: "invalidation.test-invalidate", input: never, result: number } | { key: "jobs.getHistory", input: LibraryArgs, result: JobReport[] } | { key: "jobs.getRunning", input: LibraryArgs, result: JobReport[] } | { key: "keys.getDefault", input: LibraryArgs, result: string | null } | @@ -46,6 +47,7 @@ export type Procedures = { { key: "files.setFavorite", input: LibraryArgs, result: null } | { key: "files.setNote", input: LibraryArgs, result: null } | { key: "files.updateAccessTime", input: LibraryArgs, result: null } | + { key: "invalidation.test-invalidate-mutation", input: LibraryArgs, result: null } | { key: "jobs.clear", input: LibraryArgs, result: null } | { key: "jobs.clearAll", input: LibraryArgs, result: null } | { key: "jobs.generateThumbsForLocation", input: LibraryArgs, result: null } |