From ce6d2d2fda72f68e9514bd08939482673f4423cc Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Fri, 21 Apr 2023 06:05:57 +0800 Subject: [PATCH] Eng 494 adding a location does not update the UI (#728) * `sd_init.json` support * bruh * Add `sd_init.json` to developer docs * Ran cargo clippy --fix to fix warnings * Dafuq, cargo clippy --fix messed up cargo fmt --------- Co-authored-by: Ericson Soares --- .gitignore | 1 + apps/landing/server/index.ts | 2 +- core/src/api/libraries.rs | 2 +- core/src/api/utils/invalidate.rs | 33 ++-- core/src/lib.rs | 16 +- core/src/library/manager.rs | 13 +- core/src/util/debug_initializer.rs | 163 ++++++++++++++++++ core/src/util/mod.rs | 2 + .../prerequisites/environment-setup.md | 27 +++ 9 files changed, 236 insertions(+), 23 deletions(-) create mode 100644 core/src/util/debug_initializer.rs diff --git a/.gitignore b/.gitignore index 30dd0b2f9..5f9bc75fa 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,4 @@ dev.db-journal .build/ .swiftpm /core/migration_test +sd_init.json diff --git a/apps/landing/server/index.ts b/apps/landing/server/index.ts index c4926e476..a1f15fe29 100644 --- a/apps/landing/server/index.ts +++ b/apps/landing/server/index.ts @@ -29,7 +29,7 @@ async function startServer() { app.get('*', async (req, res, next) => { const url = req.originalUrl; const pageContextInit = { - url + urlOriginal: url }; const pageContext = await renderPage(pageContextInit); const { httpResponse } = pageContext; diff --git a/core/src/api/libraries.rs b/core/src/api/libraries.rs index 9d9796de7..8569bf3a0 100644 --- a/core/src/api/libraries.rs +++ b/core/src/api/libraries.rs @@ -168,7 +168,7 @@ pub(crate) fn mount() -> RouterBuilder { }) }) .mutation("delete", |t| { - t(|ctx, id: Uuid| async move { Ok(ctx.library_manager.delete_library(id).await?) }) + t(|ctx, id: Uuid| async move { Ok(ctx.library_manager.delete(id).await?) }) }) // .yolo_merge("peer.guest.", peer_guest_router()) // .yolo_merge("peer.host.", peer_host_router()) diff --git a/core/src/api/utils/invalidate.rs b/core/src/api/utils/invalidate.rs index 497387aaf..72362d613 100644 --- a/core/src/api/utils/invalidate.rs +++ b/core/src/api/utils/invalidate.rs @@ -232,22 +232,25 @@ pub fn mount_invalidate() -> RouterBuilder { tokio::spawn(async move { let mut buf = HashMap::with_capacity(100); - tokio::select! { - event = event_bus_rx.recv() => { - if let Ok(event) = event { - if let CoreEvent::InvalidateOperation(op) = event { - // Newer data replaces older data in the buffer - buf.insert(to_key(&(op.key, &op.arg)).unwrap(), op); + loop { + tokio::select! { + event = event_bus_rx.recv() => { + if let Ok(event) = event { + if let CoreEvent::InvalidateOperation(op) = event { + // Newer data replaces older data in the buffer + buf.insert(to_key(&(op.key, &op.arg)).unwrap(), op); + } + } else { + warn!("Shutting down invalidation manager thread due to the core event bus being droppped!"); + } + }, + // Given human reaction time of ~250 milli this should be a good ballance. + _ = tokio::time::sleep(Duration::from_millis(200)) => { + let x = buf.drain().map(|(_k, v)| v).collect::>(); + match tx.send(x) { + Ok(_) => {}, + Err(_) => warn!("Error emitting invalidation manager events!"), } - } else { - warn!("Shutting down invalidation manager thread due to the core event bus being droppped!"); - } - }, - // Given human reaction time of ~250 milli this should be a good ballance. - _ = tokio::time::sleep(Duration::from_millis(200)) => { - match tx.send(buf.drain().map(|(_k, v)| v).collect::>()) { - Ok(_) => {}, - Err(_) => warn!("Error emitting invalidation manager events!"), } } } diff --git a/core/src/lib.rs b/core/src/lib.rs index d668ca41d..796741999 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -63,6 +63,9 @@ impl Node { pub async fn new(data_dir: impl AsRef) -> Result<(Arc, Arc), NodeError> { let data_dir = data_dir.as_ref(); + #[cfg(debug_assertions)] + let init_data = util::debug_initializer::InitConfig::load(data_dir).await; + // This error is ignored because it's throwing on mobile despite the folder existing. let _ = fs::create_dir_all(&data_dir).await; @@ -107,10 +110,10 @@ impl Node { .parse() .expect("Error invalid tracing directive!"), ), // .add_directive( - // "rspc=debug" - // .parse() - // .expect("Error invalid tracing directive!"), - // ), + // "rspc=debug" + // .parse() + // .expect("Error invalid tracing directive!"), + // ), ); #[cfg(not(target_os = "android"))] let subscriber = subscriber.with(tracing_subscriber::fmt::layer().with_filter(CONSOLE_LOG_FILTER)); @@ -145,6 +148,11 @@ impl Node { ) .await?; + #[cfg(debug_assertions)] + if let Some(init_data) = init_data { + init_data.apply(&library_manager).await; + } + debug!("Watching locations"); tokio::spawn({ diff --git a/core/src/library/manager.rs b/core/src/library/manager.rs index 152399c88..760324761 100644 --- a/core/src/library/manager.rs +++ b/core/src/library/manager.rs @@ -185,7 +185,16 @@ impl LibraryManager { config: LibraryConfig, km_config: OnboardingConfig, ) -> Result { - let id = Uuid::new_v4(); + self.create_with_uuid(Uuid::new_v4(), config, km_config) + .await + } + + pub(crate) async fn create_with_uuid( + &self, + id: Uuid, + config: LibraryConfig, + km_config: OnboardingConfig, + ) -> Result { LibraryConfig::save( Path::new(&self.libraries_dir).join(format!("{id}.sdlibrary")), &config, @@ -288,7 +297,7 @@ impl LibraryManager { Ok(()) } - pub async fn delete_library(&self, id: Uuid) -> Result<(), LibraryManagerError> { + pub async fn delete(&self, id: Uuid) -> Result<(), LibraryManagerError> { let mut libraries = self.libraries.write().await; let library = libraries diff --git a/core/src/util/debug_initializer.rs b/core/src/util/debug_initializer.rs new file mode 100644 index 000000000..a0f4b80ae --- /dev/null +++ b/core/src/util/debug_initializer.rs @@ -0,0 +1,163 @@ +// ! A system for loading a default set of data on startup. This is ONLY enabled in development builds. + +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; + +use crate::{ + library::LibraryConfig, + location::{delete_location, scan_location, LocationCreateArgs}, + prisma::location, +}; +use sd_crypto::{ + types::{Algorithm, HashingAlgorithm, OnboardingConfig, Params}, + Protected, +}; +use serde::Deserialize; +use tokio::{ + fs::{self, metadata}, + time::sleep, +}; +use tracing::{info, warn}; +use uuid::Uuid; + +use crate::library::LibraryManager; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LocationInitConfig { + path: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LibraryInitConfig { + id: Uuid, + name: String, + description: Option, + password: String, + #[serde(default)] + reset_locations_on_startup: bool, + locations: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct InitConfig { + #[serde(default)] + reset_on_startup: bool, + libraries: Vec, + #[serde(skip, default)] + path: PathBuf, +} + +impl InitConfig { + pub async fn load(data_dir: &Path) -> Option { + let path = std::env::current_dir() + .unwrap() + .join(std::env::var("SD_INIT_DATA").unwrap_or("sd_init.json".to_string())); + + if metadata(&path).await.is_ok() { + let config = fs::read_to_string(&path).await.unwrap(); + let mut config: InitConfig = serde_json::from_str(&config).unwrap(); + config.path = path; + + if config.reset_on_startup && data_dir.exists() { + warn!("previous 'SD_DATA_DIR' was removed on startup!"); + fs::remove_dir_all(&data_dir).await.unwrap(); + } + + return Some(config); + } + + None + } + + pub async fn apply(self, library_manager: &LibraryManager) { + info!("Initializing app from file: {:?}", self.path); + + for lib in self.libraries { + let name = lib.name.clone(); + let handle = tokio::spawn(async move { + loop { + info!("Initializing library '{name}' from 'sd_init.json'..."); + sleep(Duration::from_secs(1)).await; + } + }); + + let library = match library_manager.get_ctx(lib.id).await { + Some(lib) => lib, + None => { + let library = library_manager + .create_with_uuid( + lib.id, + LibraryConfig { + name: lib.name, + description: lib.description.unwrap_or("".to_string()), + }, + OnboardingConfig { + password: Protected::new(lib.password), + algorithm: Algorithm::XChaCha20Poly1305, + hashing_algorithm: HashingAlgorithm::BalloonBlake3( + Params::Standard, + ), + }, + ) + .await + .unwrap(); + + library_manager.get_ctx(library.uuid).await.unwrap() + } + }; + + if lib.reset_locations_on_startup { + let locations = library + .db + .location() + .find_many(vec![]) + .exec() + .await + .unwrap(); + + for location in locations { + warn!("deleting location: {:?}", location.path); + delete_location(&library, location.id).await.unwrap(); + } + } + + for loc in lib.locations { + if let Some(location) = library + .db + .location() + .find_first(vec![location::path::equals(loc.path.clone())]) + .exec() + .await + .unwrap() + { + warn!("deleting location: {:?}", location.path); + delete_location(&library, location.id).await.unwrap(); + } + + let sd_file = PathBuf::from(&loc.path).join(".spacedrive"); + if sd_file.exists() { + fs::remove_file(sd_file).await.unwrap(); + } + + let location = LocationCreateArgs { + path: loc.path.into(), + indexer_rules_ids: Vec::new(), + } + .create(&library) + .await + .unwrap(); + + scan_location(&library, location).await.unwrap(); + } + + handle.abort(); + } + + info!("Initialized app from file: {:?}", self.path); + } +} diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs index 7e4cdd3ac..bae7fe639 100644 --- a/core/src/util/mod.rs +++ b/core/src/util/mod.rs @@ -1,4 +1,6 @@ pub mod db; +#[cfg(debug_assertions)] +pub mod debug_initializer; pub mod migrator; pub mod secure_temp_keystore; pub mod seeder; diff --git a/docs/developers/prerequisites/environment-setup.md b/docs/developers/prerequisites/environment-setup.md index 4b1e39814..b0067ace7 100644 --- a/docs/developers/prerequisites/environment-setup.md +++ b/docs/developers/prerequisites/environment-setup.md @@ -69,3 +69,30 @@ If you are having issues ensure you are using the following versions of Rust and - Rust version: **1.68.2** - Node version: **18** + +### Seeding data on startup + +::: slot note +You may loose data if your using this feature so please be careful! This only works on development builds for this reason. +::: + +You can add a file called `sd_init.json` in the same folder where you start Spacedrive and it can automatically seed data on startup. + +```json +{ + "resetOnStartup": false, + "libraries": [ + { + "id": "26697dc0-ef06-4b39-ad72-ffe5d5205b61", + "name": "Oscar's Library", + "password": "password", + "resetLocationsOnStartup": true, + "locations": [ + { + "path": "/Users/oscar/Pictures/assets" + } + ] + } + ] +} +```