From bcbbe58141c39fa769d5575bdd8f70e9064d8ec4 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Thu, 17 Aug 2023 13:37:10 +0800 Subject: [PATCH] [ENG-974] DB Backup prototype (#1216) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * DB Backup prototype * Put backups behind feature flag * Warning for data folder * nit * Clippy --------- Co-authored-by: Utku <74243531+utkubakir@users.noreply.github.com> Co-authored-by: Vítor Vasconcellos --- Cargo.lock | Bin 245138 -> 245157 bytes apps/desktop/src/App.tsx | 2 + apps/web/src/App.tsx | 2 +- core/Cargo.toml | 4 +- core/src/api/backups.rs | 371 ++++++++++++++++++ core/src/api/mod.rs | 2 + core/src/api/utils/invalidate.rs | 27 ++ core/src/library/manager/mod.rs | 6 +- core/src/location/manager/mod.rs | 2 +- .../app/$libraryId/overview/Statistics.tsx | 2 +- interface/app/$libraryId/settings/Sidebar.tsx | 8 +- .../$libraryId/settings/client/backups.tsx | 87 ++++ .../$libraryId/settings/client/general.tsx | 20 +- .../app/$libraryId/settings/client/index.ts | 3 +- .../settings/library/locations/ListItem.tsx | 4 +- interface/components/Folder.tsx | 37 +- interface/util/Platform.tsx | 3 +- packages/client/src/core.ts | 10 + packages/client/src/hooks/useFeatureFlag.tsx | 11 +- 19 files changed, 563 insertions(+), 38 deletions(-) create mode 100644 core/src/api/backups.rs create mode 100644 interface/app/$libraryId/settings/client/backups.tsx diff --git a/Cargo.lock b/Cargo.lock index 0b35dbf160fba661972f9affd098e06b46a72abf..2e497d44205acde622dc1c5ceeab31f8783abf96 100644 GIT binary patch delta 53 zcmV-50LuT8`wpf14uFIKv;wVx1#Dq-Wiq#?fdVxG0%3BO8Hxfcm!FOT1()660SSi> LgaWq@gad;)JmnLz delta 44 zcmZ4bn{U!@zJ?aYElg`0r)QZkac@7|$fUwF-KdL6b^475jC}1%Elk^$T9}(H0hzfF Ay8r+H diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 0ad90af20..8a563af92 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -1,5 +1,6 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { dialog, invoke, os, shell } from '@tauri-apps/api'; +import { confirm } from '@tauri-apps/api/dialog'; import { listen } from '@tauri-apps/api/event'; import { convertFileSrc } from '@tauri-apps/api/tauri'; import { appWindow } from '@tauri-apps/api/window'; @@ -75,6 +76,7 @@ const platform: Platform = { openFilePickerDialog: () => dialog.open(), saveFilePickerDialog: () => dialog.save(), showDevtools: () => invoke('show_devtools'), + confirm: (msg, cb) => confirm(msg).then(cb), ...commands }; diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 567ea3a81..92132ea52 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -41,7 +41,7 @@ const platform: Platform = { locationLocalId )}/${encodeURIComponent(filePathId)}`, openLink: (url) => window.open(url, '_blank')?.focus(), - demoMode: import.meta.env.VITE_SD_DEMO_MODE === 'true' + confirm: (message, cb) => cb(window.confirm(message)) }; const queryClient = new QueryClient({ diff --git a/core/Cargo.toml b/core/Cargo.toml index 52f99f4a4..b028e512c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -102,6 +102,9 @@ async-channel = "1.9" tokio-util = "0.7" slotmap = "1.0.6" aovec = "1.1.0" +flate2 = "1.0.26" +tar = "0.4.40" +tempfile = "^3.5.0" [target.'cfg(target_os = "macos")'.dependencies] plist = "1" @@ -110,5 +113,4 @@ plist = "1" version = "0.1.5" [dev-dependencies] -tempfile = "^3.5.0" tracing-test = "^0.2.4" diff --git a/core/src/api/backups.rs b/core/src/api/backups.rs new file mode 100644 index 000000000..ce3f0aa4e --- /dev/null +++ b/core/src/api/backups.rs @@ -0,0 +1,371 @@ +use std::{ + cmp, + fs::{self, File}, + io::{self, BufReader, BufWriter, Read, Write}, + path::PathBuf, + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; + +use flate2::{bufread::GzDecoder, write::GzEncoder, Compression}; +use futures::executor::block_on; +use rspc::{alpha::AlphaRouter, ErrorCode}; +use serde::{Serialize, Serializer}; +use specta::Type; +use tar::Archive; +use tempfile::tempdir; +use thiserror::Error; +use tokio::task::spawn_blocking; +use tracing::{error, info}; +use uuid::Uuid; + +use crate::{ + invalidate_query, + library::{Library, LibraryManagerError}, + Node, +}; + +use super::{utils::library, Ctx, R}; + +pub(crate) fn mount() -> AlphaRouter { + R.router() + .procedure("getAll", { + #[derive(Serialize, Type)] + pub struct Backup { + #[serde(flatten)] + header: Header, + path: String, + } + + #[derive(Serialize, Type)] + pub struct GetAll { + backups: Vec, + directory: String, + } + + R.query(|node, _: ()| async move { + let directory = node.data_dir.join("backups"); + + Ok(GetAll { + backups: if !directory.exists() { + vec![] + } else { + spawn_blocking(move || { + fs::read_dir(node.data_dir.join("backups")) + .map(|dir| { + dir.filter_map(|entry| { + match entry.and_then(|e| Ok((e.metadata()?, e))) { + Ok((metadata, entry)) if metadata.is_file() => { + File::open(entry.path()) + .ok() + .and_then(|mut file| { + Header::read(&mut file).ok() + }) + .map(|header| Backup { + header, + // TODO: Lossy strings are bad + path: entry + .path() + .to_string_lossy() + .to_string(), + }) + } + _ => None, + } + }) + .collect::>() + }) + .map_err(|e| { + rspc::Error::with_cause( + ErrorCode::InternalServerError, + "Failed to fetch backups".to_string(), + e, + ) + }) + }) + .await + .map_err(|e| { + rspc::Error::with_cause( + ErrorCode::InternalServerError, + "Failed to fetch backups".to_string(), + e, + ) + })?? + }, + directory: directory.to_string_lossy().to_string(), + }) + }) + }) + .procedure("backup", { + R.with2(library()) + .mutation(|(node, library), _: ()| start_backup(node, library)) + }) + .procedure("restore", { + R + // TODO: Paths as strings is bad but here we want the flexibility of the frontend allowing any path + .mutation(|node, path: String| start_restore(node, path.into())) + }) + .procedure("delete", { + R + // TODO: Paths as strings is bad but here we want the flexibility of the frontend allowing any path + .mutation(|node, path: String| async move { + tokio::fs::remove_file(path) + .await + .map(|_| { + invalidate_query!(node; node, "backups.getAll"); + }) + .map_err(|_| { + rspc::Error::new( + ErrorCode::InternalServerError, + "Error deleting backup!".to_string(), + ) + }) + }) + }) +} + +async fn start_backup(node: Arc, library: Arc) -> Uuid { + let bkp_id = Uuid::new_v4(); + + spawn_blocking(move || { + match do_backup(bkp_id, &node, &library) { + Ok(path) => { + info!( + "Backup '{bkp_id}' for library '{}' created at '{path:?}'!", + library.id + ); + invalidate_query!(library, "backups.getAll"); + } + Err(e) => { + error!( + "Error with backup '{bkp_id}' for library '{}': {e:?}", + library.id + ); + + // TODO: Alert user something went wrong + } + } + }); + + bkp_id +} + +#[derive(Error, Debug)] +enum BackupError { + #[error("io error: {0}")] + Io(#[from] io::Error), + #[error("library manager error: {0}")] + LibraryManager(#[from] LibraryManagerError), + #[error("malformed header")] + MalformedHeader, + #[error("Library already exists, please remove it and try again!")] + LibraryAlreadyExists, +} + +#[derive(Debug)] +pub struct MustRemoveLibraryErr; + +// This is intended to be called in a `spawn_blocking` task. +// Async is pure overhead for an IO bound operation like this. +fn do_backup(id: Uuid, node: &Node, library: &Library) -> Result { + let backups_dir = node.data_dir.join("backups"); + fs::create_dir_all(&backups_dir)?; + + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_millis(); + + let bkp_path = backups_dir.join(format!("{id}.bkp")); + let mut bkp_file = BufWriter::new(File::create(&bkp_path)?); + + // Header. We do this so the file is self-sufficient. + Header { + id, + timestamp, + library_id: library.id, + library_name: library.config.name.to_string(), + } + .write(&mut bkp_file)?; + + // Regular tar.gz encoded data + let mut tar = tar::Builder::new(GzEncoder::new(bkp_file, Compression::default())); + + tar.append_file( + "library.sdlibrary", + &mut File::open( + node.libraries + .libraries_dir + .join(format!("{}.sdlibrary", library.id)), + )?, + )?; + tar.append_file( + "library.db", + &mut File::open( + node.libraries + .libraries_dir + .join(format!("{}.db", library.id)), + )?, + )?; + + Ok(bkp_path) +} + +fn start_restore(node: Arc, path: PathBuf) { + spawn_blocking(move || { + match restore_backup(&node, path.clone()) { + Ok(header) => { + info!( + "Restored to '{}' for library '{}'!", + header.id, header.library_id + ); + } + Err(e) => { + error!("Error restoring backup '{path:?}': {e:?}"); + + // TODO: Alert user something went wrong + } + } + }); +} + +fn restore_backup(node: &Arc, path: PathBuf) -> Result { + let mut file = BufReader::new(fs::File::open(path)?); + let header = Header::read(&mut file)?; + + // TODO: Actually handle restoring into a library that exists. For now it's easier to error out. + let None = block_on(node.libraries.get_library(&header.library_id)) else { + return Err(BackupError::LibraryAlreadyExists) + }; + + let temp_dir = tempdir()?; + + let mut archive = Archive::new(GzDecoder::new(file)); + archive.unpack(&temp_dir)?; + + let library_path = temp_dir.path().join("library.sdlibrary"); + let db_path = temp_dir.path().join("library.db"); + + fs::copy( + library_path, + node.libraries + .libraries_dir + .join(format!("{}.sdlibrary", header.library_id)), + )?; + fs::copy( + db_path, + node.libraries + .libraries_dir + .join(format!("{}.db", header.library_id)), + )?; + + let config_path = node + .libraries + .libraries_dir + .join(format!("{}.sdlibrary", header.library_id)); + let db_path = config_path.with_extension("db"); + block_on( + node.libraries + .load(header.library_id, &db_path, config_path, None, true, node), + )?; + + Ok(header) +} + +#[derive(Debug, PartialEq, Eq, Serialize, Type)] +struct Header { + // Backup unique id + id: Uuid, + // Time since epoch the backup was created at + #[specta(type = String)] + #[serde(serialize_with = "as_string")] + timestamp: u128, + // Library id + library_id: Uuid, + // Library display name + library_name: String, +} + +fn as_string(x: &T, s: S) -> Result +where + S: Serializer, +{ + s.serialize_str(&x.to_string()) +} + +impl Header { + fn write(&self, file: &mut impl Write) -> Result<(), io::Error> { + // For future versioning we can bump `1` to `2` and match on it in the decoder. + file.write_all(b"sdbkp1")?; + file.write_all(&self.id.to_bytes_le())?; + file.write_all(&self.timestamp.to_le_bytes())?; + file.write_all(&self.library_id.to_bytes_le())?; + { + let bytes = &self.library_name.as_bytes() + [..cmp::min(u32::MAX as usize, self.library_name.len())]; + file.write_all(&(bytes.len() as u32).to_le_bytes())?; + file.write_all(bytes)?; + } + + Ok(()) + } + + fn read(file: &mut impl Read) -> Result { + let mut buf = vec![0u8; 6 + 16 + 16 + 16 + 4]; + file.read_exact(&mut buf)?; + if &buf[..6] != b"sdbkp1" { + return Err(BackupError::MalformedHeader); + } + + Ok(Self { + id: Uuid::from_bytes_le( + buf[6..22] + .try_into() + .map_err(|_| BackupError::MalformedHeader)?, + ), + timestamp: u128::from_le_bytes( + buf[22..38] + .try_into() + .map_err(|_| BackupError::MalformedHeader)?, + ), + library_id: Uuid::from_bytes_le( + buf[38..54] + .try_into() + .map_err(|_| BackupError::MalformedHeader)?, + ), + + library_name: { + let len = u32::from_le_bytes( + buf[54..58] + .try_into() + .map_err(|_| BackupError::MalformedHeader)?, + ); + + let mut name = vec![0; len as usize]; + file.read_exact(&mut name)?; + String::from_utf8(name).map_err(|_| BackupError::MalformedHeader)? + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_backup_header() { + let original = Header { + id: Uuid::new_v4(), + timestamp: 1234567890, + library_id: Uuid::new_v4(), + library_name: "Test Library".to_string(), + }; + + let mut buf = Vec::new(); + original.write(&mut buf).unwrap(); + + let decoded = Header::read(&mut buf.as_slice()).unwrap(); + assert_eq!(original, decoded); + } +} diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs index 0619083d5..0bca93862 100644 --- a/core/src/api/mod.rs +++ b/core/src/api/mod.rs @@ -21,6 +21,7 @@ pub enum CoreEvent { InvalidateOperation(InvalidateOperationEvent), } +mod backups; mod categories; mod files; mod jobs; @@ -113,6 +114,7 @@ pub(crate) fn mount() -> Arc { .merge("sync.", sync::mount()) .merge("preferences.", preferences::mount()) .merge("notifications.", notifications::mount()) + .merge("backups.", backups::mount()) .merge("invalidation.", utils::mount_invalidate()) .build( #[allow(clippy::let_and_return)] diff --git a/core/src/api/utils/invalidate.rs b/core/src/api/utils/invalidate.rs index d62a369c4..bd7d93ccd 100644 --- a/core/src/api/utils/invalidate.rs +++ b/core/src/api/utils/invalidate.rs @@ -143,6 +143,33 @@ macro_rules! invalidate_query { $crate::api::utils::InvalidateOperationEvent::dangerously_create($key, serde_json::Value::Null, None) )) }}; + (node; $ctx:expr, $key:literal) => {{ + let ctx: &$crate::Node = &$ctx; // Assert the context is the correct type + + #[cfg(debug_assertions)] + { + #[ctor::ctor] + fn invalidate() { + $crate::api::utils::INVALIDATION_REQUESTS + .lock() + .unwrap() + .queries + .push($crate::api::utils::InvalidationRequest { + key: $key, + arg_ty: None, + result_ty: None, + macro_src: concat!(file!(), ":", line!()), + }) + } + } + + ::tracing::trace!(target: "sd_core::invalidate-query", "invalidate_query!(\"{}\") at {}", $key, concat!(file!(), ":", line!())); + + // The error are ignored here because they aren't mission critical. If they fail the UI might be outdated for a bit. + ctx.event_bus.0.send($crate::api::CoreEvent::InvalidateOperation( + $crate::api::utils::InvalidateOperationEvent::dangerously_create($key, serde_json::Value::Null, None) + )).ok(); + }}; ($ctx:expr, $key:literal: $arg_ty:ty, $arg:expr $(,)?) => {{ let _: $arg_ty = $arg; // Assert the type the user provided is correct let ctx: &$crate::library::Library = &$ctx; // Assert the context is the correct type diff --git a/core/src/library/manager/mod.rs b/core/src/library/manager/mod.rs index 8bb26e2bd..d5d3a9e9c 100644 --- a/core/src/library/manager/mod.rs +++ b/core/src/library/manager/mod.rs @@ -49,7 +49,7 @@ pub enum LibraryManagerEvent { /// is a singleton that manages all libraries for a node. pub struct Libraries { /// libraries_dir holds the path to the directory where libraries are stored. - libraries_dir: PathBuf, + pub libraries_dir: PathBuf, /// libraries holds the list of libraries which are currently loaded into the node. libraries: RwLock>>, // Transmit side of `self.rx` channel @@ -304,8 +304,8 @@ impl Libraries { self.libraries.read().await.get(library_id).is_some() } - /// load the library from a given path - async fn load( + /// load the library from a given path. + pub async fn load( self: &Arc, id: Uuid, db_path: impl AsRef, diff --git a/core/src/location/manager/mod.rs b/core/src/location/manager/mod.rs index 7efd6f968..b96b475fc 100644 --- a/core/src/location/manager/mod.rs +++ b/core/src/location/manager/mod.rs @@ -166,7 +166,7 @@ impl LocationManagerActor { LibraryManagerEvent::InstancesModified(_) => {} LibraryManagerEvent::Delete(_) => { #[cfg(debug_assertions)] - todo!("TODO: Remove locations from location manager"); // TODO + error!("TODO: Remove locations from location manager"); // TODO } } } diff --git a/interface/app/$libraryId/overview/Statistics.tsx b/interface/app/$libraryId/overview/Statistics.tsx index 48c88eaa3..90d789508 100644 --- a/interface/app/$libraryId/overview/Statistics.tsx +++ b/interface/app/$libraryId/overview/Statistics.tsx @@ -116,7 +116,7 @@ export default () => { key={`${library.uuid} ${key}`} title={StatItemNames[key as keyof Statistics]!} bytes={BigInt(value)} - isLoading={platform.demoMode ? false : stats.isLoading} + isLoading={stats.isLoading} info={StatDescriptions[key as keyof Statistics]} /> ); diff --git a/interface/app/$libraryId/settings/Sidebar.tsx b/interface/app/$libraryId/settings/Sidebar.tsx index 197324343..61e74f64d 100644 --- a/interface/app/$libraryId/settings/Sidebar.tsx +++ b/interface/app/$libraryId/settings/Sidebar.tsx @@ -1,6 +1,7 @@ import { Books, Cloud, + Database, FlyingSaucer, GearSix, HardDrive, @@ -25,7 +26,8 @@ const Section = tw.div`space-y-0.5`; export default () => { const os = useOperatingSystem(); - const isPairingEnabled = useFeatureFlag('p2pPairing'); + // const isPairingEnabled = useFeatureFlag('p2pPairing'); + const isBackupsEnabled = useFeatureFlag('backups'); return (
@@ -56,6 +58,10 @@ export default () => { Appearance + + + Backups + Keybinds diff --git a/interface/app/$libraryId/settings/client/backups.tsx b/interface/app/$libraryId/settings/client/backups.tsx new file mode 100644 index 000000000..6255ad2a4 --- /dev/null +++ b/interface/app/$libraryId/settings/client/backups.tsx @@ -0,0 +1,87 @@ +import dayjs from 'dayjs'; +import { useBridgeMutation, useBridgeQuery, useLibraryMutation } from '@sd/client'; +import { Button, Card } from '@sd/ui'; +import { Database } from '~/components'; +import { usePlatform } from '~/util/Platform'; +import { Heading } from '../Layout'; + +// TODO: This is a non-library page but does a library query for backup. That will be confusing UX. +// TODO: Should this be a library or node page? If it's a library page how can a user view all their backups across libraries (say they wanted to save some storage cause their SSD is full)? +// TODO: If it were a library page what do we do when restoring a backup? It can't be a `useLibraryQuery` to restore cause we are gonna have to unload the library from the backend. + +export const Component = () => { + const platform = usePlatform(); + const backups = useBridgeQuery(['backups.getAll']); + const doBackup = useLibraryMutation('backups.backup'); + const doRestore = useBridgeMutation('backups.restore'); + const doDelete = useBridgeMutation('backups.delete'); + + console.log(doRestore.isLoading); + + return ( + <> + + + +
+ } + /> + + {backups.data?.backups.map((backup) => ( + + +
+

+ {dayjs(backup.timestamp).toString()} +

+

+ For library '{backup.library_name}' +

+
+
+
+ + +
+ + ))} + + ); +}; diff --git a/interface/app/$libraryId/settings/client/general.tsx b/interface/app/$libraryId/settings/client/general.tsx index 0dde58606..ec9b1ab40 100644 --- a/interface/app/$libraryId/settings/client/general.tsx +++ b/interface/app/$libraryId/settings/client/general.tsx @@ -90,20 +90,20 @@ export const Component = () => {
Data Folder
- { - /* TODO */ - }} - disabled - /> +