From 6e07435a1522fdb30f7ea5e3fb3dc8b8cb86d462 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Thu, 22 Sep 2022 15:50:21 +0800 Subject: [PATCH] Use PCR for migrations (#385) * use pcr migrations branch * use 0.6.2 branch with migrations * use latest prisma stuff * allow force reset of db in dev * remove .spacedrive file * update rspc in apps/server * use rspc 0.0.5 in all crates * add os to prisma client cache key * add runner os to clippy prisma cache --- .github/workflows/ci.yml | 2 +- .github/workflows/clippy.yml | 4 +- .gitignore | 1 + Cargo.lock | Bin 188030 -> 196099 bytes Cargo.toml | 12 ++- apps/desktop/src-tauri/Cargo.toml | 2 +- apps/mobile/rust/Cargo.toml | 15 +++- apps/server/Cargo.toml | 2 +- core/Cargo.toml | 8 +- core/prisma/Cargo.toml | 1 + core/src/api/locations.rs | 4 +- core/src/encode/thumb.rs | 2 +- core/src/library/library_manager.rs | 15 ++-- core/src/location/indexer/indexer_job.rs | 12 +-- core/src/location/mod.rs | 10 +-- core/src/util/db.rs | 91 +++++------------------ 16 files changed, 70 insertions(+), 111 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d78d7b382..8a00de5af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,7 +122,7 @@ jobs: uses: actions/cache@v3 with: path: ./core/src/prisma.rs - key: prisma-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }} + key: prisma-${{ runner.os }}-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }} - name: Generate Prisma client working-directory: core diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index 3de8c04df..849990b0f 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -3,6 +3,8 @@ name: Rust Clippy check on: pull_request: push: + branches: + - main paths: - '**.rs' - '**.toml' @@ -42,7 +44,7 @@ jobs: uses: actions/cache@v3 with: path: ./core/src/prisma.rs - key: prisma-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }} + key: prisma-${{ runner.os }}-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }} - name: Generate Prisma client working-directory: core diff --git a/.gitignore b/.gitignore index 500449750..f5d2776cd 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ examples/*/*.lock /target /sdserver_data +.spacedrive diff --git a/Cargo.lock b/Cargo.lock index 0051b75141258f54b51621c025f56fd685105eae..95f13fc047cab1ef9163ce1d6a5b02038a3d995d 100644 GIT binary patch delta 3288 zcmcguZERQN70-D<`U<5|C=@n2D5Zbs%YA?DwLpuYqA0YCVR#!n_YEp-=}Ox{GGt^i zW}9x{RZk)otS@A4WCCSgr-%$gHt@@UCCipcXtJ=EY$9eEEEwFm&%gIi%f9TB_tSIF zbKakG&hOklS@gZni;n(reYF2ZO|E4~b{f zGi30B3T4sL?y{5=joY*~5uvNnOYqg`5wA79f=o29c1HB%`p5Cg8?Yp2Zt05(D;#VU z7g!>QiUm#4ws7ekF;p8^SyU2i7*_nP!!UX9&7gRk^ljgbMT67ZPv@mScwjCojvni2 zmxQ67)?ul{9o+l|R1Nm`UV&k58FX26^V`dYxw>8^hG%zn%qj4;?r0NrURQTidVd)z z8(?L#P^^%1-_$Hm6_%p9+%r7ickIwKTr&{D4MR^gd^6NSMRTk$aD9)5y2(Au*Ih9< z)c>=!XQ53n{$LL5O1UVDP+1W_Sp|%xG*Yz)mA7Fc zUaW@N2zE9NM{~QH-l3q5$^WsLS+#}ufeBy9k@6E#<*cq9_$wQlM<%LcE|llkNXfu> zHVwBQgFNJC z8|C$sOE3*nb6JVQ<0NF=)6A2k_l-W{(J2_x3s&n($Gct~h1IMyT zqt?D$Kaz>k6W~gYqqQu94UKR_{w@9m%)=6e5^gSl&1n=#re5q~d3bpNs5nu9yqK=5 z3K9{S#mccPKiW`Mg3Z;G;eS2=YZ|1wyL&Mj60>npAACKzB`>b;hUs{vhjKyLkDdry zu&{>BNH|v9d9W2b;lo*Yqyom{?pdrHKkbLiN<$BQ!*)F1R(TM5ilr)!<#1ck70pmx zH{`$8)3&{{&fDG{9qVgMvzE)%cT_;jwW+M^_?std_2JJ;zG%Kx4LR zQ0D#Thp3?5>8JMi5`x`xFgqjg+CjMbB^ZpZJyC)UGpP#?y^^&YmkhweqXy{~Su`tN zgN|%{L_?X$QRRQ8VoM`Tz@rb*NcrR?$is!N!@7inYOgu|;&u3SUYhf}n=dJ_xaSC5 z%guT`HwZr+E4xk3oZEq68H5DF z@H8a>flwg~pmEPFRdaA)AxtsD-W-&4ZjmL}-sXDp&@DRsO;mP4rBbigt67?#P*K?i zbK;IuuwX3C>t?H(xUfP$aBbaHxEa`*W%-_Gb4&9yjdLrsY)&k?N}r~NW@vldAWuy< z;6dOU(c-I9aOPPk1|!u#V3&-Fu7i5ecP7@=^_3AQS}bp|Hp-y=hbuAdi87PYo% z1TN>AZMmju+NvuI-O?1#b5xa(Q_;9-(>ZkX(BKZYxCneDbQHt1UA+De$ofBq@`M8? zsVs^_bzdO5UrN;A{bU9<(en$SQfS`nIi*J=X{{?t6hv zCS6Cdw9vLhps9xAIG(MUq3@`a8?O77X(~E??Jrctd&aYZc+F*4Tq3RAjW)?rgwK2e zPo=k`7Ml2P*WhTj+(rq;yFP|Arq4CawQP;sR9i+E_%=7gz@#n6^?Zet z27%)lv>DmB_%uw3AFpH^#N!AbT&w3IFF1YRJud^*^UH$o|ARdRma|xhsE{mN2yyeio)prAcPgTiHn#bZqaFK)Xq_M{4o(`0{G@z7pN~Yf)h`+xGpvESWUMOLn3Jg(fwmZiw@DE?_b`&Tw2a@nNI(zq&O>z0=ku3z6Jdc3Xmy`8-~ L0v~_)F}(F(3O1CH delta 5465 zcmcgwYfw~m6rXc=EiAhWBe-Fbyfha<_QftMio9GEg+a$O#Mj+df*|H9kv2kxeW?uj z&u|(W6(^^`6kFJwJC%t<=0G{omtOW_#_&T^LLbsfssApM&e)fGr@4On-#h2r|9Sk* z@0@dAxTBuBpbmUfTkUas1+zQfVs^XicDvwnnH@&KX0rI~HiyY-v{~(f$yQ*rW=a&K z#A`T{6>r+TK%#=xkdZ^s4wdtzDzKwM84K))lCCf&ydq11CQC7ib!@4Uens@w8tK9l z&8alH9G^5ub=km9*34!HYB#X%hEg8gMhcWe97VRnvhXR`EgVE;a!I3yV41w=jTS*N z(|I?^aBotd6zuV8qLuNXQ;Z6xHM}3A%-88+3C6R^Jq^+UFt%rnbm0(MUOEr+g2(p$ z2+Fv?A(hN-KeIxzQNiskopXZ&$2)66h@=DksAVI!3xmJ@F_Ob3_Eyru$DwQ*q=(Bu zM{Nf9JuImkghVRJg9X%+2L?(oLYgWgCoU~xm%o8cHqQ>#MKl?6VFpd6Lke9Qy z6DYd^liAH}YPNb$Ed8LE~3RyJ%<#xLRnPy*G)VH~~wF}xui1~llvwes5k z_!YQrlgC<(O*@}z^0;zoau|}BZAeQU8jPe@ov5QL=^THlC;4EoY8YB1XM=biO^4H$~0ZC{~=u9ouJdhIUmAn#V3A(s6H6*`45D1Y`0O_I~()i?~X z%sE3(J&0QsvQPFHA^s470fo4pWJgPEipGnP%rullZE75f56=VsXZ>|MeNA0;nBA4Z zcbf2Gb0PsW;@9!0okuy=2QiRD110?3Q45}U5FDRdakEnUe!zilSvU%}>yQO4L$UKklT8D%7#R z^ys(%%NQYVX_Q!atFUED#i;q~2FDrxG`(5jQ zY%`mkcuWS59O*|tbuGtE=}c9<0#8eqPG5X}K;VFfd7%^}TTa_wx{vYG6yiJ5b+AR967Zb&w`*7o`e*hWcIgbDU diff --git a/Cargo.toml b/Cargo.toml index de118205c..9eab39037 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,5 +14,13 @@ openssl-sys = { git = "https://github.com/spacedriveapp/rust-openssl" } rspc = { git = "https://github.com/oscartbeaumont/rspc", rev = "1b2a299e9061c81ff90706923a6d2389ea7c107e" } [patch."https://github.com/Brendonovich/prisma-client-rust.git"] -prisma-client-rust = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "8447fe493414471a23a38d780b3db246266f558f" } -prisma-client-rust-cli = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "8447fe493414471a23a38d780b3db246266f558f" } +prisma-client-rust = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "43fd489cd817efc061096978030241bbf7ad3fb9", features = [ + "migrations", + "rspc", + "sqlite-create-many", +] } +prisma-client-rust-cli = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "43fd489cd817efc061096978030241bbf7ad3fb9", features = [ + "migrations", + "rspc", + "sqlite-create-many", +] } diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index c073182cc..94835c29a 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -11,7 +11,7 @@ build = "build.rs" [dependencies] tauri = { version = "1.0.4", features = ["api-all", "macos-private-api"] } -rspc = { version = "0.0.4", features = ["tauri"] } +rspc = { version = "0.0.5", features = ["tauri"] } sdcore = { path = "../../../core" } tokio = { version = "1.17.0", features = ["sync"] } window-shadows = "0.1.2" diff --git a/apps/mobile/rust/Cargo.toml b/apps/mobile/rust/Cargo.toml index d2140b3c6..ebc5d4e53 100644 --- a/apps/mobile/rust/Cargo.toml +++ b/apps/mobile/rust/Cargo.toml @@ -10,12 +10,19 @@ crate-type = ["staticlib", "cdylib"] # staticlib for IOS and cdylib for Android [dependencies] once_cell = "1.13.0" -sdcore = { path = "../../../core", features = ["mobile", "p2p"], default-features = false } -rspc = { version = "0.0.4", features = [] } +sdcore = { path = "../../../core", features = [ + "mobile", + "p2p", +], default-features = false } +rspc = { version = "0.0.5", features = [] } serde_json = "1.0.83" tokio = "1.20.1" -openssl = { version = "0.10.41", features = ["vendored"] } # Override features of transitive dependencies -openssl-sys = { version = "0.9.75", features = ["vendored"] } # Override features of transitive dependencies to support IOS Simulator on M1 +openssl = { version = "0.10.41", features = [ + "vendored", +] } # Override features of transitive dependencies +openssl-sys = { version = "0.9.75", features = [ + "vendored", +] } # Override features of transitive dependencies to support IOS Simulator on M1 [target.'cfg(target_os = "ios")'.dependencies] objc = "0.2.7" diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index 4a8dc55cf..99bd2e887 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] sdcore = { path = "../../core", features = [] } -rspc = { version = "0.0.4", features = ["axum"] } +rspc = { version = "0.0.5", features = ["axum"] } axum = "0.5.13" tokio = { version = "1.17.0", features = ["sync", "rt-multi-thread", "signal"] } tracing = "0.1.35" diff --git a/core/Cargo.toml b/core/Cargo.toml index 8679a6d17..b5e419a58 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -33,11 +33,15 @@ rmp-serde = "^1.1.0" prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust.git", tag = "0.6.0", features = [ "rspc", "sqlite-create-many", + "migrations", +] } +quaint = { git = "https://github.com/prisma/quaint.git", features = [ + "sqlite", + "uuid", ] } -quaint = { git = "https://github.com/prisma/quaint.git", features = ["sqlite", "uuid"] } migration-core = { git = "https://github.com/Brendonovich/prisma-engines.git" } sql-migration-connector = { git = "https://github.com/Brendonovich/prisma-engines.git" } -rspc = { version = "0.0.4", features = ["uuid", "chrono", "tracing"] } +rspc = { version = "0.0.5", features = ["uuid", "chrono", "tracing"] } uuid = { version = "1.1.2", features = ["v4", "serde"] } sysinfo = "0.23.9" thiserror = "1.0.30" diff --git a/core/prisma/Cargo.toml b/core/prisma/Cargo.toml index 5adceea2f..5e1ccf661 100644 --- a/core/prisma/Cargo.toml +++ b/core/prisma/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust.git", tag = "0.6.0", features = [ "rspc", "sqlite-create-many", + "migrations", ] } diff --git a/core/src/api/locations.rs b/core/src/api/locations.rs index a042ef5f1..cd276de40 100644 --- a/core/src/api/locations.rs +++ b/core/src/api/locations.rs @@ -29,8 +29,8 @@ pub enum ExplorerContext { // Space(object_in_space::Data), } -file_path::include!(pub file_path_with_file { file }); -file::include!(pub file_with_paths { paths }); +file_path::include!(file_path_with_file { file }); +file::include!(file_with_paths { paths }); #[derive(Serialize, Deserialize, Type, Debug)] #[serde(tag = "type")] diff --git a/core/src/encode/thumb.rs b/core/src/encode/thumb.rs index fcc4a761d..21fbf2de0 100644 --- a/core/src/encode/thumb.rs +++ b/core/src/encode/thumb.rs @@ -37,7 +37,7 @@ pub struct ThumbnailJobState { root_path: PathBuf, } -file_path::include!(pub image_path_with_file { file }); +file_path::include!(image_path_with_file { file }); #[async_trait::async_trait] impl StatefulJob for ThumbnailJob { diff --git a/core/src/library/library_manager.rs b/core/src/library/library_manager.rs index f81400d2e..f21237151 100644 --- a/core/src/library/library_manager.rs +++ b/core/src/library/library_manager.rs @@ -225,17 +225,12 @@ impl LibraryManager { ) -> Result { let db_path = db_path.as_ref(); let db = Arc::new( - load_and_migrate( - db_path.parent().ok_or_else(|| { + load_and_migrate(&format!( + "file:{}", + db_path.as_os_str().to_str().ok_or_else(|| { LibraryManagerError::InvalidDatabasePath(db_path.to_path_buf()) - })?, - &format!( - "file:{}", - db_path.as_os_str().to_str().ok_or_else(|| { - LibraryManagerError::InvalidDatabasePath(db_path.to_path_buf()) - })? - ), - ) + })? + )) .await .unwrap(), ); diff --git a/core/src/location/indexer/indexer_job.rs b/core/src/location/indexer/indexer_job.rs index 7ab290547..dc804838f 100644 --- a/core/src/location/indexer/indexer_job.rs +++ b/core/src/location/indexer/indexer_job.rs @@ -32,7 +32,7 @@ pub enum ScanProgress { /// batches of [`BATCH_SIZE`]. Then for each chunk it write the file metadata to the database. pub struct IndexerJob; -location::include!(pub indexer_job_location { +location::include!(indexer_job_location { indexer_rules: select { indexer_rule } }); @@ -226,11 +226,13 @@ impl StatefulJob for IndexerJob { ctx: WorkerContext, state: &mut JobState, ) -> Result<(), JobError> { - let location_path = &state + let data = &state .data .as_ref() - .expect("critical error: missing data on job state") - .location_path; + .expect("critical error: missing data on job state"); + + let location_path = &data.location_path; + let location_id = state.init.location.id; let count = ctx .library_ctx() @@ -262,12 +264,12 @@ impl StatefulJob for IndexerJob { file_path::create( entry.file_id, + location_id, materialized_path, name, vec![ file_path::is_dir::set(entry.is_dir), file_path::extension::set(Some(extension)), - file_path::location_id::set(state.init.location.id), file_path::parent_id::set(entry.parent_id), file_path::date_created::set(entry.created_at.into()), ], diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index f7893c017..b51143b58 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -4,7 +4,7 @@ use crate::{ invalidate_query, job::Job, library::LibraryContext, - prisma::{indexer_rule, indexer_rules_in_location, location, node}, + prisma::{indexer_rules_in_location, location, node}, }; use rspc::Type; @@ -222,13 +222,7 @@ async fn link_location_and_indexer_rules( .create_many( rules_ids .iter() - .map(|id| { - indexer_rules_in_location::create( - location::id::equals(location_id), - indexer_rule::id::equals(*id), - vec![], - ) - }) + .map(|id| indexer_rules_in_location::create(location_id, *id, vec![])) .collect(), ) .exec() diff --git a/core/src/util/db.rs b/core/src/util/db.rs index 9e651e38d..ceeee7cdb 100644 --- a/core/src/util/db.rs +++ b/core/src/util/db.rs @@ -1,92 +1,37 @@ use crate::prisma::{self, PrismaClient}; -use enumflags2::BitFlags; -use include_dir::{include_dir, Dir}; -use migration_core::{ - commands::apply_migrations, - json_rpc::types::ApplyMigrationsInput, - migration_connector::{ConnectorError, ConnectorParams}, -}; -use prisma_client_rust::NewClientError; -use quaint::prelude::*; -use sql_migration_connector::SqlMigrationConnector; -use std::path::Path; +use prisma_client_rust::{migrations::*, NewClientError}; use thiserror::Error; -use tokio::fs::{create_dir, remove_dir_all}; -use tracing::debug; - -static MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/prisma/migrations"); /// MigrationError represents an error that occurring while opening a initialising and running migrations on the database. #[derive(Error, Debug)] pub enum MigrationError { #[error("An error occurred while initialising a new database connection: {0}")] NewClient(#[from] Box), - #[error("The temporary file path for the database migrations is invalid.")] - InvalidDirectory, - #[error("An error occurred creating the temporary directory for the migrations: {0}")] - CreateDir(std::io::Error), - #[error("An error occurred extracting the migrations to the temporary directory: {0}")] - ExtractMigrations(std::io::Error), - #[error("An error occurred creating the database connection for migrations: {0}")] - Quiant(#[from] quaint::error::Error), - #[error("An error occurred running the migrations: {0}")] - Connector(#[from] ConnectorError), - #[error("An error occurred removing the temporary directory for the migrations: {0}")] - RemoveDir(std::io::Error), + #[cfg(debug_assertions)] + #[error("An error occured during migartion: {0}")] + MigrateFailed(#[from] DbPushError), + #[cfg(not(debug_assertions))] + #[error("An error occured during migration: {0}")] + MigrateFailed(#[from] MigrateDeployError), } /// load_and_migrate will load the database from the given path and migrate it to the latest version of the schema. -pub async fn load_and_migrate( - base_path: &Path, - db_url: &str, -) -> Result { +pub async fn load_and_migrate(db_url: &str) -> Result { let client = prisma::new_client_with_url(db_url) .await .map_err(Box::new)?; - let temp_migrations_dir = base_path.join("./migrations_temp"); - let migrations_directory_path = temp_migrations_dir - .to_str() - .ok_or(MigrationError::InvalidDirectory)? - .to_string(); - if temp_migrations_dir.exists() { - remove_dir_all(&migrations_directory_path) - .await - .map_err(MigrationError::RemoveDir)?; - } + #[cfg(debug_assertions)] + client + ._db_push( + std::env::var("SD_FORCE_RESET_DB") + .map(|v| v == "true") + .unwrap_or(false), + ) + .await?; - create_dir(&temp_migrations_dir) - .await - .map_err(MigrationError::CreateDir)?; - MIGRATIONS_DIR - .extract(&temp_migrations_dir) - .map_err(MigrationError::ExtractMigrations)?; - - let mut connector = match &ConnectionInfo::from_url(db_url)? { - ConnectionInfo::Sqlite { .. } => SqlMigrationConnector::new_sqlite(), - ConnectionInfo::InMemorySqlite { .. } => unreachable!(), // This is how it is in the Prisma Rust tests - }; - connector.set_params(ConnectorParams { - connection_string: db_url.to_string(), - preview_features: BitFlags::empty(), - shadow_database_connection_string: None, - })?; - - let output = apply_migrations( - ApplyMigrationsInput { - migrations_directory_path, - }, - &mut connector, - ) - .await?; - - remove_dir_all(temp_migrations_dir) - .await - .map_err(MigrationError::RemoveDir)?; - - for migration in output.applied_migration_names { - debug!("Applied migration '{}'", migration); - } + #[cfg(not(debug_assertions))] + client._migrate_deploy().await?; Ok(client) }