Use PCR for migrations (#385)

* use pcr migrations branch

* use 0.6.2 branch with migrations

* use latest prisma stuff

* allow force reset of db in dev

* remove .spacedrive file

* update rspc in apps/server

* use rspc 0.0.5 in all crates

* add os to prisma client cache key

* add runner os to clippy prisma cache

.github/workflows/ci.yml

@@ -122,7 +122,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ./core/src/prisma.rs
-          key: prisma-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }}
+          key: prisma-${{ runner.os }}-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }}
 
       - name: Generate Prisma client
         working-directory: core

.github/workflows/clippy.yml

@@ -3,6 +3,8 @@ name: Rust Clippy check
 on:
   pull_request:
   push:
+    branches:
+      - main
     paths:
       - '**.rs'
       - '**.toml'
@@ -42,7 +44,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ./core/src/prisma.rs
-          key: prisma-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }}
+          key: prisma-${{ runner.os }}-${{ hashFiles('./core/prisma/Cargo.toml', './core/prisma/schema.prisma', './core/prisma/src/main.rs') }}
 
       - name: Generate Prisma client
         working-directory: core

.gitignore

@@ -62,3 +62,4 @@ examples/*/*.lock
 /target
 
 /sdserver_data
+.spacedrive

Cargo.toml

@@ -14,5 +14,13 @@ openssl-sys = { git = "https://github.com/spacedriveapp/rust-openssl" }
 rspc = { git = "https://github.com/oscartbeaumont/rspc", rev = "1b2a299e9061c81ff90706923a6d2389ea7c107e" }
 
 [patch."https://github.com/Brendonovich/prisma-client-rust.git"]
-prisma-client-rust = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "8447fe493414471a23a38d780b3db246266f558f" }
-prisma-client-rust-cli = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "8447fe493414471a23a38d780b3db246266f558f" }
+prisma-client-rust = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "43fd489cd817efc061096978030241bbf7ad3fb9", features = [
+  "migrations",
+  "rspc",
+  "sqlite-create-many",
+] }
+prisma-client-rust-cli = { git = "https://github.com//Brendonovich/prisma-client-rust.git", rev = "43fd489cd817efc061096978030241bbf7ad3fb9", features = [
+  "migrations",
+  "rspc",
+  "sqlite-create-many",
+] }

apps/desktop/src-tauri/Cargo.toml

@@ -11,7 +11,7 @@ build = "build.rs"
 
 [dependencies]
 tauri = { version = "1.0.4", features = ["api-all", "macos-private-api"] }
-rspc = { version = "0.0.4", features = ["tauri"] }
+rspc = { version = "0.0.5", features = ["tauri"] }
 sdcore = { path = "../../../core" }
 tokio = { version = "1.17.0", features = ["sync"] }
 window-shadows = "0.1.2"

apps/mobile/rust/Cargo.toml

@@ -10,12 +10,19 @@ crate-type = ["staticlib", "cdylib"] # staticlib for IOS and cdylib for Android
 
 [dependencies]
 once_cell = "1.13.0"
-sdcore = { path = "../../../core", features = ["mobile", "p2p"], default-features = false }
-rspc = { version = "0.0.4", features = [] }
+sdcore = { path = "../../../core", features = [
+  "mobile",
+  "p2p",
+], default-features = false }
+rspc = { version = "0.0.5", features = [] }
 serde_json = "1.0.83"
 tokio = "1.20.1"
-openssl = { version = "0.10.41", features = ["vendored"] } # Override features of transitive dependencies
-openssl-sys = { version = "0.9.75", features = ["vendored"] } # Override features of transitive dependencies to support IOS Simulator on M1
+openssl = { version = "0.10.41", features = [
+  "vendored",
+] } # Override features of transitive dependencies
+openssl-sys = { version = "0.9.75", features = [
+  "vendored",
+] } # Override features of transitive dependencies to support IOS Simulator on M1
 
 [target.'cfg(target_os = "ios")'.dependencies]
 objc = "0.2.7"

apps/server/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 sdcore = { path = "../../core", features = [] }
-rspc = { version = "0.0.4", features = ["axum"] }
+rspc = { version = "0.0.5", features = ["axum"] }
 axum = "0.5.13"
 tokio = { version = "1.17.0", features = ["sync", "rt-multi-thread", "signal"] }
 tracing = "0.1.35"

core/Cargo.toml

@@ -33,11 +33,15 @@ rmp-serde = "^1.1.0"
 prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust.git", tag = "0.6.0", features = [
   "rspc",
   "sqlite-create-many",
+  "migrations",
+] }
+quaint = { git = "https://github.com/prisma/quaint.git", features = [
+  "sqlite",
+  "uuid",
 ] }
-quaint = { git = "https://github.com/prisma/quaint.git", features = ["sqlite", "uuid"] }
 migration-core = { git = "https://github.com/Brendonovich/prisma-engines.git" }
 sql-migration-connector = { git = "https://github.com/Brendonovich/prisma-engines.git" }
-rspc = { version = "0.0.4", features = ["uuid", "chrono", "tracing"] }
+rspc = { version = "0.0.5", features = ["uuid", "chrono", "tracing"] }
 uuid = { version = "1.1.2", features = ["v4", "serde"] }
 sysinfo = "0.23.9"
 thiserror = "1.0.30"

core/prisma/Cargo.toml

@@ -7,4 +7,5 @@ edition = "2021"
 prisma-client-rust-cli = { git = "https://github.com/Brendonovich/prisma-client-rust.git", tag = "0.6.0", features = [
   "rspc",
   "sqlite-create-many",
+  "migrations",
 ] }

core/src/api/locations.rs

@@ -29,8 +29,8 @@ pub enum ExplorerContext {
 	// Space(object_in_space::Data),
 }
 
-file_path::include!(pub file_path_with_file { file });
-file::include!(pub file_with_paths { paths });
+file_path::include!(file_path_with_file { file });
+file::include!(file_with_paths { paths });
 
 #[derive(Serialize, Deserialize, Type, Debug)]
 #[serde(tag = "type")]

core/src/encode/thumb.rs

@@ -37,7 +37,7 @@ pub struct ThumbnailJobState {
 	root_path: PathBuf,
 }
 
-file_path::include!(pub image_path_with_file { file });
+file_path::include!(image_path_with_file { file });
 
 #[async_trait::async_trait]
 impl StatefulJob for ThumbnailJob {

core/src/library/library_manager.rs

@@ -225,17 +225,12 @@ impl LibraryManager {
 	) -> Result<LibraryContext, LibraryManagerError> {
 		let db_path = db_path.as_ref();
 		let db = Arc::new(
-			load_and_migrate(
-				db_path.parent().ok_or_else(|| {
+			load_and_migrate(&format!(
+				"file:{}",
+				db_path.as_os_str().to_str().ok_or_else(|| {
 					LibraryManagerError::InvalidDatabasePath(db_path.to_path_buf())
-				})?,
-				&format!(
-					"file:{}",
-					db_path.as_os_str().to_str().ok_or_else(|| {
-						LibraryManagerError::InvalidDatabasePath(db_path.to_path_buf())
-					})?
-				),
-			)
+				})?
+			))
 			.await
 			.unwrap(),
 		);

core/src/location/indexer/indexer_job.rs

@@ -32,7 +32,7 @@ pub enum ScanProgress {
 /// batches of [`BATCH_SIZE`]. Then for each chunk it write the file metadata to the database.
 pub struct IndexerJob;
 
-location::include!(pub indexer_job_location {
+location::include!(indexer_job_location {
 	indexer_rules: select { indexer_rule }
 });
 
@@ -226,11 +226,13 @@ impl StatefulJob for IndexerJob {
 		ctx: WorkerContext,
 		state: &mut JobState<Self::Init, Self::Data, Self::Step>,
 	) -> Result<(), JobError> {
-		let location_path = &state
+		let data = &state
 			.data
 			.as_ref()
-			.expect("critical error: missing data on job state")
-			.location_path;
+			.expect("critical error: missing data on job state");
+
+		let location_path = &data.location_path;
+		let location_id = state.init.location.id;
 
 		let count = ctx
 			.library_ctx()
@@ -262,12 +264,12 @@ impl StatefulJob for IndexerJob {
 
 						file_path::create(
 							entry.file_id,
+							location_id,
 							materialized_path,
 							name,
 							vec![
 								file_path::is_dir::set(entry.is_dir),
 								file_path::extension::set(Some(extension)),
-								file_path::location_id::set(state.init.location.id),
 								file_path::parent_id::set(entry.parent_id),
 								file_path::date_created::set(entry.created_at.into()),
 							],

core/src/location/mod.rs

@@ -4,7 +4,7 @@ use crate::{
 	invalidate_query,
 	job::Job,
 	library::LibraryContext,
-	prisma::{indexer_rule, indexer_rules_in_location, location, node},
+	prisma::{indexer_rules_in_location, location, node},
 };
 
 use rspc::Type;
@@ -222,13 +222,7 @@ async fn link_location_and_indexer_rules(
 		.create_many(
 			rules_ids
 				.iter()
-				.map(|id| {
-					indexer_rules_in_location::create(
-						location::id::equals(location_id),
-						indexer_rule::id::equals(*id),
-						vec![],
-					)
-				})
+				.map(|id| indexer_rules_in_location::create(location_id, *id, vec![]))
 				.collect(),
 		)
 		.exec()

core/src/util/db.rs

@@ -1,92 +1,37 @@
 use crate::prisma::{self, PrismaClient};
-use enumflags2::BitFlags;
-use include_dir::{include_dir, Dir};
-use migration_core::{
-	commands::apply_migrations,
-	json_rpc::types::ApplyMigrationsInput,
-	migration_connector::{ConnectorError, ConnectorParams},
-};
-use prisma_client_rust::NewClientError;
-use quaint::prelude::*;
-use sql_migration_connector::SqlMigrationConnector;
-use std::path::Path;
+use prisma_client_rust::{migrations::*, NewClientError};
 use thiserror::Error;
-use tokio::fs::{create_dir, remove_dir_all};
-use tracing::debug;
-
-static MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/prisma/migrations");
 
 /// MigrationError represents an error that occurring while opening a initialising and running migrations on the database.
 #[derive(Error, Debug)]
 pub enum MigrationError {
 	#[error("An error occurred while initialising a new database connection: {0}")]
 	NewClient(#[from] Box<NewClientError>),
-	#[error("The temporary file path for the database migrations is invalid.")]
-	InvalidDirectory,
-	#[error("An error occurred creating the temporary directory for the migrations: {0}")]
-	CreateDir(std::io::Error),
-	#[error("An error occurred extracting the migrations to the temporary directory: {0}")]
-	ExtractMigrations(std::io::Error),
-	#[error("An error occurred creating the database connection for migrations: {0}")]
-	Quiant(#[from] quaint::error::Error),
-	#[error("An error occurred running the migrations: {0}")]
-	Connector(#[from] ConnectorError),
-	#[error("An error occurred removing the temporary directory for the migrations: {0}")]
-	RemoveDir(std::io::Error),
+	#[cfg(debug_assertions)]
+	#[error("An error occured during migartion: {0}")]
+	MigrateFailed(#[from] DbPushError),
+	#[cfg(not(debug_assertions))]
+	#[error("An error occured during migration: {0}")]
+	MigrateFailed(#[from] MigrateDeployError),
 }
 
 /// load_and_migrate will load the database from the given path and migrate it to the latest version of the schema.
-pub async fn load_and_migrate(
-	base_path: &Path,
-	db_url: &str,
-) -> Result<PrismaClient, MigrationError> {
+pub async fn load_and_migrate(db_url: &str) -> Result<PrismaClient, MigrationError> {
 	let client = prisma::new_client_with_url(db_url)
 		.await
 		.map_err(Box::new)?;
-	let temp_migrations_dir = base_path.join("./migrations_temp");
-	let migrations_directory_path = temp_migrations_dir
-		.to_str()
-		.ok_or(MigrationError::InvalidDirectory)?
-		.to_string();
 
-	if temp_migrations_dir.exists() {
-		remove_dir_all(&migrations_directory_path)
-			.await
-			.map_err(MigrationError::RemoveDir)?;
-	}
+	#[cfg(debug_assertions)]
+	client
+		._db_push(
+			std::env::var("SD_FORCE_RESET_DB")
+				.map(|v| v == "true")
+				.unwrap_or(false),
+		)
+		.await?;
 
-	create_dir(&temp_migrations_dir)
-		.await
-		.map_err(MigrationError::CreateDir)?;
-	MIGRATIONS_DIR
-		.extract(&temp_migrations_dir)
-		.map_err(MigrationError::ExtractMigrations)?;
-
-	let mut connector = match &ConnectionInfo::from_url(db_url)? {
-		ConnectionInfo::Sqlite { .. } => SqlMigrationConnector::new_sqlite(),
-		ConnectionInfo::InMemorySqlite { .. } => unreachable!(), // This is how it is in the Prisma Rust tests
-	};
-	connector.set_params(ConnectorParams {
-		connection_string: db_url.to_string(),
-		preview_features: BitFlags::empty(),
-		shadow_database_connection_string: None,
-	})?;
-
-	let output = apply_migrations(
-		ApplyMigrationsInput {
-			migrations_directory_path,
-		},
-		&mut connector,
-	)
-	.await?;
-
-	remove_dir_all(temp_migrations_dir)
-		.await
-		.map_err(MigrationError::RemoveDir)?;
-
-	for migration in output.applied_migration_names {
-		debug!("Applied migration '{}'", migration);
-	}
+	#[cfg(not(debug_assertions))]
+	client._migrate_deploy().await?;
 
 	Ok(client)
 }