Merge branch 'main' into eng-348-async-crypto

This commit is contained in:
brxken128 committed 2023-01-25 22:02:23 +00:00
commit 3f2a62feb4
34 files changed
+237 -572

No files matched your search

Generated
+5 -7
View File
@@ -6648,9 +6648,8 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
version = "1.24.2"
source = "git+https://github.com/spacedriveapp/tokio?rev=d3936848ccb32ac7df41614acfabe5139a6a7d75#d3936848ccb32ac7df41614acfabe5139a6a7d75"
dependencies = [
"autocfg 1.1.0",
"bytes",
@@ -6663,14 +6662,13 @@ dependencies = [
"signal-hook-registry",
"socket2",
"tokio-macros",
"winapi",
"windows-sys 0.42.0",
]
[[package]]
name = "tokio-macros"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484"
version = "1.8.2"
source = "git+https://github.com/spacedriveapp/tokio?rev=d3936848ccb32ac7df41614acfabe5139a6a7d75#d3936848ccb32ac7df41614acfabe5139a6a7d75"
dependencies = [
"proc-macro2",
"quote",
+4
View File
@@ -34,6 +34,8 @@ specta = { version = "0.0.4" }
swift-rs = { git = "https://github.com/Brendonovich/swift-rs.git", rev = "833e29ba333f1dfe303eaa21de78c4f8c5a3f2ff" }
tokio = { version = "1.24.2" }
[patch.crates-io]
# We use this patch so we can compile for the IOS simulator on M1
openssl-sys = { git = "https://github.com/spacedriveapp/rust-openssl", rev = "92c3dec225a9e984884d5b30a517e5d44a24d03b" }
@@ -41,3 +43,5 @@ openssl-sys = { git = "https://github.com/spacedriveapp/rust-openssl", rev = "92
rspc = { git = "https://github.com/oscartbeaumont/rspc", rev = "6243b5b6a1376940a40318340e5eaef22e4a2c22" } # TODO: Move back to crates.io when new jsonrpc executor + `tokio::spawn` in the Tauri IPC plugin is released
normi = { git = "https://github.com/oscartbeaumont/rspc", rev = "6243b5b6a1376940a40318340e5eaef22e4a2c22" } # TODO: When normi is released on crates.io
specta = { git = "https://github.com/oscartbeaumont/rspc", rev = "6243b5b6a1376940a40318340e5eaef22e4a2c22" } # TODO: When normi is released on crates.io
tokio = { git = "https://github.com/spacedriveapp/tokio", rev = "d3936848ccb32ac7df41614acfabe5139a6a7d75" } # TODO: move back once tokio raise their internal read buffer size (ENG-351 for more)
+2 -4
View File
@@ -13,7 +13,7 @@ build = "build.rs"
tauri = { version = "1.1.1", features = ["api-all", "macos-private-api"] }
rspc = { workspace = true, features = ["tauri"] }
sd-core = { path = "../../../core", features = ["ffmpeg", "location-watcher"] }
tokio = { version = "1.21.2", features = ["sync"] }
tokio = { workspace = true, features = ["sync"] }
window-shadows = "0.2.0"
tracing = "0.1.36"
serde = "1.0.145"
@@ -25,9 +25,7 @@ swift-rs.workspace = true
tauri-build = { version = "1.1.1", features = [] }
[target.'cfg(target_os = "macos")'.build-dependencies]
swift-rs = { workspace = true, features = [
"build",
] }
swift-rs = { workspace = true, features = ["build"] }
[features]
default = ["custom-protocol"]
+3 -3
View File
@@ -6,16 +6,16 @@ rust-version = "1.64.0"
[features]
android = ["sd-core/android"]
[dependencies]
once_cell = "1.15.0"
sd-core = { path = "../../../../core", features = [
"mobile",
"p2p",
], default-features = false }
rspc.workspace= true
rspc.workspace = true
serde_json = "1.0.85"
tokio = "1.21.2"
tokio = { workspace = true }
openssl = { version = "0.10.42", features = [
"vendored",
] } # Override features of transitive dependencies
+1 -1
View File
@@ -7,6 +7,6 @@ edition = "2021"
sd-core = { path = "../../core", features = ["ffmpeg"] }
rspc = { workspace = true, features = ["axum"] }
axum = "0.5.16"
tokio = { version = "1.21.2", features = ["sync", "rt-multi-thread", "signal"] }
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "signal"] }
tracing = "0.1.36"
ctrlc = "3.2.3"
+8 -6
View File
@@ -19,9 +19,7 @@ ffmpeg = [
"dep:ffmpeg-next",
"dep:sd-ffmpeg",
] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg.
location-watcher = [
"dep:notify"
]
location-watcher = ["dep:notify"]
[dependencies]
hostname = "0.3.1"
@@ -46,10 +44,12 @@ uuid = { version = "1.1.2", features = ["v4", "serde"] }
sysinfo = "0.26.4"
thiserror = "1.0.37"
tokio = { version = "1.21.2", features = [
tokio = { workspace = true, features = [
"sync",
"rt-multi-thread",
"io-util",
"macros",
"time",
] }
include_dir = { version = "0.7.2", features = ["glob"] }
async-trait = "^0.1.57"
@@ -58,7 +58,7 @@ webp = "0.2.2"
ffmpeg-next = { version = "5.1.1", optional = true, features = [] }
sd-ffmpeg = { path = "../crates/ffmpeg", optional = true }
sd-crypto = { path = "../crates/crypto", features = ["rspc", "serde"] }
sd-file-ext = { path = "../crates/file-ext"}
sd-file-ext = { path = "../crates/file-ext" }
sd-sync = { path = "../crates/sync" }
fs_extra = "1.2.0"
tracing = "0.1.36"
@@ -70,7 +70,9 @@ ctor = "0.1.23"
globset = { version = "^0.4.9", features = ["serde1"] }
itertools = "^0.10.5"
enumflags2 = "0.7.5"
notify = { version = "5.0.0", default-features = false, features = ["macos_fsevent"], optional = true }
notify = { version = "5.0.0", default-features = false, features = [
"macos_fsevent",
], optional = true }
uhlc = "0.5.1"
[dev-dependencies]
+8 -7
View File
@@ -2,6 +2,7 @@ use crate::{
job::{JobError, JobReportUpdate, JobResult, JobState, StatefulJob, WorkerContext},
location::indexer::rules::RuleKind,
prisma::{file_path, location},
prisma_sync,
};
use std::{
@@ -268,10 +269,12 @@ impl StatefulJob for IndexerJob {
(
(
json!({
"id": entry.file_id,
"location_id": state.init.location.pub_id,
}),
prisma_sync::file_path::SyncId {
id: entry.file_id,
location: prisma_sync::location::SyncId {
pub_id: state.init.location.pub_id.clone(),
},
},
[
("materialized_path", json!(materialized_path.clone())),
("name", json!(name.clone())),
@@ -302,9 +305,7 @@ impl StatefulJob for IndexerJob {
.sync
.write_op(
db,
ctx.library_ctx
.sync
.owned_create_many("FilePath", sync_stuff, true),
ctx.library_ctx.sync.owned_create_many(sync_stuff, true),
db.file_path().create_many(paths).skip_duplicates(),
)
.await?;
@@ -156,7 +156,7 @@ impl StatefulJob for FullFileIdentifierJob {
let (total_objects_created, total_objects_linked) = identifier_job_step(
&ctx.library_ctx,
state.init.location_id,
&data.location,
&data.location_path,
&file_paths,
)
+22 -22
View File
@@ -2,12 +2,14 @@ use crate::{
job::JobError,
library::LibraryContext,
object::cas::generate_cas_id,
prisma::{file_path, object},
prisma::{file_path, location, object},
prisma_sync,
};
use chrono::{DateTime, FixedOffset};
use serde_json::json;
use std::{
collections::{HashMap, HashSet},
ffi::OsStr,
path::{Path, PathBuf},
};
@@ -56,18 +58,15 @@ pub async fn assemble_object_metadata(
let metadata = fs::metadata(&path).await?;
// derive Object kind
let object_kind = match path.extension() {
Some(ext) => match ext.to_str() {
Some(ext) => {
let mut file = fs::File::open(&path).await?;
let object_kind = match path.extension().and_then(OsStr::to_str) {
Some(ext) => {
let mut file = fs::File::open(&path).await?;
Extension::resolve_conflicting(&ext.to_lowercase(), &mut file, false)
.await
.map(Into::into)
.unwrap_or(ObjectKind::Unknown)
}
None => ObjectKind::Unknown,
},
Extension::resolve_conflicting(&ext.to_lowercase(), &mut file, false)
.await
.map(Into::into)
.unwrap_or(ObjectKind::Unknown)
}
None => ObjectKind::Unknown,
};
@@ -87,7 +86,7 @@ pub async fn assemble_object_metadata(
async fn batch_update_file_paths(
library: &LibraryContext,
location_id: i32,
location: &location::Data,
objects: &[object::Data],
cas_id_lookup: &HashMap<String, Vec<i32>>,
) -> Result<Vec<file_path::Data>, QueryError> {
@@ -105,15 +104,16 @@ async fn batch_update_file_paths(
(
sync.owned_update(
"FilePath",
json!({
"id": file_path_id,
"location_id": location_id
}),
prisma_sync::file_path::SyncId {
id: *file_path_id,
location: prisma_sync::location::SyncId {
pub_id: location.pub_id.clone(),
},
},
[("object", json!({ "cas_id": object.cas_id }))],
),
library.db.file_path().update(
file_path::location_id_id(location_id, *file_path_id),
file_path::location_id_id(location.id, *file_path_id),
vec![file_path::object::connect(object::id::equals(object.id))],
),
)
@@ -179,7 +179,7 @@ async fn generate_provisional_objects(
async fn identifier_job_step(
library: &LibraryContext,
location_id: i32,
location: &location::Data,
location_path: impl AsRef<Path>,
file_paths: &[file_path::Data],
) -> Result<(usize, usize), JobError> {
@@ -223,7 +223,7 @@ async fn identifier_job_step(
let existing_objects_linked = if !existing_objects.is_empty() {
// link file_path.object_id to existing objects
batch_update_file_paths(library, location_id, &existing_objects, &cas_id_lookup)
batch_update_file_paths(library, location, &existing_objects, &cas_id_lookup)
.await?
.len()
} else {
@@ -289,7 +289,7 @@ async fn identifier_job_step(
);
if !created_files.is_empty() {
batch_update_file_paths(library, location_id, &created_files, &cas_id_lookup).await?;
batch_update_file_paths(library, &location, &created_files, &cas_id_lookup).await?;
}
}
+32 -21
View File
@@ -443,15 +443,17 @@ impl SyncManager {
.collect(),
}))
}
pub fn owned_create_many<const SIZE: usize>(
pub fn owned_create_many<
const SIZE: usize,
TSyncId: SyncId<ModelTypes = TModel>,
TModel: SyncType<Marker = SharedSyncType>,
>(
&self,
model: &str,
data: impl IntoIterator<Item = (Value, [(&'static str, Value); SIZE])>,
data: impl IntoIterator<Item = (TSyncId, [(&'static str, Value); SIZE])>,
skip_duplicates: bool,
) -> CRDTOperation {
self.new_op(CRDTOperationType::Owned(OwnedOperation {
model: model.to_string(),
model: TModel::MODEL.to_string(),
items: vec![OwnedOperationItem {
id: Value::Null,
data: OwnedOperationData::CreateMany {
@@ -459,7 +461,7 @@ impl SyncManager {
.into_iter()
.map(|(id, data)| {
(
id,
json!(id),
data.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
)
})
@@ -469,19 +471,21 @@ impl SyncManager {
}],
}))
}
pub fn owned_update<const SIZE: usize>(
pub fn owned_update<
const SIZE: usize,
TSyncId: SyncId<ModelTypes = TModel>,
TModel: SyncType<Marker = SharedSyncType>,
>(
&self,
model: &str,
id: Value,
id: TSyncId,
values: [(&'static str, Value); SIZE],
) -> CRDTOperation {
self.new_op(CRDTOperationType::Owned(OwnedOperation {
model: model.to_string(),
model: TModel::MODEL.to_string(),
items: [(id, values)]
.into_iter()
.map(|(id, data)| OwnedOperationItem {
id,
id: json!(id),
data: OwnedOperationData::Update(
data.into_iter().map(|(k, v)| (k.to_string(), v)).collect(),
),
@@ -490,24 +494,31 @@ impl SyncManager {
}))
}
pub fn shared_create(&self, model: &str, record_id: Value) -> CRDTOperation {
pub fn shared_create<
TSyncId: SyncId<ModelTypes = TModel>,
TModel: SyncType<Marker = SharedSyncType>,
>(
&self,
id: TSyncId,
) -> CRDTOperation {
self.new_op(CRDTOperationType::Shared(SharedOperation {
model: model.to_string(),
record_id,
model: TModel::MODEL.to_string(),
record_id: json!(id),
data: SharedOperationData::Create(SharedOperationCreateData::Atomic),
}))
}
pub fn shared_update(
pub fn shared_update<
TSyncId: SyncId<ModelTypes = TModel>,
TModel: SyncType<Marker = SharedSyncType>,
>(
&self,
model: &str,
record_id: Value,
id: TSyncId,
field: &str,
value: Value,
) -> CRDTOperation {
self.new_op(CRDTOperationType::Shared(SharedOperation {
model: model.to_string(),
record_id,
model: TModel::MODEL.to_string(),
record_id: json!(id),
data: SharedOperationData::Update {
field: field.to_string(),
value,
+1 -1
View File
@@ -42,7 +42,7 @@ specta = { workspace = true, optional = true }
hex = "0.4.3"
tokio = { version = "1.21.2", features = ["io-util", "rt-multi-thread"] }
tokio = { workspace = true, features = ["io-util", "rt-multi-thread"] }
[dev-dependencies]
tokio = { version = "1.21.2", features = [
+2 -2
View File
@@ -15,8 +15,8 @@ resolver = "2"
ffmpeg-sys-next = "5.1.1"
thiserror = "1.0.37"
webp = "0.2.2"
tokio = { version = "1.21.2", features = ["fs", "rt"] }
tokio = { workspace = true, features = ["fs", "rt"] }
[dev-dependencies]
tempfile = "3.3.0"
tokio = { version = "1.21.2", features = ["fs", "rt", "macros"] }
tokio = { workspace = true, features = ["fs", "rt", "macros"] }
+8 -5
View File
@@ -2,16 +2,19 @@
name = "sd-file-ext"
version = "0.0.0"
edition = "2021"
authors = ["Brendan Allen <brendan@spacedrive.com>", "Jamie Pine <jamie@spacedrive.com>"]
authors = [
"Brendan Allen <brendan@spacedrive.com>",
"Jamie Pine <jamie@spacedrive.com>",
]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
int-enum = "0.5.0"
serde = {version = "1.0.145", features = ["derive"]}
serde = { version = "1.0.145", features = ["derive"] }
serde_json = "1.0.85"
strum = { version = "0.24", features = ["derive"]}
strum = { version = "0.24", features = ["derive"] }
strum_macros = "0.24"
tokio = { version = "1.21.2", features = ["fs", "rt", "io-util"] }
tokio = { workspace = true, features = ["fs", "rt", "io-util"] }
[dev-dependencies]
tokio = { version = "1.21.2", features = ["fs", "rt", "macros"] }
tokio = { workspace = true, features = ["fs", "rt", "macros"] }
+2 -2
View File
@@ -9,7 +9,7 @@ sd-tunnel-utils = { path = "./tunnel/utils" }
dashmap = "5.3.4"
rcgen = "0.9.2"
rustls = "0.20.6"
tokio = { version = "1.19.2", features = ["macros", "sync"] }
tokio = { workspace = true, features = ["macros", "sync"] }
if-watch = "1.1.1"
thiserror = "1.0.31"
mdns-sd = "0.5.5"
@@ -25,4 +25,4 @@ tracing = "0.1.35"
specta = "0.0.2"
[dev-dependencies]
tokio = { version = "1.19.2", features = ["rt-multi-thread"] }
tokio = { workspace = true, features = ["rt-multi-thread"] }
+4 -2
View File
@@ -14,11 +14,13 @@ quinn = "0.8.3"
rcgen = "0.9.2"
rustls = "0.20.6"
serde = { version = "1.0.137", features = ["derive"] }
tokio = { version = "1.19.2", features = ["rt-multi-thread", "macros"] }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
tracing = "0.1.35"
tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }
metrics = "0.19.0"
metrics-exporter-prometheus = { version = "0.10.0", features = ["http-listener"] }
metrics-exporter-prometheus = { version = "0.10.0", features = [
"http-listener",
] }
thiserror = "1.0.31"
rmp-serde = "1.1.0"
bb8-redis = "0.11.0"
+1 -1
View File
@@ -10,7 +10,7 @@ serde_json = "1.0.85"
serde = { version = "1.0.145", features = ["derive"] }
axum = "0.5.16"
rspc = { workspace = true, features = ["axum"] }
tokio = { version = "1.21.2", features = ["full"] }
tokio = { workspace = true, features = ["full"] }
prisma-client-rust = { workspace = true }
dotenv = "0.15.0"
tower-http = { version = "0.3.4", features = ["cors"] }
-373
View File
@@ -1,373 +0,0 @@
use std::{collections::HashMap, time::Duration};
use rspc::Type;
use serde::*;
use serde_json::*;
use uhlc::{HLCBuilder, Timestamp, HLC, NTP64};
use uuid::Uuid;
use super::crdt::*;
// Bytes
#[derive(Default, Debug, Serialize, Type, Clone)]
pub struct Color {
pub red: u8,
pub green: u8,
pub blue: u8,
}
// Unique Shared
#[derive(Default, Debug, Serialize, Type, Clone)]
pub struct Tag {
pub color: Color,
pub name: String,
}
#[derive(Default, Debug, Serialize, Type, Clone)]
pub struct TagOnObject {
pub tag_id: Uuid,
pub object_id: Uuid,
}
// Atomic Shared
#[derive(Default, Debug, Serialize, Type, Clone)]
pub struct Object {
pub id: Uuid,
pub name: String,
}
// Owned
#[derive(Serialize, Deserialize, Debug, Type, Clone)]
pub struct FilePath {
pub id: Uuid,
pub path: String,
pub file: Option<Uuid>,
}
pub struct Db {
pub objects: HashMap<Uuid, Object>,
pub file_paths: HashMap<Uuid, FilePath>,
pub tags: HashMap<Uuid, Tag>,
pub tags_on_objects: HashMap<(Uuid, Uuid), TagOnObject>,
pub _operations: Vec<CRDTOperation>,
pub _clocks: HashMap<Uuid, NTP64>,
_clock: HLC,
_node: Uuid,
}
impl std::fmt::Debug for Db {
fn fmt(&self, f: &mut __private::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Db")
.field("files", &self.objects)
.field("file_paths", &self.file_paths)
.finish()
}
}
impl Db {
pub fn new(node: Uuid) -> Self {
Self {
objects: Default::default(),
file_paths: Default::default(),
tags: Default::default(),
tags_on_objects: Default::default(),
_clocks: Default::default(),
_node: node,
_clock: HLCBuilder::new().with_id(node.into()).build(),
_operations: Default::default(),
}
}
pub fn register_node(&mut self, id: Uuid) {
self._clocks
.entry(id)
.or_insert_with(|| Duration::from_millis(0).into());
}
pub fn create_crdt_operation(&mut self, typ: CRDTOperationType) -> CRDTOperation {
let hlc_timestamp = self._clock.new_timestamp();
CRDTOperation {
node: self._node,
timestamp: *hlc_timestamp.get_time(),
id: Uuid::new_v4(),
typ,
}
}
fn compare_messages(&self, operations: Vec<CRDTOperation>) -> Vec<(CRDTOperation, bool)> {
operations
.into_iter()
.map(|op| (op.id, op))
.collect::<HashMap<_, _>>()
.into_iter()
.filter_map(|(_, op)| {
match &op.typ {
CRDTOperationType::Owned(_) => {
self._operations.iter().find(|find_op| match &find_op.typ {
CRDTOperationType::Owned(_) => {
find_op.timestamp >= op.timestamp && find_op.node == op.node
}
_ => false,
})
}
CRDTOperationType::Shared(shared_op) => {
self._operations.iter().find(|find_op| match &find_op.typ {
CRDTOperationType::Shared(find_shared_op) => {
shared_op.model == find_shared_op.model
&& shared_op.record_id == find_shared_op.record_id
&& find_op.timestamp >= op.timestamp
}
_ => false,
})
}
CRDTOperationType::Relation(relation_op) => {
self._operations.iter().find(|find_op| match &find_op.typ {
CRDTOperationType::Relation(find_relation_op) => {
relation_op.relation == find_relation_op.relation
&& relation_op.relation_item == find_relation_op.relation_item
&& relation_op.relation_group == find_relation_op.relation_group
}
_ => false,
})
}
}
.map(|old_op| (old_op.timestamp != op.timestamp).then_some(true))
.unwrap_or(Some(false))
.map(|old| (op, old))
})
.collect()
}
pub fn receive_crdt_operations(&mut self, ops: Vec<CRDTOperation>) {
for op in &ops {
self._clock
.update_with_timestamp(&Timestamp::new(op.timestamp, op.node.into()))
.ok();
self._clocks.insert(op.node, op.timestamp);
}
for (op, old) in self.compare_messages(ops) {
let push_op = op.clone();
if !old {
match op.typ {
CRDTOperationType::Shared(shared_op) => match shared_op.model.as_str() {
"Object" => {
let id = shared_op.record_id;
match shared_op.data {
SharedOperationData::Create(SharedOperationCreateData::Atomic) => {
self.objects.insert(
id,
Object {
id,
..Default::default()
},
);
}
SharedOperationData::Update { field, value } => {
let mut file = self.objects.get_mut(&id).unwrap();
match field.as_str() {
"name" => {
file.name = from_value(value).unwrap();
}
_ => unreachable!(),
}
}
SharedOperationData::Delete => {
self.objects.remove(&id).unwrap();
}
_ => {}
}
}
_ => unreachable!(),
},
CRDTOperationType::Owned(owned_op) => match owned_op.model.as_str() {
"FilePath" => {
for item in owned_op.items {
let id = from_value(item.id).unwrap();
match item.data {
OwnedOperationData::Create(data) => {
self.file_paths.insert(
id,
from_value(Value::Object(data.into_iter().collect()))
.unwrap(),
);
}
OwnedOperationData::Update(data) => {
let obj = self.file_paths.get_mut(&id).unwrap();
for (key, value) in data {
match key.as_str() {
"path" => obj.path = from_value(value).unwrap(),
"file" => obj.file = from_value(value).unwrap(),
_ => unreachable!(),
}
}
}
OwnedOperationData::Delete => {
self.file_paths.remove(&id);
}
}
}
}
_ => unreachable!(),
},
CRDTOperationType::Relation(relation_op) => match relation_op.relation.as_str()
{
"TagOnObject" => match relation_op.data {
RelationOperationData::Create => {
self.tags_on_objects.insert(
(relation_op.relation_item, relation_op.relation_group),
TagOnObject {
object_id: relation_op.relation_item,
tag_id: relation_op.relation_group,
},
);
}
RelationOperationData::Update { field: _, value: _ } => {
// match field.as_str() {
// _ => unreachable!(),
// }
}
RelationOperationData::Delete => {
self.tags_on_objects
.remove(&(
relation_op.relation_item,
relation_op.relation_group,
))
.unwrap();
}
},
_ => unreachable!(),
},
}
self._operations.push(push_op)
}
}
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use super::*;
fn to_map(v: &impl serde::Serialize) -> BTreeMap<String, Value> {
match to_value(&v).unwrap() {
Value::Object(m) => m.into_iter().collect(),
_ => unreachable!(),
}
}
#[test]
fn test() {
let mut dbs = vec![];
for _ in 0..3 {
let id = Uuid::new_v4();
dbs.push(Db::new(id.clone()));
let ids = dbs.iter().map(|db| db._node.clone()).collect::<Vec<_>>();
for db in &mut dbs {
for id in &ids {
db.register_node(id.clone());
}
}
}
for db in &mut dbs {
let id = Uuid::new_v4();
let file_path = FilePath {
id,
path: String::new(),
file: None,
};
let op = db.create_crdt_operation(CRDTOperationType::Owned(OwnedOperation {
model: "FilePath".to_string(),
items: vec![OwnedOperationItem {
id: serde_json::to_value(id).unwrap(),
data: OwnedOperationData::Create(to_map(&file_path)),
}],
}));
db.receive_crdt_operations(vec![op]);
}
for db in &dbs {
assert_eq!(db._operations.len(), 1);
}
let ops = dbs
.iter()
.flat_map(|db| db._operations.clone())
.collect::<Vec<_>>();
for db in &mut dbs {
db.receive_crdt_operations(ops.clone())
}
for db in &dbs {
assert_eq!(db.file_paths.len(), 3);
assert_eq!(db._operations.len(), 3);
}
for _ in 0..2 {
let db = &mut dbs[0];
let id = Uuid::new_v4();
let file_path = FilePath {
id,
path: String::new(),
file: None,
};
let op = db.create_crdt_operation(CRDTOperationType::Owned(OwnedOperation {
model: "FilePath".to_string(),
items: vec![OwnedOperationItem {
id: serde_json::to_value(id).unwrap(),
data: OwnedOperationData::Create(to_map(&file_path)),
}],
}));
db.receive_crdt_operations(vec![op]);
}
let ops = dbs
.iter()
.flat_map(|db| db._operations.clone())
.collect::<Vec<_>>();
for db in &mut dbs {
db.receive_crdt_operations(ops.clone());
}
for db in &dbs {
assert_eq!(db.file_paths.len(), 5);
assert_eq!(db._operations.len(), 5);
}
for _ in 0..4 {
let ops = dbs
.iter()
.flat_map(|db| db._operations.clone())
.collect::<Vec<_>>();
dbs[0].receive_crdt_operations(ops);
}
for db in &dbs {
assert_eq!(db.file_paths.len(), 5);
assert_eq!(db._operations.len(), 5);
}
}
}
-53
View File
@@ -1,13 +1,9 @@
mod crdt;
// mod db;
pub use crdt::*;
// pub use db::*;
use prisma_client_rust::ModelTypes;
use serde::{de::DeserializeOwned, Serialize};
use serde_value::Value;
use std::collections::BTreeMap;
pub trait SyncId: Serialize + DeserializeOwned {
type ModelTypes: SyncType;
@@ -31,52 +27,3 @@ impl SyncTypeMarker for SharedSyncType {}
pub struct RelationSyncType;
impl SyncTypeMarker for RelationSyncType {}
pub trait CreateCRDTMutation<T: ModelTypes> {
fn operation_from_data(
d: &BTreeMap<String, Value>,
typ: CreateOperationType,
) -> CRDTOperationType;
}
pub enum CreateOperationType {
Owned,
SharedUnique,
SharedAtomic,
Relation,
}
impl<T: ModelTypes> CreateCRDTMutation<T> for prisma_client_rust::Create<'_, T> {
fn operation_from_data(
_: &BTreeMap<String, Value>,
typ: CreateOperationType,
) -> CRDTOperationType {
match typ {
CreateOperationType::Owned => {
todo!()
// let id = serde_json::to_value(
// d.iter()
// .filter(|(field, _)| T::id_fields().iter().any(|f| f == field))
// .collect::<BTreeMap<_, _>>(),
// )
// .unwrap();
// CRDTOperationType::Owned(OwnedOperation {
// model: T::MODEL.to_string(),
// items: [OwnedOperationItem {
// id,
// data: OwnedOperationData::Create(
// d.clone()
// .into_iter()
// .filter(|(field, _)| T::id_fields().iter().all(|f| f != field))
// .map(|(k, v)| (k, serde_json::to_value(v).unwrap()))
// .collect(),
// ),
// }]
// .to_vec(),
// })
}
_ => todo!(),
}
}
}
-1
View File
@@ -1,4 +1,3 @@
# Privacy Policy
_This page has yet to be written out in full. Please check back later!_
+8 -3
View File
@@ -7,13 +7,18 @@ index: 10
### Schema
Our data schema is defined using Prisma, you can [view it on GitHub](https://github.com/spacedriveapp/spacedrive/blob/main/core/prisma/schema.prisma).
![A cool screenshot of the Spacedrive schema](/schema.webp)
![A cool screenshot of the Spacedrive schema](/schema.webp)
### Prisma Client Rust
We use Prisma Client Rust as a database ORM, it allows us to use Prisma to define our schema and generate migrations based on modifications to that schema.
We use Prisma Client Rust as a database ORM, it allows us to use Prisma to define our schema and generate migrations based on modifications to that schema.
### Migrations
Migrations are run by the Prisma migration engine on app launch.
### Database file
The databases file is SQLite and can be opened in any SQL viewer.
![A Spacedrive library database file open in Table Plus](/database-table-plus.webp)
![A Spacedrive library database file open in Table Plus](/database-table-plus.webp)
+2 -2
View File
@@ -4,14 +4,14 @@ index: 4
# Explorer
### Grid view
### List View
### Columns View
### Media View
### Timeline View
### Configuration
### Configuration
+1 -1
View File
@@ -4,4 +4,4 @@ index: 10
# Extensions
Extensions are planned but nothing is yet set in stone.
Extensions are planned but nothing is yet set in stone.
+1 -2
View File
@@ -8,5 +8,4 @@ A library is the database that Spacedrive stores all file structures and metadat
To learn how data is synchronized check out the documentation on [Sync](/docs/developers/architecture/sync).
Libraries can be encrypted with a passphrase set by the user.
Libraries can be encrypted with a passphrase set by the user.
+1 -1
View File
@@ -4,4 +4,4 @@ index: 10
# CLI
A CLI is planned, it would allow a user or application to connect and control a running Spacedrive node on a local or remote system.
A CLI is planned, it would allow a user or application to connect and control a running Spacedrive node on a local or remote system.
+1 -1
View File
@@ -4,4 +4,4 @@ index: 10
# Rust
This doc should just show how to import the core and use it in a Rust context.
This doc should just show how to import the core and use it in a Rust context.
+1 -1
View File
@@ -8,4 +8,4 @@ index: 0
pub struct DeveloperDocumentation;
```
This documentation is a work in progress, you will find unfinished or empty sections.
This documentation is a work in progress, you will find unfinished or empty sections.
+6 -3
View File
@@ -4,22 +4,25 @@ index: 0
---
# Terminology
Some useful Spacedrive related terminology.
### `Library`
A Library is the database that powers Spacedrive, all metadata and directory structures are saved in the Library. Libraries can be synced between devices and a user can have multiple libraries loaded in the Spacedrive app at once. [Learn more →](/docs/developers/architecture/libraries)
### `Object`
Objects are a fancy name for files, we call them Objects because we identify them uniquely based on a cryptographic hash of the contents. Objects in Spacedrive can come in a wide variety of kinds to provide a broad range of context. [Learn more →](/docs/developers/architecture/objects)
### `Location`
Locations are places Spacedrive will look for files, usually a directory on a mounted volume, but could also be a cloud service. [Learn more →](/docs/developers/architecture/locations)
Locations are places Spacedrive will look for files, usually a directory on a mounted volume, but could also be a cloud service. [Learn more →](/docs/developers/architecture/locations)
### `Node`
A Node is a device or server running the Spacedrive core. Nodes can host libraries and communicate with other Nodes to sync data. [Learn more →](/docs/developers/architecture/nodes)
### `Preview Media`
Preview media refers to highly compressed image or video content for an Object, it is generated by Spacedrive and synced between your devices. [Learn more →](/docs/developers/architecture/preview-media)
@@ -15,7 +15,7 @@ import {
import { forwardRef, useEffect, useRef } from 'react';
import { useForm } from 'react-hook-form';
import { useNavigate } from 'react-router-dom';
import { Button, Input, OverlayPanel, cva } from '@sd/ui';
import { Button, Input, Popover, cva } from '@sd/ui';
import { getExplorerStore, useExplorerStore } from '../../hooks/useExplorerStore';
import { useOperatingSystem } from '../../hooks/useOperatingSystem';
import { KeybindEvent } from '../../util/keybind';
@@ -265,7 +265,7 @@ export const TopBar: React.FC<TopBarProps> = (props) => {
<div className="flex mx-8 space-x-2">
<Tooltip label="Key Manager">
<OverlayPanel
<Popover
className="focus:outline-none"
trigger={
// <Tooltip label="Major Key Alert">
@@ -278,7 +278,7 @@ export const TopBar: React.FC<TopBarProps> = (props) => {
<div className="block w-[350px]">
<KeyManager className={TOP_BAR_ICON_STYLE} />
</div>
</OverlayPanel>
</Popover>
</Tooltip>
<Tooltip label="Tag Assign Mode">
<TopBarButton
@@ -305,7 +305,7 @@ export const TopBar: React.FC<TopBarProps> = (props) => {
</div>
<div className="flex mr-3 space-x-2">
<Tooltip label="File display options" position="left">
<OverlayPanel
<Popover
className="focus:outline-none"
trigger={
// <Tooltip label="Major Key Alert">
@@ -318,7 +318,7 @@ export const TopBar: React.FC<TopBarProps> = (props) => {
<div className="block w-[250px] ">
<ExplorerOptionsPanel />
</div>
</OverlayPanel>
</Popover>
</Tooltip>
<Tooltip
@@ -8,7 +8,6 @@ import {
Eye,
Fingerprint,
Folder,
IconProps,
LockSimple,
LockSimpleOpen,
Pause,
@@ -18,9 +17,8 @@ import {
TrashSimple,
X
} from 'phosphor-react';
import { useLibraryMutation, useLibraryQuery } from '@sd/client';
import { JobReport } from '@sd/client';
import { Button, CategoryHeading, tw } from '@sd/ui';
import { JobReport, useLibraryMutation, useLibraryQuery } from '@sd/client';
import { Button, CategoryHeading, Popover, PopoverClose, tw } from '@sd/ui';
import ProgressBar from '../primitive/ProgressBar';
import { Tooltip } from '../tooltip/Tooltip';
@@ -117,11 +115,13 @@ export function JobsManager() {
<Trash className="w-5 h-5" />
</Tooltip>
</Button>
<Button size="icon">
<Tooltip label="Close">
<X className="w-5 h-5" />
</Tooltip>
</Button>
<PopoverClose asChild>
<Button size="icon">
<Tooltip label="Close">
<X className="w-5 h-5" />
</Tooltip>
</Button>
</PopoverClose>
</HeaderContainer>
<div className="h-full mr-1 overflow-x-hidden custom-scroll inspector-scroll">
<div className="">
@@ -18,7 +18,7 @@ import {
CategoryHeading,
Dropdown,
Loader,
OverlayPanel,
Popover,
Select,
SelectOption,
Switch,
@@ -132,14 +132,13 @@ export function Sidebar() {
<Gear className="w-5 h-5" />
</Tooltip>
</ButtonLink>
<OverlayPanel
transformOrigin="bottom left"
disabled={!library}
<Popover
trigger={
<Button
size="icon"
variant="outline"
className="radix-state-open:bg-sidebar-selected/50 text-ink-faint ring-offset-sidebar"
disabled={!library}
>
{library && (
<Tooltip label="Recent Jobs">
@@ -152,7 +151,7 @@ export function Sidebar() {
<div className="block w-[430px] h-96">
<JobsManager />
</div>
</OverlayPanel>
</Popover>
</div>
{debugState.enabled && <DebugPanel />}
</SidebarFooter>
@@ -177,7 +176,7 @@ function DebugPanel() {
const platform = usePlatform();
return (
<OverlayPanel
<Popover
className="p-4 focus:outline-none"
transformOrigin="bottom left"
trigger={
@@ -246,7 +245,7 @@ function DebugPanel() {
</InputContainer>
)} */}
</div>
</OverlayPanel>
</Popover>
);
}
+1
View File
@@ -26,6 +26,7 @@
"@radix-ui/react-context-menu": "^1.0.0",
"@radix-ui/react-dialog": "^1.0.0",
"@radix-ui/react-dropdown-menu": "^1.0.0",
"@radix-ui/react-popover": "^1.0.3",
"@radix-ui/react-select": "^1.1.2",
"@radix-ui/react-switch": "^1.0.1",
"@radix-ui/react-tabs": "^1.0.1",
@@ -1,15 +1,15 @@
import * as DropdownMenu from '@radix-ui/react-dropdown-menu';
import * as Radix from '@radix-ui/react-popover';
import clsx from 'clsx';
import { PropsWithChildren, useState } from 'react';
import { animated, config, useTransition } from 'react-spring';
interface Props extends DropdownMenu.MenuContentProps {
interface Props extends Radix.PopoverContentProps {
trigger: React.ReactNode;
transformOrigin?: string;
disabled?: boolean;
className?: string;
}
export const OverlayPanel = ({
export const Popover = ({
trigger,
children,
disabled,
@@ -31,13 +31,13 @@ export const OverlayPanel = ({
// });
return (
<DropdownMenu.Root open={open} onOpenChange={setOpen}>
<DropdownMenu.Trigger disabled={disabled} asChild>
<Radix.Root open={open} onOpenChange={setOpen}>
<Radix.Trigger disabled={disabled} asChild>
{trigger}
</DropdownMenu.Trigger>
</Radix.Trigger>
{open && (
<DropdownMenu.Portal forceMount>
<DropdownMenu.Content forceMount asChild>
<Radix.Portal forceMount>
<Radix.Content forceMount asChild>
<div
className={clsx(
'flex flex-col',
@@ -53,9 +53,11 @@ export const OverlayPanel = ({
>
{children}
</div>
</DropdownMenu.Content>
</DropdownMenu.Portal>
</Radix.Content>
</Radix.Portal>
)}
</DropdownMenu.Root>
</Radix.Root>
);
};
export { Close as PopoverClose } from '@radix-ui/react-popover';
+8 -8
View File
@@ -1,16 +1,16 @@
export { cva, cx } from 'class-variance-authority';
export * from './Button';
export * as Dropdown from './Dropdown';
export * from './Dialog';
export * from './Loader';
export * from './Layout';
export * from './CheckBox';
export * as ContextMenu from './ContextMenu';
export * from './OverlayPanel';
export * from './Dialog';
export * as Dropdown from './Dropdown';
export * from './Input';
export * from './Layout';
export * from './Loader';
export * from './Popover';
export * from './Select';
export * from './Switch';
export * as Tabs from './Tabs';
export * from './Typography';
export * from './utils';
export * from './CheckBox';
export * as forms from './forms';
export { cva, cx } from 'class-variance-authority';
export * from './utils';
+69 -5
View File
@@ -531,6 +531,7 @@ importers:
'@radix-ui/react-context-menu': ^1.0.0
'@radix-ui/react-dialog': ^1.0.0
'@radix-ui/react-dropdown-menu': ^1.0.0
'@radix-ui/react-popover': ^1.0.3
'@radix-ui/react-select': ^1.1.2
'@radix-ui/react-switch': ^1.0.1
'@radix-ui/react-tabs': ^1.0.1
@@ -578,6 +579,7 @@ importers:
'@radix-ui/react-context-menu': 1.0.0_5ndqzdd6t4rivxsukjv3i3ak2q
'@radix-ui/react-dialog': 1.0.2_5ndqzdd6t4rivxsukjv3i3ak2q
'@radix-ui/react-dropdown-menu': 1.0.0_5ndqzdd6t4rivxsukjv3i3ak2q
'@radix-ui/react-popover': 1.0.3_5ndqzdd6t4rivxsukjv3i3ak2q
'@radix-ui/react-select': 1.2.0_5ndqzdd6t4rivxsukjv3i3ak2q
'@radix-ui/react-switch': 1.0.1_biqbaboplfbrettd7655fr4n2y
'@radix-ui/react-tabs': 1.0.2_biqbaboplfbrettd7655fr4n2y
@@ -619,7 +621,7 @@ importers:
storybook: 6.5.15_o4scbtliisanygemawej7x2d6i
storybook-tailwind-dark-mode: 1.0.15_biqbaboplfbrettd7655fr4n2y
style-loader: 3.3.1
tailwindcss: 3.2.4
tailwindcss: 3.2.4_postcss@8.4.21
typescript: 4.9.4
packages:
@@ -4048,7 +4050,7 @@ packages:
peerDependencies:
tailwindcss: ^3.0
dependencies:
tailwindcss: 3.2.4
tailwindcss: 3.2.4_postcss@8.4.21
dev: false
/@hookform/resolvers/2.9.10_react-hook-form@7.42.1:
@@ -4653,6 +4655,34 @@ packages:
- '@types/react'
dev: false
/@radix-ui/react-popover/1.0.3_5ndqzdd6t4rivxsukjv3i3ak2q:
resolution: {integrity: sha512-YwedSukfWsyJs3/yP3yXUq44k4/JBe3jqU63Z8v2i19qZZ3dsx32oma17ztgclWPNuqp3A+Xa9UiDlZHyVX8Vg==}
peerDependencies:
react: ^16.8 || ^17.0 || ^18.0
react-dom: ^16.8 || ^17.0 || ^18.0
dependencies:
'@babel/runtime': 7.20.7
'@radix-ui/primitive': 1.0.0
'@radix-ui/react-compose-refs': 1.0.0_react@18.2.0
'@radix-ui/react-context': 1.0.0_react@18.2.0
'@radix-ui/react-dismissable-layer': 1.0.2_biqbaboplfbrettd7655fr4n2y
'@radix-ui/react-focus-guards': 1.0.0_react@18.2.0
'@radix-ui/react-focus-scope': 1.0.1_biqbaboplfbrettd7655fr4n2y
'@radix-ui/react-id': 1.0.0_react@18.2.0
'@radix-ui/react-popper': 1.1.0_5ndqzdd6t4rivxsukjv3i3ak2q
'@radix-ui/react-portal': 1.0.1_biqbaboplfbrettd7655fr4n2y
'@radix-ui/react-presence': 1.0.0_biqbaboplfbrettd7655fr4n2y
'@radix-ui/react-primitive': 1.0.1_biqbaboplfbrettd7655fr4n2y
'@radix-ui/react-slot': 1.0.1_react@18.2.0
'@radix-ui/react-use-controllable-state': 1.0.0_react@18.2.0
aria-hidden: 1.2.2_3stiutgnnbnfnf3uowm5cip22i
react: 18.2.0
react-dom: 18.2.0_react@18.2.0
react-remove-scroll: 2.5.5_3stiutgnnbnfnf3uowm5cip22i
transitivePeerDependencies:
- '@types/react'
dev: false
/@radix-ui/react-popper/1.0.0_5ndqzdd6t4rivxsukjv3i3ak2q:
resolution: {integrity: sha512-k2dDd+1Wl0XWAMs9ZvAxxYsB9sOsEhrFQV4CINd7IUZf0wfdye4OHen9siwxvZImbzhgVeKTJi68OQmPRvVdMg==}
peerDependencies:
@@ -7410,7 +7440,7 @@ packages:
tailwindcss: '>=3.0.0 || >= 3.0.0-alpha.1'
dependencies:
mini-svg-data-uri: 1.4.4
tailwindcss: 3.2.4
tailwindcss: 3.2.4_postcss@8.4.21
dev: false
/@tailwindcss/line-clamp/0.4.2:
@@ -7424,7 +7454,7 @@ packages:
peerDependencies:
tailwindcss: '>=2.0.0 || >=3.0.0 || >=3.0.0-alpha.1'
dependencies:
tailwindcss: 3.2.4
tailwindcss: 3.2.4_postcss@8.4.21
dev: true
/@tailwindcss/typography/0.5.9:
@@ -7447,7 +7477,7 @@ packages:
lodash.isplainobject: 4.0.6
lodash.merge: 4.6.2
postcss-selector-parser: 6.0.10
tailwindcss: 3.2.4
tailwindcss: 3.2.4_postcss@8.4.21
dev: true
/@tanstack/match-sorter-utils/8.7.6:
@@ -19258,6 +19288,40 @@ packages:
resolve: 1.22.1
transitivePeerDependencies:
- ts-node
dev: false
/tailwindcss/3.2.4_postcss@8.4.21:
resolution: {integrity: sha512-AhwtHCKMtR71JgeYDaswmZXhPcW9iuI9Sp2LvZPo9upDZ7231ZJ7eA9RaURbhpXGVlrjX4cFNlB4ieTetEb7hQ==}
engines: {node: '>=12.13.0'}
hasBin: true
peerDependencies:
postcss: ^8.0.9
dependencies:
arg: 5.0.2
chokidar: 3.5.3
color-name: 1.1.4
detective: 5.2.1
didyoumean: 1.2.2
dlv: 1.1.3
fast-glob: 3.2.12
glob-parent: 6.0.2
is-glob: 4.0.3
lilconfig: 2.0.6
micromatch: 4.0.5
normalize-path: 3.0.0
object-hash: 3.0.0
picocolors: 1.0.0
postcss: 8.4.21
postcss-import: 14.1.0_postcss@8.4.21
postcss-js: 4.0.0_postcss@8.4.21
postcss-load-config: 3.1.4_postcss@8.4.21
postcss-nested: 6.0.0_postcss@8.4.21
postcss-selector-parser: 6.0.11
postcss-value-parser: 4.2.0
quick-lru: 5.1.1
resolve: 1.22.1
transitivePeerDependencies:
- ts-node
/tapable/1.1.3:
resolution: {integrity: sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==}