From 00ea5d9cadcc480e2fecff0d427d85b2be6cb56f Mon Sep 17 00:00:00 2001 From: Benjamin Kampmann Date: Fri, 18 Feb 2022 18:46:59 +0100 Subject: [PATCH] moving sled cryptostore, too --- crates/matrix-sdk-crypto/Cargo.toml | 6 +- .../src/identities/device.rs | 3 +- crates/matrix-sdk-sled/Cargo.toml | 8 +- .../src/cryptostore.rs} | 115 +++++++++--------- crates/matrix-sdk-sled/src/lib.rs | 6 +- 5 files changed, 68 insertions(+), 70 deletions(-) rename crates/{matrix-sdk-crypto/src/store/sled.rs => matrix-sdk-sled/src/cryptostore.rs} (88%) diff --git a/crates/matrix-sdk-crypto/Cargo.toml b/crates/matrix-sdk-crypto/Cargo.toml index 6b6a0ed5c..d71e4aff0 100644 --- a/crates/matrix-sdk-crypto/Cargo.toml +++ b/crates/matrix-sdk-crypto/Cargo.toml @@ -19,8 +19,8 @@ rustdoc-args = ["--cfg", "docsrs"] default = [] qrcode = ["matrix-qrcode"] backups_v1 = [] -sled_cryptostore = ["sled"] -docsrs = ["sled_cryptostore"] +sled_cryptostore = [] +docsrs = [] indexeddb_cryptostore = [] # Testing helpers for implementations based upon this @@ -45,7 +45,6 @@ rand = "0.8.4" serde = { version = "1.0.126", features = ["derive", "rc"] } serde_json = "1.0.64" sha2 = "0.10.1" -sled = { version = "0.34.6", optional = true } thiserror = "1.0.25" tracing = "0.1.26" zeroize = { version = "1.3.0", features = ["zeroize_derive"] } @@ -80,7 +79,6 @@ tokio = { version = "1.7.1", default-features = false, features = [ "rt-multi-thread", "macros", ] } -lazy_static = "1.4" [target.'cfg(target_os = "linux")'.dev-dependencies] pprof = { version = "0.6.2", features = ["flamegraph", "criterion"] } diff --git a/crates/matrix-sdk-crypto/src/identities/device.rs b/crates/matrix-sdk-crypto/src/identities/device.rs index 3ce546a29..230c86c7c 100644 --- a/crates/matrix-sdk-crypto/src/identities/device.rs +++ b/crates/matrix-sdk-crypto/src/identities/device.rs @@ -374,8 +374,7 @@ impl ReadOnlyDevice { /// Create a new Device, this constructor skips signature verification of /// the keys, `TryFrom` should be used for completely new devices we /// receive. - #[cfg(feature = "sled_cryptostore")] - pub(crate) fn new(device_keys: DeviceKeys, trust_state: LocalTrust) -> Self { + pub fn new(device_keys: DeviceKeys, trust_state: LocalTrust) -> Self { Self { inner: device_keys.into(), trust_state: Arc::new(Atomic::new(trust_state)), diff --git a/crates/matrix-sdk-sled/Cargo.toml b/crates/matrix-sdk-sled/Cargo.toml index af3669b88..cc77f75ec 100644 --- a/crates/matrix-sdk-sled/Cargo.toml +++ b/crates/matrix-sdk-sled/Cargo.toml @@ -3,8 +3,6 @@ name = "matrix-sdk-sled" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] futures-core = "0.3.15" futures-util = { version = "0.3.15", default-features = false } @@ -18,11 +16,15 @@ thiserror = "1.0.25" tokio = { version = "1.7.1", default-features = false, features = ["sync", "fs"] } tracing = "0.1.26" anyhow = "1" +dashmap = "4.0.2" [dev-dependencies] +lazy_static = "1.4" +tempfile = "3.2.0" matrix-sdk-test = { version = "0.4.0", path = "../matrix-sdk-test" } +matrix-sdk-crypto = { path = "../matrix-sdk-crypto", features = ["testing"] } matrix-sdk-base = { path = "../matrix-sdk-base", features = ["testing"] } tokio = { version = "1.7.1", default-features = false, features = [ "rt-multi-thread", "macros", -] } \ No newline at end of file +] } diff --git a/crates/matrix-sdk-crypto/src/store/sled.rs b/crates/matrix-sdk-sled/src/cryptostore.rs similarity index 88% rename from crates/matrix-sdk-crypto/src/store/sled.rs rename to crates/matrix-sdk-sled/src/cryptostore.rs index b7d3f2863..8627e87bc 100644 --- a/crates/matrix-sdk-crypto/src/store/sled.rs +++ b/crates/matrix-sdk-sled/src/cryptostore.rs @@ -21,8 +21,7 @@ use std::{ use dashmap::DashSet; use matrix_sdk_common::{async_trait, locks::Mutex}; -use olm_rs::{account::IdentityKeys, PicklingMode}; -use ruma::{ +use matrix_sdk_common::ruma::{ encryption::DeviceKeys, events::{room_key_request::RequestedKeyInfo, secret::request::SecretName}, DeviceId, DeviceKeyId, EventEncryptionAlgorithm, RoomId, TransactionId, UserId, @@ -35,17 +34,21 @@ use sled::{ }; use tracing::debug; -use super::{ - caches::SessionStore, BackupKeys, Changes, CryptoStore, CryptoStoreError, InboundGroupSession, - PickleKey, ReadOnlyAccount, Result, RoomKeyCounts, Session, -}; -use crate::{ - gossiping::{GossipRequest, SecretInfo}, - identities::{ReadOnlyDevice, ReadOnlyUserIdentities}, - olm::{OutboundGroupSession, PickledInboundGroupSession, PrivateCrossSigningIdentity}, +use matrix_sdk_crypto::{ + store::{ + caches::SessionStore, BackupKeys, Changes, CryptoStore, CryptoStoreError, + PickleKey, Result, RoomKeyCounts, IdentityKeys, PicklingMode, + }, + GossipRequest, SecretInfo, ReadOnlyAccount, + ReadOnlyDevice, ReadOnlyUserIdentities, + olm::{ + InboundGroupSession, Session, OutboundGroupSession, PickledInboundGroupSession, PrivateCrossSigningIdentity + }, LocalTrust, }; +use anyhow::anyhow; + /// This needs to be 32 bytes long since AES-GCM requires it, otherwise we will /// panic once we try to pickle a Signing object. const DEFAULT_PICKLE: &str = "DEFAULT_PICKLE_PASSPHRASE_123456"; @@ -204,28 +207,19 @@ impl std::fmt::Debug for SledStore { } } -impl From> for CryptoStoreError { - fn from(e: TransactionError) -> Self { - match e { - TransactionError::Abort(e) => CryptoStoreError::Serialization(e), - TransactionError::Storage(e) => CryptoStoreError::Database(e), - } - } -} - impl SledStore { /// Open the sled based cryptostore at the given path using the given /// passphrase to encrypt private data. - pub fn open_with_passphrase(path: impl AsRef, passphrase: Option<&str>) -> Result { + pub fn open_with_passphrase(path: impl AsRef, passphrase: Option<&str>) -> Result { let path = path.as_ref().join("matrix-sdk-crypto"); - let db = Config::new().temporary(false).path(&path).open()?; + let db = Config::new().temporary(false).path(&path).open().map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; SledStore::open_helper(db, Some(path), passphrase) } /// Create a sled based cryptostore using the given sled database. /// The given passphrase will be used to encrypt private data. - pub fn open_with_database(db: Db, passphrase: Option<&str>) -> Result { + pub fn open_with_database(db: Db, passphrase: Option<&str>) -> Result { SledStore::open_helper(db, None, passphrase) } @@ -238,7 +232,7 @@ impl SledStore { .inbound_group_sessions .iter() .map(|p| { - let item = p?; + let item = p.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; Ok(( item.0, serde_json::from_slice(&item.1).map_err(CryptoStoreError::Serialization)?, @@ -262,9 +256,9 @@ impl SledStore { Ok(()) }); - ret?; + ret.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; - self.inner.flush_async().await?; + self.inner.flush_async().await.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; Ok(()) } @@ -272,7 +266,7 @@ impl SledStore { fn upgrade(&self) -> Result<()> { let version = self .inner - .get("store_version")? + .get("store_version").map_err(|e| CryptoStoreError::Backend(anyhow!(e)))? .map(|v| { let (version_bytes, _) = v.split_at(std::mem::size_of::()); u8::from_be_bytes(version_bytes.try_into().unwrap_or_default()) @@ -287,7 +281,7 @@ impl SledStore { // We changed the schema but migrating this isn't important since we // rotate the group sessions relatively often anyways so we just // clear the tree. - self.outbound_group_sessions.clear()?; + self.outbound_group_sessions.clear().map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; } if version <= 1 { @@ -322,7 +316,7 @@ impl SledStore { let devices: Vec = self .devices .iter() - .map(|d| serde_json::from_slice(&d?.1).map_err(CryptoStoreError::Serialization)) + .map(|d| serde_json::from_slice(&d.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.1).map_err(CryptoStoreError::Serialization)) .map(|d| { let d: OldReadOnlyDevice = d?; Ok(d.into()) @@ -338,30 +332,30 @@ impl SledStore { } Ok(()) - })?; + }).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; } if version <= 2 { // We're treating our own device now differently, we're checking if // the keys match to what we have locally, remove the unchecked // device and mark our own user as dirty. - if let Some(pickle) = self.account.get("account".encode())? { + if let Some(pickle) = self.account.get("account".encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))? { let pickle = serde_json::from_slice(&pickle)?; let account = ReadOnlyAccount::from_pickle(pickle, self.get_pickle_mode())?; self.devices - .remove((account.user_id().as_str(), account.device_id.as_str()).encode())?; - self.tracked_users.insert(account.user_id().as_str(), &[true as u8])?; + .remove((account.user_id().as_str(), account.device_id.as_str()).encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; + self.tracked_users.insert(account.user_id().as_str(), &[true as u8]).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; } } - self.inner.insert("store_version", DATABASE_VERSION.to_be_bytes().as_ref())?; - self.inner.flush()?; + self.inner.insert("store_version", DATABASE_VERSION.to_be_bytes().as_ref()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; + self.inner.flush().map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; Ok(()) } - fn open_helper(db: Db, path: Option, passphrase: Option<&str>) -> Result { + fn open_helper(db: Db, path: Option, passphrase: Option<&str>) -> Result { let account = db.open_tree("account")?; let private_identity = db.open_tree("private_identity")?; @@ -418,14 +412,14 @@ impl SledStore { fn get_or_create_pickle_key(passphrase: &str, database: &Db) -> Result { let key = if let Some(key) = - database.get("pickle_key".encode())?.map(|v| serde_json::from_slice(&v)) + database.get("pickle_key".encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.map(|v| serde_json::from_slice(&v)) { PickleKey::from_encrypted(passphrase, key?) .map_err(|_| CryptoStoreError::UnpicklingError)? } else { let key = PickleKey::new(); let encrypted = key.encrypt(passphrase); - database.insert("pickle_key".encode(), serde_json::to_vec(&encrypted)?)?; + database.insert("pickle_key".encode(), serde_json::to_vec(&encrypted)?).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; key }; @@ -442,7 +436,7 @@ impl SledStore { async fn load_tracked_users(&self) -> Result<()> { for value in &self.tracked_users { - let (user, dirty) = value?; + let (user, dirty) = value.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; let user = UserId::parse(String::from_utf8_lossy(&user).to_string())?; let dirty = dirty.get(0).map(|d| *d == 1).unwrap_or(true); @@ -463,7 +457,7 @@ impl SledStore { let account_info = self.get_account_info().ok_or(CryptoStoreError::AccountUnset)?; self.outbound_group_sessions - .get(room_id.encode())? + .get(room_id.encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))? .map(|p| serde_json::from_slice(&p).map_err(CryptoStoreError::Serialization)) .transpose()? .map(|p| { @@ -671,8 +665,8 @@ impl SledStore { }, ); - ret?; - self.inner.flush_async().await?; + ret.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; + self.inner.flush_async().await.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; Ok(()) } @@ -680,12 +674,12 @@ impl SledStore { async fn get_outgoing_key_request_helper(&self, id: &[u8]) -> Result> { let request = self .outgoing_secret_requests - .get(id)? + .get(id).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))? .map(|r| serde_json::from_slice(&r)) .transpose()?; let request = if request.is_none() { - self.unsent_secret_requests.get(id)?.map(|r| serde_json::from_slice(&r)).transpose()? + self.unsent_secret_requests.get(id).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.map(|r| serde_json::from_slice(&r)).transpose()? } else { request }; @@ -697,7 +691,7 @@ impl SledStore { #[async_trait] impl CryptoStore for SledStore { async fn load_account(&self) -> Result> { - if let Some(pickle) = self.account.get("account".encode())? { + if let Some(pickle) = self.account.get("account".encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))? { let pickle = serde_json::from_slice(&pickle)?; self.load_tracked_users().await?; @@ -733,7 +727,7 @@ impl CryptoStore for SledStore { } async fn load_identity(&self) -> Result> { - if let Some(i) = self.private_identity.get("identity".encode())? { + if let Some(i) = self.private_identity.get("identity".encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))? { let pickle = serde_json::from_slice(&i)?; Ok(Some( PrivateCrossSigningIdentity::from_pickle(pickle, self.get_pickle_key()) @@ -756,7 +750,7 @@ impl CryptoStore for SledStore { let sessions: Result> = self .sessions .scan_prefix(sender_key.encode()) - .map(|s| serde_json::from_slice(&s?.1).map_err(CryptoStoreError::Serialization)) + .map(|s| serde_json::from_slice(&s.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.1).map_err(CryptoStoreError::Serialization)) .map(|p| { Session::from_pickle( account_info.user_id.clone(), @@ -782,7 +776,7 @@ impl CryptoStore for SledStore { session_id: &str, ) -> Result> { let key = (room_id.as_str(), sender_key, session_id).encode(); - let pickle = self.inbound_group_sessions.get(&key)?.map(|p| serde_json::from_slice(&p)); + let pickle = self.inbound_group_sessions.get(&key).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.map(|p| serde_json::from_slice(&p)); if let Some(pickle) = pickle { Ok(Some(InboundGroupSession::from_pickle(pickle?, self.get_pickle_mode())?)) @@ -795,7 +789,7 @@ impl CryptoStore for SledStore { let pickles: Result> = self .inbound_group_sessions .iter() - .map(|p| serde_json::from_slice(&p?.1).map_err(CryptoStoreError::Serialization)) + .map(|p| serde_json::from_slice(&p.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.1).map_err(CryptoStoreError::Serialization)) .collect(); Ok(pickles? @@ -809,7 +803,7 @@ impl CryptoStore for SledStore { .inbound_group_sessions .iter() .map(|p| { - let item = p?; + let item = p.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; serde_json::from_slice(&item.1).map_err(CryptoStoreError::Serialization) }) .collect::>()?; @@ -828,7 +822,7 @@ impl CryptoStore for SledStore { .inbound_group_sessions .iter() .map(|p| { - let item = p?; + let item = p.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; serde_json::from_slice(&item.1).map_err(CryptoStoreError::from) }) .filter_map(|p: Result| match p { @@ -887,7 +881,7 @@ impl CryptoStore for SledStore { self.users_for_key_query_cache.remove(user); } - self.tracked_users.insert(user.as_str(), &[dirty as u8])?; + self.tracked_users.insert(user.as_str(), &[dirty as u8]).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; Ok(already_added) } @@ -898,7 +892,7 @@ impl CryptoStore for SledStore { device_id: &DeviceId, ) -> Result> { let key = (user_id.as_str(), device_id.as_str()).encode(); - Ok(self.devices.get(key)?.map(|d| serde_json::from_slice(&d)).transpose()?) + Ok(self.devices.get(key).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.map(|d| serde_json::from_slice(&d)).transpose()?) } async fn get_user_devices( @@ -907,7 +901,7 @@ impl CryptoStore for SledStore { ) -> Result, ReadOnlyDevice>> { self.devices .scan_prefix(user_id.encode()) - .map(|d| serde_json::from_slice(&d?.1).map_err(CryptoStoreError::Serialization)) + .map(|d| serde_json::from_slice(&d.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.1).map_err(CryptoStoreError::Serialization)) .map(|d| { let d: ReadOnlyDevice = d?; Ok((d.device_id().to_owned(), d)) @@ -918,13 +912,13 @@ impl CryptoStore for SledStore { async fn get_user_identity(&self, user_id: &UserId) -> Result> { Ok(self .identities - .get(user_id.encode())? + .get(user_id.encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))? .map(|i| serde_json::from_slice(&i)) .transpose()?) } - async fn is_message_known(&self, message_hash: &crate::olm::OlmMessageHash) -> Result { - Ok(self.olm_hashes.contains_key(serde_json::to_vec(message_hash)?)?) + async fn is_message_known(&self, message_hash: &matrix_sdk_crypto::olm::OlmMessageHash) -> Result { + Ok(self.olm_hashes.contains_key(serde_json::to_vec(message_hash)?).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?) } async fn get_outgoing_secret_requests( @@ -940,7 +934,7 @@ impl CryptoStore for SledStore { &self, key_info: &SecretInfo, ) -> Result> { - let id = self.secret_requests_by_info.get(key_info.encode())?; + let id = self.secret_requests_by_info.get(key_info.encode()).map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; if let Some(id) = id { self.get_outgoing_key_request_helper(&id).await @@ -953,7 +947,7 @@ impl CryptoStore for SledStore { let requests: Result> = self .unsent_secret_requests .iter() - .map(|i| serde_json::from_slice(&i?.1).map_err(CryptoStoreError::from)) + .map(|i| serde_json::from_slice(&i.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?.1).map_err(CryptoStoreError::from)) .collect(); requests @@ -991,8 +985,8 @@ impl CryptoStore for SledStore { }, ); - ret?; - self.inner.flush_async().await?; + ret.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; + self.inner.flush_async().await.map_err(|e| CryptoStoreError::Backend(anyhow!(e)))?; Ok(()) } @@ -1032,6 +1026,7 @@ impl CryptoStore for SledStore { mod test { use lazy_static::lazy_static; use tempfile::{tempdir, TempDir}; + use matrix_sdk_crypto::cryptostore_integration_tests; use super::SledStore; lazy_static! { diff --git a/crates/matrix-sdk-sled/src/lib.rs b/crates/matrix-sdk-sled/src/lib.rs index 75d3297c0..6c26dbc07 100644 --- a/crates/matrix-sdk-sled/src/lib.rs +++ b/crates/matrix-sdk-sled/src/lib.rs @@ -1 +1,5 @@ -pub mod state_store; \ No newline at end of file +mod cryptostore; +mod state_store; + +pub use cryptostore::SledStore as CryptoStore; +pub use state_store::SledStore as StateStore; \ No newline at end of file