diff --git a/crates/matrix-sdk-crypto/src/file_encryption/attachments.rs b/crates/matrix-sdk-crypto/src/file_encryption/attachments.rs index ca571c9d6..fff72b69c 100644 --- a/crates/matrix-sdk-crypto/src/file_encryption/attachments.rs +++ b/crates/matrix-sdk-crypto/src/file_encryption/attachments.rs @@ -147,7 +147,7 @@ impl<'a, R: Read + 'a> AttachmentDecryptor<'a, R> { } /// A wrapper that transparently encrypts anything that implements `Read`. -pub struct AttachmentEncryptor<'a, R: Read + 'a> { +pub struct AttachmentEncryptor<'a, R: Read + ?Sized + 'a> { finished: bool, inner: &'a mut R, web_key: JsonWebKey, @@ -157,7 +157,7 @@ pub struct AttachmentEncryptor<'a, R: Read + 'a> { sha: Sha256, } -impl<'a, R: 'a + Read + std::fmt::Debug> std::fmt::Debug for AttachmentEncryptor<'a, R> { +impl<'a, R: 'a + Read + std::fmt::Debug + ?Sized> std::fmt::Debug for AttachmentEncryptor<'a, R> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("AttachmentEncryptor") .field("inner", &self.inner) @@ -166,7 +166,7 @@ impl<'a, R: 'a + Read + std::fmt::Debug> std::fmt::Debug for AttachmentEncryptor } } -impl<'a, R: Read + 'a> Read for AttachmentEncryptor<'a, R> { +impl<'a, R: Read + ?Sized + 'a> Read for AttachmentEncryptor<'a, R> { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { let read_bytes = self.inner.read(buf)?; @@ -185,7 +185,7 @@ impl<'a, R: Read + 'a> Read for AttachmentEncryptor<'a, R> { } } -impl<'a, R: Read + 'a> AttachmentEncryptor<'a, R> { +impl<'a, R: Read + ?Sized + 'a> AttachmentEncryptor<'a, R> { /// Wrap the given reader encrypting all the data we read from it. /// /// After all the reads are done, and all the data is encrypted that we wish diff --git a/crates/matrix-sdk/Cargo.toml b/crates/matrix-sdk/Cargo.toml index 2adc756dc..aafcd3357 100644 --- a/crates/matrix-sdk/Cargo.toml +++ b/crates/matrix-sdk/Cargo.toml @@ -36,13 +36,16 @@ rustls-tls = ["reqwest/rustls-tls"] socks = ["reqwest/socks"] sso_login = ["warp", "rand", "tokio-stream"] appservice = ["ruma/appservice-api-s", "ruma/appservice-api-helper"] +image_proc = ["image"] +image_rayon = ["image_proc", "image/jpeg_rayon"] docsrs = [ "encryption", "sled_cryptostore", "sled_state_store", "sso_login", - "qrcode" + "qrcode", + "image_proc", ] [dependencies] @@ -66,6 +69,26 @@ url = "2.2.2" zeroize = "1.3.0" async-stream = "0.3.2" +[dependencies.image] +version = "0.24.0" +default-features = false +features = [ + "gif", + "jpeg", + "ico", + "png", + "pnm", + "tga", + "tiff", + "webp", + "bmp", + "hdr", + "dxt", + "dds", + "farbfeld", +] +optional = true + [dependencies.matrix-sdk-base] version = "0.4.0" path = "../matrix-sdk-base" @@ -78,7 +101,7 @@ default_features = false [dependencies.ruma] git = "https://github.com/ruma/ruma/" rev = "b9f32bc6327542d382d4eb42ec43623495c50e66" -features = ["client-api-c", "compat", "rand"] +features = ["client-api-c", "compat", "rand", "unstable-msc2448"] [dependencies.tokio-stream] version = "0.1.6" diff --git a/crates/matrix-sdk/README.md b/crates/matrix-sdk/README.md index 0d2a98dc0..634f0a935 100644 --- a/crates/matrix-sdk/README.md +++ b/crates/matrix-sdk/README.md @@ -64,6 +64,8 @@ The following crate feature flags are available: | `anyhow` | No | Better logging for event handlers that return `anyhow::Result` | | `encryption` | Yes | End-to-end encryption support | | `eyre` | No | Better logging for event handlers that return `eyre::Result` | +| `image_proc` | No | Enables image processing to generate thumbnails | +| `image_rayon` | No | Enables faster image processing | | `markdown` | No | Support to send Markdown-formatted messages | | `qrcode` | Yes | QR code verification support | | `sled_cryptostore` | Yes | Persistent storage for E2EE related data | diff --git a/crates/matrix-sdk/examples/image_bot.rs b/crates/matrix-sdk/examples/image_bot.rs index 103b1cc10..abf7281e7 100644 --- a/crates/matrix-sdk/examples/image_bot.rs +++ b/crates/matrix-sdk/examples/image_bot.rs @@ -9,6 +9,7 @@ use std::{ use matrix_sdk::{ self, + attachment::AttachmentConfig, config::SyncSettings, room::Room, ruma::events::room::message::{ @@ -39,7 +40,9 @@ async fn on_room_message(event: SyncRoomMessageEvent, room: Room, image: Arc, + /// The width of the image in pixels. + pub width: Option, + /// The file size of the image in bytes. + pub size: Option, + /// The [BlurHash](https://blurha.sh/) for this image. + pub blurhash: Option, +} + +/// Base metadata about a video. +#[derive(Debug, Clone)] +pub struct BaseVideoInfo { + /// The duration of the video in milliseconds. + pub duration: Option, + /// The height of the video in pixels. + pub height: Option, + /// The width of the video in pixels. + pub width: Option, + /// The file size of the video in bytes. + pub size: Option, + /// The [BlurHash](https://blurha.sh/) for this video. + pub blurhash: Option, +} + +/// Base metadata about an audio clip. +#[derive(Debug, Clone)] +pub struct BaseAudioInfo { + /// The duration of the audio clip in milliseconds. + pub duration: Option, + /// The file size of the audio clip in bytes. + pub size: Option, +} + +/// Base metadata about a file. +#[derive(Debug, Clone)] +pub struct BaseFileInfo { + /// The size of the file in bytes. + pub size: Option, +} + +/// Types of metadata for an attachment. +#[derive(Debug)] +pub enum AttachmentInfo { + /// The metadata of an image. + Image(BaseImageInfo), + /// The metadata of a video. + Video(BaseVideoInfo), + /// The metadata of an audio clip. + Audio(BaseAudioInfo), + /// The metadata of a file. + File(BaseFileInfo), +} + +impl From for ImageInfo { + fn from(info: AttachmentInfo) -> Self { + match info { + AttachmentInfo::Image(info) => assign!(ImageInfo::new(), { + height: info.height, + width: info.width, + size: info.size, + blurhash: info.blurhash, + }), + _ => ImageInfo::new(), + } + } +} + +impl From for VideoInfo { + fn from(info: AttachmentInfo) -> Self { + match info { + AttachmentInfo::Video(info) => assign!(VideoInfo::new(), { + duration: info.duration, + height: info.height, + width: info.width, + size: info.size, + blurhash: info.blurhash, + }), + _ => VideoInfo::new(), + } + } +} + +impl From for AudioInfo { + fn from(info: AttachmentInfo) -> Self { + match info { + AttachmentInfo::Audio(info) => assign!(AudioInfo::new(), { + duration: info.duration, + size: info.size, + }), + _ => AudioInfo::new(), + } + } +} + +impl From for FileInfo { + fn from(info: AttachmentInfo) -> Self { + match info { + AttachmentInfo::File(info) => assign!(FileInfo::new(), { + size: info.size, + }), + _ => FileInfo::new(), + } + } +} + +#[derive(Debug, Clone)] +/// Base metadata about a thumbnail. +pub struct BaseThumbnailInfo { + /// The height of the thumbnail in pixels. + pub height: Option, + /// The width of the thumbnail in pixels. + pub width: Option, + /// The file size of the thumbnail in bytes. + pub size: Option, +} + +impl From for ThumbnailInfo { + fn from(info: BaseThumbnailInfo) -> Self { + assign!(ThumbnailInfo::new(), { + height: info.height, + width: info.width, + size: info.size, + }) + } +} + +/// A thumbnail to upload and send for an attachment. +#[derive(Debug)] +pub struct Thumbnail<'a, R: Read> { + /// A `Reader` that will be used to fetch the raw bytes of the thumbnail. + pub reader: &'a mut R, + /// The type of the thumbnail, this will be used as the content-type header. + pub content_type: &'a mime::Mime, + /// The metadata of the thumbnail. + pub info: Option, +} + +impl Thumbnail<'static, &'static [u8]> { + /// Typed `None` for an `>`. + pub const NONE: Option> = None; +} + +/// Configuration for sending an attachment. +#[derive(Debug)] +pub struct AttachmentConfig<'a, R: Read> { + pub(crate) txn_id: Option<&'a TransactionId>, + pub(crate) info: Option, + pub(crate) thumbnail: Option>, + #[cfg(feature = "image_proc")] + pub(crate) generate_thumbnail: bool, + #[cfg(feature = "image_proc")] + pub(crate) thumbnail_size: Option<(u32, u32)>, +} + +impl AttachmentConfig<'static, &'static [u8]> { + /// Create a new default `AttachmentConfig` without providing a thumbnail. + /// + /// To provide a thumbnail use [`AttachmentConfig::with_thumbnail()`]. + pub fn new() -> Self { + Self { + txn_id: Default::default(), + info: Default::default(), + thumbnail: None, + #[cfg(feature = "image_proc")] + generate_thumbnail: Default::default(), + #[cfg(feature = "image_proc")] + thumbnail_size: Default::default(), + } + } + + /// Generate the thumbnail to send for this media. + /// + /// Uses [`generate_image_thumbnail()`]. + /// + /// Thumbnails can only be generated for supported image attachments. For + /// more information, see the [image](https://github.com/image-rs/image) + /// crate. + /// + /// # Arguments + /// + /// * `size` - The size of the thumbnail in pixels as a `(width, height)` + /// tuple. If set to `None`, defaults to `(800, 600)`. + #[cfg(feature = "image_proc")] + #[must_use] + pub fn generate_thumbnail(mut self, size: Option<(u32, u32)>) -> Self { + self.generate_thumbnail = true; + self.thumbnail_size = size; + self + } +} + +impl Default for AttachmentConfig<'static, &'static [u8]> { + fn default() -> Self { + Self::new() + } +} + +impl<'a, R: Read> AttachmentConfig<'a, R> { + /// Create a new default `AttachmentConfig` with a `thumbnail`. + /// + /// # Arguments + /// + /// * `thumbnail` - The thumbnail of the media. If the `content_type` does + /// not support it (eg audio clips), it is ignored. + /// + /// To generate automatically a thumbnail from an image, use + /// [`AttachmentConfig::new()`] and + /// [`AttachmentConfig::generate_thumbnail()`]. + pub fn with_thumbnail(thumbnail: Thumbnail<'a, R>) -> Self { + Self { + txn_id: Default::default(), + info: Default::default(), + thumbnail: Some(thumbnail), + #[cfg(feature = "image_proc")] + generate_thumbnail: Default::default(), + #[cfg(feature = "image_proc")] + thumbnail_size: Default::default(), + } + } + + /// Set the transaction ID to send. + /// + /// # Arguments + /// + /// * `txn_id` - A unique ID that can be attached to a `MessageEvent` held + /// in its unsigned field as `transaction_id`. If not given, one is created + /// for the message. + #[must_use] + pub fn txn_id(mut self, txn_id: &'a TransactionId) -> Self { + self.txn_id = Some(txn_id); + self + } + + /// Set the media metadata to send. + /// + /// # Arguments + /// + /// * `info` - The metadata of the media. If the `AttachmentInfo` type + /// doesn't match the `content_type`, it is ignored. + #[must_use] + pub fn info(mut self, info: AttachmentInfo) -> Self { + self.info = Some(info); + self + } +} + +/// Generate a thumbnail for an image. +/// +/// This is a convenience method that uses the +/// [image](https://github.com/image-rs/image) crate. +/// +/// # Arguments +/// * `content_type` - The type of the media, this will be used as the +/// content-type header. +/// +/// * `reader` - A `Reader` that will be used to fetch the raw bytes of the +/// media. +/// +/// * `size` - The size of the thumbnail in pixels as a `(width, height)` tuple. +/// If set to `None`, defaults to `(800, 600)`. +/// +/// # Examples +/// +/// ```no_run +/// # use std::{path::PathBuf, fs::File, io::{BufReader, Cursor, Read, Seek}}; +/// # use matrix_sdk::{ +/// # Client, +/// # attachment::{AttachmentConfig, Thumbnail, generate_image_thumbnail}, +/// # ruma::room_id +/// # }; +/// # use url::Url; +/// # use mime; +/// # use futures::executor::block_on; +/// # block_on(async { +/// # let homeserver = Url::parse("http://localhost:8080")?; +/// # let mut client = Client::new(homeserver).await?; +/// # let room_id = room_id!("!test:localhost"); +/// let path = PathBuf::from("/home/example/my-cat.jpg"); +/// let mut image = BufReader::new(File::open(path)?); +/// +/// let (thumbnail_data, thumbnail_info) = generate_image_thumbnail( +/// &mime::IMAGE_JPEG, +/// &mut image, +/// None +/// )?; +/// let mut cursor = Cursor::new(thumbnail_data); +/// let config = AttachmentConfig::with_thumbnail(Thumbnail { +/// reader: &mut cursor, +/// content_type: &mime::IMAGE_JPEG, +/// info: Some(thumbnail_info), +/// }); +/// +/// image.rewind()?; +/// +/// if let Some(room) = client.get_joined_room(&room_id) { +/// room.send_attachment( +/// "My favorite cat", +/// &mime::IMAGE_JPEG, +/// &mut image, +/// config, +/// ).await?; +/// } +/// # Result::<_, matrix_sdk::Error>::Ok(()) }); +/// ``` +#[cfg(feature = "image_proc")] +pub fn generate_image_thumbnail( + content_type: &mime::Mime, + reader: &mut R, + size: Option<(u32, u32)>, +) -> Result<(Vec, BaseThumbnailInfo), ImageError> { + let image_format = image::ImageFormat::from_mime_type(content_type); + if image_format.is_none() { + return Err(ImageError::FormatNotSupported); + } + + let image_format = image_format.unwrap(); + + let image = image::load(reader, image_format)?; + let (original_width, original_height) = image.dimensions(); + + let (width, height) = size.unwrap_or((800, 600)); + + // Don't generate a thumbnail if it would be bigger than or equal to the + // original. + if height >= original_height && width >= original_width { + return Err(ImageError::ThumbnailBiggerThanOriginal); + } + + let thumbnail = image.thumbnail(width, height); + let (thumbnail_width, thumbnail_height) = thumbnail.dimensions(); + + let mut data: Vec = vec![]; + thumbnail.write_to(&mut Cursor::new(&mut data), image_format)?; + let data_size = data.len() as u32; + + Ok(( + data, + BaseThumbnailInfo { + width: Some(thumbnail_width.into()), + height: Some(thumbnail_height.into()), + size: Some(data_size.into()), + }, + )) +} diff --git a/crates/matrix-sdk/src/client.rs b/crates/matrix-sdk/src/client.rs index f7e6dcdb8..8d1e964eb 100644 --- a/crates/matrix-sdk/src/client.rs +++ b/crates/matrix-sdk/src/client.rs @@ -43,6 +43,7 @@ use ruma::{ client::{ r0::{ account::{register, whoami}, + capabilities::{get_capabilities, Capabilities}, device::{delete_devices, get_devices}, directory::{get_public_rooms, get_public_rooms_filtered}, filter::{create_filter::Request as FilterUploadRequest, FilterDefinition}, @@ -68,6 +69,7 @@ use tracing::{error, info, instrument, warn}; use url::Url; use crate::{ + attachment::{AttachmentInfo, Thumbnail}, config::{ClientConfig, RequestConfig}, error::{HttpError, HttpResult}, event_handler::{EventHandler, EventHandlerData, EventHandlerResult, EventKind, SyncEvent}, @@ -345,6 +347,33 @@ impl Client { .await } + /// Get the capabilities of the homeserver. + /// + /// This method should be used to check what features are supported by the + /// homeserver. + /// + /// # Example + /// ```no_run + /// # use futures::executor::block_on; + /// # use matrix_sdk::Client; + /// # use url::Url; + /// # block_on(async { + /// # let homeserver = Url::parse("http://example.com")?; + /// let client = Client::new(homeserver).await?; + /// + /// let capabilities = client.get_capabilities().await?; + /// + /// if capabilities.change_password.enabled { + /// // Change password + /// } + /// + /// # Result::<_, anyhow::Error>::Ok(()) }); + /// ``` + pub async fn get_capabilities(&self) -> HttpResult { + let res = self.send(get_capabilities::Request::new(), None).await?; + Ok(res.capabilities) + } + /// Process a [transaction] received from the homeserver /// /// # Arguments @@ -1474,7 +1503,7 @@ impl Client { pub async fn upload( &self, content_type: &Mime, - reader: &mut impl Read, + reader: &mut (impl Read + ?Sized), ) -> Result { let mut data = Vec::new(); reader.read_to_end(&mut data)?; @@ -2173,42 +2202,94 @@ impl Client { } /// Upload the file to be read from `reader` and construct an attachment - /// message with `body` and the specified `content_type`. - pub(crate) async fn prepare_attachment_message( + /// message with `body`, `content_type`, `info` and `thumbnail`. + pub(crate) async fn prepare_attachment_message( &self, body: &str, content_type: &Mime, reader: &mut R, + info: Option, + thumbnail: Option>, ) -> Result { + let (thumbnail_url, thumbnail_info) = if let Some(thumbnail) = thumbnail { + let response = self.upload(thumbnail.content_type, thumbnail.reader).await?; + let url = response.content_uri; + + use ruma::events::room::ThumbnailInfo; + let thumbnail_info = assign!( + thumbnail.info.as_ref().map(|info| ThumbnailInfo::from(info.clone())).unwrap_or_default(), + { mimetype: Some(thumbnail.content_type.as_ref().to_owned()) } + ); + + (Some(url), Some(Box::new(thumbnail_info))) + } else { + (None, None) + }; + let response = self.upload(content_type, reader).await?; let url = response.content_uri; - use ruma::events::room::message; + use ruma::events::room::{self, message}; Ok(match content_type.type_() { mime::IMAGE => { - // TODO create a thumbnail using the image crate?. + let info = assign!( + info.map(room::ImageInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_url, + thumbnail_info + } + ); message::MessageType::Image(message::ImageMessageEventContent::plain( body.to_owned(), url, - None, + Some(Box::new(info)), + )) + } + mime::AUDIO => { + let info = assign!( + info.map(message::AudioInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + } + ); + message::MessageType::Audio(message::AudioMessageEventContent::plain( + body.to_owned(), + url, + Some(Box::new(info)), + )) + } + mime::VIDEO => { + let info = assign!( + info.map(message::VideoInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_url, + thumbnail_info + } + ); + message::MessageType::Video(message::VideoMessageEventContent::plain( + body.to_owned(), + url, + Some(Box::new(info)), + )) + } + _ => { + let info = assign!( + info.map(message::FileInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_url, + thumbnail_info + } + ); + message::MessageType::File(message::FileMessageEventContent::plain( + body.to_owned(), + url, + Some(Box::new(info)), )) } - mime::AUDIO => message::MessageType::Audio(message::AudioMessageEventContent::plain( - body.to_owned(), - url, - None, - )), - mime::VIDEO => message::MessageType::Video(message::VideoMessageEventContent::plain( - body.to_owned(), - url, - None, - )), - _ => message::MessageType::File(message::FileMessageEventContent::plain( - body.to_owned(), - url, - None, - )), }) } } @@ -2258,6 +2339,10 @@ pub(crate) mod test { use super::{Client, Session, Url}; use crate::{ + attachment::{ + AttachmentConfig, AttachmentInfo, BaseImageInfo, BaseThumbnailInfo, BaseVideoInfo, + Thumbnail, + }, config::{ClientConfig, RequestConfig, SyncSettings}, HttpError, RoomMember, }; @@ -3040,6 +3125,11 @@ pub(crate) mod test { let _m = mock("PUT", Matcher::Regex(r"^/_matrix/client/r0/rooms/.*/send/".to_string())) .with_status(200) .match_header("authorization", "Bearer 1234") + .match_body(Matcher::PartialJson(json!({ + "info": { + "mimetype": "image/jpeg" + } + }))) .with_body(test_json::EVENT_ID.to_string()) .create(); @@ -3068,12 +3158,200 @@ pub(crate) mod test { let mut media = Cursor::new("Hello world"); - let response = - room.send_attachment("image", &mime::IMAGE_JPEG, &mut media, None).await.unwrap(); + let response = room + .send_attachment("image", &mime::IMAGE_JPEG, &mut media, AttachmentConfig::new()) + .await + .unwrap(); assert_eq!(event_id!("$h29iv0s8:example.com"), response.event_id) } + #[async_test] + async fn room_attachment_send_info() { + let client = logged_in_client().await; + + let _m = mock("PUT", Matcher::Regex(r"^/_matrix/client/r0/rooms/.*/send/".to_string())) + .with_status(200) + .match_header("authorization", "Bearer 1234") + .match_body(Matcher::PartialJson(json!({ + "info": { + "mimetype": "image/jpeg", + "h": 600, + "w": 800, + } + }))) + .with_body(test_json::EVENT_ID.to_string()) + .create(); + + let upload_mock = mock("POST", Matcher::Regex(r"^/_matrix/media/r0/upload".to_string())) + .with_status(200) + .match_header("content-type", "image/jpeg") + .with_body( + json!({ + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + }) + .to_string(), + ) + .create(); + + let _m = mock("GET", Matcher::Regex(r"^/_matrix/client/r0/sync\?.*$".to_string())) + .with_status(200) + .match_header("authorization", "Bearer 1234") + .with_body(test_json::SYNC.to_string()) + .create(); + + let sync_settings = SyncSettings::new().timeout(Duration::from_millis(3000)); + + let _response = client.sync_once(sync_settings).await.unwrap(); + + let room = client.get_joined_room(room_id!("!SVkFJHzfwvuaIEawgC:localhost")).unwrap(); + + let mut media = Cursor::new("Hello world"); + + let config = AttachmentConfig::new().info(AttachmentInfo::Image(BaseImageInfo { + height: Some(uint!(600)), + width: Some(uint!(800)), + size: None, + blurhash: None, + })); + + let response = + room.send_attachment("image", &mime::IMAGE_JPEG, &mut media, config).await.unwrap(); + + upload_mock.assert(); + assert_eq!(event_id!("$h29iv0s8:example.com"), response.event_id) + } + + #[async_test] + async fn room_attachment_send_wrong_info() { + let client = logged_in_client().await; + + let _m = mock("PUT", Matcher::Regex(r"^/_matrix/client/r0/rooms/.*/send/".to_string())) + .with_status(200) + .match_header("authorization", "Bearer 1234") + .match_body(Matcher::PartialJson(json!({ + "info": { + "mimetype": "image/jpeg", + "h": 600, + "w": 800, + } + }))) + .with_body(test_json::EVENT_ID.to_string()) + .create(); + + let _m = mock("POST", Matcher::Regex(r"^/_matrix/media/r0/upload".to_string())) + .with_status(200) + .match_header("content-type", "image/jpeg") + .with_body( + json!({ + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + }) + .to_string(), + ) + .create(); + + let _m = mock("GET", Matcher::Regex(r"^/_matrix/client/r0/sync\?.*$".to_string())) + .with_status(200) + .match_header("authorization", "Bearer 1234") + .with_body(test_json::SYNC.to_string()) + .create(); + + let sync_settings = SyncSettings::new().timeout(Duration::from_millis(3000)); + + let _response = client.sync_once(sync_settings).await.unwrap(); + + let room = client.get_joined_room(room_id!("!SVkFJHzfwvuaIEawgC:localhost")).unwrap(); + + let mut media = Cursor::new("Hello world"); + + let config = AttachmentConfig::new().info(AttachmentInfo::Video(BaseVideoInfo { + height: Some(uint!(600)), + width: Some(uint!(800)), + duration: Some(uint!(3600)), + size: None, + blurhash: None, + })); + + let response = room.send_attachment("image", &mime::IMAGE_JPEG, &mut media, config).await; + + assert!(response.is_err()) + } + + #[async_test] + async fn room_attachment_send_info_thumbnail() { + let client = logged_in_client().await; + + let _m = mock("PUT", Matcher::Regex(r"^/_matrix/client/r0/rooms/.*/send/".to_string())) + .with_status(200) + .match_header("authorization", "Bearer 1234") + .match_body(Matcher::PartialJson(json!({ + "info": { + "mimetype": "image/jpeg", + "h": 600, + "w": 800, + "thumbnail_info": { + "h": 360, + "w": 480, + "mimetype":"image/jpeg", + "size": 3600, + }, + "thumbnail_url": "mxc://example.com/AQwafuaFswefuhsfAFAgsw", + } + }))) + .with_body(test_json::EVENT_ID.to_string()) + .create(); + + let upload_mock = mock("POST", Matcher::Regex(r"^/_matrix/media/r0/upload".to_string())) + .with_status(200) + .match_header("content-type", "image/jpeg") + .with_body( + json!({ + "content_uri": "mxc://example.com/AQwafuaFswefuhsfAFAgsw" + }) + .to_string(), + ) + .expect(2) + .create(); + + let _m = mock("GET", Matcher::Regex(r"^/_matrix/client/r0/sync\?.*$".to_string())) + .with_status(200) + .match_header("authorization", "Bearer 1234") + .with_body(test_json::SYNC.to_string()) + .create(); + + let sync_settings = SyncSettings::new().timeout(Duration::from_millis(3000)); + + let _response = client.sync_once(sync_settings).await.unwrap(); + + let room = client.get_joined_room(room_id!("!SVkFJHzfwvuaIEawgC:localhost")).unwrap(); + + let mut media = Cursor::new("Hello world"); + + let mut thumbnail_reader = Cursor::new("Thumbnail"); + + let config = AttachmentConfig::with_thumbnail(Thumbnail { + reader: &mut thumbnail_reader, + content_type: &mime::IMAGE_JPEG, + info: Some(BaseThumbnailInfo { + height: Some(uint!(360)), + width: Some(uint!(480)), + size: Some(uint!(3600)), + }), + }) + .info(AttachmentInfo::Image(BaseImageInfo { + height: Some(uint!(600)), + width: Some(uint!(800)), + size: None, + blurhash: None, + })); + + let response = + room.send_attachment("image", &mime::IMAGE_JPEG, &mut media, config).await.unwrap(); + + upload_mock.assert(); + assert_eq!(event_id!("$h29iv0s8:example.com"), response.event_id) + } + #[async_test] async fn room_redact() { let client = logged_in_client().await; diff --git a/crates/matrix-sdk/src/encryption/mod.rs b/crates/matrix-sdk/src/encryption/mod.rs index 90c05e667..376594b90 100644 --- a/crates/matrix-sdk/src/encryption/mod.rs +++ b/crates/matrix-sdk/src/encryption/mod.rs @@ -283,6 +283,7 @@ use ruma::{ use tracing::{debug, instrument, trace, warn}; use crate::{ + attachment::{AttachmentInfo, Thumbnail}, encryption::{ identities::{Device, UserDevices}, verification::{SasVerification, Verification, VerificationRequest}, @@ -729,14 +730,45 @@ impl Client { } /// Encrypt and upload the file to be read from `reader` and construct an - /// attachment message with `body` and the specified `content_type`. + /// attachment message with `body`, `content_type`, `info` and `thumbnail`. #[cfg(feature = "encryption")] - pub(crate) async fn prepare_encrypted_attachment_message( + pub(crate) async fn prepare_encrypted_attachment_message( &self, body: &str, content_type: &mime::Mime, reader: &mut R, + info: Option, + thumbnail: Option>, ) -> Result { + let (thumbnail_file, thumbnail_info) = + if let Some(thumbnail) = thumbnail { + let mut reader = matrix_sdk_base::crypto::AttachmentEncryptor::new(thumbnail.reader); + + let response = self.upload(thumbnail.content_type, &mut reader).await?; + + let file: ruma::events::room::EncryptedFile = { + let keys = reader.finish(); + ruma::events::room::EncryptedFileInit { + url: response.content_uri, + key: keys.web_key, + iv: keys.iv, + hashes: keys.hashes, + v: keys.version, + } + .into() + }; + + use ruma::events::room::ThumbnailInfo; + let thumbnail_info = assign!( + thumbnail.info.as_ref().map(|info| ThumbnailInfo::from(info.clone())).unwrap_or_default(), + { mimetype: Some(thumbnail.content_type.as_ref().to_owned()) } + ); + + (Some(Box::new(file)), Some(Box::new(thumbnail_info))) + } else { + (None, None) + }; + let mut reader = matrix_sdk_base::crypto::AttachmentEncryptor::new(reader); let response = self.upload(content_type, &mut reader).await?; @@ -753,18 +785,66 @@ impl Client { .into() }; - use ruma::events::room::message; + use ruma::events::room::{self, message}; Ok(match content_type.type_() { mime::IMAGE => { - message::MessageType::Image(message::ImageMessageEventContent::encrypted(body.to_owned(), file)) + let info = assign!( + info.map(room::ImageInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_file, + thumbnail_info + } + ); + let content = assign!( + message::ImageMessageEventContent::encrypted(body.to_owned(), file), + { info: Some(Box::new(info)) } + ); + message::MessageType::Image(content) } mime::AUDIO => { - message::MessageType::Audio(message::AudioMessageEventContent::encrypted(body.to_owned(), file)) + let info = assign!( + info.map(message::AudioInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + } + ); + let content = assign!( + message::AudioMessageEventContent::encrypted(body.to_owned(), file), + { info: Some(Box::new(info)) } + ); + message::MessageType::Audio(content) } mime::VIDEO => { - message::MessageType::Video(message::VideoMessageEventContent::encrypted(body.to_owned(), file)) + let info = assign!( + info.map(message::VideoInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_file, + thumbnail_info + } + ); + let content = assign!( + message::VideoMessageEventContent::encrypted(body.to_owned(), file), + { info: Some(Box::new(info)) } + ); + message::MessageType::Video(content) + } + _ => { + let info = assign!( + info.map(message::FileInfo::from).unwrap_or_default(), + { + mimetype: Some(content_type.as_ref().to_owned()), + thumbnail_file, + thumbnail_info + } + ); + let content = assign!( + message::FileMessageEventContent::encrypted(body.to_owned(), file), + { info: Some(Box::new(info)) } + ); + message::MessageType::File(content) } - _ => message::MessageType::File(message::FileMessageEventContent::encrypted(body.to_owned(), file)), }) } diff --git a/crates/matrix-sdk/src/error.rs b/crates/matrix-sdk/src/error.rs index 423563984..00a0e7595 100644 --- a/crates/matrix-sdk/src/error.rs +++ b/crates/matrix-sdk/src/error.rs @@ -157,6 +157,11 @@ pub enum Error { /// An error encountered when trying to parse a user tag name. #[error(transparent)] UserTagName(#[from] InvalidUserTagName), + + /// An error while processing images. + #[cfg(feature = "image_proc")] + #[error(transparent)] + ImageError(#[from] ImageError), } /// Error for the room key importing functionality. @@ -257,3 +262,20 @@ impl From for Error { Error::Http(HttpError::Reqwest(e)) } } + +/// All possible errors that can happen during image processing. +#[cfg(feature = "image_proc")] +#[derive(Error, Debug)] +pub enum ImageError { + /// Error processing the image data. + #[error(transparent)] + Proc(#[from] image::ImageError), + + /// The image format is not supported. + #[error("the image format is not supported")] + FormatNotSupported, + + /// The thumbnail size is bigger than the original image. + #[error("the thumbnail size is bigger than the original image size")] + ThumbnailBiggerThanOriginal, +} diff --git a/crates/matrix-sdk/src/lib.rs b/crates/matrix-sdk/src/lib.rs index 669d01e8e..2d1c4a48b 100644 --- a/crates/matrix-sdk/src/lib.rs +++ b/crates/matrix-sdk/src/lib.rs @@ -36,6 +36,9 @@ compile_error!("only one of 'native-tls' or 'rustls-tls' features can be enabled #[cfg(all(feature = "sso_login", target_arch = "wasm32"))] compile_error!("'sso_login' cannot be enabled on 'wasm32' arch"); +#[cfg(all(feature = "image_rayon", target_arch = "wasm32"))] +compile_error!("'image_rayon' cannot be enabled on 'wasm32' arch"); + pub use bytes; pub use matrix_sdk_base::{ media, Room as BaseRoom, RoomInfo, RoomMember as BaseRoomMember, RoomType, Session, @@ -47,6 +50,8 @@ pub use reqwest; pub use ruma; mod account; +/// Types and traits for attachments. +pub mod attachment; mod client; pub mod config; mod error; @@ -62,6 +67,8 @@ pub mod encryption; pub use account::Account; pub use client::{Client, LoopCtrl}; +#[cfg(feature = "image_proc")] +pub use error::ImageError; pub use error::{Error, HttpError, HttpResult, Result}; pub use http_client::HttpSend; pub use room_member::RoomMember; diff --git a/crates/matrix-sdk/src/room/joined.rs b/crates/matrix-sdk/src/room/joined.rs index 979e513d8..87bdeb2f0 100644 --- a/crates/matrix-sdk/src/room/joined.rs +++ b/crates/matrix-sdk/src/room/joined.rs @@ -1,6 +1,11 @@ +#[cfg(feature = "image_proc")] +use std::io::Cursor; #[cfg(feature = "encryption")] use std::sync::Arc; -use std::{io::Read, ops::Deref}; +use std::{ + io::{BufReader, Read, Seek}, + ops::Deref, +}; use matrix_sdk_common::instant::{Duration, Instant}; #[cfg(feature = "encryption")] @@ -31,7 +36,14 @@ use tracing::debug; #[cfg(feature = "encryption")] use tracing::instrument; -use crate::{error::HttpResult, room::Common, BaseRoom, Client, Result, RoomType}; +#[cfg(feature = "image_proc")] +use crate::{attachment::generate_image_thumbnail, error::ImageError}; +use crate::{ + attachment::{AttachmentConfig, Thumbnail}, + error::HttpResult, + room::Common, + BaseRoom, Client, Result, RoomType, +}; const TYPING_NOTICE_TIMEOUT: Duration = Duration::from_secs(4); const TYPING_NOTICE_RESEND_TIMEOUT: Duration = Duration::from_secs(3); @@ -597,15 +609,13 @@ impl Joined { /// * `reader` - A `Reader` that will be used to fetch the raw bytes of the /// media. /// - /// * `txn_id` - A unique ID that can be attached to a `MessageEvent` - /// held in its unsigned field as `transaction_id`. If not given one is - /// created for the message. + /// * `config` - Metadata and configuration for the attachment. /// /// # Examples /// /// ```no_run /// # use std::{path::PathBuf, fs::File, io::Read}; - /// # use matrix_sdk::{Client, ruma::room_id}; + /// # use matrix_sdk::{Client, ruma::room_id, attachment::AttachmentConfig}; /// # use url::Url; /// # use mime; /// # use futures::executor::block_on; @@ -621,29 +631,127 @@ impl Joined { /// "My favorite cat", /// &mime::IMAGE_JPEG, /// &mut image, - /// None, + /// AttachmentConfig::new(), /// ).await?; /// } /// # Result::<_, matrix_sdk::Error>::Ok(()) }); /// ``` - pub async fn send_attachment( + pub async fn send_attachment( &self, body: &str, content_type: &Mime, reader: &mut R, - txn_id: Option<&TransactionId>, + config: AttachmentConfig<'_, T>, + ) -> Result { + let reader = &mut BufReader::new(reader); + + #[cfg(feature = "image_proc")] + let mut cursor; + + if config.thumbnail.is_some() { + self.prepare_and_send_attachment(body, content_type, reader, config).await + } else { + #[cfg(not(feature = "image_proc"))] + let thumbnail = Thumbnail::NONE; + + #[cfg(feature = "image_proc")] + let thumbnail = if config.generate_thumbnail { + match generate_image_thumbnail(content_type, reader, config.thumbnail_size) { + Ok((thumbnail_data, thumbnail_info)) => { + reader.rewind()?; + + cursor = Cursor::new(thumbnail_data); + Some(Thumbnail { + reader: &mut cursor, + content_type: &mime::IMAGE_JPEG, + info: Some(thumbnail_info), + }) + } + Err( + ImageError::ThumbnailBiggerThanOriginal | ImageError::FormatNotSupported, + ) => { + reader.rewind()?; + None + } + Err(error) => return Err(error.into()), + } + } else { + None + }; + + let config = AttachmentConfig { + txn_id: config.txn_id, + info: config.info, + thumbnail, + #[cfg(feature = "image_proc")] + generate_thumbnail: false, + #[cfg(feature = "image_proc")] + thumbnail_size: None, + }; + + self.prepare_and_send_attachment(body, content_type, reader, config).await + } + } + + /// Prepare and send an attachment to this room. + /// + /// This will upload the given data that the reader produces using the + /// [`upload()`](#method.upload) method and post an event to the given room. + /// If the room is encrypted and the encryption feature is enabled the + /// upload will be encrypted. + /// + /// This is a convenience method that calls the + /// [`Client::upload()`](#Client::method.upload) and afterwards the + /// [`send()`](#method.send). + /// + /// # Arguments + /// * `body` - A textual representation of the media that is going to be + /// uploaded. Usually the file name. + /// + /// * `content_type` - The type of the media, this will be used as the + /// content-type header. + /// + /// * `reader` - A `Reader` that will be used to fetch the raw bytes of the + /// media. + /// + /// * `config` - Metadata and configuration for the attachment. + async fn prepare_and_send_attachment( + &self, + body: &str, + content_type: &Mime, + reader: &mut R, + config: AttachmentConfig<'_, T>, ) -> Result { #[cfg(feature = "encryption")] let content = if self.is_encrypted() { - self.client.prepare_encrypted_attachment_message(body, content_type, reader).await? + self.client + .prepare_encrypted_attachment_message( + body, + content_type, + reader, + config.info, + config.thumbnail, + ) + .await? } else { - self.client.prepare_attachment_message(body, content_type, reader).await? + self.client + .prepare_attachment_message( + body, + content_type, + reader, + config.info, + config.thumbnail, + ) + .await? }; #[cfg(not(feature = "encryption"))] - let content = self.client.prepare_attachment_message(body, content_type, reader).await?; + let content = self + .client + .prepare_attachment_message(body, content_type, reader, config.info, config.thumbnail) + .await?; - self.send(RoomMessageEventContent::new(content), txn_id).await + self.send(RoomMessageEventContent::new(content), config.txn_id).await } /// Send a room state event to the homeserver.