From 7edcbb15d609ab583bc9a1355069e298a0f0b65d Mon Sep 17 00:00:00 2001 From: jake <77554505+brxken128@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:08:17 +0100 Subject: [PATCH] [ENG-621][ENG-1074] Trait-based image conversion overhaul (#1307) * sd-images crate which will support raw/dng, bmp, etc * more work on the image formatter * re-work `sd-images`, add svg support, r/g/b and r/g/b/a HEIF image support (will all be async again soon) * remove `ImageFormatter`, add note about r/g/b/(a) heif impl * implement the image formatter * rename the conversion trait and minor cleanups * isolate heif feature and major cleanup * very untested raw support * change fn name to `from_path` (a lot more idiomatic) * clean up orientation fixing * heif is no longer forbidden (linux has good heif) also all extensions are correctly matched in lowercase * fix builds, ext matching, feature gating * attempt to fix svg handling? * raw attempt, quite a few errors * add comment * new (untested) attempt * remove `raw` stuff for now * replace `sd-svg` with a `ToImage` `SvgHandler` impl * add some simple math to appropriately scale thumbnails (and bmp/ico support) * add comments regarding how the math works for image thumbs * rename the trait to `ImageHandler` --- Cargo.lock | Bin 252860 -> 254022 bytes core/Cargo.toml | 6 +- core/src/object/media/thumbnail/mod.rs | 142 +++++------------- crates/heif/Cargo.toml | 15 -- crates/heif/src/lib.rs | 90 ----------- crates/images/Cargo.toml | 20 +++ crates/images/src/consts.rs | 26 ++++ crates/images/src/error.rs | 37 +++++ crates/images/src/formatter.rs | 47 ++++++ crates/images/src/generic.rs | 22 +++ crates/images/src/heif.rs | 122 +++++++++++++++ crates/images/src/lib.rs | 60 ++++++++ crates/images/src/svg.rs | 63 ++++++++ .../media-metadata/src/image/orientation.rs | 2 +- crates/svg/Cargo.toml | 14 -- crates/svg/src/lib.rs | 76 ---------- 16 files changed, 442 insertions(+), 300 deletions(-) delete mode 100644 crates/heif/Cargo.toml delete mode 100644 crates/heif/src/lib.rs create mode 100644 crates/images/Cargo.toml create mode 100644 crates/images/src/consts.rs create mode 100644 crates/images/src/error.rs create mode 100644 crates/images/src/formatter.rs create mode 100644 crates/images/src/generic.rs create mode 100644 crates/images/src/heif.rs create mode 100644 crates/images/src/lib.rs create mode 100644 crates/images/src/svg.rs delete mode 100644 crates/svg/Cargo.toml delete mode 100644 crates/svg/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6fa48998f62d295c55efc9699ad0560e12ea37f5..358ae6633c00ba35db7489b463c120e690083b34 100644 GIT binary patch delta 5717 zcmZ8l35;IFeb;->_SzWZb1!V1wUdS;W-~KyPJ(x_kP3|id}|Y^R0y8aav=t62&Wa( zLjjVKxGykJQiWAR3ABLSFX$<48e`1Sv@zI7A*CV4CbVh^bwE{8+ED24?UMA6rPa4@ z{msn#|Np;hzTGZ<_Q}P2E-jkP_idW6HE}Gm+L=g|HOUaksaMuf!Ho~bah{lwq4wUT znwY?}tbK~L(U$n;?fZuM?-fT+%-6hGF3Rs0#krHj*GdPaIjd8!K@)8xl}R%tn2fQm zJvE7?5L^%;QppSz%o|~9XSvdOFE57jyBo@}{+}$qwaQ}y#l}fTlhB${pRBJrH=HDD zIU!C)$!h9jEvTYNQXE$D+C)u~_LNKEh4-YHIy}@oyL?mg{NNeQrWH$@_lJ5DRCw;0 z*LBiV)GP|kg;jNJjCRssHR~;_xgj+tmQx*#Q_R-U8=RUoZ%vIhM~CM3mk<7Pk?$bI z#gjrO!2=~;hGYe0y0%10AFacesCB3ct&@#}Sj!FLwdEpmft{Q(A?3F(E{FPGA7%rE zZjSxhs^*%tbNlbCu8Uv&I7+sO+8AWfBp{!2DI@Q9N5Lh5F7)GNFJfT}ud#0o& zfm`7Ml?rFTsqvp2B|_J^xD}gyM0{di?w?fl&f$~?ACvR44oZ1+$|~ty-Y9XMbU7v+-k76P_i7${C`ZqKrurC3nQ8$dcyL zFqcc<^e+%g1}#E9F$;eg-elSd!QI&Pu-Gyx!*p_7RSX==+7HOvWLqp#Rh z%?x(l`%p359KLdO{^U#eu;QwfSa!`r#l{I#Fn8E;HQDK>hG7&;az<%CZ zX{^Vz1?dweFzSaZ0sO zffF=8aZI_>Y8GI%1u6)zhOG(EVP!Hhujy4I{WrJmSybr!{JL^ob75m-79GS(QzjnGAXiogb;(YQK!51kdi+WfowbwSO;802th4dyVFq#X1x#1=T z(1s)uFM$Rau0rI*(T*<`_^Y?=xbDoWw}!mrmg1ugdt|sj^6-5_^Tx=y9H-6ReP_-c zqxhJMe=Alt@9cZ_|Ngq7WaD|wBh|)9Nie5OQE%&-DB-mx40T1%c@X$0>L6mT~dGig$Xdb+!Skqtl)SngeXDhUJ zUGD8G2b+Tj&hKhw#!2^on~y&|mEU*|Q=9^^;C-h!vqlp%AuuL74z{{>Le=@jOy zW4I`5MW7Nj@loQ$&D0Nvn_Y(nn(=2uA!aLZ+cQU->0S@7EVZkF5+f~P;8-9?P)?-M zNr@msMC)rCg|H$hVq)wllxPad0wp*5&l}3U!D>N&@pIo9$amd_F{L#&1y^Z7bY#NI z6hSX0D8p1pAXgG=P!HGI0hXXVrJ3M@>^(OfBOqhtOh;yCV{hrdK(Mn)1WdI zYA~d>Oc0e~^w7BAK)f|rG8zbxB3uSZRo%@??6`ZlfAIMDz)~~QkQ<`A=E|+8F=P4I zrRA#Tr$3$FCqF;9EWh>jYSTn=Gjg1mvOYoB1Vo)A&jS$Qg9e=g={5+w5fmR9)j{8D zXqQOD=l!dzk@im)6yF;tM7v{Y5eJ%MW@>_VuVyW!pbN->jEO5qIa5$(#cfDfM)Mk! ztf_QVi$HZqMo47{^k+up$F3|EwzrQKznGK1cz4yC#Fl71IEE2RAZ}e11cbZ;5D*GE zQw&sZmBG582peh+zM~Et2ff3r4&}|Kl?&!DI$pQuf2cU0mXx+2pWj|v)a|?5i#v+^ zo!y;$LftT9Q9+7C(1{WPivxqc9J?Hx3+JVC44WwXs-?n0){mHGx531QG%NPBx4vo|H$GeIF$DF&KH zcMi9&KU4f-em?pFIv@cgB{7Q-Pyhm7lq_h&LGJLPFj?qGG#XY8qU!>@LNXK}4KNNq z_?ces`D$@Od)EuaV+%8R6}41269_DzNzrQUC9wj};15w-P;W{>7`PV4u%k5!Q)*oD z!bS({d=R4@x_(%GGwAW6Un!FqxfXBZl*os>6}rJY?8@0CbxIA)U7NEvlE zh9c6a4KZ-{bq!1BJ3FrtKOE>3xq-vy)I2&bX53+Z?DxvW?Il&YYw_}5idd($v+N4M zK2@yBBg@Ja`9lvCngdt`bPrkfO145;Gwyf zoYV+$*dTIA@feMuff)`z!I{Z>AE`$3&K2czMV;;PVkBQaSQ41Dz3&w(+lL3sYtIBv ztvt6pFTZwaxw74PZu#I+{CVB&)yaANr^GaRu_5&c7sg? zlSYj==Nu**At%DOW2&MAR4tqi;cx{Mv)YBm|9^xv&1+v-nNR$5`C9YnJfH|939k+^7k~`F1$7oND>QFUjkI@PQ9fTn4wfz~rY0g3 zwoXyOAp|HJ7a$6Zb&c;JN25$|0R|XN;3>ol34>u3P>XCHrJWi0|53IfUp`XgmoF?< zw?{(xnZf*r-!G;nJ(71oHia?*E^wF7M-S_-jK>dTJK&TeDsqiz0MpqeFXg3-9I1ed z8A)zIP`SuEHiAhYMZjQ)ULc%(Y&NBPi-%Iv^;GdfVN=|z1FmZjOyR}BmAJO?oo0X&0*sx6ZeYlR@gST;R4o`@#3l>T05v+8*jZQi^8@8y z6}kC7nq7mrH124?*pdfK251f82>gry4JkV~SFI!0^Q6k?rq$aA>9 z0`I_JD*zT;EfDiGpSfXJvU1UaUy*Mvc7ypY{z3V@!TdWfm!~wZ{^fA)9WR%(;a|&} zmSJR%ykFKOy0QCT%Hj6(vbv>2haQ|$eXG#z6?3c4odBmf6LZ@oORCEUnnOPUf^ee& zA`wVi3=+ZM?~ur?Mddoq_UN1q5D^JcCx9V%3^L54g7H(4H^)9U)GiyTRFTjBS%DB@ zLFZ@??x{{Eh`4fb1os`NIYz;|sBy(0X+&to1qJ0$xT!70aC>~Tx?>2h zvG>d6So89447ZPyDy_m*OL6a}^|(fY#wJ{Ub!HSQqanT=_y!e77!ceMqJ*A=z#~O~ zW*p9&<0cD1TF>CU{q_ab&PDm@H*lW-s_im*qMD%s7R%UxTSd(@T$z^`D>$f#av**O z-lYVf1y?o5XBq~snLaqw9{H^*R4DXxLVG@aMOBYBU%PSRde|>qMRt}5n*+YQQ&=z@ z0RfB7D!{lS;Aut;F2OvMP_WpU47d`K&sl-Ey7}g6UORbH_4+{WFUQ0ta3t6l6)>So zp#c|oz%vY*hS`DD3yB3-6a+Biag)$xK$r@szLJ=~8SnV&uIl6Mzu#81xU*{?|J`cM zoN|mdPu{trJ^K0TT+`H_@2&6de{emA%6SfdgWQGNIZz2MR^WNz#W}ngVHR{!a=z;2I`LP4lU_sHksR>g?D=X<^}**;UaY>h;J*Rx^}nY8 delta 5204 zcmYjV3y@ybU7vezHoI(-ERgJOLK4^nYKtX!-VZ3*Eigb|unAFzR;1^NnE)Y#209jD zg_cr;5iadPD}~4aN?S<$gexOTm?BwfK^~)ymKK<53*#UUA2WTSzxxFoGnwq>?tbT< z^Z)-||MTR5iATGMeZO9{H=ne1HjC`ENz?=-a!5obOPGr(MD1iskxG0ajtg$H_9Rhd zGhvc3DvXk;EKF5XgLkTzmz9}V&Ce<=OfoTLj;~QfDy&u68A7w5jA2#9bIy`u4HIG@ zJS*Xu%*ICUWV^6)s$A04hn3|oRnvpJCqCblH*c-B&XVAa$XqeQH3`y2uSH7Ogy&50 zC{4D`#b~|ANheKdjz-5IB=b5`RbJ86>UMGEmgU=bZ@N5PcJcO0+(jEtX#a9@Q+wK~ zBg)Q`s`)wYIEhXMnOT%cGb30c#0M|6lA0>)-uaY?^3|%GG0_~+&ef;Q2Bo$0C@is53T9P|T)X}QLJ39!)!u5#li>=L3J%rEffysH z(jkko|1MPGE7NqeeeSlk?Qd=z9lUxpuipL7*kJPvUDa;L^QQ<-Eh8p6B5V*?jB0Li z7&Mc4M-9WyBaMMEYf&|34JyvjPv)sE9~rAx4_3*Om$u*j%KV%nf@+ghkbzP3I|eDG zHo{PnV)h0f6)v!#xo62GO01Gn37eAUA@RNx>jytS`NkvKi4XUnum*66A!MI3$u0>R zwA3Q9{;;%1i)Gn6{3s-znD1?p*m}_sjaR`d{mDet)S5~U*+f^So*m?G!O_zoL zs^@1@04N+vD5YoEGI2) zN{({QD8U(m>sbP5>^QnSa8&dDcI2Y9|F?2BAt^AzD{X@HhB!^U@+yLKE^>-DGOZZL zW9$|rV1bhmrRE96v+e$i_FygAzID_5tQ5|ApcQNgGJ*)qMC{92L6xB?09zzvmRT!MWNQPY#;9O~6JR>z15l$PDzf^vbn7%K%)iA0u!V*~`Ev0b=*W^mcw zcC=zcKnHz<@}rKZLRtC6=m@U^)cm&d+Ukz ziw8FhPWzkttMbNU^<+DC@MC4}oO-(5d2m}hxd$o<5lMzQMHpiljP^N7618N`21plF zBP8_QrpSTbFxeW@BPtDdiqY_DBBwms+VquVDIUmo_2Xo~V?2%5pU zI26uH10~QxWdaD%6oC=d+Uz!V# z)@iFiFRH+E%_EScLIBkyi;#N_;zRnu4?X|T(Pdz;98d#p zJMVB9#sraEcm4R-(J=_AW4r&e3%Ji1RT0059t8lh7ZisO}X0jkI|6(AOv z-9sa2L2)!_RW><#kpPJvZ{2Ils_%|gqI-B%l}6j@-?nVl&fTJ4@xy&O?3y>RZB<8FMkA^Ad}TMcr`l7M2Myk8VLQ?YEh!AdGl!lysVKq5@z+9g3_uDY6{+`@%v@F+wjlab z<s6_~j!2b=f51Y(Dbq?@rsZ=cVo7L*> z?5|h2+PVuYB2Hgv_l{!%55K7^f9<31$3WiF1%Hc=E;0Z{{hDa?l zU#Jf6e*9o{%Ifm=t<{#zIYO@@K57_^%4LeRRRXoN484Rqg>HZu6c~e|M2T>sopo?W zL?N*C`10anb!7ST>*`J2$WztN#@c5a?A3Cp9y9$OxxkK`&5%O4A!HjmL^$IdLa1TF z5G|Jje8)J3?{Q3dXs<87QIB^IK3_fXzIMxNTW7%_r<9I1GcSqkk0c#bHDHz|s5(!zB zC{w`q$z_lB;30PHoap~ht^+D8;5$YRgKW8Aq+U_3TvDI3&M>F!H+lnpP`Ys)dy_4nGX ztG3SeI{^M6UN({BHOxg@1keo0SHvP^^|f$)+!Tz0XqGDwO#~k)vQl|&S+%;{y}CYF zcHdjADc7#4Ngp=5XV%o0ZGtp>10f+X!FBwTZ2kgBv(oLOWwCR9|i=iyX2 zctL$cCCWctP#=rrSNVFXeg44OGUMv?<>Q~J|Fiw!OY@sE=9G|frW6$J&z>X+wkNZ& z91Md#CL2MWUa~V{GN&{_281<<`||j*X8OGs{kN+(y!VEsUFgcEhXdx+UA|iznN+ADiDy5lJxO5eOl?$G{6PGe>4o~(A(G0>RrQkYj)nRc6BX;ec1!((YRPYJaivzPAuO&xtZ>++FM`I)EEKx-ZVLK>5j15w09QTee$0AjHP9l z!#QE6US*8C$ly^T+zYTxxHrdNO#IN z>ld3tOJa<&SSy!lW)UB3TFy{&w!s~v27;xFo}-uuaTdFm0U_~^s+wrl=;)KG;aZYsgL>ZEmgc8$UU$ zd1PD;yZqVDZja^S^QvDe7c6N;x*v}>A0K}&QKiY|{Lyz?3o(qtYXgs=PT@+zBNjlj z1ui$x1J)}C^NARSB-3X?21Bz5dAPuk=)=yb?!xtruG(*?`8jATl@uqp$l0(L0(wO0 zv(=b!(OrvP!)^QF8K%JuQZqs$YOAsb9s83{cBdWF+%N?Vy>wr-u6s>33+u|p(rlfD z?_-Kh80NKweD{gZW4a_hiOhf|7!qEGdkGjOLcPWe0r$YANR)8b+-})BRnC6~l)3p= zo3U>0+~(}%?N9e&R)qC{#~k*JcnCX?xCKB?iUc}@25Up1-U8YMor?|`1Zj>!pW)8N zg!1@<$O!j7R8Mwix#pY(ZMi)*XRj&u--z291&>986~{dWrr_W(`s2eG2ml6+VOazu zW`Xx$m|(C0OdW;@4_xR#^w8*j|5}VoKU;_V@%7(n{&}>_p4C9;kq7#%S%&c&s>|^Y zR8k{nfPP3y80^sl4dS54n2{yMq^NO4gaAUzmi>Uy`sWQoR7`!#ZK`lp5E2c|SKJN#WqYKS!9O76AgH-v=^+89hi38XFF&V+=X6 z-GBZ6yTvMcv6(D?JyxGPXGrXmInw|bIE3tJ1un!4QZ2&;R)9S-1x7|pF^Dg?6u~_W z!(|i36QgV_yY6XLmk0i?T7PKK&mCW#JFAD8LE}=5K(PcdxCjCWtcGYogK;CFvq_j0 z`r9|_i2~ (u32, u32) { + let sf = (TAGRET_PX / (w * h)).sqrt(); + ((w * sf).round() as u32, (h * sf).round() as u32) } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] @@ -106,91 +120,30 @@ pub struct ThumbnailerMetadata { pub skipped: u32, } -static HEIF_EXTENSIONS: Lazy> = Lazy::new(|| { - ["heif", "heifs", "heic", "heics", "avif", "avci", "avcs"] - .into_iter() - .map(|s| s.to_string()) - .collect() -}); - -// The maximum file size that an image can be in order to have a thumbnail generated. -const MAXIMUM_FILE_SIZE: u64 = 50 * 1024 * 1024; // 50MB - pub async fn generate_image_thumbnail>( file_path: P, output_path: P, -) -> Result<(), Box> { - let file_path = file_path.as_ref(); +) -> Result<(), ThumbnailerError> { + let file_path = file_path.as_ref().to_path_buf(); - let ext = file_path - .extension() - .and_then(|ext| ext.to_str()) - .unwrap_or_default() - .to_ascii_lowercase(); - let ext = ext.as_str(); + let webp = tokio::task::block_in_place(move || -> Result<_, ThumbnailerError> { + let img = format_image(&file_path).map_err(|_| ThumbnailerError::Encoding)?; - let metadata = fs::metadata(file_path) - .await - .map_err(|e| FileIOError::from((file_path, e)))?; - - if metadata.len() - > (match ext { - "svg" => sd_svg::MAXIMUM_FILE_SIZE, - #[cfg(all(feature = "heif", not(target_os = "linux")))] - _ if HEIF_EXTENSIONS.contains(ext) => sd_heif::MAXIMUM_FILE_SIZE, - _ => MAXIMUM_FILE_SIZE, - }) { - return Err(ThumbnailerError::TooLarge.into()); - } - - #[cfg(all(feature = "heif", not(target_os = "linux")))] - if metadata.len() > sd_heif::MAXIMUM_FILE_SIZE && HEIF_EXTENSIONS.contains(ext) { - return Err(ThumbnailerError::TooLarge.into()); - } - - let data = Arc::new( - fs::read(file_path) - .await - .map_err(|e| FileIOError::from((file_path, e)))?, - ); - - let img = match ext { - "svg" => sd_svg::svg_to_dynamic_image(data.clone()).await?, - _ if HEIF_EXTENSIONS.contains(ext) => { - #[cfg(not(all(feature = "heif", not(target_os = "linux"))))] - return Err("HEIF not supported".into()); - #[cfg(all(feature = "heif", not(target_os = "linux")))] - sd_heif::heif_to_dynamic_image(data.clone()).await? - } - _ => image::load_from_memory_with_format( - &fs::read(file_path).await?, - ImageFormat::from_path(file_path)?, - )?, - }; - - let webp = spawn_blocking(move || -> Result<_, ThumbnailerError> { let (w, h) = img.dimensions(); + let (w_scale, h_scale) = calculate_factor(w as f32, h as f32); // Optionally, resize the existing photo and convert back into DynamicImage let mut img = DynamicImage::ImageRgba8(imageops::resize( &img, - // FIXME : Think of a better heuristic to get the thumbnail size - (w as f32 * THUMBNAIL_SIZE_FACTOR) as u32, - (h as f32 * THUMBNAIL_SIZE_FACTOR) as u32, + w_scale, + h_scale, imageops::FilterType::Triangle, )); - match ExifReader::from_slice(data.as_ref()) { - Ok(exif_reader) => { - // this corrects the rotation/flip of the image based on the available exif data - if let Some(orientation) = Orientation::from_reader(&exif_reader) { - img = orientation.correct_thumbnail(img); - } - } - Err(sd_media_metadata::Error::NoExifDataOnSlice) => { - // No can do if we don't have exif data - } - Err(e) => warn!("Unable to extract EXIF: {:?}", e), + // this corrects the rotation/flip of the image based on the *available* exif data + // not all images have exif data, so we don't error + if let Some(orientation) = Orientation::from_path(file_path) { + img = orientation.correct_thumbnail(img); } // Create the WebP encoder for the above image @@ -198,14 +151,11 @@ pub async fn generate_image_thumbnail>( return Err(ThumbnailerError::Encoding); }; - // Encode the image at a specified quality 0-100 - // Type WebPMemory is !Send, which makes the Future in this function !Send, // this make us `deref` to have a `&[u8]` and then `to_owned` to make a Vec // which implies on a unwanted clone... - Ok(encoder.encode(THUMBNAIL_QUALITY).deref().to_owned()) - }) - .await??; + Ok(encoder.encode(TARGET_QUALITY).deref().to_owned()) + })?; let output_path = output_path.as_ref(); @@ -214,11 +164,7 @@ pub async fn generate_image_thumbnail>( .await .map_err(|e| FileIOError::from((shard_dir, e)))?; } else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Cannot determine parent shard directory for thumbnail", - ) - .into()); + return Err(ThumbnailerError::Encoding); } fs::write(output_path, &webp) @@ -234,7 +180,7 @@ pub async fn generate_video_thumbnail + Send>( ) -> Result<(), Box> { use sd_ffmpeg::to_thumbnail; - to_thumbnail(file_path, output_path, 256, THUMBNAIL_QUALITY).await?; + to_thumbnail(file_path, output_path, 256, TARGET_QUALITY).await?; Ok(()) } @@ -249,16 +195,10 @@ pub const fn can_generate_thumbnail_for_video(video_extension: &VideoExtension) pub const fn can_generate_thumbnail_for_image(image_extension: &ImageExtension) -> bool { use ImageExtension::*; - #[cfg(all(feature = "heif", not(target_os = "linux")))] - let res = matches!( + matches!( image_extension, - Jpg | Jpeg | Png | Webp | Gif | Svg | Heic | Heics | Heif | Heifs | Avif - ); - - #[cfg(not(all(feature = "heif", not(target_os = "linux"))))] - let res = matches!(image_extension, Jpg | Jpeg | Png | Webp | Gif | Svg); - - res + Jpg | Jpeg | Png | Webp | Gif | Svg | Heic | Heics | Heif | Heifs | Avif | Bmp | Ico + ) } pub(super) async fn process( diff --git a/crates/heif/Cargo.toml b/crates/heif/Cargo.toml deleted file mode 100644 index 64dd597fc..000000000 --- a/crates/heif/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "sd-heif" -version = "0.1.0" -authors = ["Jake Robinson "] -license = { workspace = true } -repository = { workspace = true } -edition = { workspace = true } - -[dependencies] -libheif-rs = "0.19.2" -libheif-sys = "=1.14.2" -image = "0.24.6" -once_cell = "1.17.2" -tokio = { workspace = true, features = ["fs", "io-util"] } -thiserror = "1.0.40" diff --git a/crates/heif/src/lib.rs b/crates/heif/src/lib.rs deleted file mode 100644 index e7af00f08..000000000 --- a/crates/heif/src/lib.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::{ - io::{Cursor, SeekFrom}, - sync::Arc, -}; - -use image::DynamicImage; -use libheif_rs::{ColorSpace, HeifContext, LibHeif, RgbChroma}; -use once_cell::sync::Lazy; -use thiserror::Error; -use tokio::{ - io::{AsyncReadExt, AsyncSeekExt, BufReader}, - task::{spawn_blocking, JoinError}, -}; - -type HeifResult = Result; - -// The maximum file size that an image can be in order to have a thumbnail generated. -pub const MAXIMUM_FILE_SIZE: u64 = 20 * 1024 * 1024; // 20MB - -#[derive(Error, Debug)] -pub enum HeifError { - #[error("error with libheif: {0}")] - LibHeif(#[from] libheif_rs::HeifError), - #[error("error while loading the image (via the `image` crate): {0}")] - Image(#[from] image::ImageError), - #[error("Blocking task failed to execute to completion.")] - Join(#[from] JoinError), - #[error("there was an error while converting the image to an `RgbImage`")] - RgbImageConversion, - #[error("the image provided is unsupported")] - Unsupported, - #[error("the provided bit depth is invalid")] - InvalidBitDepth, - #[error("invalid path provided (non UTF-8)")] - InvalidPath, -} - -static HEIF: Lazy = Lazy::new(LibHeif::new); - -pub async fn heif_to_dynamic_image(data: Arc>) -> HeifResult { - let (img_data, stride, height, width) = spawn_blocking(move || -> Result<_, HeifError> { - let ctx = HeifContext::read_from_bytes(&data)?; - let handle = ctx.primary_image_handle()?; - let img = HEIF.decode(&handle, ColorSpace::Rgb(RgbChroma::Rgb), None)?; - - // TODO(brxken128): add support for images with individual r/g/b channels - // i'm unable to find a sample to test with, but it should follow the same principles as this one - let Some(planes) = img.planes().interleaved else { - return Err(HeifError::Unsupported); - }; - - if planes.bits_per_pixel != 8 { - return Err(HeifError::InvalidBitDepth); - } - - Ok(( - planes.data.to_vec(), - planes.stride, - img.height(), - img.width(), - )) - }) - .await??; - - let mut buffer = [0u8; 3]; // [r, g, b] - let mut reader = BufReader::new(Cursor::new(img_data)); - let mut sequence = vec![]; - - // this is the interpolation stuff, it essentially just makes the image correct - // in regards to stretching/resolution, etc - for y in 0..height { - reader - .seek(SeekFrom::Start((stride * y as usize) as u64)) - .await - .map_err(|_| HeifError::RgbImageConversion)?; - - for _ in 0..width { - reader - .read_exact(&mut buffer) - .await - .map_err(|_| HeifError::RgbImageConversion)?; - sequence.extend_from_slice(&buffer); - } - } - - let rgb_img = - image::RgbImage::from_raw(width, height, sequence).ok_or(HeifError::RgbImageConversion)?; - - Ok(DynamicImage::ImageRgb8(rgb_img)) -} diff --git a/crates/images/Cargo.toml b/crates/images/Cargo.toml new file mode 100644 index 000000000..38a4b1db5 --- /dev/null +++ b/crates/images/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sd-images" +version = "0.0.0" +authors = [ + "Jake Robinson ", + "Vítor Vasconcellos ", +] +license = { workspace = true } +repository = { workspace = true } +edition = { workspace = true } + +[features] +heif = ["dep:libheif-rs", "dep:libheif-sys"] + +[dependencies] +libheif-rs = { version = "0.19.2", optional = true } +libheif-sys = { version = "=1.14.2", optional = true } +image = "0.24.7" +thiserror = "1.0.45" +resvg = "0.35.0" diff --git a/crates/images/src/consts.rs b/crates/images/src/consts.rs new file mode 100644 index 000000000..6aeb8ed51 --- /dev/null +++ b/crates/images/src/consts.rs @@ -0,0 +1,26 @@ +/// The size of 1MiB in bytes +const MIB: u64 = 1_048_576; + +pub const HEIF_EXTENSIONS: [&str; 7] = ["heif", "heifs", "heic", "heics", "avif", "avci", "avcs"]; + +/// The maximum file size that an image can be in order to have a thumbnail generated. +/// +/// This value is in MiB. +pub const HEIF_MAXIMUM_FILE_SIZE: u64 = MIB * 32; + +pub const SVG_EXTENSIONS: [&str; 2] = ["svg", "svgz"]; + +/// The maximum file size that an image can be in order to have a thumbnail generated. +/// +/// This value is in MiB. +pub const SVG_MAXIMUM_FILE_SIZE: u64 = MIB * 24; + +/// The size that SVG images are rendered at, assuming they are square. +// TODO(brxken128): check for non-1:1 SVG images and create a function to resize +// them while maintaining the aspect ratio. +pub const SVG_RENDER_SIZE: u32 = 512; + +/// The maximum file size that an image can be in order to have a thumbnail generated. +/// +/// This value is in MiB. +pub const GENERIC_MAXIMUM_FILE_SIZE: u64 = MIB * 64; diff --git a/crates/images/src/error.rs b/crates/images/src/error.rs new file mode 100644 index 000000000..e95a6dbdc --- /dev/null +++ b/crates/images/src/error.rs @@ -0,0 +1,37 @@ +use std::num::TryFromIntError; + +pub type Result = std::result::Result; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[cfg(feature = "heif")] + #[error("error with libheif: {0}")] + LibHeif(#[from] libheif_rs::HeifError), + + #[error("error with usvg: {0}")] + USvg(#[from] resvg::usvg::Error), + #[error("failed to allocate `Pixbuf` while converting an SVG")] + Pixbuf, + #[error("error while loading the image (via the `image` crate): {0}")] + Image(#[from] image::ImageError), + #[error("there was an i/o error: {0}")] + Io(#[from] std::io::Error), + #[error("there was an error while converting the image to an `RgbImage`")] + RgbImageConversion, + #[error("the image provided is unsupported")] + Unsupported, + #[error("the image provided is too large (over 20MiB)")] + TooLarge, + #[error("the provided bit depth is invalid")] + InvalidBitDepth, + #[error("invalid path provided (non UTF-8)")] + InvalidPath, + #[error("the image has an invalid length to be RGB")] + InvalidLength, + #[error("invalid path provided (it had no file extension)")] + NoExtension, + #[error("error while converting from raw")] + RawConversion, + #[error("error while parsing integers")] + TryFromInt(#[from] TryFromIntError), +} diff --git a/crates/images/src/formatter.rs b/crates/images/src/formatter.rs new file mode 100644 index 000000000..04516f8ed --- /dev/null +++ b/crates/images/src/formatter.rs @@ -0,0 +1,47 @@ +use crate::{ + consts, + error::{Error, Result}, + generic::GenericHandler, + svg::SvgHandler, + ImageHandler, +}; +use image::DynamicImage; +use std::{ + ffi::{OsStr, OsString}, + path::Path, +}; + +#[cfg(feature = "heif")] +use crate::heif::HeifHandler; + +pub fn format_image(path: impl AsRef) -> Result { + let ext = path + .as_ref() + .extension() + .map_or_else(|| Err(Error::NoExtension), |e| Ok(e.to_ascii_lowercase()))?; + match_to_handler(&ext).handle_image(path.as_ref()) +} + +#[allow(clippy::useless_let_if_seq)] +fn match_to_handler(ext: &OsStr) -> Box { + let mut handler: Box = Box::new(GenericHandler {}); + + #[cfg(feature = "heif")] + if consts::HEIF_EXTENSIONS + .iter() + .map(OsString::from) + .any(|x| x == ext) + { + handler = Box::new(HeifHandler {}); + } + + if consts::SVG_EXTENSIONS + .iter() + .map(OsString::from) + .any(|x| x == ext) + { + handler = Box::new(SvgHandler {}); + } + + handler +} diff --git a/crates/images/src/generic.rs b/crates/images/src/generic.rs new file mode 100644 index 000000000..9d401bb54 --- /dev/null +++ b/crates/images/src/generic.rs @@ -0,0 +1,22 @@ +use crate::consts::GENERIC_MAXIMUM_FILE_SIZE; +pub use crate::error::{Error, Result}; +use crate::ImageHandler; +use image::DynamicImage; +use std::path::Path; + +pub struct GenericHandler {} + +impl ImageHandler for GenericHandler { + fn maximum_size(&self) -> u64 { + GENERIC_MAXIMUM_FILE_SIZE + } + + fn validate_image(&self, _bits_per_pixel: u8, _length: usize) -> Result<()> { + Ok(()) + } + + fn handle_image(&self, path: &Path) -> Result { + let data = self.get_data(path)?; // this also makes sure the file isn't above the maximum size + Ok(image::load_from_memory(&data)?) + } +} diff --git a/crates/images/src/heif.rs b/crates/images/src/heif.rs new file mode 100644 index 000000000..d7a817b80 --- /dev/null +++ b/crates/images/src/heif.rs @@ -0,0 +1,122 @@ +pub use crate::consts::HEIF_EXTENSIONS; +use crate::consts::HEIF_MAXIMUM_FILE_SIZE; +pub use crate::error::{Error, Result}; +use crate::ImageHandler; +use image::DynamicImage; +use libheif_rs::{ColorSpace, HeifContext, LibHeif, RgbChroma}; +use std::io::{Cursor, SeekFrom}; +use std::io::{Read, Seek}; +use std::path::Path; + +pub struct HeifHandler {} + +impl ImageHandler for HeifHandler { + fn maximum_size(&self) -> u64 { + HEIF_MAXIMUM_FILE_SIZE + } + + fn validate_image(&self, bits_per_pixel: u8, length: usize) -> Result<()> { + if bits_per_pixel != 8 { + return Err(Error::InvalidBitDepth); + } else if length % 3 != 0 || length % 4 != 0 { + return Err(Error::InvalidLength); + } + + Ok(()) + } + + fn handle_image(&self, path: &Path) -> Result { + let img = { + let data = self.get_data(path)?; + let handle = HeifContext::read_from_bytes(&data)?.primary_image_handle()?; + LibHeif::new().decode(&handle, ColorSpace::Rgb(RgbChroma::Rgb), None) + }?; + + let planes = img.planes(); + + if let Some(i) = planes.interleaved { + self.validate_image(i.bits_per_pixel, i.data.len())?; + + let mut reader = Cursor::new(i.data); + let mut sequence = vec![]; + let mut buffer = [0u8; 3]; // [r, g, b] + + // this is the interpolation stuff, it essentially just makes the image correct + // in regards to stretching/resolution, etc + (0..img.height()).try_for_each(|x| { + let x: usize = x.try_into()?; + let start: u64 = (i.stride * x).try_into()?; + reader.seek(SeekFrom::Start(start))?; + (0..img.width()).try_for_each(|_| { + reader.read_exact(&mut buffer)?; + sequence.extend_from_slice(&buffer); + Ok::<(), Error>(()) + })?; + Ok::<(), Error>(()) + })?; + + image::RgbImage::from_raw(img.width(), img.height(), sequence).map_or_else( + || Err(Error::RgbImageConversion), + |x| Ok(DynamicImage::ImageRgb8(x)), + ) + } else if let (Some(r), Some(g), Some(b)) = (planes.r, planes.g, planes.b) { + // This implementation is **ENTIRELY** untested, as I'm unable to source + // a HEIF image that has separate r/g/b channels, let alone r/g/b/a. + // This was hand-crafted using my best judgement, and I think it should work. + // I'm sure we'll get a GH issue opened regarding it if not - brxken128 + + self.validate_image(r.bits_per_pixel, r.data.len())?; + self.validate_image(g.bits_per_pixel, g.data.len())?; + self.validate_image(b.bits_per_pixel, b.data.len())?; + + let mut red = Cursor::new(r.data); + let mut green = Cursor::new(g.data); + let mut blue = Cursor::new(b.data); + + let (mut alpha, has_alpha) = if let Some(a) = planes.a { + self.validate_image(a.bits_per_pixel, a.data.len())?; + (Cursor::new(a.data), true) + } else { + (Cursor::new([].as_ref()), false) + }; + + let mut sequence = vec![]; + let mut buffer: [u8; 4] = [0u8; 4]; + + // this is the interpolation stuff, it essentially just makes the image correct + // in regards to stretching/resolution, etc + (0..img.height()).try_for_each(|x| { + let x: usize = x.try_into()?; + let start: u64 = (r.stride * x).try_into()?; + red.seek(SeekFrom::Start(start))?; + (0..img.width()).try_for_each(|_| { + red.read_exact(&mut buffer[0..1])?; + green.read_exact(&mut buffer[1..2])?; + blue.read_exact(&mut buffer[2..3])?; + sequence.extend_from_slice(&buffer[..3]); + + if has_alpha { + alpha.read_exact(&mut buffer[3..4])?; + sequence.extend_from_slice(&buffer[3..4]); + } + Ok::<(), Error>(()) + })?; + Ok::<(), Error>(()) + })?; + + if has_alpha { + image::RgbaImage::from_raw(img.width(), img.height(), sequence).map_or_else( + || Err(Error::RgbImageConversion), + |x| Ok(DynamicImage::ImageRgba8(x)), + ) + } else { + image::RgbImage::from_raw(img.width(), img.height(), sequence).map_or_else( + || Err(Error::RgbImageConversion), + |x| Ok(DynamicImage::ImageRgb8(x)), + ) + } + } else { + Err(Error::Unsupported) + } + } +} diff --git a/crates/images/src/lib.rs b/crates/images/src/lib.rs new file mode 100644 index 000000000..60149c0c7 --- /dev/null +++ b/crates/images/src/lib.rs @@ -0,0 +1,60 @@ +#![warn( + clippy::all, + clippy::pedantic, + clippy::correctness, + clippy::perf, + clippy::style, + clippy::suspicious, + clippy::complexity, + clippy::nursery, + clippy::unwrap_used, + unused_qualifications, + rust_2018_idioms, + clippy::expect_used, + trivial_casts, + trivial_numeric_casts, + unused_allocation, + clippy::as_conversions, + clippy::dbg_macro +)] +#![forbid(unsafe_code)] +#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] + +mod consts; +mod error; +mod formatter; +mod generic; +#[cfg(feature = "heif")] +mod heif; +mod svg; + +pub use error::{Error, Result}; +pub use formatter::format_image; +pub use image::DynamicImage; +use std::{fs, io::Read, path::Path}; + +pub trait ImageHandler { + fn maximum_size(&self) -> u64 + where + Self: Sized; // thanks vtables + + fn get_data(&self, path: &Path) -> Result> + where + Self: Sized, + { + let mut file = fs::File::open(path)?; + if file.metadata()?.len() > self.maximum_size() { + Err(Error::TooLarge) + } else { + let mut data = vec![]; + file.read_to_end(&mut data)?; + Ok(data) + } + } + + fn validate_image(&self, bits_per_pixel: u8, length: usize) -> Result<()> + where + Self: Sized; + + fn handle_image(&self, path: &Path) -> Result; +} diff --git a/crates/images/src/svg.rs b/crates/images/src/svg.rs new file mode 100644 index 000000000..975a20d5f --- /dev/null +++ b/crates/images/src/svg.rs @@ -0,0 +1,63 @@ +use std::path::Path; + +use crate::{ + consts::{SVG_MAXIMUM_FILE_SIZE, SVG_RENDER_SIZE}, + Error, ImageHandler, Result, +}; +use image::DynamicImage; +use resvg::{ + tiny_skia::{self}, + usvg, +}; +use usvg::{fontdb, TreeParsing, TreeTextToPath}; + +pub struct SvgHandler {} + +impl ImageHandler for SvgHandler { + fn maximum_size(&self) -> u64 { + SVG_MAXIMUM_FILE_SIZE + } + + fn validate_image(&self, _bits_per_pixel: u8, _length: usize) -> Result<()> { + Ok(()) + } + + fn handle_image(&self, path: &Path) -> Result { + let data = self.get_data(path)?; + let rtree = usvg::Tree::from_data(&data, &usvg::Options::default()).map(|mut tree| { + let mut fontdb = fontdb::Database::new(); + fontdb.load_system_fonts(); + tree.convert_text(&fontdb); + resvg::Tree::from_usvg(&tree) + })?; + + let size = if rtree.size.width() > rtree.size.height() { + rtree.size.to_int_size().scale_to_width(SVG_RENDER_SIZE) // make this a const + } else { + rtree.size.to_int_size().scale_to_height(SVG_RENDER_SIZE) + } + .ok_or(Error::InvalidLength)?; + + #[allow(clippy::cast_precision_loss)] + #[allow(clippy::as_conversions)] + let transform = tiny_skia::Transform::from_scale( + size.width() as f32 / rtree.size.width(), + size.height() as f32 / rtree.size.height(), + ); + + #[allow(clippy::cast_possible_truncation)] + #[allow(clippy::cast_sign_loss)] + #[allow(clippy::as_conversions)] + let Some(mut pixmap) = tiny_skia::Pixmap::new(size.width(), size.height()) else { + return Err(Error::Pixbuf); + }; + + rtree.render(transform, &mut pixmap.as_mut()); + + image::RgbaImage::from_raw(pixmap.width(), pixmap.height(), pixmap.data().into()) + .map_or_else( + || Err(Error::RgbImageConversion), + |x| Ok(DynamicImage::ImageRgba8(x)), + ) + } +} diff --git a/crates/media-metadata/src/image/orientation.rs b/crates/media-metadata/src/image/orientation.rs index 3cf0b5cff..0cf934e02 100644 --- a/crates/media-metadata/src/image/orientation.rs +++ b/crates/media-metadata/src/image/orientation.rs @@ -21,7 +21,7 @@ pub enum Orientation { impl Orientation { /// This is used for quickly sourcing [`Orientation`] data from a path, to be later used by one of the modification functions. #[allow(clippy::future_not_send)] - pub fn source_orientation(path: impl AsRef) -> Option { + pub fn from_path(path: impl AsRef) -> Option { let reader = ExifReader::from_path(path).ok()?; reader.get_tag_int(Tag::Orientation).map(Into::into) } diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml deleted file mode 100644 index 4f7046347..000000000 --- a/crates/svg/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "sd-svg" -version = "0.1.0" -authors = ["Vítor Vasconcellos "] -license = { workspace = true } -repository = { workspace = true } -edition = { workspace = true } - -[dependencies] -image = "0.24.6" -resvg = "0.35.0" -thiserror = "1.0.40" -tokio = { workspace = true, features = ["fs", "io-util"] } -tracing = "0.1.37" diff --git a/crates/svg/src/lib.rs b/crates/svg/src/lib.rs deleted file mode 100644 index aa119e587..000000000 --- a/crates/svg/src/lib.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::sync::Arc; - -use image::DynamicImage; -use resvg::{ - tiny_skia::{self, Pixmap}, - usvg, -}; -use thiserror::Error; -use tokio::task::{spawn_blocking, JoinError}; -use tracing::error; -use usvg::{fontdb, TreeParsing, TreeTextToPath}; - -type SvgResult = Result; - -const THUMB_SIZE: u32 = 512; - -// The maximum file size that an image can be in order to have a thumbnail generated. -pub const MAXIMUM_FILE_SIZE: u64 = 20 * 1024 * 1024; // 20MB - -#[derive(Error, Debug)] -pub enum SvgError { - #[error("error with usvg: {0}")] - USvg(#[from] resvg::usvg::Error), - #[error("error while loading the image (via the `image` crate): {0}")] - Image(#[from] image::ImageError), - #[error("Blocking task failed to execute to completion")] - Join(#[from] JoinError), - #[error("failed to allocate `Pixbuf`")] - Pixbuf, - #[error("there was an error while converting the image to an `RgbImage`")] - RgbImageConversion, - #[error("failed to calculate thumbnail size")] - InvalidSize, -} - -pub async fn svg_to_dynamic_image(data: Arc>) -> SvgResult { - let mut pixmap = spawn_blocking(move || -> Result { - let rtree = usvg::Tree::from_data(&data, &usvg::Options::default()).map(|mut tree| { - let mut fontdb = fontdb::Database::new(); - fontdb.load_system_fonts(); - - tree.convert_text(&fontdb); - - resvg::Tree::from_usvg(&tree) - })?; - - let size = if rtree.size.width() > rtree.size.height() { - rtree.size.to_int_size().scale_to_width(THUMB_SIZE) - } else { - rtree.size.to_int_size().scale_to_height(THUMB_SIZE) - } - .ok_or(SvgError::InvalidSize)?; - - let transform = tiny_skia::Transform::from_scale( - size.width() as f32 / rtree.size.width(), - size.height() as f32 / rtree.size.height(), - ); - - let Some(mut pixmap) = tiny_skia::Pixmap::new(size.width(), size.height()) else { - return Err(SvgError::Pixbuf); - }; - - rtree.render(transform, &mut pixmap.as_mut()); - - Ok(pixmap) - }) - .await??; - - let Some(rgb_img) = - image::RgbaImage::from_raw(pixmap.width(), pixmap.height(), pixmap.data_mut().into()) - else { - return Err(SvgError::RgbImageConversion); - }; - - Ok(DynamicImage::ImageRgba8(rgb_img)) -}