From 598ba38a9d6dce1ec98e77d885c680be3d422922 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Fri, 7 Oct 2022 05:32:45 +0800 Subject: [PATCH 1/5] merge in P2P code --- Cargo.lock | Bin 152645 -> 172917 bytes Cargo.toml | 2 + core/src/api/p2p.rs | 78 ++++ core/src/lib.rs | 12 + core/src/node/config.rs | 6 + crates/p2p/Cargo.toml | 26 +- crates/p2p/README.md | 9 + crates/p2p/docs-wip/index.md | 87 ++++ crates/p2p/examples/basic.rs | 119 +++++ crates/p2p/src/discovery/global_discovery.rs | 86 ++++ crates/p2p/src/discovery/mdns.rs | 147 ++++++ crates/p2p/src/discovery/mod.rs | 7 + crates/p2p/src/discovery/stack.rs | 32 ++ crates/p2p/src/lib.rs | 17 + crates/p2p/src/main.rs | 3 - crates/p2p/src/network_manager/mod.rs | 13 + crates/p2p/src/network_manager/nm.rs | 439 ++++++++++++++++++ crates/p2p/src/network_manager/nm_config.rs | 16 + crates/p2p/src/network_manager/nm_error.rs | 23 + crates/p2p/src/network_manager/nm_internal.rs | 256 ++++++++++ crates/p2p/src/network_manager/nm_server.rs | 239 ++++++++++ crates/p2p/src/network_manager/proto.rs | 28 ++ crates/p2p/src/p2p_manager.rs | 76 +++ crates/p2p/src/peer/mod.rs | 8 + crates/p2p/src/peer/peer.rs | 106 +++++ crates/p2p/src/peer/peer_candidate.rs | 19 + crates/p2p/src/peer/peer_metadata.rs | 94 ++++ crates/p2p/src/utils/identity.rs | 47 ++ crates/p2p/src/utils/mod.rs | 3 + crates/p2p/tunnel/Cargo.toml | 24 + crates/p2p/tunnel/Dockerfile | 29 ++ crates/p2p/tunnel/README.md | 12 + crates/p2p/tunnel/fly.toml | 21 + crates/p2p/tunnel/src/bin/generate-env.rs | 34 ++ crates/p2p/tunnel/src/main.rs | 300 ++++++++++++ crates/p2p/tunnel/utils/Cargo.toml | 15 + crates/p2p/tunnel/utils/src/client.rs | 80 ++++ crates/p2p/tunnel/utils/src/lib.rs | 10 + crates/p2p/tunnel/utils/src/peer_id.rs | 64 +++ crates/p2p/tunnel/utils/src/proto.rs | 34 ++ crates/p2p/tunnel/utils/src/quic.rs | 101 ++++ crates/p2p/tunnel/utils/src/rmp_quic.rs | 43 ++ 42 files changed, 2759 insertions(+), 6 deletions(-) create mode 100644 core/src/api/p2p.rs create mode 100644 crates/p2p/README.md create mode 100644 crates/p2p/docs-wip/index.md create mode 100644 crates/p2p/examples/basic.rs create mode 100644 crates/p2p/src/discovery/global_discovery.rs create mode 100644 crates/p2p/src/discovery/mdns.rs create mode 100644 crates/p2p/src/discovery/mod.rs create mode 100644 crates/p2p/src/discovery/stack.rs create mode 100644 crates/p2p/src/lib.rs delete mode 100644 crates/p2p/src/main.rs create mode 100644 crates/p2p/src/network_manager/mod.rs create mode 100644 crates/p2p/src/network_manager/nm.rs create mode 100644 crates/p2p/src/network_manager/nm_config.rs create mode 100644 crates/p2p/src/network_manager/nm_error.rs create mode 100644 crates/p2p/src/network_manager/nm_internal.rs create mode 100644 crates/p2p/src/network_manager/nm_server.rs create mode 100644 crates/p2p/src/network_manager/proto.rs create mode 100644 crates/p2p/src/p2p_manager.rs create mode 100644 crates/p2p/src/peer/mod.rs create mode 100644 crates/p2p/src/peer/peer.rs create mode 100644 crates/p2p/src/peer/peer_candidate.rs create mode 100644 crates/p2p/src/peer/peer_metadata.rs create mode 100644 crates/p2p/src/utils/identity.rs create mode 100644 crates/p2p/src/utils/mod.rs create mode 100644 crates/p2p/tunnel/Cargo.toml create mode 100644 crates/p2p/tunnel/Dockerfile create mode 100644 crates/p2p/tunnel/README.md create mode 100644 crates/p2p/tunnel/fly.toml create mode 100644 crates/p2p/tunnel/src/bin/generate-env.rs create mode 100644 crates/p2p/tunnel/src/main.rs create mode 100644 crates/p2p/tunnel/utils/Cargo.toml create mode 100644 crates/p2p/tunnel/utils/src/client.rs create mode 100644 crates/p2p/tunnel/utils/src/lib.rs create mode 100644 crates/p2p/tunnel/utils/src/peer_id.rs create mode 100644 crates/p2p/tunnel/utils/src/proto.rs create mode 100644 crates/p2p/tunnel/utils/src/quic.rs create mode 100644 crates/p2p/tunnel/utils/src/rmp_quic.rs diff --git a/Cargo.lock b/Cargo.lock index 6c306e8b28b5915f50c7ecdeee4135879f499e4c..212afe86fa9431c08642ec929e058b8dcd82b234 100644 GIT binary patch delta 10299 zcmb7K3y_{wd7ks{CVR<-WV4&y+_MP~ka7Qf=X{s*9cr>05k#qF6R@IOl5g0{6|wdzMWwHJhrw>nzTMgA!>P_#u0W3_l0?F@)?DuNXut)eoH^f}+Zo9qVK zP92B*`(M8AeCK`N=Y5{%Jr94n>*rtTzUM^upgtrgC++z5?jv@lUYnX6+BLMDR=MAH zab@3bnk0D|X9=T;pv=UI3a#TLPK~jNaak5+DP`P6sgsf6spgcqICGR~=2-F6F9y!l zQ0gEBN^Nx>&GVbu)WFZ*~Zk& zS!+s)Q+)48&Z!ehr8bg`S*y6swWC>2MtxS2G=S=1z;u-!mG>RP0QxQk0F7Enh zSnu+y7XlUb@2q_~^Hy!DTyg93@hRPUL=|4uCJ(;iX^NefU)b6Da&5}L?`|ST^4K!! zVws3YbEaYyX_sJ~jD{sgQIZHU z%MLHz)Kgbh27SjhWTme@OggINucoKQ#~0RIAD60pvyKO`4OYHBDsQwi!*0^c&ye%G z&pdK*|Ci-%E4ZYb8Y(EynNu>RN{NUmnn)uO%MzL8G)a*uNFi$+@=B^m>BvN!7rmJB2B2$+8C)4p-mjQ zh|^S3BQwsN6_TZ_IMKF#?xvpS+PbGEYI?YQG=Js$i5x|R#XN~pX1OVMI!d^rG>f#B zsijGV=VWn8aeoz?@LK0QQZiDxHIm8V)b7DKy?R&0_x>fh8rg`-jV(`|!9TgA%HP4i;gmXPbKi}$^_$xnWV zbom|EkU{^J4P^B+Yax6Wq;Nlf_Qu#vn6wBBMOY z6n5K%&5vxF6W&Q`>B!k6o-FHFp%Z(3jTKwC^ITuw$wdg%OHp;c7 zIm=S3S&WoM?xt9SNs-!4A*~Y!T#O`uI!dwYJkJpfo}es%m5HKdcHF;dXM4vyL=;a| zic09z3cSG;V@O@e zR1#;Dx`fln=qT2qKO{L&it{uRtaxGf`nhlKU>$A9W*<1`&zk246hj#XPewe&< zuD0{?ZvVo6k=|n9y+gkGBIzmKbmxlVnNJN4y`p(6juNf1qW5QeM0vI!hJI< z1u~b3B&FO@=bXa!$vg%QNHhlk2uc{`5vOS+j1uK`c5Aut^u^&Lo8}%9=k|0iU~Dn; z#vOB;E<4b^9Pfx~>d>!5hD1lVRobBI3rhjXASY<0xnoQx z*0fUdz;}Aj4HcqAFc;eWH8oL{`KuYGRkzFWIG z=GR}YdG4RC-P+MN1tN1~$$BGyxTA9QC0&&>tyW{WIm1NF1~T9zdx{^CATY~{|jW_2+sug zp^)flNM@!vRxCjnX{G?2nx?U%0y&cM+@wfwp)#h^JQ4zr>ypq*Iv1)Q9B@nV;LRPq z6Lvx$nkl0(EoZ=5{MbumW5;~6FHRgkckXX)*}8UqiN%8#TsZf_Z5OU-yV1B}<6o;0 zFcKLt;WCB0FrK9xyG!%j>ddBSeJ&Hggbe*Yw}1uI7>!Yrk-)kD^5V6pnMSI^!0sb8)1KegCJV+HUDb?-R1HH}k^c9^5NXoU}Kgx?zqQtMKV z9z-=T-sU#TD4;)zMDeAq16>oc0dzorDYoACh7LE3RGO*#6(2=Q)}S;X=fD}wK_UPs zpmjOTwLxQKxEjk9@Ev=D=QmM;PMQJdqr~CzR2IG88}JWpCa>wNInWZ^#((U)MC}HF z!4^=fGPG9oLOipJ6P3l%Nbm#|13dBsJvYbGFv%i|N6gTh032L++EXc6ZKXC<+?=d; z4%HJ_>^#*jU*m@5bF}`Lp0thf)%aA4$IdkH4j5vK;?|ErOH2#{k6rEH?2K$c%9LL} zTAMUeN9(BQIB78!b>AnJhx7qGo$GkunN16?pa0kMdsfI}$_~#?)+TF>;y*GunniJh zvM~rVqAYo&Bdxd)RPc-?d7MG-L@^aM&+|0W2@q0%@hJsOBy3dHRL6vwtPj_XAMQp# zEEtzkl2gzv@HMHK)aU~W)tfL-trQe3;T+YeR1`zND3&?U7|ON7PXb4xZhUs4g`P_` zX<=)YQ0I~{7mxkb#<{+y-%A!Tx6l9R&_GYQgJ`OORu{eR?)OI(auRV;A#;^UEYJg? z1hRrI<~q&5dB77*ltmULM3GY&vWf$Hoi=IEi2zQBfZ}iOSaH5O*057E#{+k_nD%=E?0{R#1W{3fr$TZG=>#Gv1Y@a{fHf2c8Yo`V z7`-07A=6GI&SV_uC#662cVH)2T#@znivM$QNU53DRxe#)fkpQ(eQS8OQ5&!OA9Pma zXz1layAoj%k)tA@IwVb_Tskf}D0(JkBvnuXJl4QW!9bzFQt~8EQd^wr8z}C+qq}eE z4o*Xt+TwS9{hFRu?hjATOf{zb1786p&LA@(ZvrBNQGmF{04%fwaI*}>r(_EL3mg_2 z9aN#g00F=-pnn(xVJ}YY3QfCr=aMQr>j_`68TDhn_X*NpJo)2J|JDs;uw(uQ#i{T2 z75jICN-!|VBoaJgoTX6`Q5Xgg2hu4Zs)E#{G&l(~k|IYdM7x$e2tWLHk%RzQxWG(`oY=~69# zN=QN95xr!vb*lEuCbS zKeUWIJpZAuc9LQLj^*Ub3vZO)czeg`Z#3}+SyX>b(M3jA&40UjY#lkgtiq;?z0ddi z<8!2MqpCrLl(Eo_nekTL7?2haBQL*01`aL&a`~Nkh$&wmo~=!`z;8w$wDYefribem zl~{gbX13lKZ^@A}Z`P+Fs9VyyK5cbF0yDuLi?99of;B_i8};Ft)-M{f!7vK>-+Xp6 z*-;5NIg+r@odk0*Tyiu>sx{aR%Tt!48G@97^F|ruU&=U-tOk~VRiYGPYmDKZo*?V| zmhEJtzs8e&&DkOHKwH?Gz!?K~4WF&>)pJR|KXM-V>8pQK9N4wbzjQwN(89-G>mwI6 z@7+Nr+sK4}@Ma=+qbT6I#9%~{K`STV1&ND;QeY`CLj#vnfSW;TK=>w-JD#{G$QvjP zSA0mU^IeaURsMIMAcM`Ji^$aj{qsJ>V&)&5pR0EZ0h+8t$pO+>^CX6Lq}0Ntf>UM^ z5Lk3Zv_~eCM9+YL122q|1S%z>Wsii`Ud;0Q{ec}&#V=o3>Fhpg1I!OQS1UKDSsx|8 zTHl7q`-A79%>WGI6yR-y3f>&r6A%(Y7L6+c+(0BlFXc{Vu>v)auoS>lp_<@SrPk#^ zy#ft(lrntS-4y8ZndEB0&Z56(&GhNS@~fX9YXJ!T;K@g=n!d3(yk_HyGAZZbYV040 z+!Z?wK@OS*#C9CwIfZsb0ivts35yKsALa? zY8F*CVH!SK*f6L4}Z zVsJc&R@iMxDkU5o$6;QguW|uYLgX?BY6*!?%jV_3|DR;d)`ecWXk%MTZCPY~&!3X@ z#eq8qo9}!F`E{ju_UU~i8Ki&#U59K#D~Bs$;X9@>RxmV_1YrhMXW>p3To%%gLl&eF z%V-SuFDb1^B;@k2tJU0ZzJYAI1vIJ^cx?V68{^D-hjjV%`4EI})#_5Vb}U3}|2u1q z#ymeL|DKu;Mfhx5mm#dSP^tX*rm30wp_*%P!K41*XUJARq)B`8ZP$}!WPKU4MK^0* z0b%s?TC%D62RD!nq`2qEzL7M|U)}&fF#!xapvbc3`}dRITUH!+ew9D4mRvHzRT5=Tf?R=QOAF>m zb1-@qTc>a}np;q39lT-014aOPAjAcr03LUc)1dhN_(9U+hps2fPv559zi7zQ%gXI6 z?m8l&Bv2Z`0Z$z?4NKv=$;=rJv7jJ<(0&DCnhIyY^hGe_fV@^Po5D$i{Cnyb8~m;b zvLif0`|?&?{6n8c)mIrPW;exr- zRH2oCb>q1rW2G`_geC%2(XGd)X5seLw!hZDcpbUu9GGgYY&0Q%TFoTf)jw{@8sD{w zT(w(aq7dQ#l*2@DAYoYoBSyIxtTJ#)9Ses_g**a!gRoR)mJ@KfAn@tYvthd9m9tQev=VC?hC7FlVrO^dMMhT*Cwk8Xegtn6z;9VhWxc zp)bI@OIy6;RnnmO5030RO-!oeWPP)GBN-!YX#U06E;WkT2r|o9YK?_i6C;yR(m70E z_)#WCUc>seF`V#NnGA75j3L}1hqZ&AHNN*+GB9i>THAOwtTt*B0V(4uP0BAEll4g% zs%@kH{c-Z6zFK{WdjL$O1rd8kQYaue-#W|DY++QJ*h1_WLlbP01&fU#t!(62pk*on z=hp)L62#enGDgMWcLfExVWpm$P_;=rl)zHWN@mnuYfMRS2Q11@udZzHLmSA7;_Ln0 zZ8OJa{peA2r?=eMRebBMoBcUcD5&Zr8EF3g1i7fE4b7sEH;s*8@&XiwwqbD=C<;fY zA)o_13pjC*Yv8a*`V6)WY#lf@aQY30mx4B$&vVFoRw2u-e;3islgG$y%Q_bXYe6=t zkr;*)a@>S$&*82*>;}d_6y7c9DUex4No-CC2(M2v90S4!?M-@ z%0ITAtbKjC{|h4UEau)4RH12|{vb$)@+XtEn*wg4Qi7k-yzmw>dWKmyZ#_jUC(3u0);sD)w=f5HD#p$h-k(Y09>MKMU;rtp@L)hJYg2d6e0_R$D9Ia z97aQt&R}f9Jj4JCX_f}}a>vxPovhc#17T5%IrH!WHwfuPiI zX#)epbka%~L>Q#N7I4s6X!w9C6(}BPrpAN_OgLndb*Ok9T%=q<`Upc|Ji&SCe3Yy% zx#r3x)WmQ5Gt%cbZ6;UlhB==|>oAaTdDv;V^#+5+z~N$HVq|SlR2E|x+?Qc83)N>G zrx|FdGXC=yqj8 zNo8u#DAD4=4qMcRmZ5L0dqrGlCmZ2ef{We!$Op*l$m#%_yc!M5@##adN@?@!A0(Gm znwNZ-tZMIXX@D|K!M!Gj>*F(7`P{`lQw?Xoakp#>cq8-Ygv za*L;j(P}UR#Fzklk|N|lb-=no7Q&^3N2(1h`G7tNR0{IcVH#^gD)dY&@`SuRNj5f5 z+)eIiZ-ZJYwtcM6|HT1vZ>5 zFxd=_as=?sF&&4o2SW*FGCDnl+6gW+S`TbS3y{yyx&Uyu--m|(_+_NCn7w+n-+dq1 z?2k9mLbl#ZE?K*_Wz#P%GFC0;@S{LJiw0?t7!fQijN<^x@EuTM&S6%GqzSWCoDcvb zV4230DgyD~*aHrizMMSy>mDVY&Dta6o9%${Mb4c+Uc=}U2MRD_K+1+PwjvD@41O*A z1We#SF(jbY29y+-mO$XkK3)x<**wf$tYVKh1!r?7`|2IfS=fbUv zpL~BqG5*7`5y&tHR+huvWDqC}bFkRPu=Rk-h*g-(P}G&e3JrS7$_7h-QXy# z_1FD~bal0g3HK>(d1ia1QTOsuGDupt2?8ZTN~#!H<19D=rp~a9Fgpi5!LSXg65cT& z1&S@lF3B`UEL|xCBzeFqR znnN2Z&#!8mzOhz}uU4ZN(WZvyKqq)X7(vANkA@kkFomILSSf}eS&V}lE&+7m%9wm$ zl83PYcbM@RzfF;W!9_7DcYkp|>A9$qws)N|MqWAxeHi2oV1P@duzoQ7#jH>>Ohs_w zA)=6ZIEI6PF%sAaR5lEM^wT(r67XOgFLFV=4EhHLf%~J?O0?Yfz6T~Wpi^Kf4x<7; z<{5>~QVvQv!9WyD6|_-_K#OonO5%U08iu(I^ck=bjixM<<5{J{Z%He?B=J#N+1UCA z9#2~7B+2bbWpnvo%wOJAY4=0C(vEM0Ll^!9RvBzgCzbm=N&+ia38=Gsu89Jh>@WVBbyAT z$TY;O!|{f2$i?9(&N_eR?nF;sBjVoM|&jF0vtpjKXCX5_zCL=13&-yJ(aED`Wsu9-tp$j*6!ltpIy7G zrJh3j!w?2aCxIQ5LDHibU{hkUh7%^SW01fih4{vh9`2rkJ%+Rcv86bulEf%^m$XLR zKX~%|j@E?T7JvMhe0>620ltok;X9-$hZTxe2}Yvf{{b67<$?ZabU4<586lXWGzk=M N7+hf5=gju*{{~f#5qkgt delta 982 zcmZ9Ie@L8l9LM>5@A15|ImMl4dZOMrr!!g4^S$SJ?w)B=x^2w0HFvdIDcC(fE?3QT zG_z>RaHGUObbX`mg3Z}5FoJIKO+H0oskxdhvLBe$A3+VIGOd9St3fv`g8urvKJWJ% z-h4i5m$PQayq8X9iMHIEASyR4xx^O*wSV6xUk4E)ld(dD#Qk3dBc8OEbl5$5U=mbYc|5hdodFMCVJT`Fv^us+c?` ziRsrj=du1tKcC-E;=()mXEtAz-s-Z3?UB^e>1Rn;rLvse6X@CmChjxR+2X{d(_QxPcVtKnOct7-H?GtFdM;w7c-xdm-+}We5H!#HBvCLD;!pIP>37PLlf}C z1z6%e7om`C-*g{tz6f96h9xODnSnn~K>hk>a^D6w9zbYz;Tw~1$%}igKrQfreWVyu zb5KoO9khc_wS$jIC-QOOD&$GHG7r8YK0X5h^en@$g!9)SmXDu%$Tt#xl0)tZ@?wP; z?7`_g>`{m>56{L(nS}2(lEZ*E_K}?u@2(>UcW9OtQNl?%X{n}Wo4OuN(1fLjE!&Kk zrmE_O9kq<4W6Ns9RBV|V)UqO`Y&q0S7_8&50{2r#+MnriSTGjgg9nI*9eF1kgK?s} zc>8#`WxmmUakND diff --git a/Cargo.toml b/Cargo.toml index 2d17a662d..21fc6d3ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,8 @@ members = [ "core", "crates/*", + "crates/p2p/tunnel", + "crates/p2p/tunnel/utils", "crates/sync/example/src-tauri", "apps/desktop/src-tauri", "apps/mobile/rust", diff --git a/core/src/api/p2p.rs b/core/src/api/p2p.rs new file mode 100644 index 000000000..71cd2420f --- /dev/null +++ b/core/src/api/p2p.rs @@ -0,0 +1,78 @@ +// use std::collections::HashMap; + +// use p2p::PeerId; +// use rspc::Type; +// use serde::Deserialize; + +// use super::{LibraryArgs, RouterBuilder}; + +// #[derive(Type, Deserialize)] +// pub struct AcceptPairingRequestArgs { +// pub peer_id: PeerId, +// pub preshared_key: String, +// } + +// pub(crate) fn mount() -> RouterBuilder { +// RouterBuilder::new() +// .query("getNodes", |ctx, arg: LibraryArgs<()>| async move { +// let (_, library) = arg.get_library(&ctx).await?; + +// Ok( +// library.db.node().find_many(vec![]).exec().await?, // TODO: Make this work +// // .into_iter() +// // .filter_map(|v| { +// // if v.id == ctx.node_local_id { +// // None +// // } else { +// // Some(v.into()) +// // } +// // }) +// // .collect::>() +// ) +// }) +// .query("connectedPeers", |ctx, _: ()| async move { +// ctx.p2p +// .nm +// .connected_peers() +// .into_iter() +// .map(|(_, v)| (v.id, v.metadata)) +// .collect::>() +// }) +// .query("discoveredPeers", |ctx, _: ()| async move { +// ctx.p2p +// .nm +// .discovered_peers() +// .into_iter() +// // TODO: Make this better +// .map(|(_, v)| v) +// .collect::>() +// }) +// .mutation("pairNode", |ctx, arg: LibraryArgs| async move { +// let (peer_id, library) = arg.get_library(&ctx).await?; + +// let preshared_key = ctx.p2p.pair(&library, peer_id).await.unwrap(); + +// // TODO: These aren't library queries so they can't be invalidated with the current system. We can fix this with the normalised cache! +// // invalidate_query!(ctx, "p2p.discoveredPeers": (), ()); +// // invalidate_query!(ctx, "p2p.connectedPeers": (), ()); + +// Ok(preshared_key) +// }) +// .mutation( +// "unpairNode", +// |_, _: LibraryArgs| async move { todo!() }, +// ) +// .mutation( +// "acceptPairingRequest", +// |ctx, arg: AcceptPairingRequestArgs| async move { +// ctx.p2p +// .pairing_requests +// .lock() +// .unwrap() +// .remove(&arg.peer_id) +// .unwrap() +// .send(Ok(arg.preshared_key)) +// .unwrap(); // TODO: Remove unwrap +// }, +// ) +// } diff --git a/core/src/lib.rs b/core/src/lib.rs index a8c3264f2..1efe8bc30 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -53,6 +53,12 @@ impl Node { // dbg!(get_object_kind_from_extension("png")); + let (non_blocking, _guard) = tracing_appender::non_blocking(rolling::daily( + Path::new(&data_dir).join("logs"), + "log", + )); + // TODO: Make logs automatically delete after x time https://github.com/tokio-rs/tracing/pull/2169 + tracing_subscriber::registry() .with( EnvFilter::from_default_env() @@ -74,6 +80,12 @@ impl Node { ), ) .with(fmt::layer().with_filter(CONSOLE_LOG_FILTER)) + // .with( + // Layer::default() + // .with_writer(non_blocking) + // .with_ansi(false) + // .with_filter(LevelFilter::DEBUG), + // ) .init(); let event_bus = broadcast::channel(1024); diff --git a/core/src/node/config.rs b/core/src/node/config.rs index 41d158e9c..21d61ef47 100644 --- a/core/src/node/config.rs +++ b/core/src/node/config.rs @@ -40,6 +40,12 @@ pub struct NodeConfig { pub name: String, // the port this node uses for peer to peer communication. By default a random free port will be chosen each time the application is started. pub p2p_port: Option, + // /// The P2P identity public key + // pub p2p_cert: Vec, + // /// The P2P identity private key + // pub p2p_key: Vec, + // /// The address of the Spacetunnel discovery service being used. + // pub spacetunnel_addr: Option, } #[derive(Error, Debug)] diff --git a/crates/p2p/Cargo.toml b/crates/p2p/Cargo.toml index e59d768f9..1f064147d 100644 --- a/crates/p2p/Cargo.toml +++ b/crates/p2p/Cargo.toml @@ -1,8 +1,28 @@ [package] -name = "sd-p2p" +name = "p2p" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] +tunnel-utils = { path = "./tunnel/utils" } + +dashmap = "5.3.4" +rcgen = "0.9.2" +rustls = "0.20.6" +tokio = { version = "1.19.2", features = ["macros", "sync"] } +if-watch = "1.1.1" +thiserror = "1.0.31" +mdns-sd = "0.5.5" +quinn = "0.8.3" +futures-util = "0.3.21" +ts-rs = "6.2.0" +serde = { version = "1.0.138", features = ["derive"] } +bip39 = { version = "1.0.1", features = ["rand"] } +rmp-serde = "1.1.0" +spake2 = "0.3.1" +ctrlc = { version = "3.2.2", features = ["termination"] } +tracing = "0.1.35" +specta = "0.0.2" + +[dev-dependencies] +tokio = { version = "1.19.2", features = ["rt-multi-thread"] } diff --git a/crates/p2p/README.md b/crates/p2p/README.md new file mode 100644 index 000000000..fd14d11a8 --- /dev/null +++ b/crates/p2p/README.md @@ -0,0 +1,9 @@ +# P2P + +This is the P2P library which powers Spacedrive's P2P functionality. + +## Viewing debug logs + +```bash +RUST_LOG="p2p=debug" cargo run -p server +``` \ No newline at end of file diff --git a/crates/p2p/docs-wip/index.md b/crates/p2p/docs-wip/index.md new file mode 100644 index 000000000..861d4834c --- /dev/null +++ b/crates/p2p/docs-wip/index.md @@ -0,0 +1,87 @@ +# Spacedrive Peer to Peer + +This document outlines the peer to peer protocol used by the Spacedrive desktop application. This document is designed to outlined how the system works and also discuss the security and decision making behind the system. + +## Concepts + + - **Peer** - TODO + +### P2PManager + +The peer to peer library is designed to be general purpose. This means none of the Spacedrive code is directly tied into the peer to peer library. This makes for a very nice separation of concerns but also introduces the requirement for an abstraction to sit above the P2P library so it can properly make decisions with the data that Spacedrive holds. This is where the `P2PManager` trait comes in. You must implement the `P2PManager` trait in your application code and then the peer to peer system will call various hooks, allowing your system to react to various events. + +The `P2PManager` is implemented as a Rust trait which works very well for allowing the application to hook into the peer to peer system, however, in Rust async functions are not properly supported in traits. This is works very well when combined with an `tokio::mpsc::unbound_channel()` implemented in your application, so that you can run async code in response to a specific situtation. The only expect to syncronus methods is the `peer_paired` method as we want to be sure that the peer was properly saved into the database on both sides. This method returns a `Pin>>>`. + +It's important we maintain a good separation between the `P2PManager` and the application which is using it. This led to to the decision to make the P2P system focusing on getting a stream of bytes (`Vec`) between peers. This means the P2P system does not enforce a specific serialization method for your data, this gives you the choice in your application layer to choice whatever works best for the type of data you are going to be sending. This also means you are responsible for data compression. We use the [`rmp_serde`](https://crates.io/crates/rmp-serde) crate (which uses [msgpack](https://msgpack.org/index.html)) internally to send data between clients, and would reccomend it in your application however, this decision is entirely up to you. + +### Identity keypair + +Unpon installing Spacedrive your application with generate a public and private key paired which is called the identity keypair. These certificates facilitate secure communication and identify the client to other peers. + +### Peer ID's + +Each peer has a unique identifier which is called a peer id. This identifier is derived from a [SHA-1](https://en.wikipedia.org/wiki/SHA-1) hash of the identity keypair's public key. + +**Note: We might change from SHA-1 to SHA-255 or SHA-256 in the near future. The current limitation is the maximum size of a DNS TXT record. No known SHA-1 collisions exist for certificates but given SHA-1 has been broken it would be preferable to use something more secure.** + +## Discovery + +Discovery is the first phase of the peer to peer process. The goal of discovery is to determine a list of other peers which we could potentially pair to. This phase is made up of multiple different protocol and the result of all of the systems are combined and returned to the application. + +### LAN + +To discovery other machines running Spacedrive over your local network, we make use of [mDNS](). mDNS is a protocol which transmits DNS packets using multicast UDP which allows a DNS record to be published to your local network and read by other devices on the network. This system is used commonly by other systems with similar goals such as [libp2p]() and [Apple's Airplay](). We are using [DNS-SD]() which makes use of DNS SRV and TXT records to advertise information about the current peer. + +Spacedrive advertise a SRV record that looks like: + +_{peer_id}_spacedrive_._udp_.local. 86400 IN SRV 10 5 5223 server.example.com. + +This system will continue to passively discover clients while Spacedrive is running. + +### Global Discovery + +The global discovery system works in a different way. TODO + +#### Announcement + +TODO: Discuss proto + security + +```rust +Message::ClientAnnouncement { peer_id, addresses: vec!["192.168.0.1".to_string(), "1.1.1.1".to_string()] } +``` + +#### Query + +TODO + +```rust +Message::QueryClientAnnouncement(vec![peer_id, peer_id2]); +``` + + + + + + + + + + + +## General Overview + +This system is designed on top of the following main technologies: + - [QUIC]() - A tcp-like protocol built on top of UDP. QUIC also supports [TLS 1.3]() for encryption and pro + + + + + +## Pairing + +TODO + +# External Resources + + - TODO: Magic Wormhole talk + - TODO: Syncthing spec diff --git a/crates/p2p/examples/basic.rs b/crates/p2p/examples/basic.rs new file mode 100644 index 000000000..e5a2df5ba --- /dev/null +++ b/crates/p2p/examples/basic.rs @@ -0,0 +1,119 @@ +use std::{env, time::Duration}; + +use p2p::{Identity, NetworkManager, NetworkManagerConfig, P2PManager, Peer, PeerId, PeerMetadata}; +use quinn::{RecvStream, SendStream}; +use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; + +#[derive(Debug, Clone)] +pub enum P2PEvent { + PeerDiscovered(PeerId), + PeerExpired(PeerId), + PeerConnected(PeerId), + PeerDisconnected(PeerId), +} + +// SdP2PManager is part of your application and allows you to hook into the behavior of the P2PManager. +#[derive(Clone)] +pub struct SdP2PManager { + // peer_name is the name of the current peer. In a normal application this would be a display name set by the end user. + peer_name: String, + /// event_channel is used to send events to the application + event_channel: UnboundedSender, +} + +impl P2PManager for SdP2PManager { + const APPLICATION_NAME: &'static str = "spacedrive"; + + fn get_metadata(&self) -> PeerMetadata { + PeerMetadata { + name: self.peer_name.clone(), + version: Some(env!("CARGO_PKG_VERSION").into()), + } + } + + fn peer_discovered(&self, nm: &NetworkManager, peer_id: &PeerId) { + self.event_channel + .send(P2PEvent::PeerDiscovered(peer_id.clone())); + nm.add_known_peer(peer_id.clone()); // Be careful doing this in a production application because it will just trust all clients + } + + fn peer_expired(&self, nm: &NetworkManager, peer_id: PeerId) { + self.event_channel.send(P2PEvent::PeerExpired(peer_id)); + } + + fn peer_connected(&self, nm: &NetworkManager, peer_id: PeerId) { + self.event_channel.send(P2PEvent::PeerConnected(peer_id)); + } + + fn peer_disconnected(&self, nm: &NetworkManager, peer_id: PeerId) { + self.event_channel.send(P2PEvent::PeerDisconnected(peer_id)); + } + + fn accept_stream(&self, peer: &Peer, (mut tx, mut rx): (SendStream, RecvStream)) { + let peer = peer.clone(); + tokio::spawn(async move { + let msg = rx.read_chunk(1024, true).await.unwrap().unwrap(); + println!("Received '{:?}' from peer '{}'", msg.bytes, peer.id); + tx.write(b"Pong").await.unwrap(); + }); + } +} + +#[tokio::main] +async fn main() { + let identity = Identity::new().unwrap(); + let peer_id = PeerId::from_cert(&identity.clone().into_rustls().0); + let mut event_channel = unbounded_channel(); + let nm = NetworkManager::new( + identity, + SdP2PManager { + peer_name: format!( + "{}-{}", + peer_id + .to_string() + .chars() + .into_iter() + .take(5) + .collect::(), + env::consts::OS + ), + event_channel: event_channel.0, + }, + NetworkManagerConfig { + known_peers: Default::default(), + listen_port: None, + }, + ) + .await + .unwrap(); + println!( + "Peer '{}' listening on: {:?}", + nm.peer_id(), + nm.listen_addr() + ); + + loop { + tokio::select! { + event = event_channel.1.recv() => { + if let Some(event) = event { + println!("{:?}", event); + + match event { + P2PEvent::PeerConnected(peer_id) => { + nm.send_to(peer_id, b"Ping on Connection").await.unwrap(); + } + _ => {} + } + } + } + _ = tokio::time::sleep(Duration::from_secs(5)) => { + println!(""); + for (peer_id, peer) in nm.connected_peers() { + println!("Sending ping to '{:?}'", peer_id); + let resp = nm.send_to(peer_id, b"Ping").await.unwrap(); + println!("Peer '{}' responded to ping with message '{:?}'", peer.id, resp.bytes); + } + } + }; + } +} diff --git a/crates/p2p/src/discovery/global_discovery.rs b/crates/p2p/src/discovery/global_discovery.rs new file mode 100644 index 000000000..5dc722132 --- /dev/null +++ b/crates/p2p/src/discovery/global_discovery.rs @@ -0,0 +1,86 @@ +/// The functions in this file are predominantly useless in the current system. This will be fixed in a future PR's. +use std::sync::Arc; + +use tracing::warn; +use tunnel_utils::{Client, Message}; + +use crate::{NetworkManager, NetworkManagerError, P2PManager}; + +/// GlobalDiscovery is the discovery system for discovering devices which are not on the same local network as you. +/// This is done through the Spacetunnel server hosted by Spacedrive Inc. it could however be hosted by anyone and documentation for doing so will be released in the future once we are confident in the current design. +pub(crate) struct GlobalDiscovery { + nm: Arc>, + client: Client, +} + +impl GlobalDiscovery { + pub fn init(nm: &Arc>) -> Result { + tracing::debug!("Starting mdns discovery service"); + + if let Some(url) = &nm.spacetunnel_url { + Ok(Self { + nm: nm.clone(), + client: Client::new(url.clone(), nm.endpoint.clone(), nm.identity.clone()), + }) + } else { + panic!("Why no Spacetunnel? (~_^)"); + // TODO: Refactor to allow the system to work without Spacetunnel enabled. + } + } + + pub async fn poll(&self) { + tracing::debug!("Polling global discovery service"); + + // TODO: Allow the tunnel server to accept a list of PeerId's instead of doing heaps of requests + let peers = self.nm.known_peers.iter().map(|v| v.clone()).collect(); + match self + .client + .send_message(Message::QueryClientAnnouncement(peers)) + .await + { + Ok(_) => { + tracing::debug!("Successfully sent query announcement"); + } + Err(err) => { + warn!( + "[TODO: WIP FEATURE REPORTED ERROR] Spacetunnel failed lookup peers with error: {:?}", + err + ); + // TODO: Handle error when this is implemented. + } + } + + // TODO: Handle error from discovery service + // self.nm.discovered_peers.insert(key, value); // TODO: make this work + // TODO: Open connection to peers if they are not already connected + } + + pub async fn register(&self) { + // TODO: Send the metadata along with the discovery payload + // TODO: Only do announcement if data has changed or it's been over 10 minutes since last packet + + let announcement = Message::ClientAnnouncement { + peer_id: self.nm.peer_id.clone(), + addresses: self.nm.lan_addrs.iter().map(|v| v.to_string()).collect(), // TODO: Include STUN address in this list + }; + tracing::debug!( + "Registering self with global discovery service: {:?}", + announcement + ); + + match self.client.send_message(announcement).await { + Ok(_) => tracing::debug!("Successfully registered with global discovery service"), + Err(err) => { + warn!("[TODO: WIP FEATURE REPORTED ERROR] Spacetunnel failed announcement with error: {:?}", err); + // TODO: Handle error when this is implemented. + } + } + + // TODO: Handle error from discovery service + } + + pub(crate) fn shutdown(&self) { + tracing::debug!("Shutting down gloval discovery service"); + // TODO: Remove the announcement from the tunnel + } +} diff --git a/crates/p2p/src/discovery/mdns.rs b/crates/p2p/src/discovery/mdns.rs new file mode 100644 index 000000000..0c72a1256 --- /dev/null +++ b/crates/p2p/src/discovery/mdns.rs @@ -0,0 +1,147 @@ +use std::{net::Ipv4Addr, sync::Arc}; + +use mdns_sd::{Receiver, ServiceDaemon, ServiceEvent, ServiceInfo}; +use tracing::warn; +use tunnel_utils::PeerId; + +use crate::{NetworkManager, NetworkManagerError, P2PManager, PeerCandidate, PeerMetadata}; + +/// MDNS is the discovery system used for over local networks. It makes use of Multicast DNS (mDNS) to discover peers. +/// It should also conforms to the mDNS SD specification. +pub(crate) struct Mdns { + nm: Arc>, + mdns: ServiceDaemon, + browser: Receiver, + service_type: String, +} + +impl Mdns { + pub fn init(nm: &Arc>) -> Result { + tracing::debug!("Starting mdns discovery service"); + let mdns = ServiceDaemon::new()?; + let service_type = format!("_{}._udp.local.", TP2PManager::APPLICATION_NAME); + + Ok(Self { + nm: nm.clone(), + browser: mdns.browse(&service_type)?, + mdns, + service_type, + }) + } + + pub async fn handle_mdns_event(&self) { + match self.browser.recv_async().await { + Ok(event) => { + tracing::debug!("Handling incoming mdns event: {:?}", event); + match event { + ServiceEvent::SearchStarted(_) => {} + ServiceEvent::ServiceFound(_, _) => {} + ServiceEvent::ServiceResolved(info) => { + let raw_peer_id = info + .get_fullname() + .replace(&format!(".{}", self.service_type), ""); + match PeerId::from_string(raw_peer_id.clone()) { + Ok(peer_id) => { + // Prevent discovery of the current peer. + if peer_id == self.nm.peer_id { + return; + } + + let peer = PeerCandidate { + id: peer_id.clone(), + metadata: PeerMetadata::from_hashmap( + &peer_id, + info.get_properties(), + ), + addresses: info.get_addresses().iter().copied().collect(), + port: info.get_port(), + }; + + self.nm.add_discovered_peer(peer); + } + Err(_) => { + warn!( + "resolved peer advertising itself with an invalid peer_id '{}'", + raw_peer_id + ); + } + } + } + ServiceEvent::ServiceRemoved(_, fullname) => { + let raw_peer_id = fullname.replace(&format!(".{}", self.service_type), ""); + match PeerId::from_string(raw_peer_id.clone()) { + Ok(peer_id) => { + // Prevent discovery of the current peer. + if peer_id == self.nm.peer_id { + return; + } + + self.nm.remove_discovered_peer(peer_id); + } + Err(_) => { + warn!( + "resolved peer advertising itself with an invalid peer_id '{}'", + raw_peer_id + ); + } + } + } + ServiceEvent::SearchStopped(_) => {} + } + } + Err(err) => { + tracing::warn!( + "Error receiving MDNS event as the ServiceDaemon has been shut down: {:?}", + err + ); + tracing::info!("Error receiving MDNS event as the ServiceDaemon has been shut down. Local discovery has been disabled, please restart your app to re-enable local discovery!"); + } + } + } + + pub async fn register(&self) { + let peer_id_str = &self.nm.peer_id.to_string(); + let service_info = ServiceInfo::new( + &self.service_type, + peer_id_str, + &format!("{}.", peer_id_str), + &(self + .nm + .lan_addrs + .iter() + .map(|v| *v) + .collect::>())[..], + self.nm.listen_addr.port(), + Some(self.nm.manager.get_metadata().to_hashmap()), + ); + tracing::debug!("Registering mdns service entry: {:?}", service_info); + + match service_info { + Ok(service_info) => match self.mdns.register(service_info) { + Ok(_) => {} + Err(err) => { + warn!("failed to register mdns service: {}", err); + } + }, + Err(err) => { + warn!("failed to register mdns service: {}", err); + } + } + } + + /// shutdown shuts down the MDNS service. This will advertise the current peer as unavailable to the rest of the network. + pub(crate) fn shutdown(&self) { + tracing::debug!("Shutting down mdns discovery service"); + + // The panics caused by `.expect` are acceptable here because they are run during shutdown where nothing can be done if they were to fail. + self.mdns + .unregister(&format!("{}.{}", self.nm.peer_id, self.service_type)) + .expect("Error unregistering the mDNS service") + .recv() + .expect("Error unregistering the mDNS service"); + + self.mdns + .shutdown() + .expect("Error shutting down mDNS service"); + } +} diff --git a/crates/p2p/src/discovery/mod.rs b/crates/p2p/src/discovery/mod.rs new file mode 100644 index 000000000..ff33f241e --- /dev/null +++ b/crates/p2p/src/discovery/mod.rs @@ -0,0 +1,7 @@ +mod global_discovery; +mod mdns; +mod stack; + +pub(crate) use global_discovery::*; +pub(crate) use mdns::*; +pub(crate) use stack::*; diff --git a/crates/p2p/src/discovery/stack.rs b/crates/p2p/src/discovery/stack.rs new file mode 100644 index 000000000..451dfdfe1 --- /dev/null +++ b/crates/p2p/src/discovery/stack.rs @@ -0,0 +1,32 @@ +use std::sync::Arc; + +use crate::{GlobalDiscovery, Mdns, NetworkManager, NetworkManagerError, P2PManager}; + +/// Represents a stack of all of the different discovery mechanisms that are used by the P2P library. +/// Traits are not used due to Rust's current lack of proper support for async traits. +pub(crate) struct DiscoveryStack { + pub mdns: Arc>, + pub global: Arc>, +} + +impl DiscoveryStack { + pub async fn new(nm: &Arc>) -> Result { + let global = Arc::new(GlobalDiscovery::init(nm)?); + global.poll().await; + + Ok(Self { + mdns: Arc::new(Mdns::init(nm)?), + global, + }) + } + + pub async fn register(&self) { + self.mdns.register().await; + self.global.register().await; + } + + pub fn shutdown(&self) { + self.mdns.shutdown(); + self.global.shutdown(); + } +} diff --git a/crates/p2p/src/lib.rs b/crates/p2p/src/lib.rs new file mode 100644 index 000000000..21820a660 --- /dev/null +++ b/crates/p2p/src/lib.rs @@ -0,0 +1,17 @@ +mod discovery; +mod network_manager; +mod p2p_manager; +mod peer; +mod utils; + +pub(crate) use discovery::*; +pub use network_manager::*; +pub use p2p_manager::*; +pub use peer::*; +pub use tunnel_utils::{read_value, write_value, PeerId}; +pub use utils::*; + +/// We reexport some types from `quinn` to avoid the user needing to add `quinn` and keep its version in sync with the p2p library. +pub mod quinn { + pub use quinn::{RecvStream, SendStream}; +} diff --git a/crates/p2p/src/main.rs b/crates/p2p/src/main.rs deleted file mode 100644 index a30eb952c..000000000 --- a/crates/p2p/src/main.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/crates/p2p/src/network_manager/mod.rs b/crates/p2p/src/network_manager/mod.rs new file mode 100644 index 000000000..a9e2d89fa --- /dev/null +++ b/crates/p2p/src/network_manager/mod.rs @@ -0,0 +1,13 @@ +mod nm; +mod nm_config; +mod nm_error; +mod nm_internal; +mod nm_server; +mod proto; + +pub use nm::*; +pub use nm_config::*; +pub use nm_error::*; +pub use nm_internal::*; +pub use nm_server::*; +pub use proto::*; diff --git a/crates/p2p/src/network_manager/nm.rs b/crates/p2p/src/network_manager/nm.rs new file mode 100644 index 000000000..9d3e84fe8 --- /dev/null +++ b/crates/p2p/src/network_manager/nm.rs @@ -0,0 +1,439 @@ +use std::{ + collections::HashMap, + net::{Ipv4Addr, SocketAddr}, + sync::Arc, + time::Duration, +}; + +use bip39::{Language, Mnemonic}; +use dashmap::{DashMap, DashSet}; +use futures_util::future::join_all; +use quinn::{Chunk, Endpoint, NewConnection, RecvStream, SendStream, ServerConfig}; +use rustls::{Certificate, PrivateKey}; +use spake2::{Ed25519Group, Password, Spake2}; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; +use tracing::{debug, error, warn}; +use tunnel_utils::{quic, write_value, PeerId, UtilError}; + +use crate::{ + ConnectError, ConnectionEstablishmentPayload, ConnectionType, Identity, NetworkManagerConfig, + NetworkManagerError, NetworkManagerInternalEvent, P2PManager, PairingParticipantType, + PairingPayload, Peer, PeerCandidate, +}; + +/// Is the core of the P2P Library. It manages listening for and creating P2P network connections and also provides a nice API for the application embedding this library to interface with. +pub struct NetworkManager { + /// PeerId is the unique identifier of the current peer. + pub(crate) peer_id: PeerId, + /// identity is the TLS identity of the current peer. + pub(crate) identity: (Certificate, PrivateKey), + /// known_peers contains a list of all peers which are known to the network. These will be automatically connected if found. + /// We store these so when making a request to the global discovery server we know who to lookup. + pub(crate) known_peers: DashSet, + /// discovered_peers contains a list of all peers which have been discovered by any discovery mechanism. + discovered_peers: DashMap, + /// connected_peers + connected_peers: DashMap>, + /// lan_addrs contains a list of all local addresses which exists on the current peer. + pub(crate) lan_addrs: DashSet, + /// listen_addr contains the address which the current peer is listening on. This peer will listening on IPv4 and IPv6 on a random port if none was provided at startup. + pub(crate) listen_addr: SocketAddr, + /// manager is a trait which implements P2PManager and is used so the NetworkManager can interact with the host application. + pub(crate) manager: TP2PManager, + /// endpoint is the QUIC endpoint that is used to send and receive network traffic between peers. + pub(crate) endpoint: Endpoint, + /// spacetunnel_server is the URL used to lookup information about the Spacetunnel server to establish a connection with. + pub(crate) spacetunnel_url: Option, + /// internal_channel is a channel which is used to communicate with the main internal event loop. + internal_channel: mpsc::UnboundedSender, +} + +impl NetworkManager { + /// Initalise a new network manager for your application. + /// Be aware this will create a separate thread running the P2P manager event loop so this should really only be run once per application. + pub async fn new( + identity: Identity, + manager: TP2PManager, + config: NetworkManagerConfig, + ) -> Result, NetworkManagerError> { + debug!("Creating new NetworkManager..."); + + if !TP2PManager::APPLICATION_NAME + .chars() + .all(char::is_alphanumeric) + { + return Err(NetworkManagerError::InvalidAppName); + } + + let identity = identity.into_rustls(); + let (endpoint, incoming) = Endpoint::server( + ServerConfig::with_crypto(Arc::new(quic::server_config( + vec![identity.0.clone()], + identity.1.clone(), + )?)), + format!("[::]:{}", config.listen_port.unwrap_or(0)) + .parse() + .expect("unreachable error: invalid connection address. Please report if you encounter this error!"), + ) + .map_err(NetworkManagerError::Server)?; + + let internal_channel = mpsc::unbounded_channel(); + let this = Arc::new(Self { + peer_id: PeerId::from_cert(&identity.0), + identity, + known_peers: config.known_peers.into_iter().collect(), + discovered_peers: DashMap::new(), + connected_peers: DashMap::new(), + lan_addrs: DashSet::new(), + listen_addr: endpoint.local_addr().map_err(NetworkManagerError::Server)?, + manager, + endpoint, + spacetunnel_url: config.spacetunnel_url, + internal_channel: internal_channel.0, + }); + Self::event_loop(&this, incoming, internal_channel.1).await?; + Ok(this) + } + + pub(crate) fn add_discovered_peer(&self, peer: PeerCandidate) { + debug!("Discovered peer: {:?}", peer); + self.discovered_peers.insert(peer.id.clone(), peer.clone()); + self.manager.peer_discovered(self, &peer.id); + + if self.known_peers.contains(&peer.id) { + match self + .internal_channel + .send(NetworkManagerInternalEvent::Connect(peer)) + { + Ok(_) => {} + Err(err) => { + error!("Failed to send on internal_channel: {:?}", err); + } + } + } + } + + pub(crate) fn remove_discovered_peer(&self, peer_id: PeerId) { + debug!("Removing discovered peer: {:?}", peer_id); + self.discovered_peers.remove(&peer_id); + self.manager.peer_expired(self, peer_id); + } + + pub(crate) fn get_discovered_peer(&self, peer_id: &PeerId) -> Option { + self.discovered_peers.get(peer_id).map(|v| v.clone()) + } + + pub(crate) fn is_peer_connected(&self, peer_id: &PeerId) -> bool { + self.connected_peers.contains_key(peer_id) + } + + pub(crate) fn add_connected_peer(&self, peer: Peer) { + debug!("Connected with peer: {:?}", peer); + let peer_id = peer.id.clone(); + self.connected_peers.insert(peer.id.clone(), peer); + self.manager.peer_connected(self, peer_id); + } + + pub(crate) fn remove_connected_peer(&self, peer_id: PeerId) { + debug!("Disconnected with peer: {:?}", peer_id); + self.connected_peers.remove(&peer_id); + self.manager.peer_disconnected(self, peer_id); + } + + /// returns the peer ID of the current peer. These are unique identifier derived from the peers public key. + pub fn peer_id(&self) -> PeerId { + self.peer_id.clone() + } + + /// returns the address that the NetworkManager will listen on for incoming connections from other peers. + pub fn listen_addr(&self) -> SocketAddr { + self.listen_addr + } + + /// adds a new peer to the known peers list. This will cause the NetworkManager to attempt to connect to the peer if it is discovered. + pub fn add_known_peer(&self, peer_id: PeerId) { + debug!("Adding '{:?}' as a known peer", peer_id); + self.known_peers.insert(peer_id.clone()); + + match self + .internal_channel + .send(NetworkManagerInternalEvent::NewKnownPeer(peer_id)) + { + Ok(_) => {} + Err(err) => { + error!("Failed to send on internal_channel: {:?}", err); + } + } + } + + /// send a single message to a peer and await a single response. This is good for quick one-off communications but any longer term communication should be done with a stream. + /// TODO: Error type + pub async fn send_to(&self, peer_id: PeerId, data: &[u8]) -> Result { + debug!("Sending message to '{:?}'", peer_id); + + tokio::time::sleep(Duration::from_millis(500)).await; // TODO: Fix this issue. This workaround is because DashMap is eventually consistent + + let peer = self + .connected_peers + .get(&peer_id) + .ok_or(NMError::PeerNotConnected)? + .value() + .clone(); + let (mut tx, mut rx) = peer.conn.open_bi().await?; + tx.write(data).await?; + let (oneshot_tx, oneshot_rx) = oneshot::channel(); + tokio::spawn(async move { + // TODO: Max length of packet should be a constant in tunnel-utils::quic + match rx.read_chunk(64 * 1024, true).await { + Ok(Some(data)) => match oneshot_tx.send(data) { + Ok(_) => match tx.finish().await { + Ok(_) => {} + Err(err) => { + warn!("Failed to finish connection: {:?}", err); + } + }, + Err(_) => { + error!("Failed to transmit result back to `NetworkManager::send_to` using oneshot! `send_to` will timeout and this error can be ignored."); + } + }, + Ok(None) => {} + Err(err) => { + warn!( + "Failed to read from stream with peer '{}': {:?}", + peer.id, err + ); + } + } + }); + // TODO: add timeout for oneshot + Ok(oneshot_rx.await?) + } + + pub fn broadcast(self: &Arc, data: Vec) { + let mut connections = Vec::with_capacity(self.connected_peers.len()); + for peer in self.connected_peers.iter() { + connections.push(( + peer.key().clone(), + peer.value().conn.open_bi(), + data.clone(), + )); + } + let connections = connections + .into_iter() + .map(move |(peer_id, conn, data)| async move { + match conn.await { + Ok((mut tx, _)) => match tx.write(&data).await { + Ok(_) => {} + Err(err) => { + warn!( + "Failed to write to stream with peer '{}': {:?}", + peer_id, err + ); + } + }, + Err(err) => { + warn!( + "Failed to write to stream with peer '{}': {:?}", + peer_id, err + ); + } + } + }); + + tokio::spawn(join_all(connections)); + } + + /// stream will return the tx and rx channel to a new stream with a remote peer. + /// Be aware that when you drop the rx channel, the stream will be closed and any data in transit will be lost. + pub async fn stream(&self, peer_id: &PeerId) -> Result<(SendStream, RecvStream), NMError> { + debug!("Opening stream with peer '{:?}'", peer_id); + + Ok(self + .connected_peers + .get(peer_id) + .ok_or(NMError::PeerNotConnected)? + .conn + .open_bi() + .await?) + } + + /// returns a list of the connected peers. + pub fn connected_peers(&self) -> HashMap> { + self.connected_peers.clone().into_iter().collect() + } + + /// returns a list of the discovered peers. + pub fn discovered_peers(&self) -> HashMap { + self.discovered_peers.clone().into_iter().collect() + } + + // initiate_pairing_with_peer will initiate a pairing with a peer. + // This will cause the NetworkManager to attempt to connect to the peer if it is discovered and if it is, verify the preshared_key using PAKE before telling the [crate::P2PManager] that the pairing is complete. + pub async fn initiate_pairing_with_peer( + self: &Arc, + remote_peer_id: PeerId, + extra_data: HashMap, + ) -> Result { + debug!("Starting pairing with '{:?}'", remote_peer_id); + + // TODO: Ensure we are not already paired with the peer + + let candidate = self + .discovered_peers + .get(&remote_peer_id) + .ok_or(NMError::PeerNotFound)? + .clone(); + + let m = Mnemonic::generate_in( + Language::English, + 24, /* This library doesn't work with any number here for some reason */ + )?; + let preshared_key: String = m.word_iter().take(4).collect::>().join("-"); + + let (spake, pake_msg) = Spake2::::start_a( + &Password::new(preshared_key.as_bytes()), + &spake2::Identity::new(self.peer_id.as_bytes()), + &spake2::Identity::new(remote_peer_id.as_bytes()), + ); + + let NewConnection { + connection, + bi_streams, + .. + } = Self::connect_to_peer_internal(&self.clone(), candidate).await?; + + let (mut tx, mut rx) = connection.open_bi().await?; + + write_value( + &mut tx, + &ConnectionEstablishmentPayload::PairingRequest { + pake_msg, + metadata: self.manager.get_metadata(), + extra_data: extra_data.clone(), + }, + ) + .await?; + + let nm = self.clone(); + tokio::spawn(async move { + // TODO: Timeout if reading chunk is not quick + + // TODO: Get max chunk size from constant. + let data = match rx.read_chunk(64 * 1024, true).await { + Ok(Some(data)) => data, + Ok(None) => { + warn!("connection closed before we could read from it!"); + return; + } + Err(err) => { + warn!("error reading from connection: {}", err); + return; + } + }; + + let payload = match rmp_serde::decode::from_read(&data.bytes[..]) { + Ok(payload) => payload, + Err(err) => { + warn!("error decoding pairing payload: {}", err); + return; + } + }; + + match payload { + PairingPayload::PairingAccepted { pake_msg, metadata } => { + match spake.finish(&pake_msg) { + Ok(_) => {} // We only use SPAKE2 to ensure the current connection is to the peer we expect, hence we don't use the key which is returned. + Err(err) => { + warn!( + "error pairing with peer. Connection has been tampered with! err: {:?}", + err + ); + return; + } + }; + + let resp = match nm + .manager + .peer_paired( + &nm, + PairingParticipantType::Initiator, + &remote_peer_id, + &metadata, + &extra_data, + ) + .await + { + Ok(_) => PairingPayload::PairingComplete, + Err(err) => { + warn!("p2p manager error: {:?}", err); + PairingPayload::PairingFailed + } + }; + + match write_value(&mut tx, &resp).await { + Ok(_) => {} + Err(err) => { + warn!( + "error encoding and sending pairing response to connection: {}", + err + ); + return; + } + } + + match Peer::new( + ConnectionType::Client, + remote_peer_id, + connection, + metadata, + nm, + ) + .await + { + Ok(peer) => { + tokio::spawn(peer.handler(bi_streams)); + } + Err(err) => { + warn!("error creating peer: {:?}", err); + } + } + } + PairingPayload::PairingFailed => { + panic!("Pairing failed"); + + // TODO + // self.manager + // .peer_paired_rollback(&self, &remote_peer_id, &extra_data) + // .await; + + // TODO: emit event to frontend + } + _ => panic!("Invalid request!"), + } + }); + + Ok(preshared_key) + } +} + +// TODO: rename + docs +#[derive(Error, Debug)] +pub enum NMError { + #[error("The peer is not currently connected")] + PeerNotConnected, + #[error("The peer could not be found")] + PeerNotFound, + #[error("Error communicating with peer")] + ConnectionError(#[from] quinn::ConnectionError), + #[error("Error communicating with peer")] + UtilError(#[from] UtilError), + #[error("Internal error receiving result from oneshot")] + RecvError(#[from] oneshot::error::RecvError), + #[error("Error writing message to peer")] + WriteError(#[from] quinn::WriteError), + #[error("Error connecting to peer")] + ConnectError(#[from] ConnectError), + #[error("Error generating preshared key")] + GeneratePresharedKeyError(#[from] bip39::Error), +} diff --git a/crates/p2p/src/network_manager/nm_config.rs b/crates/p2p/src/network_manager/nm_config.rs new file mode 100644 index 000000000..775ed2be0 --- /dev/null +++ b/crates/p2p/src/network_manager/nm_config.rs @@ -0,0 +1,16 @@ +use std::collections::HashSet; + +use tunnel_utils::PeerId; + +/// Stores configuration which is given to the [crate::NetworkManager] at startup so it can resume from it's previous state. +#[derive(Clone)] +pub struct NetworkManagerConfig { + /// known_peers contains a list of all the peers that were connected last time the application was running. + /// These are used to know who to lookup when using the global discovery service. + pub known_peers: HashSet, + /// listen_port allows the user to specify which port to listen on for incoming connections. + /// By default the network manager will listen on a random free port which changes every time the application is restarted. + pub listen_port: Option, + /// TODO + pub spacetunnel_url: Option, +} diff --git a/crates/p2p/src/network_manager/nm_error.rs b/crates/p2p/src/network_manager/nm_error.rs new file mode 100644 index 000000000..2a4571cac --- /dev/null +++ b/crates/p2p/src/network_manager/nm_error.rs @@ -0,0 +1,23 @@ +use std::io; + +use thiserror::Error; + +/// Represents an error that occurs while initalising the [crate::NetworkManager]. +#[derive(Error, Debug)] +pub enum NetworkManagerError { + // TODO: Cleanup the names of the errors + #[error("the application name your provided is invalid. Ensure it is alphanumeric!")] + InvalidAppName, + #[error("error starting the mDNS service")] + MDNSDaemon(#[from] mdns_sd::Error), + #[error("error attaching the shutdown handler")] + ShutdownHandler(#[from] ctrlc::Error), + #[error("error starting the if_watch service")] + IfWatch(io::Error), + #[error("error configuring certificates for the P2P server")] + Crypto(#[from] rustls::Error), + #[error("error starting P2P server")] + Server(io::Error), + #[error("error generating P2P identity")] + RcGen(#[from] rcgen::RcgenError), +} diff --git a/crates/p2p/src/network_manager/nm_internal.rs b/crates/p2p/src/network_manager/nm_internal.rs new file mode 100644 index 000000000..5276f1072 --- /dev/null +++ b/crates/p2p/src/network_manager/nm_internal.rs @@ -0,0 +1,256 @@ +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddrV4}, + sync::Arc, + time::Duration, +}; + +use futures_util::StreamExt; +use if_watch::{IfEvent, IfWatcher}; +use quinn::{ClientConfig, Incoming, NewConnection, VarInt}; +use thiserror::Error; +use tokio::{select, sync::mpsc, time::sleep}; +use tracing::{debug, error, warn}; +use tunnel_utils::{quic::client_config, PeerId}; + +use crate::{ + ConnectionType, DiscoveryStack, NetworkManager, NetworkManagerError, P2PManager, Peer, + PeerCandidate, +}; + +/// Represents an event that should be handled by the [NetworkManager] event loop. +#[derive(Debug, Clone)] +pub(crate) enum NetworkManagerInternalEvent { + Connect(PeerCandidate), + NewKnownPeer(PeerId), +} + +impl NetworkManager { + // this event_loop is run in a tokio task and is responsible for handling events emitted by components of the P2P library. + pub(crate) async fn event_loop( + nm: &Arc, + mut quic_incoming: Incoming, + mut internal_channel: mpsc::UnboundedReceiver, + ) -> Result<(), NetworkManagerError> { + debug!("Starting P2P event loop"); + let mut if_watcher = IfWatcher::new() + .await + .map_err(NetworkManagerError::IfWatch)?; + let discovery = DiscoveryStack::new(nm).await?; + let (shutdown_signal_tx, mut shutdown_signal_rx) = mpsc::unbounded_channel(); // This should be able to be a oneshot but ctrlc is cringe + ctrlc::set_handler(move || { + debug!("Shutdown signal captured. Sending shutdown signal..."); + match shutdown_signal_tx.send(()) { + Ok(_) => {} + Err(err) => { + error!( + "Failed to send shutdown signal. Falling back to hard shutdown. {:?}", + err + ); + } + } + })?; + + for iface in if_watcher.iter() { + Self::handle_ifwatch_event(nm, IfEvent::Up(*iface)); + } + + discovery.register().await; + + debug!( + "Network adapters discovered on startup: {:?}", + nm.lan_addrs + .iter() + .map(|a| a.to_string()) + .collect::>() + .join(",") + ); + + let nm = nm.clone(); + tokio::spawn(async move { + loop { + // TODO: Deal with `discovery.register`'s network calls blocking the main event loop + select! { + conn = quic_incoming.next() => match conn { + Some(conn) => { + debug!("Handling incoming QUIC connection"); + nm.clone().handle_connection(conn) + }, + None => break, + }, + event = Pin::new(&mut if_watcher) => { + match event { + Ok(event) => { + debug!("Handling ifwatch event: {:?}", event); + if Self::handle_ifwatch_event(&nm, event) { + discovery.register().await; + } + }, + Err(_) => break, + } + } + _ = discovery.mdns.handle_mdns_event() => {} + _ = sleep(Duration::from_secs(15 * 60 /* 15 Minutes */)) => { + debug!("Discovery service registration timer reached"); + discovery.register().await; + } + // TODO: Maybe use subscription system instead of polling or review this timeout! + _ = sleep(Duration::from_secs(60 /* 1 minute */)) => { + debug!("Discovery service pool timer reached"); + discovery.global.poll().await; // TODO: this does network calls and blocks. Is this ok? + } + event = internal_channel.recv() => { + debug!("Received internal event: {:?}", event); + let event = match event { + Some(event) => event, + None => { + error!("internal_channel has been closed, stopping p2p event loop!"); + break; + }, + }; + + match event { + NetworkManagerInternalEvent::Connect(peer) => { + Self::connect_to_peer(&nm, peer).await; + } + NetworkManagerInternalEvent::NewKnownPeer(peer_id) => { + if let Some(peer) = nm.get_discovered_peer(&peer_id) { + Self::connect_to_peer(&nm, peer).await; + } + } + } + } + _ = shutdown_signal_rx.recv() => { + debug!("Event loop received shutdown signal. Shutting down..."); + nm.endpoint.close(VarInt::from_u32(69 /* TODO */), b"BRUH"); + discovery.shutdown(); + debug!("P2P event loop shutdown"); + return; // Shutdown p2p manager thread as program is exitting + } + }; + } + }); + Ok(()) + } + + fn handle_ifwatch_event(nm: &Arc, event: IfEvent) -> bool { + match event { + IfEvent::Up(iface) => { + let ip = match iface.addr() { + IpAddr::V4(ip) if ip != Ipv4Addr::LOCALHOST => ip, + _ => return false, // Currently IPv6 is not supported. Support will likely be added in the future. + }; + nm.lan_addrs.insert(ip) + } + IfEvent::Down(iface) => { + let ip = match iface.addr() { + IpAddr::V4(ip) if ip != Ipv4Addr::LOCALHOST => ip, + _ => return false, // Currently IPv6 is not supported. Support will likely be added in the future. + }; + nm.lan_addrs.remove(&ip).is_some() + } + } + } + + async fn connect_to_peer(nm: &Arc, peer: PeerCandidate) { + tracing::debug!("Connecting to peer: {:?}", peer); + let metadata = peer.metadata.clone(); + let peer_id = peer.id.clone(); + if nm.is_peer_connected(&peer.id) && nm.peer_id <= peer.id { + return; + } + + let NewConnection { + connection, + bi_streams, + .. + } = match Self::connect_to_peer_internal(nm, peer).await { + Ok(connection) => connection, + Err(e) => { + warn!("failed to connect to peer {:?}: {:?}", peer_id, e); + return; + } + }; + + if nm.is_peer_connected(&peer_id) && nm.peer_id <= peer_id { + debug!( + "Closing new connection to peer '{}' as we are already connect!", + peer_id + ); + connection.close(VarInt::from_u32(0), b"DUP_CONN"); + return; + } + + match Peer::new( + ConnectionType::Client, + peer_id, + connection, + metadata, + nm.clone(), + ) + .await + { + Ok(peer) => { + tokio::spawn(peer.handler(bi_streams)); + } + Err(e) => { + error!("failed to create peer: {:?}", e); + } + } + } + + // TODO: Error type + pub(crate) async fn connect_to_peer_internal( + nm: &Arc, + peer: PeerCandidate, + ) -> Result { + tracing::debug!("Attempting connection to {:?}", peer); + // TODO: Guess the best default IP. + + let mut i = 0; + let identity = nm.identity.clone(); + let client_config = + ClientConfig::new(Arc::new(client_config(vec![identity.0], identity.1)?)); + loop { + let address = match peer.addresses.get(i) { + Some(address) => address, + None => break None, + }; + debug!( + "Attempting connection to peer '{}' at address {:?}", + peer.id, address + ); + + // TODO: Shorter timeout for connections! + let conn = match nm.endpoint.connect_with( + client_config.clone(), + SocketAddrV4::new(*address, peer.port).into(), + &peer.id.to_string(), + ) { + Ok(conn) => conn, + Err(e) => { + debug!("failed to connect to addr '{:?}': {}", address, e); + i += 1; + continue; + } + }; + + match conn.await { + Ok(conn) => break Some(conn), + Err(e) => { + debug!("failed to connect to addr '{:?}': {}", address, e); + i += 1; + continue; + } + } + } + .ok_or(ConnectError::UnableToConnect) + } +} + +#[derive(Error, Debug)] +pub enum ConnectError { + #[error("Unable to connect to peer")] + UnableToConnect, + #[error("error setting up client TLS")] + TlsError(#[from] rustls::Error), +} diff --git a/crates/p2p/src/network_manager/nm_server.rs b/crates/p2p/src/network_manager/nm_server.rs new file mode 100644 index 000000000..aba1f5904 --- /dev/null +++ b/crates/p2p/src/network_manager/nm_server.rs @@ -0,0 +1,239 @@ +use std::{sync::Arc, time::Duration}; + +use futures_util::StreamExt; +use quinn::{Connecting, NewConnection, VarInt}; +use rustls::Certificate; +use spake2::{Ed25519Group, Password, Spake2}; +use tokio::{sync::oneshot, time::sleep}; +use tracing::{debug, error, info, warn}; +use tunnel_utils::{read_value, write_value, PeerId}; + +use crate::{ + ConnectionEstablishmentPayload, ConnectionType, NetworkManager, P2PManager, + PairingParticipantType, PairingPayload, Peer, +}; + +impl NetworkManager { + /// is called when a new connection is received from the 'QUIC' server listener to handle the connection. + pub(crate) fn handle_connection(self: Arc, conn: Connecting) { + tokio::spawn(async move { + let NewConnection { + connection, + mut bi_streams, + .. + } = match conn.await { + Ok(conn) => conn, + Err(err) => { + warn!("error accepting connection: {:?}", err); + return; + } + }; + + // let handshake_data = connection + // .handshake_data()? + // .downcast::()?; + + let peer_id = match connection + .peer_identity() + .map(|v| v.downcast::>()) + { + Some(Ok(certs)) if certs.len() == 1 => PeerId::from_cert(&certs[0]), + Some(Ok(_)) => { + warn!("client presenting an invalid number of certificates!"); + return; + } + Some(Err(_)) => { + warn!("error decoding certificates from connection!"); + return; + } + _ => unimplemented!(), + }; + + // TODO: Reenable this + // if let Some(server_name) = handshake_data.server_name { + // if server_name != peer_id.to_string() { + // println!("{} {}", server_name, peer_id.to_string()); // TODO: BRUH + // println!( + // "p2p warning: client presented a certificate and servername which don't match!" + // ); + // return; + // } + // } else { + // println!( + // "p2p warning: client presented a certificate and servername which don't match!" + // ); + // return; + // } + + // TODO: Do this check again before adding to array because the `ConnectionEstablishmentPayload` adds delay + if self.is_peer_connected(&peer_id) && self.peer_id > peer_id { + debug!( + "Closing new connection to peer '{}' as we are already connect!", + peer_id + ); + connection.close(VarInt::from_u32(0), b"DUP_CONN"); + return; + } + + let stream = tokio::select! { + stream = bi_streams.next() => { + match stream { + Some(stream) => stream, + None => { + warn!("connection closed before we could read from it!"); + return; + } + } + } + _ = sleep(Duration::from_secs(1)) => { + warn!("Connection create connection establishment stream in expected time."); + return; + } + + }; + + match stream { + Ok((mut tx, mut rx)) => { + let payload = match read_value(&mut rx).await { + Ok(msg) => msg, + Err(err) => { + warn!("error decoding connection establishment payload: {}", err); + return; + } + }; + + match payload { + ConnectionEstablishmentPayload::ConnectionRequest => { + debug!("ConnectionRequest from peer '{}'", peer_id); + // TODO: Only allow peers we trust to get pass this point + } + ConnectionEstablishmentPayload::PairingRequest { + pake_msg, + metadata, + extra_data, + } => { + debug!("PairingRequest from peer '{}'", peer_id); + // TODO: Ensure we are not already paired with the peer + + let (oneshot_tx, oneshot_rx) = oneshot::channel(); + self.manager.peer_pairing_request( + &self, + &peer_id, + &metadata, + &extra_data, + oneshot_tx, + ); + + // TODO: Have a timeout and console warning if the P2PManager doesn't respond + let preshared_key = match oneshot_rx.await { + Ok(Ok(preshared_key)) => preshared_key, + Ok(Err(err)) => { + warn!("P2PManager reported error pairing: {:?}", err); + return; + } + Err(err) => { + warn!("error receiving response for P2PManager: {:?}", err); + return; + } + }; + + let (spake, outgoing_pake_msg) = Spake2::::start_b( + &Password::new(preshared_key.as_bytes()), + &spake2::Identity::new(peer_id.as_bytes()), + &spake2::Identity::new(self.peer_id.as_bytes()), + ); + match spake.finish(&pake_msg) { + Ok(_) => {} // We only use SPAKE2 to ensure the current connection is to the peer we expect, hence we don't use the key which is returned. + Err(err) => { + warn!( + "error pairing with peer. Connection has been tampered with! err: {:?}", + err + ); + return; + } + }; + + let resp = match self + .manager + .peer_paired( + &self, + PairingParticipantType::Accepter, + &peer_id, + &metadata, + &extra_data, + ) + .await + { + Ok(_) => PairingPayload::PairingAccepted { + pake_msg: outgoing_pake_msg, + metadata: self.manager.get_metadata(), + }, + Err(err) => { + warn!("p2p manager error: {:?}", err); + PairingPayload::PairingFailed + } + }; + + match write_value(&mut tx, &resp).await { + Ok(_) => {} + Err(err) => { + warn!("error encoding and sending pairing response: {}", err); + return; + } + }; + + let payload = match read_value(&mut rx).await { + Ok(payload) => payload, + Err(err) => { + warn!("error reading and decoding pairing payload: {}", err); + return; + } + }; + + match payload { + PairingPayload::PairingAccepted { .. } => { + todo!("invalid") // TODO: Remove this + } + PairingPayload::PairingComplete { .. } => { + info!("Pairing with peer '{}' complete.", peer_id); + } + PairingPayload::PairingFailed => { + error!("Pairing with peer '{}' complete.", peer_id); + + // TODO + // self.manager + // .peer_paired_rollback(&self, &remote_peer_id, &extra_data) + // .await; + + // TODO: emit event to frontend + + return; + } + } + + match Peer::new( + ConnectionType::Server, + peer_id.clone(), + connection, + metadata, + self, + ) + .await + { + Ok(peer) => { + tokio::spawn(peer.handler(bi_streams)); + } + Err(err) => { + error!("p2p warning: error creating peer: {:?}", err); + } + } + } + } + } + _ => { + error!("connection from peer '{}' didn't send establishment payload fast enough. Closing connection", peer_id); + } + } + }); + } +} diff --git a/crates/p2p/src/network_manager/proto.rs b/crates/p2p/src/network_manager/proto.rs new file mode 100644 index 000000000..d376d29df --- /dev/null +++ b/crates/p2p/src/network_manager/proto.rs @@ -0,0 +1,28 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::PeerMetadata; + +/// Is sent as the first payload in each connection to establish the information and intent of the remote device. +/// This is sent by the QUIC client to the QUIC server. +#[derive(Debug, Serialize, Deserialize)] +pub enum ConnectionEstablishmentPayload { + PairingRequest { + pake_msg: Vec, + metadata: PeerMetadata, + extra_data: HashMap, + }, + ConnectionRequest, // TODO: Add `PeerMetadata` as argument to this. +} + +/// PairingPayload are exchanged during the pairing process to establish a secure long term relationship. +#[derive(Debug, Serialize, Deserialize)] +pub enum PairingPayload { + PairingAccepted { + pake_msg: Vec, + metadata: PeerMetadata, + }, + PairingComplete, + PairingFailed, +} diff --git a/crates/p2p/src/p2p_manager.rs b/crates/p2p/src/p2p_manager.rs new file mode 100644 index 000000000..144ccd74e --- /dev/null +++ b/crates/p2p/src/p2p_manager.rs @@ -0,0 +1,76 @@ +use std::{collections::HashMap, future::Future, pin::Pin}; + +use quinn::{RecvStream, SendStream}; +use tokio::sync::oneshot; +use tunnel_utils::PeerId; + +use crate::{NetworkManager, Peer, PeerMetadata}; + +/// Represents the type of the peer participating in pairing. This is useful for the P2PManager application to know but is not used in the P2PManager itself. +pub enum PairingParticipantType { + // This peer initiated the pairing request + Initiator, + // This peer accepted a pairing request initiated by another device + Accepter, +} + +/// Is implement by the application which is embedding this P2P library. +/// This trait allows your application which holds the users state to hook into the P2P lifecycle and make decisions from the state it holds. +#[allow(unused_variables)] +pub trait P2PManager: Clone + Send + Sync + Sized + 'static { + const APPLICATION_NAME: &'static str; + + /// Called to get the metadata of the application. This metadata is sent as part of the discovery payload. + fn get_metadata(&self) -> PeerMetadata; + + /// Called when a peer is discovered using any of the available discovery mechanisms . + fn peer_discovered(&self, nm: &NetworkManager, peer_id: &PeerId) {} + + /// Called when a peer that had previously been discovered is now unavailable. + /// This could be due to the peer announcing it is going offline or due to a timeout. + fn peer_expired(&self, nm: &NetworkManager, peer_id: PeerId) {} + + /// Called when a connection is established with a peer. + /// This will happen after pairing or if a peer that is in the [NetworkManager]'s `known_peers` list is discovered. + fn peer_connected(&self, nm: &NetworkManager, peer_id: PeerId) {} + + /// Called when a connection to a peer is disconnected. + /// This could occur due to the remote peer announcing it is going offline, or the device not responding to network activity for a certain timeout. + fn peer_disconnected(&self, nm: &NetworkManager, peer_id: PeerId) {} + + /// Called when a peer request to pair with you. The application should accept or reject the pairing request by returning the preshared_key enter by the user through the `password_resp` oneshot channel. + /// The application MUST respond to the channel regardless of result. + fn peer_pairing_request( + &self, + nm: &NetworkManager, + peer_id: &PeerId, + metadata: &PeerMetadata, + extra_data: &HashMap, + password_resp: oneshot::Sender>, + ) { + } + + /// Called when a peer has been paired with you. This function will block the pairing process until it is complete. + /// Pairing MAY fail after this function is completed due to the nature of having to run it on both machines. It is expected any changes will be reverted in `peer_paired_rollback`. + fn peer_paired<'a>( + &'a self, + nm: &'a NetworkManager, + direction: PairingParticipantType, + peer_id: &'a PeerId, + peer_metadata: &'a PeerMetadata, + extra_data: &'a HashMap, + ) -> Pin> + Send + 'a>>; + + /// Called when pairing failed but `peer_paired` was called. This function will undo any changes that may of been made by `peer_paired`. + fn peer_paired_rollback<'a>( + &'a self, + nm: &'a NetworkManager, + direction: PairingParticipantType, + peer_id: &'a PeerId, + peer_metadata: &'a PeerMetadata, + extra_data: &'a HashMap, + ) -> Pin + Send + Sync + 'a>>; + + /// Called when a network stream is created. This will contain your application code to communicate with the remote device. + fn accept_stream(&self, peer: &Peer, stream: (SendStream, RecvStream)) {} +} diff --git a/crates/p2p/src/peer/mod.rs b/crates/p2p/src/peer/mod.rs new file mode 100644 index 000000000..609293fa3 --- /dev/null +++ b/crates/p2p/src/peer/mod.rs @@ -0,0 +1,8 @@ +#[allow(clippy::module_inception)] +mod peer; +mod peer_candidate; +mod peer_metadata; + +pub use peer::*; +pub use peer_candidate::*; +pub use peer_metadata::*; diff --git a/crates/p2p/src/peer/peer.rs b/crates/p2p/src/peer/peer.rs new file mode 100644 index 000000000..5ea9830cb --- /dev/null +++ b/crates/p2p/src/peer/peer.rs @@ -0,0 +1,106 @@ +use std::{ + fmt::{self, Formatter}, + sync::Arc, +}; + +use futures_util::StreamExt; +use quinn::{ApplicationClose, Connection, IncomingBiStreams}; +use tracing::{debug, error}; +use tunnel_utils::PeerId; + +use crate::{NetworkManager, P2PManager, PeerMetadata}; + +/// This emum represents the type of the connection to the current peer. +/// QUIC is a client/server protocol so when doing P2P communication one client will be the server and one will be the client from a QUIC perspective. +/// The protocol is bi-directional so this doesn't matter a huge amount and the P2P library does it's best to hide this detail from the embedding application as thinking about this can be very confusing. +/// The decision for who is the client and server should be treated as arbitrary and shouldn't affect how the protocol operates. +#[derive(Debug, Clone, PartialEq)] +pub enum ConnectionType { + /// I am the QUIC server. + Server, + /// I am the QUIC client. + Client, +} + +/// Represents a currently connected peer. This struct holds the connection as well as any information the network manager may required about the remote peer. +/// It also stores a reference to the network manager for communication back to the [P2PManager]. +/// The [Peer] acts as an abstraction above the QUIC connection which could be a client or server so that when building code we don't have to think about the technicalities of the connection. +#[derive(Clone)] +pub struct Peer { + /// peer_id holds the id of the remote peer. This is their unique identifier. + pub id: PeerId, + /// conn_type holds the type of connection that is being established. + pub conn_type: ConnectionType, + /// metadata holds the metadata of the remote peer. This includes information such as their display name and version. + pub metadata: PeerMetadata, + /// conn holds the quinn::Connection that is being used to communicate with the remote peer. This allows creating new streams. + pub(crate) conn: Connection, + /// nm is a reference to the network manager. This is used to send messages back to the P2PManager. + nm: Arc>, +} + +impl fmt::Debug for Peer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Peer") + .field("id", &self.id) + .field("conn_type", &self.conn_type) + .field("metadata", &self.metadata) + .finish() + } +} + +impl Peer { + /// create a new peer from a [quinn::Connection]. + pub(crate) async fn new( + conn_type: ConnectionType, + id: PeerId, + conn: Connection, + metadata: PeerMetadata, + nm: Arc>, + ) -> Result { + Ok(Self { + id, + conn_type, + metadata, + conn, + nm, + }) + } + + /// handler is run in a separate thread for each peer connection and is responsible for keep the connection alive and handling incoming streams. + pub(crate) async fn handler(self, mut bi_streams: IncomingBiStreams) { + debug!( + "Started handler thread for connection with remote peer '{}'", + self.id + ); + self.nm.add_connected_peer(self.clone()); + while let Some(stream) = bi_streams.next().await { + match stream { + Err(quinn::ConnectionError::ApplicationClosed(ApplicationClose { + reason, .. + })) => { + debug!("Connection with peer closed due to '{:?}'", reason); + + // TODO: This is hacky, fix! + if reason != "DUP_CONN" { + self.nm.remove_connected_peer(self.id); + } + + break; + } + Err(err) => { + error!( + "Connection error when communicating with peer '{:?}': {:?}", + self.id, err + ); + self.nm.remove_connected_peer(self.id); + break; + } + Ok(stream) => { + debug!("Accepting stream from peer '{:?}'", self.id); + self.nm.manager.accept_stream(&self, stream); + } + } + } + } +} diff --git a/crates/p2p/src/peer/peer_candidate.rs b/crates/p2p/src/peer/peer_candidate.rs new file mode 100644 index 000000000..b67ee3c6f --- /dev/null +++ b/crates/p2p/src/peer/peer_candidate.rs @@ -0,0 +1,19 @@ +use std::net::Ipv4Addr; + +use serde::{Deserialize, Serialize}; +use specta::Type; +use tunnel_utils::PeerId; + +use crate::PeerMetadata; + +/// Represents a peer that has been discovered but not paired with. +/// It is called a candidate as it contains all of the information required to connection and pair with the peer. +/// A peer candidate discovered through mDNS may have been modified by an attacker on your local network but this is deemed acceptable as the attacker can only modify primitive metadata such a name or Spacedrive version which is used for pairing. +/// When we initiated communication with the device we will ensure we are talking to the correct device using PAKE (specially SPAKE2) for pairing and verifying the TLS certificate for general communication. +#[derive(Debug, Clone, Type, Serialize, Deserialize)] +pub struct PeerCandidate { + pub id: PeerId, + pub metadata: PeerMetadata, + pub addresses: Vec, + pub port: u16, +} diff --git a/crates/p2p/src/peer/peer_metadata.rs b/crates/p2p/src/peer/peer_metadata.rs new file mode 100644 index 000000000..6e3fe3abb --- /dev/null +++ b/crates/p2p/src/peer/peer_metadata.rs @@ -0,0 +1,94 @@ +use std::{collections::HashMap, env, str::FromStr}; + +use serde::{Deserialize, Serialize}; +use specta::Type; +use tunnel_utils::PeerId; + +/// Represents the operating system which the remote peer is running. +/// This is not used internally and predominantly is designed to be used for display purposes by the embedding application. +#[derive(Debug, Clone, Type, Serialize, Deserialize)] +pub enum OperationSystem { + Windows, + Linux, + MacOS, + IOS, + Android, + Other(String), +} + +impl OperationSystem { + pub fn get_os() -> Self { + match env::consts::OS { + "windows" => OperationSystem::Windows, + "macos" => OperationSystem::MacOS, + "linux" => OperationSystem::Linux, + "ios" => OperationSystem::IOS, + "android" => OperationSystem::Android, + platform => OperationSystem::Other(platform.into()), + } + } +} + +impl From for String { + fn from(os: OperationSystem) -> Self { + match os { + OperationSystem::Windows => "Windows".into(), + OperationSystem::Linux => "Linux".into(), + OperationSystem::MacOS => "MacOS".into(), + OperationSystem::IOS => "IOS".into(), + OperationSystem::Android => "Android".into(), + OperationSystem::Other(s) => { + let mut chars = s.chars(); + chars.next(); + chars.as_str().to_string() + } + } + } +} + +impl FromStr for OperationSystem { + type Err = (); + + fn from_str(s: &str) -> Result { + let mut chars = s.chars(); + match chars.next() { + Some('w') => Ok(OperationSystem::Windows), + Some('l') => Ok(OperationSystem::Linux), + Some('m') => Ok(OperationSystem::MacOS), + Some('i') => Ok(OperationSystem::IOS), + Some('a') => Ok(OperationSystem::Android), + _ => Ok(OperationSystem::Other(chars.as_str().to_string())), + } + } +} + +/// Represents public metadata about a peer. This is designed to hold information which is required among all applications using the P2P library. +/// This metadata is discovered through the discovery process or sent by the connecting device when establishing a new P2P connection. +#[derive(Debug, Clone, Type, Serialize, Deserialize)] +pub struct PeerMetadata { + pub name: String, + pub operating_system: Option, + pub version: Option, +} + +impl PeerMetadata { + pub fn from_hashmap(peer_id: &PeerId, hashmap: &HashMap) -> Self { + Self { + name: hashmap + .get("name") + .map(|v| v.to_string()) + .unwrap_or_else(|| peer_id.to_string()), + operating_system: hashmap.get("os").map(|v| v.parse().ok()).unwrap_or(None), + version: hashmap.get("version").map(|v| v.to_string()), + } + } + + pub fn to_hashmap(self) -> HashMap { + let mut hashmap = HashMap::new(); + hashmap.insert("name".to_string(), self.name); + if let Some(version) = self.version { + hashmap.insert("version".to_string(), version); + } + hashmap + } +} diff --git a/crates/p2p/src/utils/identity.rs b/crates/p2p/src/utils/identity.rs new file mode 100644 index 000000000..c3aa7231b --- /dev/null +++ b/crates/p2p/src/utils/identity.rs @@ -0,0 +1,47 @@ +use std::net::Ipv4Addr; + +use rcgen::{CertificateParams, DistinguishedName, DnType, RcgenError, SanType}; + +/// The common name of the identity certificate generated by sd-p2p. +const CERTIFICATE_COMMON_NAME: &str = "sd-p2p-identity"; + +/// Is the identity which respresents the current peer. An Identity is made from a public key and a private key combo. [crate::PeerId]'s are derived from the public key portion of a peer's [Identity]. +/// The public key is safe to share while the private key must remain private to ensure the connections between peers are secure. +#[derive(Clone)] +pub struct Identity { + cert: Vec, + key: Vec, +} + +impl Identity { + /// Create a new Identity for the current peer. + pub fn new() -> Result { + let mut params: CertificateParams = Default::default(); + params.distinguished_name = DistinguishedName::new(); + params + .distinguished_name + .push(DnType::CommonName, CERTIFICATE_COMMON_NAME); + params.subject_alt_names = vec![SanType::IpAddress(Ipv4Addr::LOCALHOST.into())]; + let cert = rcgen::Certificate::from_params(params)?; + + Ok(Self { + cert: cert.serialize_der()?, + key: cert.serialize_private_key_der(), + }) + } + + /// Load the current identity from it's raw form. + pub fn from_raw(cert: Vec, key: Vec) -> Result { + Ok(Self { cert, key }) + } + + /// Convert this identity into it's raw form so it can be saved. + pub fn to_raw(&self) -> (Vec, Vec) { + (self.cert.clone(), self.key.clone()) + } + + /// Convert this identity into rustls compatible form so it can be used for the QUIC TLS handshake. + pub fn into_rustls(self) -> (rustls::Certificate, rustls::PrivateKey) { + (rustls::Certificate(self.cert), rustls::PrivateKey(self.key)) + } +} diff --git a/crates/p2p/src/utils/mod.rs b/crates/p2p/src/utils/mod.rs new file mode 100644 index 000000000..ab3c470e5 --- /dev/null +++ b/crates/p2p/src/utils/mod.rs @@ -0,0 +1,3 @@ +mod identity; + +pub use identity::*; diff --git a/crates/p2p/tunnel/Cargo.toml b/crates/p2p/tunnel/Cargo.toml new file mode 100644 index 000000000..28e81c66e --- /dev/null +++ b/crates/p2p/tunnel/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "tunnel" +version = "0.1.0" +edition = "2021" +default-run = "tunnel" + +[dependencies] +tunnel-utils = { path = "./utils" } + +base64 = "0.13.0" +dotenv = "0.15.0" +futures = "0.3.21" +quinn = "0.8.3" +rcgen = "0.9.2" +rustls = "0.20.6" +serde = { version = "1.0.137", features = ["derive"] } +tokio = { version = "1.19.2", features = ["rt-multi-thread", "macros"] } +tracing = "0.1.35" +tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } +metrics = "0.19.0" +metrics-exporter-prometheus = { version = "0.10.0", features = ["http-listener"] } +thiserror = "1.0.31" +rmp-serde = "1.1.0" +bb8-redis = "0.11.0" diff --git a/crates/p2p/tunnel/Dockerfile b/crates/p2p/tunnel/Dockerfile new file mode 100644 index 000000000..71d6a6113 --- /dev/null +++ b/crates/p2p/tunnel/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1.3-labs +FROM rust:latest as build + +# Set working directory for build container +WORKDIR /app + +# Cache the Rust build between Docker builds +RUN --mount=type=cache,target=/usr/local/cargo/registry +RUN --mount=type=cache,target=/app/target + +# Copy prerequisites and install dependencies +# TODO: Make this work - it's difficult cause the Cargo.lock is outside the Docker build context +# RUN mkdir src && echo "fn main(){}" > src/main.rs +# COPY Cargo.toml ../../Cargo.lock /app/ +# RUN mkdir .cargo && cargo vendor > .cargo/config + +# Copy in code and build it +COPY . /app +RUN cargo build --release + +# Create minimal non-root production container +FROM gcr.io/distroless/cc:nonroot + +# Expose ports +EXPOSE 9000 + +# Copy in binary and set it as startup command +COPY --from=build /app/target/release/tunnel / +CMD ["/tunnel"] \ No newline at end of file diff --git a/crates/p2p/tunnel/README.md b/crates/p2p/tunnel/README.md new file mode 100644 index 000000000..b7cea7f33 --- /dev/null +++ b/crates/p2p/tunnel/README.md @@ -0,0 +1,12 @@ +**Warning: Deploying this application is currently at your own risk! It hasn't been battle tested and support will not be provided!** + +# Spacetunnel + +TODO: Write some docs + +## Usage + +```bash +cargo run -p tunnel --bin generate-env +cargo run -p tunnel +``` \ No newline at end of file diff --git a/crates/p2p/tunnel/fly.toml b/crates/p2p/tunnel/fly.toml new file mode 100644 index 000000000..b48236dcb --- /dev/null +++ b/crates/p2p/tunnel/fly.toml @@ -0,0 +1,21 @@ +app = "sdtunnel" + +[env] +SD_PORT = 9000 +SD_BIND_ADDR = "fly-global-services" + +[metrics] +port = 9000 +path = "/metrics" + +[experimental] + allowed_public_ports = [] + auto_rollback = true + +[[services]] + protocol = "udp" + internal_port = 9000 + + [[services.ports]] + handlers = [] + port = 443 diff --git a/crates/p2p/tunnel/src/bin/generate-env.rs b/crates/p2p/tunnel/src/bin/generate-env.rs new file mode 100644 index 000000000..a7c4333d0 --- /dev/null +++ b/crates/p2p/tunnel/src/bin/generate-env.rs @@ -0,0 +1,34 @@ +use std::{fs, path::Path}; + +use base64::encode; +use rcgen::generate_simple_self_signed; + +fn main() { + println!("Issuing sdtunnel certificate..."); + + let env_file = Path::new("./.env"); + if env_file.exists() { + println!("File '{}' already exists. Exiting...", env_file.display()); + return; + } + + // TODO: Replace 'generate_simple_self_signed' with full code so we have full control over generated certificate. + let cert = + generate_simple_self_signed(vec!["sdtunnel.spacedrive.com".into()]).unwrap(); + + match fs::write( + env_file, + format!( + r#"SD_ROOT_CERTIFICATE={} +SD_ROOT_CERTIFICATE_KEY={} +SD_REDIS_URL=redis://127.0.0.1/"#, + encode(cert.serialize_der().unwrap()), + encode(cert.serialize_private_key_der()) + ), + ) { + Ok(_) => {}, + Err(err) => println!("Error writing to '{}': {}", env_file.display(), err), + } + + println!("Generate keypair!"); +} diff --git a/crates/p2p/tunnel/src/main.rs b/crates/p2p/tunnel/src/main.rs new file mode 100644 index 000000000..163297373 --- /dev/null +++ b/crates/p2p/tunnel/src/main.rs @@ -0,0 +1,300 @@ +use base64::decode; +use bb8_redis::{ + bb8::Pool, + redis::{cmd, RedisError}, + RedisConnectionManager, +}; +use dotenv::dotenv; +use futures::StreamExt; +use metrics::increment_counter; +use metrics_exporter_prometheus::PrometheusBuilder; +use quinn::{ApplicationClose, Endpoint, ServerConfig}; +use rustls::Certificate; +use sd_tunnel_utils::{ + quic, ClientAnnouncementResponse, Message, MessageError, PeerId, MAX_MESSAGE_SIZE, +}; +use std::{ + collections::HashMap, + env, + net::{Ipv4Addr, ToSocketAddrs}, + sync::Arc, +}; +use thiserror::Error; + +use tracing::{debug, error, info}; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; + +#[tokio::main] +async fn main() { + dotenv().ok(); + + tracing_subscriber::registry() + .with(fmt::layer()) + .with( + EnvFilter::from_default_env() + .add_directive("info".parse().expect("Error invalid tracing directive!")) + .add_directive( + "tunnel=debug" + .parse() + .expect("Error invalid tracing directive!"), + ), + ) + .init(); + + let certificate = match env::var("SD_ROOT_CERTIFICATE") { + Ok(certificate) => rustls::Certificate( + decode(certificate).expect("Error decoding 'SD_ROOT_CERTIFICATE'"), + ), + Err(_) => { + error!("Error: 'SD_ROOT_CERTIFICATE' env var is not set!"); + return; + }, + }; + let priv_key = match env::var("SD_ROOT_CERTIFICATE_KEY") { + Ok(key) => rustls::PrivateKey( + decode(key).expect("Error decoding 'SD_ROOT_CERTIFICATE_KEY'"), + ), + Err(_) => { + error!("Error: 'SD_ROOT_CERTIFICATE_KEY' env var is not set!"); + return; + }, + }; + let redis_url = match env::var("SD_REDIS_URL") { + Ok(redis_url) => redis_url, + Err(_) => { + error!("Error: 'SD_REDIS_URL' env var is not set!"); + return; + }, + }; + let server_port = env::var("SD_PORT") + .map(|port| port.parse::().unwrap_or(9000)) + .unwrap_or(9000); + let bind_addr = env::var("SD_BIND_ADDR").unwrap_or(Ipv4Addr::UNSPECIFIED.to_string()); + + let manager = RedisConnectionManager::new(redis_url) + .expect("Error creating Redis connection manager!"); + let redis_pool = Pool::builder() + .build(manager) + .await + .expect("Error creating Redis pool!"); + + let builder = PrometheusBuilder::new(); + builder + .install() + .expect("failed to install recorder/exporter"); + + let addr = format!("{}:{}", bind_addr, server_port) + .to_socket_addrs() + .expect("Error looking up bind address") + .into_iter() + .next() + .expect("Error no bind addresses were found"); + let server_config = ServerConfig::with_crypto(Arc::new( + quic::server_config(vec![certificate], priv_key) + .expect("Error initialising 'ServerConfig'!"), + )); + let (endpoint, mut incoming) = + Endpoint::server(server_config, addr).expect("Error creating endpoint!"); + info!( + "Listening on {}", + endpoint.local_addr().expect("Error passing local address!") + ); + + while let Some(conn) = incoming.next().await { + let remote_addr = conn.remote_address(); + debug!("accepted connection from '{}'", remote_addr); + increment_counter!("spacetunnel_connections_accepted"); + + let fut = handle_connection(redis_pool.clone(), conn); + tokio::spawn(async move { + if let Err(e) = fut.await { + error!( + "'handle_connection' from remote '{}' threw error: {}", + remote_addr, + e.to_string() + ); + increment_counter!("spacetunnel_connections_errored"); + } else { + debug!("closed connection from '{}'", remote_addr); + } + }); + } +} + +async fn handle_connection( + redis_pool: Pool, + conn: quinn::Connecting, +) -> Result<(), ConnectionError> { + let quinn::NewConnection { + connection, + mut bi_streams, + .. + } = conn.await?; + + let peer_id = match connection + .peer_identity() + .unwrap() + .downcast::>() + { + Ok(certs) if certs.len() == 1 => PeerId::from_cert(&certs[0]), + Ok(_) => { + error!("Error: peer has multiple client certificates!"); + increment_counter!("spacetunnel_connections_invalid"); + return Ok(()); + }, + Err(_) => { + error!("Error: peer did not provide a client certificates!"); + increment_counter!("spacetunnel_connections_invalid"); + return Ok(()); + }, + }; + info!( + "established connection with peer '{}' from addr '{}'", + peer_id, + connection.remote_address() + ); + + // TODO: Ensure connections are closed automatically after an inactivity timeout + // TODO: Ensure streams are closed automatically after an inactivity timeout + + let peer_id = &peer_id; + while let Some(stream) = bi_streams.next().await { + let stream = match stream { + Err(quinn::ConnectionError::ApplicationClosed(ApplicationClose { + error_code, + reason, + })) => { + debug!("closed connection with peer '{}' with error_code '{}' and reason '{:?}' ", peer_id, error_code, reason); + return Ok(()); + }, + Err(e) => return Err(e.into()), + Ok(s) => s, + }; + + debug!("accepted stream from peer '{}'", peer_id); + increment_counter!("spacetunnel_streams_accepted"); + + let peer_id = peer_id.clone(); + let redis_pool = redis_pool.clone(); + tokio::spawn(async move { + let (mut tx, mut rx) = stream; + let fut = handle_stream(redis_pool, &peer_id, (&mut tx, &mut rx)); + if let Err(err) = fut.await { + error!("'handle_stream' threw error: {}", err.to_string()); + if matches!(err, ConnectionError::RedisErr(_)) { + increment_counter!("spacetunnel_redis_error", "error_src" => "handle_stream"); + } else { + increment_counter!("spacetunnel_stream_errored"); + } + match Message::Error(MessageError::InternalServerErr).encode() { + Ok(msg) => { + let _ = tx.write_all(&msg).await; + }, + Err(e) => { + error!("Error encoding error error message: {}", e.to_string()); + increment_counter!("spacetunnel_stream_errored"); + }, + } + } else { + debug!("closed stream from peer '{}'", peer_id); + } + }); + } + + Ok(()) +} + +async fn handle_stream( + redis_pool: Pool, + authenticated_peer_id: &PeerId, + (send, recv): (&mut quinn::SendStream, &mut quinn::RecvStream), +) -> Result<(), ConnectionError> { + let mut redis = match redis_pool.get().await { + Ok(conn) => conn, + Err(err) => { + error!("Error getting Redis connection: {}", err); + increment_counter!("spacetunnel_redis_error", "error_src" => "get"); + return Ok(()); + }, + }; + + while let Some(chunk) = recv.read_chunk(MAX_MESSAGE_SIZE, true).await? { + let mut bytes: &[u8] = &chunk.bytes; + let msg = match Message::read(&mut bytes)? { + Message::ClientAnnouncement { peer_id, addresses } => { + if authenticated_peer_id != peer_id { + Message::Error(MessageError::InvalidAuthErr) + } else { + increment_counter!("spacetunnel_discovery_announcements"); + let redis_key = format!("peer:announcement:{}", peer_id.to_string()); + cmd("HSET") + .arg(&redis_key) + .arg("addresses") + .arg(addresses.join(",")) + .query_async(&mut *redis) + .await?; + cmd("EXPIRE") + .arg(&redis_key) + .arg(60 * 60u32 /* 1 Hour in seconds */) + .query_async(&mut *redis) + .await?; + + Message::ClientAnnouncementOk + } + }, + Message::QueryClientAnnouncement(peer_ids) => { + increment_counter!("spacetunnel_discovery_announcement_queries"); + + // TODO: Rate limit number queries that can come from each each IP + // TODO: Check if peer is authorised to query this announcement. Syncthing don't do an auth check so for now it's fine being unauthorised. + + if peer_ids.len() > 15 { + error!( + "Client requested too many client announcements '{}'", + peer_ids.len() + ); + increment_counter!( + "spacetunnel_discovery_announcement_queries_invalid" + ); + Message::Error(MessageError::InvalidReqErr) + } else { + let mut peers = Vec::with_capacity(peer_ids.len()); + for peer_id in peer_ids.iter() { + let redis_key = + format!("peer:announcement:{}", peer_id.to_string()); + + let resp: HashMap = cmd("HGETALL") + .arg(&redis_key) + .query_async(&mut *redis) + .await?; + + peers.push(ClientAnnouncementResponse { + peer_id: peer_id.clone(), + addresses: resp + .get("addresses") + .unwrap_or(&"".to_string()) + .split(",") + .map(|v| v.to_string()) + .collect(), + }) + } + Message::QueryClientAnnouncementResponse(peers) + } + }, + Message::ClientAnnouncementOk + | Message::QueryClientAnnouncementResponse { .. } + | Message::Error(_) => Message::Error(MessageError::InvalidReqErr), + }; + send.write_all(&msg.encode()?).await?; + } + + Ok(()) +} + +#[derive(Error, Debug)] +pub enum ConnectionError { + #[error("connection error: {0}")] + ConnectionErr(#[from] quinn::ConnectionError), + #[error("redis error: {0}")] + RedisErr(#[from] RedisError), +} diff --git a/crates/p2p/tunnel/utils/Cargo.toml b/crates/p2p/tunnel/utils/Cargo.toml new file mode 100644 index 000000000..89faa0252 --- /dev/null +++ b/crates/p2p/tunnel/utils/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "tunnel-utils" +version = "0.1.0" +edition = "2021" + +[dependencies] +quinn = "0.8.3" +ring = "0.16.20" +rmp = "0.8.11" +rmp-serde = "1.1.0" +rustls = { version = "0.20.6", default-features = false, features = ["quic", "dangerous_configuration"] } # 'dangerous_configuration' is required to implement custom certificate verifiers which we use due to the self-signed nature of the protocol. +serde = { version = "1.0.137", features = ["derive"] } +specta = "0.0.2" +thiserror = "1.0.31" +ts-rs = "6.2.0" diff --git a/crates/p2p/tunnel/utils/src/client.rs b/crates/p2p/tunnel/utils/src/client.rs new file mode 100644 index 000000000..31ba1cee7 --- /dev/null +++ b/crates/p2p/tunnel/utils/src/client.rs @@ -0,0 +1,80 @@ +use std::{io, net::ToSocketAddrs, sync::Arc}; + +use quinn::{ClientConfig, Endpoint, NewConnection}; +use rustls::{Certificate, PrivateKey}; +use thiserror::Error; + +use crate::{ + quic::client_config, + rmp_quic::{read_value, UtilError}, + write_value, Message, +}; + +/// represents an error which can be thrown by the client. +#[derive(Error, Debug)] +pub enum ClientError { + #[error("no valid Spacetunnel addresses were provided")] + MissingServerAddr, + #[error("error resolving DNS for Spacetunnel address")] + IoError(#[from] io::Error), + #[error("error setting up client TLS")] + TlsError(#[from] rustls::Error), + #[error("error connecting to Spacetunnel")] + ConnectError(#[from] quinn::ConnectError), + #[error("error communicating with Spacetunnel")] + ConnectionError(#[from] quinn::ConnectionError), + #[error("error writing message to Spacetunnel")] + UtilError(#[from] UtilError), + #[error("error writing message to Spacetunnel connection")] + WriteError(#[from] quinn::WriteError), +} + +/// holds a connection to the Spacetunnel server and can be used to send messages to the server. +pub struct Client { + backend_url: String, + endpoint: Endpoint, + identity: (Certificate, PrivateKey), +} + +impl Client { + pub fn new( + backend_url: String, + endpoint: Endpoint, + identity: (Certificate, PrivateKey), + ) -> Self { + Self { + backend_url, + endpoint, + identity, + } + } + + /// sends a message to the Spacetunnel server and awaits a response. + pub async fn send_message(&self, msg: Message) -> Result { + let identity = self.identity.clone(); + let NewConnection { connection, .. } = self + .endpoint + .connect_with( + ClientConfig::new(Arc::new(client_config( + vec![identity.0], + identity.1.clone(), + )?)), + self.backend_url + .to_socket_addrs()? // TODO: Make this only lookup IPv4 -> Filter IPV6's + .into_iter() + .next() + .ok_or(ClientError::MissingServerAddr)?, + "todo", + )? + .await?; + + let (mut tx, mut rx) = connection.open_bi().await?; + write_value(&mut tx, &msg).await?; + let msg: Message = read_value(&mut rx).await?; + + // tx.finish().await?; + // connection.close(VarInt::from_u32(0), b"DUP_CONN"); + + Ok(msg) + } +} diff --git a/crates/p2p/tunnel/utils/src/lib.rs b/crates/p2p/tunnel/utils/src/lib.rs new file mode 100644 index 000000000..f31cc193c --- /dev/null +++ b/crates/p2p/tunnel/utils/src/lib.rs @@ -0,0 +1,10 @@ +mod client; +mod peer_id; +mod proto; +pub mod quic; +mod rmp_quic; + +pub use client::*; +pub use peer_id::*; +pub use proto::*; +pub use rmp_quic::*; diff --git a/crates/p2p/tunnel/utils/src/peer_id.rs b/crates/p2p/tunnel/utils/src/peer_id.rs new file mode 100644 index 000000000..9a736a607 --- /dev/null +++ b/crates/p2p/tunnel/utils/src/peer_id.rs @@ -0,0 +1,64 @@ +use std::{fmt, ops::Deref}; + +use ring::digest::digest; +use rustls::Certificate; +use serde::{Deserialize, Serialize}; +use specta::Type; +use thiserror::Error; + +/// is a unique identifier for a peer. These are derived from the public key of the peer. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Serialize, Deserialize, Type)] +pub struct PeerId(String); + +impl PeerId { + /// from_str attempts to load a PeerId from a string. It will return an error if the PeerId is invalid. + pub fn from_string(id: String) -> Result { + if id.len() != 40 { + return Err(PeerIdError::InvalidLength); + } else if !id.chars().all(char::is_alphanumeric) { + return Err(PeerIdError::InvalidCharacters); + } + Ok(Self(id)) + } + + /// from_cert will derive a [PeerId] from a [rustls::Certificate]. + pub fn from_cert(cert: &Certificate) -> Self { + // SHA-1 is used due to the limitation of the length of a DNS record used for mDNS local network discovery. + let peer_id = digest(&ring::digest::SHA1_FOR_LEGACY_USE_ONLY, &cert.0) + .as_ref() + .iter() + .map(|b| format!("{:02x}", b)) + .collect(); + + Self(peer_id) + } +} + +impl PartialEq for &PeerId { + fn eq(&self, other: &PeerId) -> bool { + self.0 == other.0 + } +} + +impl fmt::Display for PeerId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl Deref for PeerId { + type Target = String; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Represents an error that can occur when creating a [PeerId] from a string. +#[derive(Error, Debug)] +pub enum PeerIdError { + #[error("the PeerId must be 40 chars in length")] + InvalidLength, + #[error("the PeerId must be alphanumeric")] + InvalidCharacters, +} diff --git a/crates/p2p/tunnel/utils/src/proto.rs b/crates/p2p/tunnel/utils/src/proto.rs new file mode 100644 index 000000000..a612ea435 --- /dev/null +++ b/crates/p2p/tunnel/utils/src/proto.rs @@ -0,0 +1,34 @@ +use serde::{Deserialize, Serialize}; + +use crate::PeerId; + +/// MessageError is an error that occurs when a message is malformed. +/// NEVER REMOVE OR REORDER VARIANTS OF THIS ENUM OR YOU WILL BREAK STUFF DUE TO SUBOPTIMAL MSGPACK ENCODING. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum MessageError { + InvalidAuthErr, + InvalidReqErr, + InternalServerErr, +} + +/// ClientAnnouncementResponse is returned by the server when a client queries for an announcement. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ClientAnnouncementResponse { + pub peer_id: PeerId, + pub addresses: Vec, +} + +/// Message is a single request that is sent between a client and the Spacetunnel server. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum Message { + // Announce your current device addresses + ClientAnnouncement { + peer_id: PeerId, + addresses: Vec, + }, + ClientAnnouncementOk, + // Query for an existing client announcement + QueryClientAnnouncement(Vec), + QueryClientAnnouncementResponse(Vec), + Error(MessageError), +} diff --git a/crates/p2p/tunnel/utils/src/quic.rs b/crates/p2p/tunnel/utils/src/quic.rs new file mode 100644 index 000000000..550f49655 --- /dev/null +++ b/crates/p2p/tunnel/utils/src/quic.rs @@ -0,0 +1,101 @@ +use std::{sync::Arc, time::SystemTime}; + +use rustls::{ + client::{ServerCertVerified, ServerCertVerifier}, + server::{ClientCertVerified, ClientCertVerifier}, + Certificate, DistinguishedNames, Error, ServerName, +}; + +/// The Application-Layer Protocol Negotiation (ALPN) value for QUIC. +const ALPN_QUIC_HTTP: &[&[u8]] = &[b"hq-29"]; + +/// server_config will return a rustls::ServerConfig for a QUIC server. Ensures this matches the client config below! +pub fn server_config( + cert_chain: Vec, + key: rustls::PrivateKey, +) -> Result { + let mut cfg = rustls::ServerConfig::builder() + .with_safe_default_cipher_suites() + .with_safe_default_kx_groups() + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_client_cert_verifier(AllowAllClientCertificateVerifier::dangerously_new()) + .with_single_cert(cert_chain, key)?; + cfg.max_early_data_size = u32::MAX; + cfg.alpn_protocols = ALPN_QUIC_HTTP.iter().map(|&x| x.into()).collect(); + Ok(cfg) +} + +/// client_config will return a rustls::ClientConfig for a QUIC client. Ensures this matches the server config above! +pub fn client_config( + cert_chain: Vec, + key: rustls::PrivateKey, +) -> Result { + let mut cfg = rustls::ClientConfig::builder() + .with_safe_default_cipher_suites() + .with_safe_default_kx_groups() + .with_protocol_versions(&[&rustls::version::TLS13])? + // .with_root_certificates(root_store) // TODO: Do this + .with_custom_certificate_verifier(ServerCertificateVerifier::dangerously_new()) // TODO: Remove this and use chain instead + .with_single_cert(cert_chain, key)?; + cfg.alpn_protocols = ALPN_QUIC_HTTP.iter().map(|&x| x.into()).collect(); + Ok(cfg) +} + +/// ServerCertificateVerifier is a custom certificate verifier that is responsible for verifying the server certificate when making a QUIC connection. +pub(crate) struct ServerCertificateVerifier; // TODO: Private this + +impl ServerCertificateVerifier { + // TODO: Private this + pub(crate) fn dangerously_new() -> Arc { + Arc::new(Self) + } +} + +impl ServerCertVerifier for ServerCertificateVerifier { + fn verify_server_cert( + &self, + _end_entity: &Certificate, + _intermediates: &[Certificate], + _server_name: &ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + // TODO: Verify certificate expiry + // TODO: Verify certificate algorithms match + + Ok(ServerCertVerified::assertion()) + } +} + +/// ClientCertificateVerifier is a custom certificate verifier that is responsible for verifying the client certificate when making a QUIC connection. +struct AllowAllClientCertificateVerifier; + +impl AllowAllClientCertificateVerifier { + fn dangerously_new() -> Arc { + Arc::new(Self {}) + } +} + +impl ClientCertVerifier for AllowAllClientCertificateVerifier { + fn offer_client_auth(&self) -> bool { + true + } + + fn client_auth_root_subjects(&self) -> Option { + Some(vec![]) + } + + fn verify_client_cert( + &self, + _end_entity: &Certificate, + _intermediates: &[Certificate], + _now: SystemTime, + ) -> Result { + // TODO: Verify certificate expiry + // TODO: Verify certificate algorithms match + + // We accept any client with a valid certificate because any valid certificate will have a valid PeerId. It's ok to accept all connections cause this is the public service. + Ok(ClientCertVerified::assertion()) + } +} diff --git a/crates/p2p/tunnel/utils/src/rmp_quic.rs b/crates/p2p/tunnel/utils/src/rmp_quic.rs new file mode 100644 index 000000000..76d8265d7 --- /dev/null +++ b/crates/p2p/tunnel/utils/src/rmp_quic.rs @@ -0,0 +1,43 @@ +use quinn::{RecvStream, SendStream}; +use serde::{de::DeserializeOwned, Serialize}; +use thiserror::Error; + +/// MAX_MESSAGE_SIZE is the maximum size of a single message. +pub const MAX_MESSAGE_SIZE: usize = 64 * 1024; + +#[derive(Error, Debug)] +pub enum UtilError { + #[error("error reading from stream as it was closed")] + StreamClosed, + #[error("error writing message")] + WriteError(#[from] quinn::WriteError), + #[error("error reading data")] + ReadError(#[from] quinn::ReadError), + #[error("error decoding message")] + DecodeError(#[from] rmp_serde::decode::Error), + #[error("error encoding message")] + EncodeError(#[from] rmp_serde::encode::Error), +} + +// write_value is a helper to write a Serde struct to a [quin::SendStream]. +pub async fn write_value(tx: &mut SendStream, value: &T) -> Result<(), UtilError> +where + T: Serialize + Unpin + ?Sized, +{ + let data = rmp_serde::encode::to_vec_named(value)?; + // rmp_serde doesn't support `AsyncWrite` so we have to allocate buffer here. + tx.write_all(&data).await?; + Ok(()) +} + +// read_value is a helper to read a Serde struct from a [quin::RecvStream]. +pub async fn read_value(rx: &mut RecvStream) -> Result +where + T: DeserializeOwned + ?Sized, +{ + let data = rx + .read_chunk(MAX_MESSAGE_SIZE, true) + .await? + .ok_or(UtilError::StreamClosed)?; + Ok(rmp_serde::decode::from_read(&data.bytes[..])?) +} From d7fddc83f74ccbd42c5b074be801cca68dbb4bc7 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Fri, 7 Oct 2022 05:51:40 +0800 Subject: [PATCH 2/5] fix major bugs in p2p PR --- .gitignore | 1 - Cargo.lock | Bin 172917 -> 172932 bytes apps/landing/stats.html | 4034 ----------------- core/src/lib.rs | 8 +- crates/p2p/Cargo.toml | 4 +- crates/p2p/examples/basic.rs | 1 + crates/p2p/src/discovery/global_discovery.rs | 2 +- crates/p2p/src/discovery/mdns.rs | 2 +- crates/p2p/src/lib.rs | 2 +- crates/p2p/src/network_manager/nm.rs | 2 +- crates/p2p/src/network_manager/nm_config.rs | 2 +- crates/p2p/src/network_manager/nm_internal.rs | 2 +- crates/p2p/src/network_manager/nm_server.rs | 2 +- crates/p2p/src/p2p_manager.rs | 2 +- crates/p2p/src/peer/peer.rs | 2 +- crates/p2p/src/peer/peer_candidate.rs | 5 +- crates/p2p/src/peer/peer_metadata.rs | 2 +- crates/p2p/tunnel/Cargo.toml | 6 +- crates/p2p/tunnel/fly.toml | 2 +- crates/p2p/tunnel/src/main.rs | 50 +- crates/p2p/tunnel/utils/Cargo.toml | 2 +- 21 files changed, 49 insertions(+), 4084 deletions(-) delete mode 100644 apps/landing/stats.html diff --git a/.gitignore b/.gitignore index fc00b6ad0..1a7673277 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ apps/*/stats.html docs/public/*.st docs/public/*.toml dev.db -stats.html !cli/cmd/turbo cli/npm/turbo-android-arm64/bin diff --git a/Cargo.lock b/Cargo.lock index 212afe86fa9431c08642ec929e058b8dcd82b234..3724a84a026c58aa92b500b5bea168b86cb75469 100644 GIT binary patch delta 147 zcmex*jH~51SHl*@GW+cl9T+`&m6q&B;%_K5?fjwh6Pl|3yXmNE#l)vyJHyz*4wago_>)l-q+>Oz;tSqPUeg~4GK%vTBO8)a47WmfdZ`an S%l62#jJs>M|MFp~4g&z{88>MF delta 115 zcmZoU&h_;eSHl*@GW+TKPctr@eCi9=^tv;Q?i>nAC8c?JsX3GTLlm}`+B16gOuv7I zv2nZqdB$C}tYE3>2TwC+LV$%!GG4i6AFn#(t#_iiV Ie3@#(06d&7+5i9m diff --git a/apps/landing/stats.html b/apps/landing/stats.html deleted file mode 100644 index 3450e2f6b..000000000 --- a/apps/landing/stats.html +++ /dev/null @@ -1,4034 +0,0 @@ - - - - - - - - RollUp Visualizer - - - -
- - - - - diff --git a/core/src/lib.rs b/core/src/lib.rs index 1efe8bc30..fae1da894 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -53,10 +53,10 @@ impl Node { // dbg!(get_object_kind_from_extension("png")); - let (non_blocking, _guard) = tracing_appender::non_blocking(rolling::daily( - Path::new(&data_dir).join("logs"), - "log", - )); + // let (non_blocking, _guard) = tracing_appender::non_blocking(rolling::daily( + // Path::new(&data_dir).join("logs"), + // "log", + // )); // TODO: Make logs automatically delete after x time https://github.com/tokio-rs/tracing/pull/2169 tracing_subscriber::registry() diff --git a/crates/p2p/Cargo.toml b/crates/p2p/Cargo.toml index 1f064147d..d6df53c99 100644 --- a/crates/p2p/Cargo.toml +++ b/crates/p2p/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "p2p" +name = "sd-p2p" version = "0.1.0" edition = "2021" [dependencies] -tunnel-utils = { path = "./tunnel/utils" } +sd-tunnel-utils = { path = "./tunnel/utils" } dashmap = "5.3.4" rcgen = "0.9.2" diff --git a/crates/p2p/examples/basic.rs b/crates/p2p/examples/basic.rs index e5a2df5ba..5f3f0e4db 100644 --- a/crates/p2p/examples/basic.rs +++ b/crates/p2p/examples/basic.rs @@ -28,6 +28,7 @@ impl P2PManager for SdP2PManager { PeerMetadata { name: self.peer_name.clone(), version: Some(env!("CARGO_PKG_VERSION").into()), + operating_system: todo!(), } } diff --git a/crates/p2p/src/discovery/global_discovery.rs b/crates/p2p/src/discovery/global_discovery.rs index 5dc722132..8dd48f6f8 100644 --- a/crates/p2p/src/discovery/global_discovery.rs +++ b/crates/p2p/src/discovery/global_discovery.rs @@ -1,8 +1,8 @@ /// The functions in this file are predominantly useless in the current system. This will be fixed in a future PR's. use std::sync::Arc; +use sd_tunnel_utils::{Client, Message}; use tracing::warn; -use tunnel_utils::{Client, Message}; use crate::{NetworkManager, NetworkManagerError, P2PManager}; diff --git a/crates/p2p/src/discovery/mdns.rs b/crates/p2p/src/discovery/mdns.rs index 0c72a1256..17910f316 100644 --- a/crates/p2p/src/discovery/mdns.rs +++ b/crates/p2p/src/discovery/mdns.rs @@ -1,8 +1,8 @@ use std::{net::Ipv4Addr, sync::Arc}; use mdns_sd::{Receiver, ServiceDaemon, ServiceEvent, ServiceInfo}; +use sd_tunnel_utils::PeerId; use tracing::warn; -use tunnel_utils::PeerId; use crate::{NetworkManager, NetworkManagerError, P2PManager, PeerCandidate, PeerMetadata}; diff --git a/crates/p2p/src/lib.rs b/crates/p2p/src/lib.rs index 21820a660..ad147eae5 100644 --- a/crates/p2p/src/lib.rs +++ b/crates/p2p/src/lib.rs @@ -8,7 +8,7 @@ pub(crate) use discovery::*; pub use network_manager::*; pub use p2p_manager::*; pub use peer::*; -pub use tunnel_utils::{read_value, write_value, PeerId}; +pub use sd_tunnel_utils::{read_value, write_value, PeerId}; pub use utils::*; /// We reexport some types from `quinn` to avoid the user needing to add `quinn` and keep its version in sync with the p2p library. diff --git a/crates/p2p/src/network_manager/nm.rs b/crates/p2p/src/network_manager/nm.rs index 9d3e84fe8..4b57a2a2e 100644 --- a/crates/p2p/src/network_manager/nm.rs +++ b/crates/p2p/src/network_manager/nm.rs @@ -10,11 +10,11 @@ use dashmap::{DashMap, DashSet}; use futures_util::future::join_all; use quinn::{Chunk, Endpoint, NewConnection, RecvStream, SendStream, ServerConfig}; use rustls::{Certificate, PrivateKey}; +use sd_tunnel_utils::{quic, write_value, PeerId, UtilError}; use spake2::{Ed25519Group, Password, Spake2}; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, error, warn}; -use tunnel_utils::{quic, write_value, PeerId, UtilError}; use crate::{ ConnectError, ConnectionEstablishmentPayload, ConnectionType, Identity, NetworkManagerConfig, diff --git a/crates/p2p/src/network_manager/nm_config.rs b/crates/p2p/src/network_manager/nm_config.rs index 775ed2be0..c7dea7555 100644 --- a/crates/p2p/src/network_manager/nm_config.rs +++ b/crates/p2p/src/network_manager/nm_config.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use tunnel_utils::PeerId; +use sd_tunnel_utils::PeerId; /// Stores configuration which is given to the [crate::NetworkManager] at startup so it can resume from it's previous state. #[derive(Clone)] diff --git a/crates/p2p/src/network_manager/nm_internal.rs b/crates/p2p/src/network_manager/nm_internal.rs index 5276f1072..fba79b12e 100644 --- a/crates/p2p/src/network_manager/nm_internal.rs +++ b/crates/p2p/src/network_manager/nm_internal.rs @@ -7,10 +7,10 @@ use std::{ use futures_util::StreamExt; use if_watch::{IfEvent, IfWatcher}; use quinn::{ClientConfig, Incoming, NewConnection, VarInt}; +use sd_tunnel_utils::{quic::client_config, PeerId}; use thiserror::Error; use tokio::{select, sync::mpsc, time::sleep}; use tracing::{debug, error, warn}; -use tunnel_utils::{quic::client_config, PeerId}; use crate::{ ConnectionType, DiscoveryStack, NetworkManager, NetworkManagerError, P2PManager, Peer, diff --git a/crates/p2p/src/network_manager/nm_server.rs b/crates/p2p/src/network_manager/nm_server.rs index aba1f5904..e543a8312 100644 --- a/crates/p2p/src/network_manager/nm_server.rs +++ b/crates/p2p/src/network_manager/nm_server.rs @@ -3,10 +3,10 @@ use std::{sync::Arc, time::Duration}; use futures_util::StreamExt; use quinn::{Connecting, NewConnection, VarInt}; use rustls::Certificate; +use sd_tunnel_utils::{read_value, write_value, PeerId}; use spake2::{Ed25519Group, Password, Spake2}; use tokio::{sync::oneshot, time::sleep}; use tracing::{debug, error, info, warn}; -use tunnel_utils::{read_value, write_value, PeerId}; use crate::{ ConnectionEstablishmentPayload, ConnectionType, NetworkManager, P2PManager, diff --git a/crates/p2p/src/p2p_manager.rs b/crates/p2p/src/p2p_manager.rs index 144ccd74e..a25a0252f 100644 --- a/crates/p2p/src/p2p_manager.rs +++ b/crates/p2p/src/p2p_manager.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, future::Future, pin::Pin}; use quinn::{RecvStream, SendStream}; +use sd_tunnel_utils::PeerId; use tokio::sync::oneshot; -use tunnel_utils::PeerId; use crate::{NetworkManager, Peer, PeerMetadata}; diff --git a/crates/p2p/src/peer/peer.rs b/crates/p2p/src/peer/peer.rs index 5ea9830cb..9f688e5aa 100644 --- a/crates/p2p/src/peer/peer.rs +++ b/crates/p2p/src/peer/peer.rs @@ -5,8 +5,8 @@ use std::{ use futures_util::StreamExt; use quinn::{ApplicationClose, Connection, IncomingBiStreams}; +use sd_tunnel_utils::PeerId; use tracing::{debug, error}; -use tunnel_utils::PeerId; use crate::{NetworkManager, P2PManager, PeerMetadata}; diff --git a/crates/p2p/src/peer/peer_candidate.rs b/crates/p2p/src/peer/peer_candidate.rs index b67ee3c6f..2ba387427 100644 --- a/crates/p2p/src/peer/peer_candidate.rs +++ b/crates/p2p/src/peer/peer_candidate.rs @@ -1,8 +1,7 @@ use std::net::Ipv4Addr; +use sd_tunnel_utils::PeerId; use serde::{Deserialize, Serialize}; -use specta::Type; -use tunnel_utils::PeerId; use crate::PeerMetadata; @@ -10,7 +9,7 @@ use crate::PeerMetadata; /// It is called a candidate as it contains all of the information required to connection and pair with the peer. /// A peer candidate discovered through mDNS may have been modified by an attacker on your local network but this is deemed acceptable as the attacker can only modify primitive metadata such a name or Spacedrive version which is used for pairing. /// When we initiated communication with the device we will ensure we are talking to the correct device using PAKE (specially SPAKE2) for pairing and verifying the TLS certificate for general communication. -#[derive(Debug, Clone, Type, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] // TODO: Type pub struct PeerCandidate { pub id: PeerId, pub metadata: PeerMetadata, diff --git a/crates/p2p/src/peer/peer_metadata.rs b/crates/p2p/src/peer/peer_metadata.rs index 6e3fe3abb..b8f909585 100644 --- a/crates/p2p/src/peer/peer_metadata.rs +++ b/crates/p2p/src/peer/peer_metadata.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, env, str::FromStr}; +use sd_tunnel_utils::PeerId; use serde::{Deserialize, Serialize}; use specta::Type; -use tunnel_utils::PeerId; /// Represents the operating system which the remote peer is running. /// This is not used internally and predominantly is designed to be used for display purposes by the embedding application. diff --git a/crates/p2p/tunnel/Cargo.toml b/crates/p2p/tunnel/Cargo.toml index 28e81c66e..cdbe7860e 100644 --- a/crates/p2p/tunnel/Cargo.toml +++ b/crates/p2p/tunnel/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "tunnel" +name = "sd-tunnel" version = "0.1.0" edition = "2021" -default-run = "tunnel" +default-run = "sd-tunnel" [dependencies] -tunnel-utils = { path = "./utils" } +sd-tunnel-utils = { path = "./utils" } base64 = "0.13.0" dotenv = "0.15.0" diff --git a/crates/p2p/tunnel/fly.toml b/crates/p2p/tunnel/fly.toml index b48236dcb..b696664a0 100644 --- a/crates/p2p/tunnel/fly.toml +++ b/crates/p2p/tunnel/fly.toml @@ -1,4 +1,4 @@ -app = "sdtunnel" +app = "sd-tunnel" [env] SD_PORT = 9000 diff --git a/crates/p2p/tunnel/src/main.rs b/crates/p2p/tunnel/src/main.rs index 163297373..63de3c82f 100644 --- a/crates/p2p/tunnel/src/main.rs +++ b/crates/p2p/tunnel/src/main.rs @@ -42,37 +42,37 @@ async fn main() { .init(); let certificate = match env::var("SD_ROOT_CERTIFICATE") { - Ok(certificate) => rustls::Certificate( - decode(certificate).expect("Error decoding 'SD_ROOT_CERTIFICATE'"), - ), + Ok(certificate) => { + rustls::Certificate(decode(certificate).expect("Error decoding 'SD_ROOT_CERTIFICATE'")) + } Err(_) => { error!("Error: 'SD_ROOT_CERTIFICATE' env var is not set!"); return; - }, + } }; let priv_key = match env::var("SD_ROOT_CERTIFICATE_KEY") { - Ok(key) => rustls::PrivateKey( - decode(key).expect("Error decoding 'SD_ROOT_CERTIFICATE_KEY'"), - ), + Ok(key) => { + rustls::PrivateKey(decode(key).expect("Error decoding 'SD_ROOT_CERTIFICATE_KEY'")) + } Err(_) => { error!("Error: 'SD_ROOT_CERTIFICATE_KEY' env var is not set!"); return; - }, + } }; let redis_url = match env::var("SD_REDIS_URL") { Ok(redis_url) => redis_url, Err(_) => { error!("Error: 'SD_REDIS_URL' env var is not set!"); return; - }, + } }; let server_port = env::var("SD_PORT") .map(|port| port.parse::().unwrap_or(9000)) .unwrap_or(9000); let bind_addr = env::var("SD_BIND_ADDR").unwrap_or(Ipv4Addr::UNSPECIFIED.to_string()); - let manager = RedisConnectionManager::new(redis_url) - .expect("Error creating Redis connection manager!"); + let manager = + RedisConnectionManager::new(redis_url).expect("Error creating Redis connection manager!"); let redis_pool = Pool::builder() .build(manager) .await @@ -141,12 +141,12 @@ async fn handle_connection( error!("Error: peer has multiple client certificates!"); increment_counter!("spacetunnel_connections_invalid"); return Ok(()); - }, + } Err(_) => { error!("Error: peer did not provide a client certificates!"); increment_counter!("spacetunnel_connections_invalid"); return Ok(()); - }, + } }; info!( "established connection with peer '{}' from addr '{}'", @@ -164,9 +164,12 @@ async fn handle_connection( error_code, reason, })) => { - debug!("closed connection with peer '{}' with error_code '{}' and reason '{:?}' ", peer_id, error_code, reason); + debug!( + "closed connection with peer '{}' with error_code '{}' and reason '{:?}' ", + peer_id, error_code, reason + ); return Ok(()); - }, + } Err(e) => return Err(e.into()), Ok(s) => s, }; @@ -189,11 +192,11 @@ async fn handle_connection( match Message::Error(MessageError::InternalServerErr).encode() { Ok(msg) => { let _ = tx.write_all(&msg).await; - }, + } Err(e) => { error!("Error encoding error error message: {}", e.to_string()); increment_counter!("spacetunnel_stream_errored"); - }, + } } } else { debug!("closed stream from peer '{}'", peer_id); @@ -215,7 +218,7 @@ async fn handle_stream( error!("Error getting Redis connection: {}", err); increment_counter!("spacetunnel_redis_error", "error_src" => "get"); return Ok(()); - }, + } }; while let Some(chunk) = recv.read_chunk(MAX_MESSAGE_SIZE, true).await? { @@ -241,7 +244,7 @@ async fn handle_stream( Message::ClientAnnouncementOk } - }, + } Message::QueryClientAnnouncement(peer_ids) => { increment_counter!("spacetunnel_discovery_announcement_queries"); @@ -253,15 +256,12 @@ async fn handle_stream( "Client requested too many client announcements '{}'", peer_ids.len() ); - increment_counter!( - "spacetunnel_discovery_announcement_queries_invalid" - ); + increment_counter!("spacetunnel_discovery_announcement_queries_invalid"); Message::Error(MessageError::InvalidReqErr) } else { let mut peers = Vec::with_capacity(peer_ids.len()); for peer_id in peer_ids.iter() { - let redis_key = - format!("peer:announcement:{}", peer_id.to_string()); + let redis_key = format!("peer:announcement:{}", peer_id.to_string()); let resp: HashMap = cmd("HGETALL") .arg(&redis_key) @@ -280,7 +280,7 @@ async fn handle_stream( } Message::QueryClientAnnouncementResponse(peers) } - }, + } Message::ClientAnnouncementOk | Message::QueryClientAnnouncementResponse { .. } | Message::Error(_) => Message::Error(MessageError::InvalidReqErr), diff --git a/crates/p2p/tunnel/utils/Cargo.toml b/crates/p2p/tunnel/utils/Cargo.toml index 89faa0252..0800d7974 100644 --- a/crates/p2p/tunnel/utils/Cargo.toml +++ b/crates/p2p/tunnel/utils/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "tunnel-utils" +name = "sd-tunnel-utils" version = "0.1.0" edition = "2021" From 162eef60abe2239019460e885d44ce0f04fff070 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Fri, 7 Oct 2022 07:36:22 +0800 Subject: [PATCH 3/5] toast notification hook --- packages/interface/package.json | 1 + packages/interface/src/AppLayout.tsx | 92 +++++++++++++++++++++- packages/interface/src/hooks/useToasts.ts | 34 ++++++++ pnpm-lock.yaml | Bin 681757 -> 683033 bytes 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 packages/interface/src/hooks/useToasts.ts diff --git a/packages/interface/package.json b/packages/interface/package.json index 9563c9469..9891553a8 100644 --- a/packages/interface/package.json +++ b/packages/interface/package.json @@ -24,6 +24,7 @@ "@radix-ui/react-progress": "^1.0.0", "@radix-ui/react-slider": "^1.0.0", "@radix-ui/react-tabs": "^1.0.0", + "@radix-ui/react-toast": "^1.0.0", "@radix-ui/react-tooltip": "^1.0.0", "@sd/assets": "workspace:*", "@sd/client": "workspace:*", diff --git a/packages/interface/src/AppLayout.tsx b/packages/interface/src/AppLayout.tsx index 263173dae..67cb4c7d8 100644 --- a/packages/interface/src/AppLayout.tsx +++ b/packages/interface/src/AppLayout.tsx @@ -1,10 +1,12 @@ +import * as ToastPrimitive from '@radix-ui/react-toast'; import { useCurrentLibrary } from '@sd/client'; import clsx from 'clsx'; -import { Suspense } from 'react'; +import { Suspense, useEffect, useState } from 'react'; import { Outlet } from 'react-router-dom'; import { Sidebar } from './components/layout/Sidebar'; import { useOperatingSystem } from './hooks/useOperatingSystem'; +import { useToasts } from './hooks/useToasts'; export function AppLayout() { const { libraries } = useCurrentLibrary(); @@ -34,6 +36,94 @@ export function AppLayout() { + + + ); +} + +function Toasts() { + const { toasts, addToast, removeToast } = useToasts(); + + // useEffect(() => { + // setTimeout(() => { + // addToast({ + // title: 'Spacedrop', + // subtitle: 'Someone tried to send you a file. Accept it?', + // actionButton: { + // text: 'Accept', + // onClick: () => { + // console.log('Bruh'); + // } + // } + // }); + // }, 2000); + // }, []); + + return ( +
+ + <> + {toasts.map((toast) => ( + removeToast(toast)} + duration={toast.duration || 3000} + className={clsx( + 'w-80 m-4 shadow-lg rounded-lg', + 'bg-gray-800/20 backdrop-blur', + 'radix-state-open:animate-toast-slide-in-bottom md:radix-state-open:animate-toast-slide-in-right', + 'radix-state-closed:animate-toast-hide', + 'radix-swipe-end:animate-toast-swipe-out', + 'translate-x-radix-toast-swipe-move-x', + 'radix-swipe-cancel:translate-x-0 radix-swipe-cancel:duration-200 radix-swipe-cancel:ease-[ease]', + 'focus:outline-none focus-visible:ring focus-visible:ring-primary focus-visible:ring-opacity-75 border-white/10 border-2 shadow-2xl' + )} + > +
+
+
+ + {toast.title} + + {toast.subtitle && ( + + {toast.subtitle} + + )} +
+
+
+
+
+ {toast.actionButton && ( + { + e.preventDefault(); + toast.actionButton?.onClick(); + removeToast(toast); + }} + > + {toast.actionButton.text || 'Open'} + + )} +
+
+ + Dismiss + +
+
+
+
+ + ))} + + + +
); } diff --git a/packages/interface/src/hooks/useToasts.ts b/packages/interface/src/hooks/useToasts.ts new file mode 100644 index 000000000..f0b624fd6 --- /dev/null +++ b/packages/interface/src/hooks/useToasts.ts @@ -0,0 +1,34 @@ +import { proxy, useSnapshot } from 'valtio'; + +interface Toast { + id: string; + title: string; + subtitle?: string; + duration?: number; + actionButton?: { + text: string; + onClick: () => void; + }; +} + +const state = proxy({ + toasts: [] as Toast[] +}); + +const randomId = () => Math.random().toString(36).slice(2); + +export function useToasts() { + return { + toasts: useSnapshot(state).toasts, + addToast: (toast: Omit) => { + state.toasts.push({ + id: randomId(), + ...toast + }); + }, + removeToast: (toast: Toast | string) => { + const id = typeof toast === 'string' ? toast : toast.id; + state.toasts = state.toasts.filter((t) => t.id !== id); + } + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d60c0a145670d396fae2960dad66d6ea7b705544..018a6fe2eb93b69e33c0ae764e93ecf1d2d4d0e1 100644 GIT binary patch delta 442 zcmbPxPIKlB%?%A2%!$P%lLa*tH`i%&a)LP1H;OP>Pp;G8YUVfE&Tqu{|2|mM@jatk zXl_crn^{p}MtW&+MoMa#fq#UyWum#4cfN-~MU-QdQ@)XRNt%AKxw&Pqxoci_RH~1$ zzOP$mkY|{wPeD+CsdI^AaBf;ku#2&qPo8U;yJ3(=P|5THCZ^TX*K;x5onHT#(QEo@ z5vCK<7u;uLoqqo%qs{dF987xC4Zbl7g2Y&+`!X_VLb%ZiO#IWoi83WlS5RV-p6(#S z!~%3Sd%Lg@({^DY=B(c7meO3BlMl3tOuxQ~nQ!}*`OMpLr|UCva87?8%)vGN3O9?$ zZ2!!{wL%2!mF*WKxqN*Aw*#23 delta 219 zcmbPvLv!vq%?%A2lXHzYHm}pD=WNb3+Ma8~$o`=Hw;0p*-(t-9y_3&y3QqShW;C9B z;SuZhgDxOV2a+iVCbMv@rFwFmx;M z4)v=D^Kq=oPtT~d@QU&>@$)QA)s9RHjdV&XHFV4_NUw4$^DD1%(~t6t^00Kx$ Date: Fri, 7 Oct 2022 07:39:59 +0800 Subject: [PATCH 4/5] cargo fmt + fix clippy lints --- crates/p2p/src/peer/peer.rs | 2 +- crates/p2p/tunnel/src/bin/generate-env.rs | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/p2p/src/peer/peer.rs b/crates/p2p/src/peer/peer.rs index 9f688e5aa..4a9c91a4c 100644 --- a/crates/p2p/src/peer/peer.rs +++ b/crates/p2p/src/peer/peer.rs @@ -14,7 +14,7 @@ use crate::{NetworkManager, P2PManager, PeerMetadata}; /// QUIC is a client/server protocol so when doing P2P communication one client will be the server and one will be the client from a QUIC perspective. /// The protocol is bi-directional so this doesn't matter a huge amount and the P2P library does it's best to hide this detail from the embedding application as thinking about this can be very confusing. /// The decision for who is the client and server should be treated as arbitrary and shouldn't affect how the protocol operates. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum ConnectionType { /// I am the QUIC server. Server, diff --git a/crates/p2p/tunnel/src/bin/generate-env.rs b/crates/p2p/tunnel/src/bin/generate-env.rs index a7c4333d0..b2de3ca8a 100644 --- a/crates/p2p/tunnel/src/bin/generate-env.rs +++ b/crates/p2p/tunnel/src/bin/generate-env.rs @@ -13,8 +13,7 @@ fn main() { } // TODO: Replace 'generate_simple_self_signed' with full code so we have full control over generated certificate. - let cert = - generate_simple_self_signed(vec!["sdtunnel.spacedrive.com".into()]).unwrap(); + let cert = generate_simple_self_signed(vec!["sdtunnel.spacedrive.com".into()]).unwrap(); match fs::write( env_file, @@ -26,7 +25,7 @@ SD_REDIS_URL=redis://127.0.0.1/"#, encode(cert.serialize_private_key_der()) ), ) { - Ok(_) => {}, + Ok(_) => {} Err(err) => println!("Error writing to '{}': {}", env_file.display(), err), } From aacdf33216fa4ed586d2600b52b9c3c67f37fd16 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Fri, 7 Oct 2022 07:45:15 +0800 Subject: [PATCH 5/5] merge duplicate prettier config and format repo with it --- .prettierignore | 2 + .prettierrc.js | 2 +- .prettierrc.json | 20 - README.md | 1 - apps/landing/src/App.tsx | 4 +- apps/landing/src/pages/blog/blog.ts | 52 +-- apps/landing/src/pages/index.page.tsx | 4 +- apps/landing/src/pages/team.page.tsx | 4 +- apps/landing/src/style.scss | 36 +- .../AppIcon.appiconset/Contents.json | 242 +++++------ .../Spacedrive/Images.xcassets/Contents.json | 8 +- .../SplashScreen.imageset/Contents.json | 40 +- .../Contents.json | 40 +- apps/mobile/src/types/bindings.ts | 395 ++++++++++++++---- apps/web/src/index.html | 20 +- apps/web/src/index.tsx | 6 +- apps/web/tsconfig.json | 8 +- crates/ffmpeg/README.md | 6 +- crates/p2p/README.md | 2 +- crates/p2p/docs-wip/index.md | 23 +- crates/p2p/tunnel/README.md | 2 +- crates/sync/README.md | 1 - crates/sync/docs/HLC.md | 68 ++- crates/sync/example/.vscode/extensions.json | 2 +- crates/sync/example/index.html | 24 +- crates/sync/example/package.json | 60 +-- crates/sync/example/src-tauri/tauri.conf.json | 128 +++--- crates/sync/example/src/App.tsx | 114 +++-- crates/sync/example/src/bindings.ts | 25 +- crates/sync/example/src/index.tsx | 24 +- crates/sync/example/src/rspc.ts | 26 +- crates/sync/example/tsconfig.json | 26 +- crates/sync/example/vite.config.ts | 44 +- docs/changelog/beta/0.1.0.md | 20 +- docs/company/legal/terms.md | 2 +- docs/developers/architecture/albums.md | 5 +- docs/developers/architecture/database.md | 2 +- docs/developers/architecture/explorer.md | 3 +- docs/developers/architecture/extensions.md | 2 +- docs/developers/architecture/jobs.md | 2 +- docs/developers/architecture/libraries.md | 3 +- docs/developers/architecture/locations.md | 2 +- docs/developers/architecture/nodes.md | 3 +- docs/developers/architecture/objects.md | 48 +-- docs/developers/architecture/preview-media.md | 2 +- docs/developers/architecture/search.md | 1 - docs/developers/architecture/spaces.md | 3 +- docs/developers/architecture/tags.md | 2 +- .../architecture/virtual-filesystem.md | 1 - docs/developers/clients/cli.md | 2 +- docs/developers/clients/javascript.md | 14 +- .../prerequisites/environment-setup.md | 53 +-- docs/developers/prerequisites/welcome.md | 2 +- docs/developers/technology/open-source.md | 2 +- docs/developers/technology/tech-stack.md | 2 +- docs/package.json | 6 +- docs/product/getting-started/features.md | 2 +- docs/product/getting-started/introduction.md | 2 +- docs/product/getting-started/setup.md | 2 +- docs/product/getting-started/terminology.md | 1 - docs/product/guides/adding-locations.md | 12 +- docs/product/guides/connecting-nodes.md | 7 +- docs/product/guides/converting-files.md | 2 +- docs/product/guides/creating-spaces.md | 2 +- docs/product/guides/discover-duplicates.md | 2 +- docs/product/guides/file-viewer.md | 2 +- docs/product/guides/generating-thumbails.md | 2 +- docs/product/guides/library-setup.md | 3 +- docs/product/guides/managing-jobs.md | 2 +- docs/product/guides/managing-tags.md | 2 +- docs/product/guides/photo-albums.md | 2 +- docs/product/resources/privacy.md | 2 +- docs/product/resources/security.md | 2 +- lefthook.yml | 2 +- package.json | 2 +- packages/client/src/core.ts | 395 ++++++++++++++---- packages/interface/package.json | 188 ++++----- packages/interface/src/AppRouter.tsx | 7 +- .../src/components/explorer/FileThumb.tsx | 5 +- .../src/components/explorer/Inspector.tsx | 15 +- .../components/explorer/VirtualizedList.tsx | 7 +- packages/interface/src/style.scss | 26 +- turbo.json | 26 +- 83 files changed, 1415 insertions(+), 948 deletions(-) create mode 100644 .prettierignore delete mode 100644 .prettierrc.json diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 000000000..69a352b81 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +/target +.build \ No newline at end of file diff --git a/.prettierrc.js b/.prettierrc.js index f75b3d3d2..a522c7ba5 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -7,7 +7,7 @@ module.exports = { bracketSameLine: false, semi: true, quoteProps: 'consistent', - importOrder: ['^@sd/interface/(.*)$', '^@sd/client/(.*)$', '^@sd/ui/(.*)$', '^[./]'], + importOrder: ['^[./]', '^@sd/interface/(.*)$', '^@sd/client/(.*)$', '^@sd/ui/(.*)$'], importOrderSeparation: true, importOrderSortSpecifiers: true, plugins: ['@trivago/prettier-plugin-sort-imports'] diff --git a/.prettierrc.json b/.prettierrc.json deleted file mode 100644 index 3e5b2cdae..000000000 --- a/.prettierrc.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "pluginSearchDirs": [ - "." - ], - "useTabs": true, - "printWidth": 100, - "singleQuote": true, - "trailingComma": "none", - "bracketSameLine": false, - "semi": true, - "quoteProps": "consistent", - "importOrder": [ - "^[./]", - "^@sd/ui/(.*)$", - "^@sd/client/(.*)$", - "^@sd/interface/(.*)$" - ], - "importOrderSeparation": true, - "importOrderSortSpecifiers": true -} diff --git a/README.md b/README.md index f386104c6..7c554606e 100644 --- a/README.md +++ b/README.md @@ -109,4 +109,3 @@ This project is using what I'm calling the **"PRRTT"** stack (Prisma, Rust, Reac - `ios`: A [Swift](https://developer.apple.com/swift/) Native binary (planned). - `windows`: A [C#](https://docs.microsoft.com/en-us/dotnet/csharp/) Native binary (planned). - `android`: A [Kotlin](https://kotlinlang.org/) Native binary (planned). - diff --git a/apps/landing/src/App.tsx b/apps/landing/src/App.tsx index 33822c31d..ca83abe21 100644 --- a/apps/landing/src/App.tsx +++ b/apps/landing/src/App.tsx @@ -2,13 +2,13 @@ import { Button } from '@sd/ui'; import React from 'react'; import { PageContextBuiltIn } from 'vite-plugin-ssr'; -import '@sd/ui/style'; - import { Footer } from './components/Footer'; import NavBar from './components/NavBar'; import { PageContextProvider } from './renderer/usePageContext'; import './style.scss'; +import '@sd/ui/style'; + export default function App({ children, pageContext diff --git a/apps/landing/src/pages/blog/blog.ts b/apps/landing/src/pages/blog/blog.ts index 64b298abf..3f2451d87 100644 --- a/apps/landing/src/pages/blog/blog.ts +++ b/apps/landing/src/pages/blog/blog.ts @@ -7,35 +7,35 @@ const ghostURL = import.meta.env.VITE_API_URL; export const blogEnabled = !!(ghostURL && ghostKey); export const api = blogEnabled - ? new GhostContentAPI({ - url: ghostURL, - key: ghostKey, - version: 'v5.0' - }) - : null; + ? new GhostContentAPI({ + url: ghostURL, + key: ghostKey, + version: 'v5.0' + }) + : null; export async function getPosts() { - if (!api) { - return []; - } - const posts = await api.posts - .browse({ - include: ['tags', 'authors'] - }) - .catch(() => []); - return posts; + if (!api) { + return []; + } + const posts = await api.posts + .browse({ + include: ['tags', 'authors'] + }) + .catch(() => []); + return posts; } export async function getPost(slug: string) { - if (!api) { - return null; - } - return await api.posts - .read( - { slug }, - { - include: ['tags', 'authors'] - } - ) - .catch(() => null); + if (!api) { + return null; + } + return await api.posts + .read( + { slug }, + { + include: ['tags', 'authors'] + } + ) + .catch(() => null); } diff --git a/apps/landing/src/pages/index.page.tsx b/apps/landing/src/pages/index.page.tsx index 43afe1d3c..cc61aed14 100644 --- a/apps/landing/src/pages/index.page.tsx +++ b/apps/landing/src/pages/index.page.tsx @@ -2,8 +2,6 @@ import clsx from 'clsx'; import { useEffect, useState } from 'react'; import { Helmet } from 'react-helmet'; -import { ReactComponent as Info } from '@sd/interface/assets/svg/info.svg'; - import AppEmbed, { AppEmbedPlaceholder } from '../components/AppEmbed'; import { Bubbles } from '../components/Bubbles'; // import { Bubbles } from '../components/Bubbles'; @@ -12,6 +10,8 @@ import NewBanner from '../components/NewBanner'; import { usePageContext } from '../renderer/usePageContext'; import { getWindow } from '../utils'; +import { ReactComponent as Info } from '@sd/interface/assets/svg/info.svg'; + interface SectionProps { orientation: 'left' | 'right'; heading?: string; diff --git a/apps/landing/src/pages/team.page.tsx b/apps/landing/src/pages/team.page.tsx index 1b4de52bc..29689cba8 100644 --- a/apps/landing/src/pages/team.page.tsx +++ b/apps/landing/src/pages/team.page.tsx @@ -1,11 +1,11 @@ import { Helmet } from 'react-helmet'; -import { ReactComponent as ArrowRight } from '@sd/interface/assets/svg/arrow-right.svg'; - import Markdown from '../components/Markdown'; import { TeamMember, TeamMemberProps } from '../components/TeamMember'; import { resolveFilesGlob } from '../utils'; +import { ReactComponent as ArrowRight } from '@sd/interface/assets/svg/arrow-right.svg'; + const teamImages = resolveFilesGlob(import.meta.globEager('../assets/images/team/*')); const investorImages = resolveFilesGlob(import.meta.globEager('../assets/images/investors/*')); diff --git a/apps/landing/src/style.scss b/apps/landing/src/style.scss index a3a681043..452de7ba9 100644 --- a/apps/landing/src/style.scss +++ b/apps/landing/src/style.scss @@ -116,7 +116,7 @@ html { &.bloom-one { @apply left-0 -ml-[300px]; background: url('/bloom-one.png') no-repeat center center; - background-size: contain; + background-size: contain; animation-delay: 300ms; } &.bloom-two { @@ -139,7 +139,6 @@ html { background: url('/egg-bloom-two.png') no-repeat center center; background-size: contain; } - } @keyframes bloomBurst { @@ -170,47 +169,48 @@ html { } .slot-block { - @apply bg-gray-550 py-3 px-4 border-l-4 border-gray-400 rounded mb-2 + @apply bg-gray-550 py-3 px-4 border-l-4 border-gray-400 rounded mb-2; } .slot-block.note { - @apply border-yellow-400 bg-yellow-300/20 + @apply border-yellow-400 bg-yellow-300/20; } .slot-block.info { - @apply border-green-400 bg-green-400/20 + @apply border-green-400 bg-green-400/20; } .slot-block.warning { - @apply border-red-400 bg-red-400/20 + @apply border-red-400 bg-red-400/20; } .slot-block-title { @apply text-white font-bold text-sm m-0 uppercase; } .slot-block-content { - @apply my-1 mx-0 mb-0 text-white + @apply my-1 mx-0 mb-0 text-white; } -.doc-sidebar-button:hover, .doc-sidebar-button.nav-active { +.doc-sidebar-button:hover, +.doc-sidebar-button.nav-active { &.product { - color:#459EE8; + color: #459ee8; div { - background-color: #459EE8; + background-color: #459ee8; } } &.developers { - color:#48BB78; + color: #48bb78; div { - background-color: #48BB78; + background-color: #48bb78; } } &.company { - color:#bb9247; + color: #bb9247; div { background-color: #bb9247; } } &.changelog { - color:#8A47BB; + color: #8a47bb; div { - background-color: #8A47BB; + background-color: #8a47bb; } } } @@ -222,7 +222,6 @@ html { // #1D054B // #9A3F8C - .custom-scroll { -ms-overflow-style: none; /* for Internet Explorer, Edge */ scrollbar-width: none; /* for Firefox */ @@ -238,7 +237,6 @@ html { @apply bg-[#00000006] dark:bg-[#00000030] my-[10px] rounded-[6px]; } &::-webkit-scrollbar-thumb { - @apply rounded-[6px] bg-gray-300 dark:bg-gray-550 ; - + @apply rounded-[6px] bg-gray-300 dark:bg-gray-550; } -} \ No newline at end of file +} diff --git a/apps/mobile/ios/Spacedrive/Images.xcassets/AppIcon.appiconset/Contents.json b/apps/mobile/ios/Spacedrive/Images.xcassets/AppIcon.appiconset/Contents.json index f920cb0ec..53263a682 100644 --- a/apps/mobile/ios/Spacedrive/Images.xcassets/AppIcon.appiconset/Contents.json +++ b/apps/mobile/ios/Spacedrive/Images.xcassets/AppIcon.appiconset/Contents.json @@ -1,122 +1,122 @@ { - "images": [ - { - "idiom": "iphone", - "size": "20x20", - "scale": "2x", - "filename": "App-Icon-20x20@2x.png" - }, - { - "idiom": "iphone", - "size": "20x20", - "scale": "3x", - "filename": "App-Icon-20x20@3x.png" - }, - { - "idiom": "iphone", - "size": "29x29", - "scale": "1x", - "filename": "App-Icon-29x29@1x.png" - }, - { - "idiom": "iphone", - "size": "29x29", - "scale": "2x", - "filename": "App-Icon-29x29@2x.png" - }, - { - "idiom": "iphone", - "size": "29x29", - "scale": "3x", - "filename": "App-Icon-29x29@3x.png" - }, - { - "idiom": "iphone", - "size": "40x40", - "scale": "2x", - "filename": "App-Icon-40x40@2x.png" - }, - { - "idiom": "iphone", - "size": "40x40", - "scale": "3x", - "filename": "App-Icon-40x40@3x.png" - }, - { - "idiom": "iphone", - "size": "60x60", - "scale": "2x", - "filename": "App-Icon-60x60@2x.png" - }, - { - "idiom": "iphone", - "size": "60x60", - "scale": "3x", - "filename": "App-Icon-60x60@3x.png" - }, - { - "idiom": "ipad", - "size": "20x20", - "scale": "1x", - "filename": "App-Icon-20x20@1x.png" - }, - { - "idiom": "ipad", - "size": "20x20", - "scale": "2x", - "filename": "App-Icon-20x20@2x.png" - }, - { - "idiom": "ipad", - "size": "29x29", - "scale": "1x", - "filename": "App-Icon-29x29@1x.png" - }, - { - "idiom": "ipad", - "size": "29x29", - "scale": "2x", - "filename": "App-Icon-29x29@2x.png" - }, - { - "idiom": "ipad", - "size": "40x40", - "scale": "1x", - "filename": "App-Icon-40x40@1x.png" - }, - { - "idiom": "ipad", - "size": "40x40", - "scale": "2x", - "filename": "App-Icon-40x40@2x.png" - }, - { - "idiom": "ipad", - "size": "76x76", - "scale": "1x", - "filename": "App-Icon-76x76@1x.png" - }, - { - "idiom": "ipad", - "size": "76x76", - "scale": "2x", - "filename": "App-Icon-76x76@2x.png" - }, - { - "idiom": "ipad", - "size": "83.5x83.5", - "scale": "2x", - "filename": "App-Icon-83.5x83.5@2x.png" - }, - { - "idiom": "ios-marketing", - "size": "1024x1024", - "scale": "1x", - "filename": "ItunesArtwork@2x.png" - } - ], - "info": { - "version": 1, - "author": "expo" - } -} \ No newline at end of file + "images": [ + { + "idiom": "iphone", + "size": "20x20", + "scale": "2x", + "filename": "App-Icon-20x20@2x.png" + }, + { + "idiom": "iphone", + "size": "20x20", + "scale": "3x", + "filename": "App-Icon-20x20@3x.png" + }, + { + "idiom": "iphone", + "size": "29x29", + "scale": "1x", + "filename": "App-Icon-29x29@1x.png" + }, + { + "idiom": "iphone", + "size": "29x29", + "scale": "2x", + "filename": "App-Icon-29x29@2x.png" + }, + { + "idiom": "iphone", + "size": "29x29", + "scale": "3x", + "filename": "App-Icon-29x29@3x.png" + }, + { + "idiom": "iphone", + "size": "40x40", + "scale": "2x", + "filename": "App-Icon-40x40@2x.png" + }, + { + "idiom": "iphone", + "size": "40x40", + "scale": "3x", + "filename": "App-Icon-40x40@3x.png" + }, + { + "idiom": "iphone", + "size": "60x60", + "scale": "2x", + "filename": "App-Icon-60x60@2x.png" + }, + { + "idiom": "iphone", + "size": "60x60", + "scale": "3x", + "filename": "App-Icon-60x60@3x.png" + }, + { + "idiom": "ipad", + "size": "20x20", + "scale": "1x", + "filename": "App-Icon-20x20@1x.png" + }, + { + "idiom": "ipad", + "size": "20x20", + "scale": "2x", + "filename": "App-Icon-20x20@2x.png" + }, + { + "idiom": "ipad", + "size": "29x29", + "scale": "1x", + "filename": "App-Icon-29x29@1x.png" + }, + { + "idiom": "ipad", + "size": "29x29", + "scale": "2x", + "filename": "App-Icon-29x29@2x.png" + }, + { + "idiom": "ipad", + "size": "40x40", + "scale": "1x", + "filename": "App-Icon-40x40@1x.png" + }, + { + "idiom": "ipad", + "size": "40x40", + "scale": "2x", + "filename": "App-Icon-40x40@2x.png" + }, + { + "idiom": "ipad", + "size": "76x76", + "scale": "1x", + "filename": "App-Icon-76x76@1x.png" + }, + { + "idiom": "ipad", + "size": "76x76", + "scale": "2x", + "filename": "App-Icon-76x76@2x.png" + }, + { + "idiom": "ipad", + "size": "83.5x83.5", + "scale": "2x", + "filename": "App-Icon-83.5x83.5@2x.png" + }, + { + "idiom": "ios-marketing", + "size": "1024x1024", + "scale": "1x", + "filename": "ItunesArtwork@2x.png" + } + ], + "info": { + "version": 1, + "author": "expo" + } +} diff --git a/apps/mobile/ios/Spacedrive/Images.xcassets/Contents.json b/apps/mobile/ios/Spacedrive/Images.xcassets/Contents.json index ed285c2e5..9f74c33ba 100644 --- a/apps/mobile/ios/Spacedrive/Images.xcassets/Contents.json +++ b/apps/mobile/ios/Spacedrive/Images.xcassets/Contents.json @@ -1,6 +1,6 @@ { - "info" : { - "version" : 1, - "author" : "expo" - } + "info": { + "version": 1, + "author": "expo" + } } diff --git a/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreen.imageset/Contents.json b/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreen.imageset/Contents.json index 3cf848977..857c97408 100644 --- a/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreen.imageset/Contents.json +++ b/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreen.imageset/Contents.json @@ -1,21 +1,21 @@ { - "images": [ - { - "idiom": "universal", - "filename": "image.png", - "scale": "1x" - }, - { - "idiom": "universal", - "scale": "2x" - }, - { - "idiom": "universal", - "scale": "3x" - } - ], - "info": { - "version": 1, - "author": "expo" - } -} \ No newline at end of file + "images": [ + { + "idiom": "universal", + "filename": "image.png", + "scale": "1x" + }, + { + "idiom": "universal", + "scale": "2x" + }, + { + "idiom": "universal", + "scale": "3x" + } + ], + "info": { + "version": 1, + "author": "expo" + } +} diff --git a/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreenBackground.imageset/Contents.json b/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreenBackground.imageset/Contents.json index 3cf848977..857c97408 100644 --- a/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreenBackground.imageset/Contents.json +++ b/apps/mobile/ios/Spacedrive/Images.xcassets/SplashScreenBackground.imageset/Contents.json @@ -1,21 +1,21 @@ { - "images": [ - { - "idiom": "universal", - "filename": "image.png", - "scale": "1x" - }, - { - "idiom": "universal", - "scale": "2x" - }, - { - "idiom": "universal", - "scale": "3x" - } - ], - "info": { - "version": 1, - "author": "expo" - } -} \ No newline at end of file + "images": [ + { + "idiom": "universal", + "filename": "image.png", + "scale": "1x" + }, + { + "idiom": "universal", + "scale": "2x" + }, + { + "idiom": "universal", + "scale": "3x" + } + ], + "info": { + "version": 1, + "author": "expo" + } +} diff --git a/apps/mobile/src/types/bindings.ts b/apps/mobile/src/types/bindings.ts index 5ad4ffda2..7baccfece 100644 --- a/apps/mobile/src/types/bindings.ts +++ b/apps/mobile/src/types/bindings.ts @@ -2,111 +2,358 @@ // This file was generated by [rspc](https://github.com/oscartbeaumont/rspc). Do not edit this file manually. export type Procedures = { - queries: - { key: "files.readMetadata", input: LibraryArgs, result: null } | - { key: "getNode", input: never, result: NodeState } | - { key: "jobs.getHistory", input: LibraryArgs, result: Array } | - { key: "jobs.getRunning", input: LibraryArgs, result: Array } | - { key: "library.getStatistics", input: LibraryArgs, result: Statistics } | - { key: "library.list", input: never, result: Array } | - { key: "locations.getById", input: LibraryArgs, result: Location | null } | - { key: "locations.getExplorerData", input: LibraryArgs, result: ExplorerData } | - { key: "locations.indexer_rules.get", input: LibraryArgs, result: IndexerRule } | - { key: "locations.indexer_rules.list", input: LibraryArgs, result: Array } | - { key: "locations.list", input: LibraryArgs, result: Array<{ id: number, pub_id: Array, node_id: number, name: string | null, local_path: string | null, total_capacity: number | null, available_capacity: number | null, filesystem: string | null, disk_type: number | null, is_removable: boolean | null, is_online: boolean, is_archived: boolean, date_created: string, node: Node }> } | - { key: "tags.get", input: LibraryArgs, result: Tag | null } | - { key: "tags.getExplorerData", input: LibraryArgs, result: ExplorerData } | - { key: "tags.getForObject", input: LibraryArgs, result: Array } | - { key: "tags.list", input: LibraryArgs, result: Array } | - { key: "version", input: never, result: string } | - { key: "volumes.list", input: never, result: Array }, - mutations: - { key: "files.delete", input: LibraryArgs, result: null } | - { key: "files.setFavorite", input: LibraryArgs, result: null } | - { key: "files.setNote", input: LibraryArgs, result: null } | - { key: "jobs.generateThumbsForLocation", input: LibraryArgs, result: null } | - { key: "jobs.identifyUniqueFiles", input: LibraryArgs, result: null } | - { key: "library.create", input: string, result: LibraryConfigWrapped } | - { key: "library.delete", input: string, result: null } | - { key: "library.edit", input: EditLibraryArgs, result: null } | - { key: "locations.create", input: LibraryArgs, result: null } | - { key: "locations.delete", input: LibraryArgs, result: null } | - { key: "locations.fullRescan", input: LibraryArgs, result: null } | - { key: "locations.indexer_rules.create", input: LibraryArgs, result: IndexerRule } | - { key: "locations.indexer_rules.delete", input: LibraryArgs, result: null } | - { key: "locations.quickRescan", input: LibraryArgs, result: null } | - { key: "locations.update", input: LibraryArgs, result: null } | - { key: "tags.assign", input: LibraryArgs, result: null } | - { key: "tags.create", input: LibraryArgs, result: Tag } | - { key: "tags.delete", input: LibraryArgs, result: null } | - { key: "tags.update", input: LibraryArgs, result: null }, - subscriptions: - { key: "invalidateQuery", input: never, result: InvalidateOperationEvent } | - { key: "jobs.newThumbnail", input: LibraryArgs, result: string } + queries: + | { key: 'files.readMetadata'; input: LibraryArgs; result: null } + | { key: 'getNode'; input: never; result: NodeState } + | { key: 'jobs.getHistory'; input: LibraryArgs; result: Array } + | { key: 'jobs.getRunning'; input: LibraryArgs; result: Array } + | { key: 'library.getStatistics'; input: LibraryArgs; result: Statistics } + | { key: 'library.list'; input: never; result: Array } + | { key: 'locations.getById'; input: LibraryArgs; result: Location | null } + | { + key: 'locations.getExplorerData'; + input: LibraryArgs; + result: ExplorerData; + } + | { key: 'locations.indexer_rules.get'; input: LibraryArgs; result: IndexerRule } + | { key: 'locations.indexer_rules.list'; input: LibraryArgs; result: Array } + | { + key: 'locations.list'; + input: LibraryArgs; + result: Array<{ + id: number; + pub_id: Array; + node_id: number; + name: string | null; + local_path: string | null; + total_capacity: number | null; + available_capacity: number | null; + filesystem: string | null; + disk_type: number | null; + is_removable: boolean | null; + is_online: boolean; + is_archived: boolean; + date_created: string; + node: Node; + }>; + } + | { key: 'tags.get'; input: LibraryArgs; result: Tag | null } + | { key: 'tags.getExplorerData'; input: LibraryArgs; result: ExplorerData } + | { key: 'tags.getForObject'; input: LibraryArgs; result: Array } + | { key: 'tags.list'; input: LibraryArgs; result: Array } + | { key: 'version'; input: never; result: string } + | { key: 'volumes.list'; input: never; result: Array }; + mutations: + | { key: 'files.delete'; input: LibraryArgs; result: null } + | { key: 'files.setFavorite'; input: LibraryArgs; result: null } + | { key: 'files.setNote'; input: LibraryArgs; result: null } + | { + key: 'jobs.generateThumbsForLocation'; + input: LibraryArgs; + result: null; + } + | { key: 'jobs.identifyUniqueFiles'; input: LibraryArgs; result: null } + | { key: 'library.create'; input: string; result: LibraryConfigWrapped } + | { key: 'library.delete'; input: string; result: null } + | { key: 'library.edit'; input: EditLibraryArgs; result: null } + | { key: 'locations.create'; input: LibraryArgs; result: null } + | { key: 'locations.delete'; input: LibraryArgs; result: null } + | { key: 'locations.fullRescan'; input: LibraryArgs; result: null } + | { + key: 'locations.indexer_rules.create'; + input: LibraryArgs; + result: IndexerRule; + } + | { key: 'locations.indexer_rules.delete'; input: LibraryArgs; result: null } + | { key: 'locations.quickRescan'; input: LibraryArgs; result: null } + | { key: 'locations.update'; input: LibraryArgs; result: null } + | { key: 'tags.assign'; input: LibraryArgs; result: null } + | { key: 'tags.create'; input: LibraryArgs; result: Tag } + | { key: 'tags.delete'; input: LibraryArgs; result: null } + | { key: 'tags.update'; input: LibraryArgs; result: null }; + subscriptions: + | { key: 'invalidateQuery'; input: never; result: InvalidateOperationEvent } + | { key: 'jobs.newThumbnail'; input: LibraryArgs; result: string }; }; -export interface ConfigMetadata { version: string | null } +export interface ConfigMetadata { + version: string | null; +} -export interface EditLibraryArgs { id: string, name: string | null, description: string | null } +export interface EditLibraryArgs { + id: string; + name: string | null; + description: string | null; +} -export type ExplorerContext = { type: "Location" } & Location | { type: "Tag" } & Tag +export type ExplorerContext = ({ type: 'Location' } & Location) | ({ type: 'Tag' } & Tag); -export interface ExplorerData { context: ExplorerContext, items: Array } +export interface ExplorerData { + context: ExplorerContext; + items: Array; +} -export type ExplorerItem = { type: "Path" } & { id: number, is_dir: boolean, location_id: number, materialized_path: string, name: string, extension: string | null, object_id: number | null, parent_id: number | null, key_id: number | null, date_created: string, date_modified: string, date_indexed: string, object: Object | null } | { type: "Object" } & { id: number, cas_id: string, integrity_checksum: string | null, name: string | null, extension: string | null, kind: number, size_in_bytes: string, key_id: number | null, hidden: boolean, favorite: boolean, important: boolean, has_thumbnail: boolean, has_thumbstrip: boolean, has_video_preview: boolean, ipfs_id: string | null, note: string | null, date_created: string, date_modified: string, date_indexed: string, file_paths: Array } +export type ExplorerItem = + | ({ type: 'Path' } & { + id: number; + is_dir: boolean; + location_id: number; + materialized_path: string; + name: string; + extension: string | null; + object_id: number | null; + parent_id: number | null; + key_id: number | null; + date_created: string; + date_modified: string; + date_indexed: string; + object: Object | null; + }) + | ({ type: 'Object' } & { + id: number; + cas_id: string; + integrity_checksum: string | null; + name: string | null; + extension: string | null; + kind: number; + size_in_bytes: string; + key_id: number | null; + hidden: boolean; + favorite: boolean; + important: boolean; + has_thumbnail: boolean; + has_thumbstrip: boolean; + has_video_preview: boolean; + ipfs_id: string | null; + note: string | null; + date_created: string; + date_modified: string; + date_indexed: string; + file_paths: Array; + }); -export interface FilePath { id: number, is_dir: boolean, location_id: number, materialized_path: string, name: string, extension: string | null, object_id: number | null, parent_id: number | null, key_id: number | null, date_created: string, date_modified: string, date_indexed: string } +export interface FilePath { + id: number; + is_dir: boolean; + location_id: number; + materialized_path: string; + name: string; + extension: string | null; + object_id: number | null; + parent_id: number | null; + key_id: number | null; + date_created: string; + date_modified: string; + date_indexed: string; +} -export interface GenerateThumbsForLocationArgs { id: number, path: string } +export interface GenerateThumbsForLocationArgs { + id: number; + path: string; +} -export interface IdentifyUniqueFilesArgs { id: number, path: string } +export interface IdentifyUniqueFilesArgs { + id: number; + path: string; +} -export interface IndexerRule { id: number, kind: number, name: string, parameters: Array, date_created: string, date_modified: string } +export interface IndexerRule { + id: number; + kind: number; + name: string; + parameters: Array; + date_created: string; + date_modified: string; +} -export interface IndexerRuleCreateArgs { kind: RuleKind, name: string, parameters: Array } +export interface IndexerRuleCreateArgs { + kind: RuleKind; + name: string; + parameters: Array; +} -export interface InvalidateOperationEvent { key: string, arg: any } +export interface InvalidateOperationEvent { + key: string; + arg: any; +} -export interface JobReport { id: string, name: string, data: Array | null, metadata: any | null, date_created: string, date_modified: string, status: JobStatus, task_count: number, completed_task_count: number, message: string, seconds_elapsed: number } +export interface JobReport { + id: string; + name: string; + data: Array | null; + metadata: any | null; + date_created: string; + date_modified: string; + status: JobStatus; + task_count: number; + completed_task_count: number; + message: string; + seconds_elapsed: number; +} -export type JobStatus = "Queued" | "Running" | "Completed" | "Canceled" | "Failed" | "Paused" +export type JobStatus = 'Queued' | 'Running' | 'Completed' | 'Canceled' | 'Failed' | 'Paused'; -export interface LibraryArgs { library_id: string, arg: T } +export interface LibraryArgs { + library_id: string; + arg: T; +} -export interface LibraryConfig { version: string | null, name: string, description: string } +export interface LibraryConfig { + version: string | null; + name: string; + description: string; +} -export interface LibraryConfigWrapped { uuid: string, config: LibraryConfig } +export interface LibraryConfigWrapped { + uuid: string; + config: LibraryConfig; +} -export interface Location { id: number, pub_id: Array, node_id: number, name: string | null, local_path: string | null, total_capacity: number | null, available_capacity: number | null, filesystem: string | null, disk_type: number | null, is_removable: boolean | null, is_online: boolean, is_archived: boolean, date_created: string } +export interface Location { + id: number; + pub_id: Array; + node_id: number; + name: string | null; + local_path: string | null; + total_capacity: number | null; + available_capacity: number | null; + filesystem: string | null; + disk_type: number | null; + is_removable: boolean | null; + is_online: boolean; + is_archived: boolean; + date_created: string; +} -export interface LocationCreateArgs { path: string, indexer_rules_ids: Array } +export interface LocationCreateArgs { + path: string; + indexer_rules_ids: Array; +} -export interface LocationExplorerArgs { location_id: number, path: string, limit: number, cursor: string | null } +export interface LocationExplorerArgs { + location_id: number; + path: string; + limit: number; + cursor: string | null; +} -export interface LocationUpdateArgs { id: number, name: string | null, indexer_rules_ids: Array } +export interface LocationUpdateArgs { + id: number; + name: string | null; + indexer_rules_ids: Array; +} -export interface Node { id: number, pub_id: Array, name: string, platform: number, version: string | null, last_seen: string, timezone: string | null, date_created: string } +export interface Node { + id: number; + pub_id: Array; + name: string; + platform: number; + version: string | null; + last_seen: string; + timezone: string | null; + date_created: string; +} -export interface NodeConfig { version: string | null, id: string, name: string, p2p_port: number | null } +export interface NodeConfig { + version: string | null; + id: string; + name: string; + p2p_port: number | null; +} -export interface NodeState { version: string | null, id: string, name: string, p2p_port: number | null, data_path: string } +export interface NodeState { + version: string | null; + id: string; + name: string; + p2p_port: number | null; + data_path: string; +} -export interface Object { id: number, cas_id: string, integrity_checksum: string | null, name: string | null, extension: string | null, kind: number, size_in_bytes: string, key_id: number | null, hidden: boolean, favorite: boolean, important: boolean, has_thumbnail: boolean, has_thumbstrip: boolean, has_video_preview: boolean, ipfs_id: string | null, note: string | null, date_created: string, date_modified: string, date_indexed: string } +export interface Object { + id: number; + cas_id: string; + integrity_checksum: string | null; + name: string | null; + extension: string | null; + kind: number; + size_in_bytes: string; + key_id: number | null; + hidden: boolean; + favorite: boolean; + important: boolean; + has_thumbnail: boolean; + has_thumbstrip: boolean; + has_video_preview: boolean; + ipfs_id: string | null; + note: string | null; + date_created: string; + date_modified: string; + date_indexed: string; +} -export type RuleKind = "AcceptFilesByGlob" | "RejectFilesByGlob" | "AcceptIfChildrenDirectoriesArePresent" | "RejectIfChildrenDirectoriesArePresent" +export type RuleKind = + | 'AcceptFilesByGlob' + | 'RejectFilesByGlob' + | 'AcceptIfChildrenDirectoriesArePresent' + | 'RejectIfChildrenDirectoriesArePresent'; -export interface SetFavoriteArgs { id: number, favorite: boolean } +export interface SetFavoriteArgs { + id: number; + favorite: boolean; +} -export interface SetNoteArgs { id: number, note: string | null } +export interface SetNoteArgs { + id: number; + note: string | null; +} -export interface Statistics { id: number, date_captured: string, total_object_count: number, library_db_size: string, total_bytes_used: string, total_bytes_capacity: string, total_unique_bytes: string, total_bytes_free: string, preview_media_bytes: string } +export interface Statistics { + id: number; + date_captured: string; + total_object_count: number; + library_db_size: string; + total_bytes_used: string; + total_bytes_capacity: string; + total_unique_bytes: string; + total_bytes_free: string; + preview_media_bytes: string; +} -export interface Tag { id: number, pub_id: Array, name: string | null, color: string | null, total_objects: number | null, redundancy_goal: number | null, date_created: string, date_modified: string } +export interface Tag { + id: number; + pub_id: Array; + name: string | null; + color: string | null; + total_objects: number | null; + redundancy_goal: number | null; + date_created: string; + date_modified: string; +} -export interface TagAssignArgs { object_id: number, tag_id: number, unassign: boolean } +export interface TagAssignArgs { + object_id: number; + tag_id: number; + unassign: boolean; +} -export interface TagCreateArgs { name: string, color: string } +export interface TagCreateArgs { + name: string; + color: string; +} -export interface TagUpdateArgs { id: number, name: string | null, color: string | null } +export interface TagUpdateArgs { + id: number; + name: string | null; + color: string | null; +} -export interface Volume { name: string, mount_point: string, total_capacity: bigint, available_capacity: bigint, is_removable: boolean, disk_type: string | null, file_system: string | null, is_root_filesystem: boolean } +export interface Volume { + name: string; + mount_point: string; + total_capacity: bigint; + available_capacity: bigint; + is_removable: boolean; + disk_type: string | null; + file_system: string | null; + is_root_filesystem: boolean; +} diff --git a/apps/web/src/index.html b/apps/web/src/index.html index 49ea9cd2f..2ca3b4c6f 100644 --- a/apps/web/src/index.html +++ b/apps/web/src/index.html @@ -1,15 +1,13 @@ + + + Spacedrive + + - - - Spacedrive - - - - -
- - - + +
+ + diff --git a/apps/web/src/index.tsx b/apps/web/src/index.tsx index f806c6bf4..5ec95c5f6 100644 --- a/apps/web/src/index.tsx +++ b/apps/web/src/index.tsx @@ -1,8 +1,10 @@ -import App from './App'; -import '@sd/ui/style'; import React from 'react'; import ReactDOM from 'react-dom/client'; +import App from './App'; + +import '@sd/ui/style'; + const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); root.render( diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index 0cec03c37..168f12434 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -1,7 +1,5 @@ { - "extends": "../../packages/config/interface.tsconfig.json", - "compilerOptions": {}, - "include": [ - "src" - ] + "extends": "../../packages/config/interface.tsconfig.json", + "compilerOptions": {}, + "include": ["src"] } diff --git a/crates/ffmpeg/README.md b/crates/ffmpeg/README.md index 66f535fd6..a3e205406 100644 --- a/crates/ffmpeg/README.md +++ b/crates/ffmpeg/README.md @@ -1,6 +1,6 @@ # FFMPEG Thumbnailer RS -Rust implementation of a thumbnail generation for video files using ffmpeg. +Rust implementation of a thumbnail generation for video files using ffmpeg. Based on https://github.com/dirkvdb/ffmpegthumbnailer For now only implements the minimum API for Spacedrive needs. PRs are welcome @@ -32,8 +32,8 @@ async fn main() -> Result<(), ThumbnailerError> { .with_film_strip(false) .quality(80.0)? .build(); - + thumbnailer.process("input.mp4", "output.webp").await } -``` \ No newline at end of file +``` diff --git a/crates/p2p/README.md b/crates/p2p/README.md index fd14d11a8..3921329d6 100644 --- a/crates/p2p/README.md +++ b/crates/p2p/README.md @@ -6,4 +6,4 @@ This is the P2P library which powers Spacedrive's P2P functionality. ```bash RUST_LOG="p2p=debug" cargo run -p server -``` \ No newline at end of file +``` diff --git a/crates/p2p/docs-wip/index.md b/crates/p2p/docs-wip/index.md index 861d4834c..a55446488 100644 --- a/crates/p2p/docs-wip/index.md +++ b/crates/p2p/docs-wip/index.md @@ -4,7 +4,7 @@ This document outlines the peer to peer protocol used by the Spacedrive desktop ## Concepts - - **Peer** - TODO +- **Peer** - TODO ### P2PManager @@ -34,7 +34,7 @@ To discovery other machines running Spacedrive over your local network, we make Spacedrive advertise a SRV record that looks like: -_{peer_id}_spacedrive_._udp_.local. 86400 IN SRV 10 5 5223 server.example.com. +_{peer_id}\_spacedrive_._udp_.local. 86400 IN SRV 10 5 5223 server.example.com. This system will continue to passively discover clients while Spacedrive is running. @@ -58,24 +58,11 @@ TODO Message::QueryClientAnnouncement(vec![peer_id, peer_id2]); ``` - - - - - - - - - - ## General Overview This system is designed on top of the following main technologies: - - [QUIC]() - A tcp-like protocol built on top of UDP. QUIC also supports [TLS 1.3]() for encryption and pro - - - +- [QUIC]() - A tcp-like protocol built on top of UDP. QUIC also supports [TLS 1.3]() for encryption and pro ## Pairing @@ -83,5 +70,5 @@ TODO # External Resources - - TODO: Magic Wormhole talk - - TODO: Syncthing spec +- TODO: Magic Wormhole talk +- TODO: Syncthing spec diff --git a/crates/p2p/tunnel/README.md b/crates/p2p/tunnel/README.md index b7cea7f33..89d5eb4e1 100644 --- a/crates/p2p/tunnel/README.md +++ b/crates/p2p/tunnel/README.md @@ -9,4 +9,4 @@ TODO: Write some docs ```bash cargo run -p tunnel --bin generate-env cargo run -p tunnel -``` \ No newline at end of file +``` diff --git a/crates/sync/README.md b/crates/sync/README.md index 211cd3dc6..1098b660d 100644 --- a/crates/sync/README.md +++ b/crates/sync/README.md @@ -1,4 +1,3 @@ # crdt-rs Just @brendonovich experimenting with CRDT stuff. - diff --git a/crates/sync/docs/HLC.md b/crates/sync/docs/HLC.md index 4c10ef0ca..f4a947511 100644 --- a/crates/sync/docs/HLC.md +++ b/crates/sync/docs/HLC.md @@ -1,5 +1,3 @@ - - ```rust pub fn update_with_timestamp(&self, timestamp: &Timestamp) -> Result<(), String> { let mut now = (self.clock)(); @@ -31,49 +29,45 @@ pub fn update_with_timestamp(&self, timestamp: &Timestamp) -> Result<(), String> ``` ```javascript -Timestamp.recv = function(msg) { - if (!clock) { - return null; - } +Timestamp.recv = function (msg) { + if (!clock) { + return null; + } - var now = Date.now(); + var now = Date.now(); - var msg_time = msg.millis(); - var msg_time = msg.counter(); + var msg_time = msg.millis(); + var msg_time = msg.counter(); - if (msg_time - now > config.maxDrift) { - throw new Timestamp.ClockDriftError(); - } + if (msg_time - now > config.maxDrift) { + throw new Timestamp.ClockDriftError(); + } - var last_time = clock.timestamp.millis(); - var last_time = clock.timestamp.counter(); + var last_time = clock.timestamp.millis(); + var last_time = clock.timestamp.counter(); - var max_time = Math.max(Math.max(last_time, now), msg_time); + var max_time = Math.max(Math.max(last_time, now), msg_time); - var last_time = - max_time === last_time && lNew === msg_time - ? Math.max(last_time, msg_time) + 1 - : max_time === last_time - ? last_time + 1 - : max_time === msg_time - ? msg_time + 1 - : 0; + var last_time = + max_time === last_time && lNew === msg_time + ? Math.max(last_time, msg_time) + 1 + : max_time === last_time + ? last_time + 1 + : max_time === msg_time + ? msg_time + 1 + : 0; - // 3. - if (max_time - phys > config.maxDrift) { - throw new Timestamp.ClockDriftError(); - } - if (last_time > MAX_COUNTER) { - throw new Timestamp.OverflowError(); - } + // 3. + if (max_time - phys > config.maxDrift) { + throw new Timestamp.ClockDriftError(); + } + if (last_time > MAX_COUNTER) { + throw new Timestamp.OverflowError(); + } - clock.timestamp.setMillis(max_time); - clock.timestamp.setCounter(last_time); + clock.timestamp.setMillis(max_time); + clock.timestamp.setCounter(last_time); - return new Timestamp( - clock.timestamp.millis(), - clock.timestamp.counter(), - clock.timestamp.node() - ); + return new Timestamp(clock.timestamp.millis(), clock.timestamp.counter(), clock.timestamp.node()); }; ``` diff --git a/crates/sync/example/.vscode/extensions.json b/crates/sync/example/.vscode/extensions.json index 24d7cc6de..9f5433281 100644 --- a/crates/sync/example/.vscode/extensions.json +++ b/crates/sync/example/.vscode/extensions.json @@ -1,3 +1,3 @@ { - "recommendations": ["tauri-apps.tauri-vscode", "rust-lang.rust-analyzer"] + "recommendations": ["tauri-apps.tauri-vscode", "rust-lang.rust-analyzer"] } diff --git a/crates/sync/example/index.html b/crates/sync/example/index.html index 7cb13f956..35ba8f45e 100644 --- a/crates/sync/example/index.html +++ b/crates/sync/example/index.html @@ -1,17 +1,17 @@ - - - - - - Tauri + Solid + Typescript App - + + + + + + Tauri + Solid + Typescript App + - - -
+ + +
- - + + diff --git a/crates/sync/example/package.json b/crates/sync/example/package.json index 2cd744da4..7c72ed993 100644 --- a/crates/sync/example/package.json +++ b/crates/sync/example/package.json @@ -1,32 +1,32 @@ { - "name": "@sd/sync-example", - "version": "0.0.0", - "description": "", - "scripts": { - "start": "vite", - "dev": "vite", - "build": "vite build", - "serve": "vite preview", - "tauri": "tauri" - }, - "license": "MIT", - "devDependencies": { - "@tauri-apps/cli": "^1.1.1", - "@types/babel__core": "^7.1.19", - "@types/node": "^18.8.2", - "autoprefixer": "^10.4.12", - "postcss": "^8.4.17", - "tailwindcss": "^3.1.8", - "typescript": "^4.8.4", - "vite": "^3.1.4", - "vite-plugin-solid": "^2.3.9" - }, - "dependencies": { - "@rspc/client": "~0.1.2", - "@rspc/solid": "~0.1.2", - "@rspc/tauri": "~0.1.2", - "@tanstack/solid-query": "4.10.1", - "clsx": "^1.2.1", - "solid-js": "^1.5.7" - } + "name": "@sd/sync-example", + "version": "0.0.0", + "description": "", + "scripts": { + "start": "vite", + "dev": "vite", + "build": "vite build", + "serve": "vite preview", + "tauri": "tauri" + }, + "license": "MIT", + "devDependencies": { + "@tauri-apps/cli": "^1.1.1", + "@types/babel__core": "^7.1.19", + "@types/node": "^18.8.2", + "autoprefixer": "^10.4.12", + "postcss": "^8.4.17", + "tailwindcss": "^3.1.8", + "typescript": "^4.8.4", + "vite": "^3.1.4", + "vite-plugin-solid": "^2.3.9" + }, + "dependencies": { + "@rspc/client": "~0.1.2", + "@rspc/solid": "~0.1.2", + "@rspc/tauri": "~0.1.2", + "@tanstack/solid-query": "4.10.1", + "clsx": "^1.2.1", + "solid-js": "^1.5.7" + } } diff --git a/crates/sync/example/src-tauri/tauri.conf.json b/crates/sync/example/src-tauri/tauri.conf.json index 4d9d1579e..4a5454e61 100644 --- a/crates/sync/example/src-tauri/tauri.conf.json +++ b/crates/sync/example/src-tauri/tauri.conf.json @@ -1,66 +1,66 @@ { - "build": { - "beforeDevCommand": "pnpm dev", - "beforeBuildCommand": "pnpm build", - "devPath": "http://localhost:1420", - "distDir": "../dist", - "withGlobalTauri": false - }, - "package": { - "productName": "example", - "version": "0.0.0" - }, - "tauri": { - "allowlist": { - "all": true - }, - "bundle": { - "active": true, - "category": "DeveloperTool", - "copyright": "", - "deb": { - "depends": [] - }, - "externalBin": [], - "icon": [ - "icons/32x32.png", - "icons/128x128.png", - "icons/128x128@2x.png", - "icons/icon.icns", - "icons/icon.ico" - ], - "identifier": "com.tauri.dev", - "longDescription": "", - "macOS": { - "entitlements": null, - "exceptionDomain": "", - "frameworks": [], - "providerShortName": null, - "signingIdentity": null - }, - "resources": [], - "shortDescription": "", - "targets": "all", - "windows": { - "certificateThumbprint": null, - "digestAlgorithm": "sha256", - "timestampUrl": "" - } - }, - "security": { - "csp": null - }, - "updater": { - "active": false - }, - "windows": [ - { - "fullscreen": false, - "height": 600, - "resizable": true, - "title": "example", - "width": 800 - } - ] - } + "build": { + "beforeDevCommand": "pnpm dev", + "beforeBuildCommand": "pnpm build", + "devPath": "http://localhost:1420", + "distDir": "../dist", + "withGlobalTauri": false + }, + "package": { + "productName": "example", + "version": "0.0.0" + }, + "tauri": { + "allowlist": { + "all": true + }, + "bundle": { + "active": true, + "category": "DeveloperTool", + "copyright": "", + "deb": { + "depends": [] + }, + "externalBin": [], + "icon": [ + "icons/32x32.png", + "icons/128x128.png", + "icons/128x128@2x.png", + "icons/icon.icns", + "icons/icon.ico" + ], + "identifier": "com.tauri.dev", + "longDescription": "", + "macOS": { + "entitlements": null, + "exceptionDomain": "", + "frameworks": [], + "providerShortName": null, + "signingIdentity": null + }, + "resources": [], + "shortDescription": "", + "targets": "all", + "windows": { + "certificateThumbprint": null, + "digestAlgorithm": "sha256", + "timestampUrl": "" + } + }, + "security": { + "csp": null + }, + "updater": { + "active": false + }, + "windows": [ + { + "fullscreen": false, + "height": 600, + "resizable": true, + "title": "example", + "width": 800 + } + ] + } } diff --git a/crates/sync/example/src/App.tsx b/crates/sync/example/src/App.tsx index e46b91829..7b332ccd6 100644 --- a/crates/sync/example/src/App.tsx +++ b/crates/sync/example/src/App.tsx @@ -1,75 +1,69 @@ -import clsx from "clsx"; -import { createSignal, For, JSX, Suspense } from "solid-js"; -import { queryClient, rspc } from "./rspc"; +import clsx from 'clsx'; +import { For, JSX, Suspense, createSignal } from 'solid-js'; + +import { queryClient, rspc } from './rspc'; export function App() { - const dbs = rspc.createQuery(() => ["dbs"]); + const dbs = rspc.createQuery(() => ['dbs']); - const createDb = rspc.createMutation("createDatabase", { - onSuccess: () => { - queryClient.invalidateQueries(); - }, - }); - const removeDbs = rspc.createMutation("removeDatabases", { - onSuccess: () => queryClient.invalidateQueries(), - }); + const createDb = rspc.createMutation('createDatabase', { + onSuccess: () => { + queryClient.invalidateQueries(); + } + }); + const removeDbs = rspc.createMutation('removeDatabases', { + onSuccess: () => queryClient.invalidateQueries() + }); - return ( -
-
- - -
-
    - - {(id) => ( - - - - )} - -
-
- ); + return ( +
+
+ + +
+
    + + {(id) => ( + + + + )} + +
+
+ ); } interface DatabaseViewProps { - id: string; + id: string; } -const TABS = ["Tags", "Files", "File Paths", "Messages"]; +const TABS = ['Tags', 'Files', 'File Paths', 'Messages']; function DatabaseView(props: DatabaseViewProps) { - const [currentTab, setCurrentTab] = createSignal("Tags"); + const [currentTab, setCurrentTab] = createSignal('Tags'); - return ( -
-

{props.id}

-
- -
-
-
- ); + return ( +
+

{props.id}

+
+ +
+
+
+ ); } function Button(props: JSX.ButtonHTMLAttributes) { - return ( -