From c61efdcb67be0db8e72a4a70e7be960dbb418460 Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Mon, 8 Jan 2024 14:10:26 +0800 Subject: [PATCH] Hosted Locations - Part 1 (#1861) * Hosted locations CRD * Authorise + file upload demo * Configurable path for testing perms --- Cargo.lock | Bin 243850 -> 257859 bytes core/Cargo.toml | 3 + core/src/api/cloud.rs | 173 +++++++++++++++++- .../$libraryId/settings/client/account.tsx | 124 ++++++++++++- packages/client/src/core.ts | 8 + packages/client/src/hooks/useFeatureFlag.tsx | 8 +- 6 files changed, 312 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7f84ebd3b404135963eef7517fbad02473fe644..0f490af250a45738defdb23025cfdc8536c2f70c 100644 GIT binary patch delta 7534 zcmc&(Ym8mjS=BjrocK|{?YWcKo{XK~kjG5+%qw$e zoSIi+mr4*f0ryDRLMp9Le^hE6NMfV`MYI%D5Go}Efl%6#CJ_`AA(0>?C_;sG?j6ru zPwe1?O86nqJ@@$Ry}xgLYpw4)PrSA1k6+pR?1r1xJ=z)dGt+6}Sib+7!j2PC1z(9I zot7p_tB7JEq^i>R#wtdfN`fURi3&mlt*P)X2z;Nk46%Oh$lznPdFEUUae98jPL9r> zoQ<7*x2_-A*L-n&etvfO!`uRHnTpGwJK^EcPG@HI=b4G%FTOUv^1#m2#QgD-qqcK$ zx_47|{iUlHt+sUS6P=mqm8bRo_uDpd(dAJ)JF)tITzsmPPwwRK{L*og?aa^F=`g~^ z$QbQD;CTFGe({NNXvf^d^sy0KRb#6AMH7bnDlK=_r_b!&aj6J)^K++9R#2~1qJ&YJ zxbi{iC6Zwp%A#idwW_R#e&p4mUp z+@D|DS!^$4K6F=cbR28>ic^w;kEY^Gd(Q&(mUwL|?I}+cC&YMciF2uh7E%iFP&*eejNVv5Dn$EJZ#))h-6}fjz~h{E@A#!F*V^jo=SAVE^AzwEU12TwV%{D8>BR7m921jvtiU>xb`ecO%v_Q|Qq)p_Yd?FYUM=sx(da_5@4cnmCY@9DCgZyRg1H*~9_ zM~wL&kAkChBu)dPF`4L$phQxKQqm18gO5>Xktb3`>#eOE<5Wjgsmg)7y-T!y?!I?} z;>%Fu3g7ls?P{=IHl+8gCjR%vx83+H1?=GM<^D~5v4Uvi?;kDP0Twx9)=();MYy)g zC}z2Hlu=D7*p$jj7+O^ox6E@FC0E8VB{*(!`5{saJq99qkBYWho!(2d7VcgplRuhh ztsrZ@(?gbXF=`ogFpj#Yh;e}=uAQ+E7HVYV1}Kt=Gs!GLOpP^xlLU(pT`=|6H@5$O zL-okQV{z`}a+xi0?qwi+#Voq?m`fBeOwBUpyLG!SAG*E#v31=nH}utWRZcqZY><&8 z$_eAlF%~OVL3%W=IP@_Jr;KOZ2_lT+T6v;VAi={|sYV>%*;Os=@LlN070ZY_+4VC& zg`@hple6sh?e#n~c+bCdAOa2B*{ecgp(4=&%b8s1cn* zx3zY(n2=rd+1U$RV-WjXQT@WXk%6u&+Ip;DTl&hT%RJq<9uSIG6}7^`44GJQKo^!Z zL8XXmP)URrHp&!Liaa{o09X)Xs5cgHed~0)zV|z8^LHO8*EP0bPyM-9+BftqwW2HQ zPsF*-#LV=_og@3m*plbL!@1(o6Kdghc?d+x<`e9W^HtPI1o$tx+n)2p z53I=p8PmC;%Kd$d^uy+o2c@uRCekR9CqWqj!AZ(e#vFy`3~+JUXm~mpOw zz^NdR9N#rgH})-f*;C;txtJ?=^{A;wbNx2+fN(TKZDV9;LR5&AR1O4>N&!nEX%l^k2+nkx89Y!9}i{=KZ>o$XV^rfml%_0OQncGEfnLcsSfU>Ot9@Jhw$Y>E zC5KMDWIDmyXu_16Y zHtME&{q-C3o3z}}bfEow+r6pBx7u0FH99%en_@vpL$x--8tysdtx`#Wi<>=Ci`{Dh zKan7Q4qm~9`q00&pB^r+8XoOTO!ZV^C8n3HptPshT;Ee|$xrKYXA{QO+{uOf=K69l z|Iu)&I>m{1ritfZzWG>reOKK3f|9Hi2Iyqc@=7re6B$)h0B|5&H8#mA3uzA#g$&ys z;I5=joLXO{P~ZDh+cn$2tiOCGFkcaH8(pa%|Lvh}Q+$5?O=s3z)f4ZUoN3qR#@-m9 z($D9!Uo7lFUKvXy_U)BRhO-cGlt7ad2#|IR1{!WKRFK&UBZx#3Zo~)z!Bc4YWBnT4 zb^htwZfvdTCOQAof1qIjOf9S>1{I2fACzPfJQCp@VR3+Oqayqz=Y)E&4LWh^Ny2LA zX<*EIq2DsIfc`jjtL#LBpZd`Icb)$|cHgdg_Co_jCvUw94b$d*t$PnBR72zzf}|X2 zI#pQuSVbWTjX_B6g)kcI)udfU!#UIqW#)JT-B7>u@pdL3FSh3sW!XNz;loefTKIbJ zMmrvWd>xvqbX90nE(i|=1OW-m4iGp+Y!p$zVK~xEJp!pz_#wIi!6u~sYA7E2JeVToQRcoRCT9q zpFO=lf98#1UH$U8>+3sDU0ol3{4erf*X7RqJs*Ct zIC>}S3fyyS>#uyc+9md-;qg?4csnzt!5= z9@%$nZf2o3a-DtHcQlS*#z=m#E_aol*55qG&cE=?r`vh>?=c~Rso=>ow7w{20ayd! zJ~gdNg^{98z$z*-vWj5hOErqVV#sadr<0;G9~lQpKK|e3K;H5b<=Xse4P)KK;el_g zH3Q8?dh1}M>|ocd#KS?xz&UJ=tf222RR|Jt_?^H8%>-IP@EsD3!IHvE&RO3$=Q5?{ z5ZCsefALkAk>HRqBwq!~d4*O5T!DT?vI^W`x(XR^a=2N9 z#Zc4}U<@{G#%J}bAKOriGh6D>UqX~!|oATV7i1!KT+s9-29 zhCh_}NUgKRM64()j|kHWTLaJ!xS$yJN4RNp zFG#d)R0}}@*<;Ye5GTn@7h@Y{XI+Vnx#MFu)Up(PA*{F)41EV~9;N&eEJ21G<9TaM)3iLWRjz zy|)@1T*iE_aAzN$Xv)0)`qtgKzPkL{c6?G}y8la6|K*q>e%}RvEGGp zJigNUHZ$|Za}Sn}Zhv1NcshATwCp$xf`w%Or%U|eCokzOA737iM7} z90*`lZ8z4S#N1=Fer! zGz__MWM3aIf)DB9L%Do?cm2^Rs z#60dK3{5BQ!;~?oZW~sAVth;9cUO7K;;p}1es60%_59IsseHnnpfUwcz=qM}R7o`S zgf`VPiu#v$WrF@0P6)3bAfM4SCe?C={_4~NE8*2BNl;AdH73j+BR$Mb X!4KJf6v}bk&$QMo-f@5H@I(ItT+tvg delta 396 zcmV;70dxMt-VciL4zLR;v;Rkm1(Q(J7n7z<>a$Q!Cj+x$Q%)PRn{SOQvkRTf1(*Ht z0#cJcq-T@1rXjQ1q$L5D1J(ghKVmUtF=041W;8i6Gd5!~F*Rl~H!);1FgQ6fWHn}E zIW;geIWRRfF*GnRVlXmgW-~T0HaRk5GC4RgHDfn1mjSy07L$IX60^3Z7!;G9+fuU# zyipUAYtX?AA}k6ZB6DeHZeetFmw`$F9J683{QcYnK5N0vxwyHUcvK0GGaC`HezEkGB#s4GBz+~ qWH~owFgZ6iHZYfA#sL_&84d%xHn+h=0}3~{j(r2A5V!D-15;qcZI289 diff --git a/core/Cargo.toml b/core/Cargo.toml index 644c16ad8..e0b79a15e 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -119,6 +119,9 @@ slotmap = "1.0.6" static_assertions = "1.1.0" sysinfo = "0.29.10" tar = "0.4.40" +aws-sdk-s3 = { version = "1.5.0", features = ["behavior-version-latest"] } +aws-config = "1.0.3" +aws-credential-types = "1.0.3" # Override features of transitive dependencies [dependencies.openssl] diff --git a/core/src/api/cloud.rs b/core/src/api/cloud.rs index 204c4fb62..67a4444ae 100644 --- a/core/src/api/cloud.rs +++ b/core/src/api/cloud.rs @@ -19,7 +19,9 @@ async fn parse_json_body(response: Response) -> Result AlphaRouter { - R.router().merge("library.", library::mount()) + R.router() + .merge("library.", library::mount()) + .merge("locations.", locations::mount()) } mod library { @@ -104,3 +106,172 @@ mod library { }) } } + +mod locations { + use aws_config::{Region, SdkConfig}; + use aws_credential_types::provider::future; + use aws_sdk_s3::{ + config::{Credentials, ProvideCredentials, SharedCredentialsProvider}, + primitives::ByteStream, + }; + use http_body::Full; + use serde::{Deserialize, Serialize}; + use serde_json::json; + use specta::Type; + + use crate::util::http::ensure_response; + + use super::*; + + #[derive(Type, Serialize, Deserialize)] + pub struct CloudLocation { + id: String, + name: String, + } + + #[derive(Debug, Clone, Type, Deserialize)] + pub struct AuthoriseResponse { + access_key_id: String, + secret_access_key: String, + session_token: String, + } + + pub fn mount() -> AlphaRouter { + R.router() + .procedure("list", { + R.query(|node, _: ()| async move { + let api_url = &node.env.api_url; + + node.authed_api_request(node.http.get(&format!("{api_url}/api/v1/locations"))) + .await + .and_then(ensure_response) + .map(parse_json_body::>)? + .await + }) + }) + .procedure("create", { + R.mutation(|node, name: String| async move { + let api_url = &node.env.api_url; + + node.authed_api_request( + node.http + .post(&format!("{api_url}/api/v1/locations")) + .json(&json!({ + "name": name + })), + ) + .await + .and_then(ensure_response) + .map(parse_json_body::)? + .await + }) + }) + .procedure("remove", { + R.mutation(|node, id: String| async move { + let api_url = &node.env.api_url; + + node.authed_api_request( + node.http + .post(&format!("{api_url}/api/v1/locations/delete")) + .json(&json!({ + "id": id + })), + ) + .await + .and_then(ensure_response)?; + + Ok(()) + }) + }) + // TODO: Remove this + .procedure("testing", { + // // TODO: Move this off a static. This is just for debugging. + // static AUTH_TOKEN: Lazy>> = + // Lazy::new(|| Mutex::new(None)); + + #[derive(Debug)] + pub struct CredentialsProvider(AuthoriseResponse); + + impl ProvideCredentials for CredentialsProvider { + fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> + where + Self: 'a, + { + future::ProvideCredentials::ready(Ok(Credentials::new( + self.0.access_key_id.clone(), + self.0.secret_access_key.clone(), + Some(self.0.session_token.clone()), + None, // TODO: Get this from the SD Cloud backend + "sd-cloud", + ))) + } + + fn fallback_on_interrupt(&self) -> Option { + None + } + } + + #[derive(Type, Deserialize)] + pub struct TestingParams { + id: String, + path: String, + } + + R.mutation(|node, params: TestingParams| async move { + let token = { + let token = &mut None; // AUTH_TOKEN.lock().await; // TODO: Caching of the token. For now it's annoying when debugging. + if token.is_none() { + let api_url = &node.env.api_url; + + *token = Some( + node.authed_api_request( + node.http + .post(&format!("{api_url}/api/v1/locations/authorise")) + .json(&json!({ + "id": params.id + })), + ) + .await + .and_then(ensure_response) + .map(parse_json_body::)? + .await?, + ); + } + + token.clone().expect("Checked above") + }; + + println!("{token:?}"); // TODO + + // TODO: Reuse the client between procedure calls + let client = aws_sdk_s3::Client::new( + &SdkConfig::builder() + .region(Region::new("us-west-1")) // TODO: From cloud config + .credentials_provider(SharedCredentialsProvider::new( + CredentialsProvider(token), + )) + .build(), + ); + + client + .put_object() + .bucket("spacedrive-cloud") // TODO: From cloud config + .key(params.path) // TODO: Proper access control to only the current locations files + .body(ByteStream::from_body_0_4(Full::from("Hello, world!"))) + .send() + .await + .map_err(|err| { + tracing::error!("S3 error: {err:?}"); + rspc::Error::new( + rspc::ErrorCode::InternalServerError, + "Failed to upload to S3".to_string(), + ) + })?; // TODO: Error handling + + println!("Uploaded file!"); + + Ok(()) + }) + }) + } +} diff --git a/interface/app/$libraryId/settings/client/account.tsx b/interface/app/$libraryId/settings/client/account.tsx index 16fa22646..0432a6079 100644 --- a/interface/app/$libraryId/settings/client/account.tsx +++ b/interface/app/$libraryId/settings/client/account.tsx @@ -1,7 +1,8 @@ import { Envelope, User } from '@phosphor-icons/react'; import { iconNames } from '@sd/assets/util'; -import { auth, useBridgeQuery } from '@sd/client'; -import { Button, Card } from '@sd/ui'; +import { useEffect, useState } from 'react'; +import { auth, useBridgeMutation, useBridgeQuery, useFeatureFlag } from '@sd/client'; +import { Button, Card, Input, toast } from '@sd/ui'; import { Icon, TruncatedText } from '~/components'; import { AuthRequiredOverlay } from '~/components/AuthRequiredOverlay'; @@ -31,6 +32,7 @@ export const Component = () => { + {useFeatureFlag('hostedLocations') && } ); }; @@ -97,3 +99,121 @@ const Cloud = () => { ); }; + +function HostedLocationsPlayground() { + const locations = useBridgeQuery(['cloud.locations.list'], { retry: false }); + + const [locationName, setLocationName] = useState(''); + const [path, setPath] = useState(''); + const createLocation = useBridgeMutation('cloud.locations.create', { + onSuccess(data) { + // console.log('DATA', data); // TODO: Optimistic UI + + locations.refetch(); + setLocationName(''); + } + }); + const removeLocation = useBridgeMutation('cloud.locations.remove', { + onSuccess() { + // TODO: Optimistic UI + + locations.refetch(); + } + }); + const doTheThing = useBridgeMutation('cloud.locations.testing', { + onSuccess() { + toast.success('Uploaded file!'); + }, + onError(err) { + toast.error(err.message); + } + }); + + useEffect(() => { + if (path === '' && locations.data?.[0]) { + setPath(`location/${locations.data[0].id}/hello.txt`); + } + }, [path, locations.data]); + + const isLoading = createLocation.isLoading || removeLocation.isLoading || doTheThing.isLoading; + + return ( + <> + + {/* TODO: We need UI for this. I wish I could use `prompt` for now but Tauri doesn't have it :( */} +
+ setLocationName(e.currentTarget.value)} + placeholder="My sick location" + disabled={isLoading} + /> + + +
+ + } + title="Hosted Locations" + description="Augment your local storage with our cloud!" + /> + + {/* TODO: Cleanup this mess + styles */} + {locations.status === 'loading' ?
Loading!
: null} + {locations.status !== 'loading' && locations.data?.length === 0 ? ( +
Looks like you don't have any!
+ ) : ( +
+ {locations.data?.map((location) => ( +
+

{location.name}

+ + +
+ ))} +
+ )} + +
+

Path to save when clicking 'Do the thing':

+ setPath(e.currentTarget.value)} + disabled={isLoading} + /> +
+ + ); +} diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index 34dda6357..b82a49b97 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -8,6 +8,7 @@ export type Procedures = { { key: "buildInfo", input: never, result: BuildInfo } | { key: "cloud.library.get", input: LibraryArgs, result: { uuid: string; name: string; instances: CloudInstance[]; ownerId: string } | null } | { key: "cloud.library.list", input: never, result: CloudLibrary[] } | + { key: "cloud.locations.list", input: never, result: CloudLocation[] } | { key: "ephemeralFiles.getMediaData", input: string, result: ({ type: "Image" } & ImageMetadata) | ({ type: "Video" } & VideoMetadata) | ({ type: "Audio" } & AudioMetadata) | null } | { key: "files.get", input: LibraryArgs, result: { item: Reference; nodes: CacheNode[] } | null } | { key: "files.getConvertableImageExtensions", input: never, result: string[] } | @@ -56,6 +57,9 @@ export type Procedures = { { key: "backups.restore", input: string, result: null } | { key: "cloud.library.create", input: LibraryArgs, result: null } | { key: "cloud.library.join", input: string, result: LibraryConfigWrapped } | + { key: "cloud.locations.create", input: string, result: CloudLocation } | + { key: "cloud.locations.remove", input: string, result: null } | + { key: "cloud.locations.testing", input: TestingParams, result: null } | { key: "ephemeralFiles.copyFiles", input: LibraryArgs, result: null } | { key: "ephemeralFiles.createFolder", input: LibraryArgs, result: string } | { key: "ephemeralFiles.cutFiles", input: LibraryArgs, result: null } | @@ -156,6 +160,8 @@ export type CloudInstance = { id: string; uuid: string; identity: string } export type CloudLibrary = { uuid: string; name: string; instances: CloudInstance[]; ownerId: string } +export type CloudLocation = { id: string; name: string } + export type ColorProfile = "Normal" | "Custom" | "HDRNoOriginal" | "HDRWithOriginal" | "OriginalForHDR" | "Panorama" | "PortraitHDR" | "Portrait" export type Composite = @@ -559,6 +565,8 @@ export type TagUpdateArgs = { id: number; name: string | null; color: string | n export type Target = { Object: number } | { FilePath: number } +export type TestingParams = { id: string; path: string } + export type TextMatch = { contains: string } | { startsWith: string } | { endsWith: string } | { equals: string } export type ThumbnailerPreferences = { background_processing_percentage: number } diff --git a/packages/client/src/hooks/useFeatureFlag.tsx b/packages/client/src/hooks/useFeatureFlag.tsx index 39f7854d6..0890885e0 100644 --- a/packages/client/src/hooks/useFeatureFlag.tsx +++ b/packages/client/src/hooks/useFeatureFlag.tsx @@ -5,7 +5,13 @@ import type { BackendFeature } from '../core'; import { valtioPersist } from '../lib/valito'; import { nonLibraryClient, useBridgeQuery } from '../rspc'; -export const features = ['spacedrop', 'p2pPairing', 'backups', 'debugRoutes'] as const; +export const features = [ + 'spacedrop', + 'p2pPairing', + 'backups', + 'debugRoutes', + 'hostedLocations' +] as const; // This defines which backend feature flags show up in the UI. // This is kinda a hack to not having the runtime array of possible features as Specta only exports the types.