diff --git a/.github/actions/setup-pnpm/action.yml b/.github/actions/setup-pnpm/action.yml index 89cda1ffc..baf889960 100644 --- a/.github/actions/setup-pnpm/action.yml +++ b/.github/actions/setup-pnpm/action.yml @@ -9,9 +9,7 @@ runs: using: 'composite' steps: - name: Install pnpm - uses: pnpm/action-setup@v3 - with: - version: 9.0.6 + uses: pnpm/action-setup@v4 - name: Install Node.js uses: actions/setup-node@v4 diff --git a/.github/actions/setup-system/action.yml b/.github/actions/setup-system/action.yml index d6287844f..6b934c9e0 100644 --- a/.github/actions/setup-system/action.yml +++ b/.github/actions/setup-system/action.yml @@ -29,7 +29,7 @@ runs: - name: Install LLVM and Clang if: ${{ runner.os == 'Windows' }} - uses: KyleMayes/install-llvm-action@v1 + uses: KyleMayes/install-llvm-action@v2 with: cached: ${{ steps.cache-llvm-restore.outputs.cache-hit }} version: '15' diff --git a/.prettierignore b/.prettierignore index 4d06ac882..96e019a26 100644 --- a/.prettierignore +++ b/.prettierignore @@ -35,3 +35,5 @@ package*.json # Dont format locales json interface/locales + +scripts/utils/.tmp/* diff --git a/Cargo.lock b/Cargo.lock index b2f21055d..a8e328efd 100644 Binary files a/Cargo.lock and b/Cargo.lock differ diff --git a/Cargo.toml b/Cargo.toml index 8d64619b8..9ea80477d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,38 +20,38 @@ rust-version = "1.81" [workspace.dependencies] # First party dependencies -sd-cloud-schema = { git = "https://github.com/spacedriveapp/cloud-services-schema", rev = "fb41a3c4eb" } +sd-cloud-schema = { git = "https://github.com/spacedriveapp/cloud-services-schema", rev = "bbc69c5cb2" } # Third party dependencies used by one or more of our crates async-channel = "2.3" async-stream = "0.3.6" async-trait = "0.1.83" -axum = "0.6.20" # Update blocked by hyper +axum = "0.7.7" +axum-extra = "0.9.4" base64 = "0.22.1" blake3 = "1.5.4" -bytes = "1.7.1" # Update blocked by hyper +bytes = "1.7.1" # Update blocked by hyper chrono = "0.4.38" ed25519-dalek = "2.1" flume = "0.11.0" -futures = "0.3.30" +futures = "0.3.31" futures-concurrency = "7.6" globset = "0.4.15" -http = "0.2" # Update blocked by axum -hyper = "0.14" # Update blocked due to API breaking changes -image = "0.24.9" # Update blocked due to https://github.com/image-rs/image/issues/2230 +http = "1.1" +hyper = "1.5" +image = "0.25.4" itertools = "0.13.0" lending-stream = "1.0" libc = "0.2.159" mimalloc = "0.1.43" -normpath = "1.2" +normpath = "1.3" pin-project-lite = "0.2.14" rand = "0.9.0-alpha.2" regex = "1.11" -reqwest = { version = "0.11", default-features = false } # Update blocked by hyper +reqwest = { version = "0.12.8", default-features = false } rmp = "0.8.14" rmp-serde = "1.3" rmpv = { version = "1.3", features = ["with-serde"] } -rspc = "0.1.4" # Update blocked by custom patch below serde = "1.0" serde_json = "1.0" specta = "=2.0.0-rc.20" @@ -65,46 +65,48 @@ tokio-util = "0.7.12" tracing = "0.1.40" tracing-subscriber = "0.3.18" tracing-test = "0.2.5" -uhlc = "0.8.0" # Must follow version used by specta -uuid = "1.10" # Must follow version used by specta -webp = "0.2.6" # Update blocked by image +uhlc = "0.8.0" # Must follow version used by specta +uuid = "1.10" # Must follow version used by specta +webp = "0.3.0" zeroize = "1.8" +[workspace.dependencies.rspc] +git = "https://github.com/spacedriveapp/rspc.git" +rev = "6a77167495" + [workspace.dependencies.prisma-client-rust] default-features = false features = ["migrations", "specta", "sqlite", "sqlite-create-many"] -git = "https://github.com/brendonovich/prisma-client-rust" -rev = "4f9ef9d38c" +git = "https://github.com/spacedriveapp/prisma-client-rust" +rev = "b22ad7dc7d" [workspace.dependencies.prisma-client-rust-sdk] default-features = false features = ["sqlite"] -git = "https://github.com/brendonovich/prisma-client-rust" -rev = "4f9ef9d38c" +git = "https://github.com/spacedriveapp/prisma-client-rust" +rev = "b22ad7dc7d" # Proper IOS Support [patch.crates-io.if-watch] git = "https://github.com/spacedriveapp/if-watch.git" rev = "a92c17d3f8" -# We use our own version of rspc -[patch.crates-io.rspc] -git = "https://github.com/spacedriveapp/rspc.git" -rev = "bc882f4724" - # Add `Control::open_stream_with_addrs` [patch.crates-io.libp2p] -git = "https://github.com/spacedriveapp/rust-libp2p.git" -rev = "a005656df7" +git = "https://github.com/spacedriveapp/rust-libp2p" +rev = "1024411ffa" [patch.crates-io.libp2p-core] -git = "https://github.com/spacedriveapp/rust-libp2p.git" -rev = "a005656df7" +git = "https://github.com/spacedriveapp/rust-libp2p" +rev = "1024411ffa" +[patch.crates-io.libp2p-identity] +git = "https://github.com/spacedriveapp/rust-libp2p" +rev = "1024411ffa" [patch.crates-io.libp2p-swarm] -git = "https://github.com/spacedriveapp/rust-libp2p.git" -rev = "a005656df7" +git = "https://github.com/spacedriveapp/rust-libp2p" +rev = "1024411ffa" [patch.crates-io.libp2p-stream] -git = "https://github.com/spacedriveapp/rust-libp2p.git" -rev = "a005656df7" +git = "https://github.com/spacedriveapp/rust-libp2p" +rev = "1024411ffa" [profile.dev] # Make compilation faster on macOS diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 08058a5de..31b5cdee0 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -12,19 +12,19 @@ "lint": "eslint src --cache" }, "dependencies": { - "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", - "@oscartbeaumont-sd/rspc-tauri": "github:spacedriveapp/rspc#path:packages/tauri&bc882f4724", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", + "@spacedrive/rspc-tauri": "github:spacedriveapp/rspc#path:packages/tauri&6a77167495", "@remix-run/router": "=1.13.1", "@sd/client": "workspace:*", "@sd/interface": "workspace:*", "@sd/ui": "workspace:*", "@t3-oss/env-core": "^0.7.1", - "@tanstack/react-query": "^4.36.1", - "@tauri-apps/api": "=2.0.1", - "@tauri-apps/plugin-dialog": "2.0.0", - "@tauri-apps/plugin-http": "2.0.0", + "@tanstack/react-query": "^5.59", + "@tauri-apps/api": "=2.0.3", + "@tauri-apps/plugin-dialog": "2.0.1", + "@tauri-apps/plugin-http": "2.0.1", "@tauri-apps/plugin-os": "2.0.0", - "@tauri-apps/plugin-shell": "2.0.0", + "@tauri-apps/plugin-shell": "2.0.1", "consistent-hash": "^1.2.2", "immer": "^10.0.3", "react": "^18.2.0", @@ -36,12 +36,12 @@ "devDependencies": { "@sd/config": "workspace:*", "@sentry/vite-plugin": "^2.16.0", - "@tauri-apps/cli": "2.0.1", + "@tauri-apps/cli": "2.0.4", "@types/react": "^18.2.67", "@types/react-dom": "^18.2.22", "sass": "^1.72.0", "typescript": "^5.6.2", - "vite": "^5.2.0", - "vite-tsconfig-paths": "^4.3.2" + "vite": "^5.4.9", + "vite-tsconfig-paths": "^5.0.1" } } diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index ef5d46949..6963addb8 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -16,7 +16,8 @@ sd-fda = { path = "../../../crates/fda" } sd-prisma = { path = "../../../crates/prisma" } # Workspace dependencies -axum = { workspace = true, features = ["headers", "query"] } +axum = { workspace = true, features = ["query"] } +axum-extra = { workspace = true, features = ["typed-header"] } futures = { workspace = true } http = { workspace = true } hyper = { workspace = true } @@ -38,10 +39,10 @@ opener = { version = "0.7.1", features = ["reveal"], def specta-typescript = "=0.0.7" tauri-plugin-clipboard-manager = "=2.0.1" tauri-plugin-deep-link = "=2.0.1" -tauri-plugin-dialog = "=2.0.1" -tauri-plugin-http = "=2.0.1" +tauri-plugin-dialog = "=2.0.3" +tauri-plugin-http = "=2.0.3" tauri-plugin-os = "=2.0.1" -tauri-plugin-shell = "=2.0.1" +tauri-plugin-shell = "=2.0.2" tauri-plugin-updater = "=2.0.2" # memory allocator @@ -49,12 +50,12 @@ mimalloc = { workspace = true } [dependencies.tauri] features = ["linux-libxdo", "macos-private-api", "native-tls-vendored", "unstable"] -version = "=2.0.1" +version = "=2.0.6" [dependencies.tauri-specta] features = ["derive", "typescript"] git = "https://github.com/spacedriveapp/tauri-specta" -rev = "1baf68be47" +rev = "8c85d40eb9" [target.'cfg(target_os = "linux")'.dependencies] # Spacedrive Sub-crates @@ -76,7 +77,7 @@ sd-desktop-windows = { path = "../crates/windows" } [build-dependencies] # Specific Desktop dependencies -tauri-build = "=2.0.1" +tauri-build = "=2.0.2" [features] ai-models = ["sd-core/ai"] diff --git a/apps/desktop/src-tauri/src/tauri_plugins.rs b/apps/desktop/src-tauri/src/tauri_plugins.rs index 6e2fe7eb4..bd8c9db81 100644 --- a/apps/desktop/src-tauri/src/tauri_plugins.rs +++ b/apps/desktop/src-tauri/src/tauri_plugins.rs @@ -1,20 +1,18 @@ -use std::{ - net::Ipv4Addr, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; +use std::{net::Ipv4Addr, sync::Arc}; use axum::{ - extract::{Query, State, TypedHeader}, - headers::authorization::{Authorization, Bearer}, + body::Body, + extract::{Query, State}, http::{Request, StatusCode}, middleware::{self, Next}, response::Response, RequestPartsExt, }; +use axum_extra::{ + headers::authorization::{Authorization, Bearer}, + TypedHeader, +}; use http::Method; -use hyper::server::{accept::Accept, conn::AddrIncoming}; use rand::{distr::Alphanumeric, Rng}; use sd_core::{custom_uri, Node, NodeError}; use serde::Deserialize; @@ -66,29 +64,14 @@ pub async fn sd_server_plugin( .fallback(|| async { "404 Not Found: We're past the event horizon..." }); // Only allow current device to access it - let listenera = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addra = listenera.local_addr()?; - let listenerb = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addrb = listenerb.local_addr()?; - let listenerc = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addrc = listenerc.local_addr()?; - let listenerd = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let listen_addrd = listenerd.local_addr()?; - - // let listen_addr = listener.local_addr()?; // We get it from a listener so `0` is turned into a random port + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; + let listen_addr = listener.local_addr()?; // We get it from a listener so `0` is turned into a random port let (tx, mut rx) = tokio::sync::mpsc::channel(1); - info!("Internal server listening on: http://{listen_addra:?} http://{listen_addrb:?} http://{listen_addrc:?} http://{listen_addrd:?}"); - let server = axum::Server::builder(CombinedIncoming { - a: AddrIncoming::from_listener(listenera)?, - b: AddrIncoming::from_listener(listenerb)?, - c: AddrIncoming::from_listener(listenerc)?, - d: AddrIncoming::from_listener(listenerd)?, - }); + info!("Internal server listening on: http://{listen_addr:?}"); tokio::spawn(async move { - server - .serve(app.into_make_service()) - .with_graceful_shutdown(async { + axum::serve(listener, app) + .with_graceful_shutdown(async move { rx.recv().await; }) .await @@ -96,12 +79,7 @@ pub async fn sd_server_plugin( }); let script = format!( - r#"window.__SD_CUSTOM_SERVER_AUTH_TOKEN__ = "{auth_token}"; window.__SD_CUSTOM_URI_SERVER__ = [{}];"#, - [listen_addra, listen_addrb, listen_addrc, listen_addrd] - .iter() - .map(|addr| format!("'http://{addr}'")) - .collect::>() - .join(","), + r#"window.__SD_CUSTOM_SERVER_AUTH_TOKEN__ = "{auth_token}"; window.__SD_CUSTOM_URI_SERVER__ = ['http://{listen_addr}'];"#, ); Ok(tauri::plugin::Builder::new("sd-server") @@ -127,15 +105,12 @@ struct QueryParams { token: Option, } -async fn auth_middleware( +async fn auth_middleware( Query(query): Query, State(auth_token): State, - request: Request, - next: Next, -) -> Result -where - B: Send, -{ + request: Request, + next: Next, +) -> Result { let req = if query.token.as_ref() != Some(&auth_token) { let (mut parts, body) = request.into_parts(); @@ -158,38 +133,3 @@ where Ok(next.run(req).await) } - -struct CombinedIncoming { - a: AddrIncoming, - b: AddrIncoming, - c: AddrIncoming, - d: AddrIncoming, -} - -impl Accept for CombinedIncoming { - type Conn = ::Conn; - type Error = ::Error; - - fn poll_accept( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - if let Poll::Ready(Some(value)) = Pin::new(&mut self.a).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - if let Poll::Ready(Some(value)) = Pin::new(&mut self.b).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - if let Poll::Ready(Some(value)) = Pin::new(&mut self.c).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - if let Poll::Ready(Some(value)) = Pin::new(&mut self.d).poll_accept(cx) { - return Poll::Ready(Some(value)); - } - - Poll::Pending - } -} diff --git a/apps/desktop/src/patches.ts b/apps/desktop/src/patches.ts index 8fd5d4500..f1c33a0a7 100644 --- a/apps/desktop/src/patches.ts +++ b/apps/desktop/src/patches.ts @@ -1,4 +1,4 @@ -import { tauriLink } from '@oscartbeaumont-sd/rspc-tauri/src/v2'; +import { tauriLink } from '@spacedrive/rspc-tauri/src/v2'; globalThis.isDev = import.meta.env.DEV; globalThis.rspcLinks = [ diff --git a/apps/landing/package.json b/apps/landing/package.json index 23fedacdf..6cc22e363 100644 --- a/apps/landing/package.json +++ b/apps/landing/package.json @@ -49,7 +49,7 @@ "three": "^0.161.0", "tsparticles": "^3.3.0", "unist-util-visit": "^5.0.0", - "zod": "~3.22.4" + "zod": "^3.23" }, "devDependencies": { "@next/bundle-analyzer": "^13.5.6", diff --git a/apps/mobile/modules/sd-core/core/Cargo.toml b/apps/mobile/modules/sd-core/core/Cargo.toml index 2c8e8cf3b..218461d3e 100644 --- a/apps/mobile/modules/sd-core/core/Cargo.toml +++ b/apps/mobile/modules/sd-core/core/Cargo.toml @@ -7,7 +7,6 @@ license.workspace = true repository.workspace = true rust-version.workspace = true - # Spacedrive Sub-crates [target.'cfg(target_os = "ios")'.dependencies] sd-core = { default-features = false, features = [ diff --git a/apps/mobile/modules/sd-core/src/index.ts b/apps/mobile/modules/sd-core/src/index.ts index fb034782a..dfb338f36 100644 --- a/apps/mobile/modules/sd-core/src/index.ts +++ b/apps/mobile/modules/sd-core/src/index.ts @@ -1,4 +1,4 @@ -import { AlphaRSPCError, Link, RspcRequest } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { Link, RSPCError, RspcRequest } from '@spacedrive/rspc-client'; import { EventEmitter, requireNativeModule } from 'expo-modules-core'; // It loads the native module object from the JSI or falls back to @@ -15,7 +15,7 @@ export function reactNativeLink(): Link { string, { resolve: (result: any) => void; - reject: (error: Error | AlphaRSPCError) => void; + reject: (error: Error | RSPCError) => void; } >(); @@ -29,7 +29,7 @@ export function reactNativeLink(): Link { activeMap.delete(id); } else if (result.type === 'error') { const { message, code } = result.data; - activeMap.get(id)?.reject(new AlphaRSPCError(code, message)); + activeMap.get(id)?.reject(new RSPCError(code, message)); activeMap.delete(id); } else { console.error(`rspc: received event of unknown type '${result.type}'`); diff --git a/apps/mobile/package.json b/apps/mobile/package.json index 31c28d3f3..5d2ce5dec 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -21,18 +21,17 @@ "@dr.pogodin/react-native-fs": "^2.24.1", "@gorhom/bottom-sheet": "^4.6.1", "@hookform/resolvers": "^3.1.0", - "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", - "@oscartbeaumont-sd/rspc-react": "github:spacedriveapp/rspc#path:packages/react&bc882f4724", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", "@react-native-async-storage/async-storage": "~1.23.1", "@react-native-masked-view/masked-view": "^0.3.1", "@react-navigation/bottom-tabs": "^6.5.19", - "@react-navigation/drawer": "^6.6.14", + "@react-navigation/drawer": "^6.6.15", "@react-navigation/native": "^6.1.16", "@react-navigation/native-stack": "^6.9.25", "@sd/assets": "workspace:*", "@sd/client": "workspace:*", "@shopify/flash-list": "1.6.4", - "@tanstack/react-query": "^4.36.1", + "@tanstack/react-query": "^5.59", "babel-preset-solid": "^1.9.0", "class-variance-authority": "^0.7.0", "dayjs": "^1.11.10", @@ -75,8 +74,8 @@ "twrnc": "^4.1.0", "use-count-up": "^3.0.1", "use-debounce": "^9.0.4", - "valtio": "^1.11.2", - "zod": "~3.22.4" + "valtio": "^2.0", + "zod": "^3.23" }, "devDependencies": { "@babel/core": "^7.24.0", diff --git a/apps/mobile/src/components/browse/BrowseLocations.tsx b/apps/mobile/src/components/browse/BrowseLocations.tsx index 75787d7a4..4159fbe7f 100644 --- a/apps/mobile/src/components/browse/BrowseLocations.tsx +++ b/apps/mobile/src/components/browse/BrowseLocations.tsx @@ -1,4 +1,5 @@ import { useNavigation } from '@react-navigation/native'; +import { keepPreviousData } from '@tanstack/react-query'; import { Plus } from 'phosphor-react-native'; import { useRef, useState } from 'react'; import { FlatList, Text, View } from 'react-native'; @@ -22,7 +23,7 @@ const BrowseLocations = () => { const modalRef = useRef(null); const [showAll, setShowAll] = useState(false); - const result = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['locations.list'], { placeholderData: keepPreviousData }); const locations = result.data; return ( diff --git a/apps/mobile/src/components/drawer/DrawerLocations.tsx b/apps/mobile/src/components/drawer/DrawerLocations.tsx index 2e1d8995c..0a5f87559 100644 --- a/apps/mobile/src/components/drawer/DrawerLocations.tsx +++ b/apps/mobile/src/components/drawer/DrawerLocations.tsx @@ -1,5 +1,6 @@ import { DrawerNavigationHelpers } from '@react-navigation/drawer/lib/typescript/src/types'; import { useNavigation } from '@react-navigation/native'; +import { keepPreviousData } from '@tanstack/react-query'; import { useRef } from 'react'; import { Pressable, Text, View } from 'react-native'; import { @@ -73,7 +74,7 @@ const DrawerLocations = () => { const modalRef = useRef(null); - const result = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['locations.list'], { placeholderData: keepPreviousData }); const locations = result.data || []; return ( diff --git a/apps/mobile/src/components/explorer/Explorer.tsx b/apps/mobile/src/components/explorer/Explorer.tsx index 10b448d7c..d3281aa7d 100644 --- a/apps/mobile/src/components/explorer/Explorer.tsx +++ b/apps/mobile/src/components/explorer/Explorer.tsx @@ -1,6 +1,6 @@ import { useNavigation } from '@react-navigation/native'; import { FlashList } from '@shopify/flash-list'; -import { UseInfiniteQueryResult } from '@tanstack/react-query'; +import { InfiniteData, UseInfiniteQueryResult } from '@tanstack/react-query'; import * as Haptics from 'expo-haptics'; import { useRef } from 'react'; import { ActivityIndicator } from 'react-native'; @@ -32,7 +32,7 @@ type ExplorerProps = { items: ExplorerItem[] | null; /** Function to fetch next page of items. */ loadMore: () => void; - query: UseInfiniteQueryResult>; + query: UseInfiniteQueryResult>>; count?: number; empty?: never; isEmpty?: never; diff --git a/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx b/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx index 5a09e515c..2dd851b7a 100644 --- a/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx +++ b/apps/mobile/src/components/explorer/sections/FavoriteButton.tsx @@ -12,7 +12,7 @@ type Props = { const FavoriteButton = (props: Props) => { const [favorite, setFavorite] = useState(props.data.favorite); - const { mutate: toggleFavorite, isLoading } = useLibraryMutation('files.setFavorite', { + const { mutate: toggleFavorite, isPending } = useLibraryMutation('files.setFavorite', { onSuccess: () => { // TODO: Invalidate search queries setFavorite(!favorite); @@ -22,7 +22,7 @@ const FavoriteButton = (props: Props) => { return ( toggleFavorite({ id: props.data.id, favorite: !favorite })} style={props.style} > diff --git a/apps/mobile/src/components/job/JobGroup.tsx b/apps/mobile/src/components/job/JobGroup.tsx index 26dc832ac..9000a7b0a 100644 --- a/apps/mobile/src/components/job/JobGroup.tsx +++ b/apps/mobile/src/components/job/JobGroup.tsx @@ -191,7 +191,7 @@ function Options({ activeJob, group, setShowChildJobs, showChildJobs }: OptionsP const clearJob = useLibraryMutation(['jobs.clear'], { onSuccess: () => { - rspc.queryClient.invalidateQueries(['jobs.reports']); + rspc.queryClient.invalidateQueries({ queryKey: ['jobs.reports'] }); } }); diff --git a/apps/mobile/src/components/modal/AddTagModal.tsx b/apps/mobile/src/components/modal/AddTagModal.tsx index 929820c4c..ae69a74ff 100644 --- a/apps/mobile/src/components/modal/AddTagModal.tsx +++ b/apps/mobile/src/components/modal/AddTagModal.tsx @@ -35,8 +35,8 @@ const AddTagModal = forwardRef((_, ref) => { const mutation = useLibraryMutation(['tags.assign'], { onSuccess: () => { // this makes sure that the tags are updated in the UI - rspc.queryClient.invalidateQueries(['tags.getForObject']); - rspc.queryClient.invalidateQueries(['search.paths']); + rspc.queryClient.invalidateQueries({ queryKey: ['tags.getForObject'] }); + rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }); modalRef.current?.dismiss(); } }); diff --git a/apps/mobile/src/components/modal/CreateLibraryModal.tsx b/apps/mobile/src/components/modal/CreateLibraryModal.tsx index 346988473..83d005bfd 100644 --- a/apps/mobile/src/components/modal/CreateLibraryModal.tsx +++ b/apps/mobile/src/components/modal/CreateLibraryModal.tsx @@ -17,7 +17,7 @@ const CreateLibraryModal = forwardRef((_, ref) => { const submitPlausibleEvent = usePlausibleEvent(); - const { mutate: createLibrary, isLoading: createLibLoading } = useBridgeMutation( + const { mutate: createLibrary, isPending: createLibLoading } = useBridgeMutation( 'library.create', { onSuccess: (lib) => { diff --git a/apps/mobile/src/components/modal/ImportLibraryModal.tsx b/apps/mobile/src/components/modal/ImportLibraryModal.tsx index dddedcc53..81b4656a2 100644 --- a/apps/mobile/src/components/modal/ImportLibraryModal.tsx +++ b/apps/mobile/src/components/modal/ImportLibraryModal.tsx @@ -57,11 +57,11 @@ const ImportModalLibrary = forwardRef((_, ref) => { description="No cloud libraries available to join" /> } - keyExtractor={(item) => item.uuid} + keyExtractor={(item) => item.pub_id} showsVerticalScrollIndicator={false} renderItem={({ item }) => ( @@ -93,7 +93,7 @@ const CloudLibraryCard = ({ modalRef, navigation }: Props) => { diff --git a/apps/mobile/src/components/overview/Devices.tsx b/apps/mobile/src/components/overview/Devices.tsx index 715d17bf4..fdd69ddb3 100644 --- a/apps/mobile/src/components/overview/Devices.tsx +++ b/apps/mobile/src/components/overview/Devices.tsx @@ -1,5 +1,5 @@ import * as RNFS from '@dr.pogodin/react-native-fs'; -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseQueryResult } from '@tanstack/react-query'; import React, { useEffect, useState } from 'react'; import { Platform, Text, View } from 'react-native'; @@ -17,7 +17,7 @@ import StatCard from './StatCard'; interface Props { node: NodeState | undefined; - stats: UseQueryResult; + stats: UseQueryResult; } export function hardwareModelToIcon(hardwareModel: HardwareModel) { diff --git a/apps/mobile/src/components/overview/OverviewStats.tsx b/apps/mobile/src/components/overview/OverviewStats.tsx index 2a7ecd0de..3ba73a8ce 100644 --- a/apps/mobile/src/components/overview/OverviewStats.tsx +++ b/apps/mobile/src/components/overview/OverviewStats.tsx @@ -1,5 +1,5 @@ import * as RNFS from '@dr.pogodin/react-native-fs'; -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseQueryResult } from '@tanstack/react-query'; import { useEffect, useState } from 'react'; import { Platform, Text, View } from 'react-native'; @@ -47,7 +47,7 @@ const StatItem = ({ title, bytes, isLoading, style }: StatItemProps) => { }; interface Props { - stats: UseQueryResult; + stats: UseQueryResult; } const OverviewStats = ({ stats }: Props) => { diff --git a/apps/mobile/src/components/search/filters/SavedSearches.tsx b/apps/mobile/src/components/search/filters/SavedSearches.tsx index e4dacb47e..5a63ac22f 100644 --- a/apps/mobile/src/components/search/filters/SavedSearches.tsx +++ b/apps/mobile/src/components/search/filters/SavedSearches.tsx @@ -71,7 +71,7 @@ const SavedSearch = ({ search }: Props) => { const dataForSearch = useSavedSearch(search); const rspc = useRspcLibraryContext(); const deleteSearch = useLibraryMutation('search.saved.delete', { - onSuccess: () => rspc.queryClient.invalidateQueries(['search.saved.list']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.saved.list'] }) }); return ( { diff --git a/apps/mobile/src/hooks/useSavedSearch.ts b/apps/mobile/src/hooks/useSavedSearch.ts index 0bdaa7e59..cf61b0ecc 100644 --- a/apps/mobile/src/hooks/useSavedSearch.ts +++ b/apps/mobile/src/hooks/useSavedSearch.ts @@ -1,4 +1,5 @@ import { IconTypes } from '@sd/assets/util'; +import { keepPreviousData } from '@tanstack/react-query'; import { useCallback, useMemo } from 'react'; import { SavedSearch, SearchFilterArgs, Tag, useLibraryQuery } from '@sd/client'; import { kinds } from '~/components/search/filters/Kind'; @@ -44,11 +45,11 @@ export function useSavedSearch(search: SavedSearch) { }; const locations = useLibraryQuery(['locations.list'], { - keepPreviousData: true, + placeholderData: keepPreviousData, enabled: filterKeys.includes('locations') }); const tags = useLibraryQuery(['tags.list'], { - keepPreviousData: true, + placeholderData: keepPreviousData, enabled: filterKeys.includes('tags') }); diff --git a/apps/mobile/src/navigation/tabs/SettingsStack.tsx b/apps/mobile/src/navigation/tabs/SettingsStack.tsx index 07cdc3688..c07f1111d 100644 --- a/apps/mobile/src/navigation/tabs/SettingsStack.tsx +++ b/apps/mobile/src/navigation/tabs/SettingsStack.tsx @@ -14,12 +14,10 @@ import PrivacySettingsScreen from '~/screens/settings/client/PrivacySettings'; import AboutScreen from '~/screens/settings/info/About'; import DebugScreen from '~/screens/settings/info/Debug'; import SupportScreen from '~/screens/settings/info/Support'; -import CloudSettings from '~/screens/settings/library/CloudSettings/CloudSettings'; import EditLocationSettingsScreen from '~/screens/settings/library/EditLocationSettings'; import LibraryGeneralSettingsScreen from '~/screens/settings/library/LibraryGeneralSettings'; import LocationSettingsScreen from '~/screens/settings/library/LocationSettings'; import NodesSettingsScreen from '~/screens/settings/library/NodesSettings'; -import SyncSettingsScreen from '~/screens/settings/library/SyncSettings'; import TagsSettingsScreen from '~/screens/settings/library/TagsSettings'; import SettingsScreen from '~/screens/settings/Settings'; @@ -106,16 +104,6 @@ export default function SettingsStack() { component={TagsSettingsScreen} options={{ header: () =>
}} /> -
}} - /> -
}} - /> {/* { const syncEnabled = useLibraryQuery(['sync.enabled']); useEffect(() => { - (async () => { - await enableSync.mutateAsync(null); - })(); - }, []); + enableSync.mutate(null); + }, [enableSync]); return ( diff --git a/apps/mobile/src/screens/browse/Location.tsx b/apps/mobile/src/screens/browse/Location.tsx index 81f9a152c..ca1bff307 100644 --- a/apps/mobile/src/screens/browse/Location.tsx +++ b/apps/mobile/src/screens/browse/Location.tsx @@ -62,10 +62,13 @@ export default function LocationScreen({ navigation, route }: BrowseStackScreenP filters: [...defaultFilters, ...layoutFilter].filter(Boolean), take: 30 }, - order, - onSuccess: () => getExplorerStore().resetNewThumbnails() + order }); + useEffect(() => { + getExplorerStore().resetNewThumbnails(); + }, [path]); + useEffect(() => { // Set screen title to location. if (path && path !== '') { diff --git a/apps/mobile/src/screens/search/Search.tsx b/apps/mobile/src/screens/search/Search.tsx index f0f0a3ad0..bf6dd3a0f 100644 --- a/apps/mobile/src/screens/search/Search.tsx +++ b/apps/mobile/src/screens/search/Search.tsx @@ -1,6 +1,6 @@ import { useIsFocused } from '@react-navigation/native'; import { ArrowLeft, DotsThree, FunnelSimple } from 'phosphor-react-native'; -import { Suspense, useDeferredValue, useMemo, useState } from 'react'; +import { Suspense, useDeferredValue, useEffect, useMemo, useState } from 'react'; import { ActivityIndicator, Platform, Pressable, TextInput, View } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { ObjectKindEnum, useLibraryQuery, usePathsExplorerQuery } from '@sd/client'; @@ -41,10 +41,11 @@ const SearchScreen = ({ navigation }: SearchStackScreenProps<'Search'>) => { filters: [...layoutSearchFilter, ...searchStore.mergedFilters] }, enabled: isFocused && searchStore.mergedFilters.length >= 1, // only fetch when screen is focused & filters are applied - suspense: true, - onSuccess: () => getExplorerStore().resetNewThumbnails() + suspense: true }); + useEffect(() => getExplorerStore().resetNewThumbnails(), [objects]); + useFiltersSearch(deferredSearch); const appliedFiltersLength = Object.keys(searchStore.appliedFilters).length; diff --git a/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx b/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx index c72de3eac..fddd07041 100644 --- a/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx +++ b/apps/mobile/src/screens/settings/client/AccountSettings/Login.tsx @@ -1,6 +1,6 @@ -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/src/v2'; import AsyncStorage from '@react-native-async-storage/async-storage'; import { useNavigation } from '@react-navigation/native'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseMutationResult } from '@tanstack/react-query'; import { useState } from 'react'; import { Controller } from 'react-hook-form'; @@ -116,7 +116,7 @@ async function signInClicked( email: string, password: string, navigator: SettingsStackScreenProps<'AccountProfile'>['navigation'], - cloudBootstrap: UseMutationResult, // Cloud bootstrap mutation + cloudBootstrap: UseMutationResult, // Cloud bootstrap mutation updateUserStore: ReturnType ) { try { diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx deleted file mode 100644 index f121eb8c4..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/CloudSettings.tsx +++ /dev/null @@ -1,130 +0,0 @@ -import { useMemo } from 'react'; -import { ActivityIndicator, FlatList, Text, View } from 'react-native'; -import { useLibraryContext, useLibraryMutation, useLibraryQuery } from '@sd/client'; -import { Icon } from '~/components/icons/Icon'; -import Card from '~/components/layout/Card'; -import Empty from '~/components/layout/Empty'; -import ScreenContainer from '~/components/layout/ScreenContainer'; -import VirtualizedListWrapper from '~/components/layout/VirtualizedListWrapper'; -import { Button } from '~/components/primitive/Button'; -import { Divider } from '~/components/primitive/Divider'; -import { styled, tw, twStyle } from '~/lib/tailwind'; -import { useAuthStateSnapshot } from '~/stores/auth'; - -import Instance from './Instance'; -import Library from './Library'; -import Login from './Login'; -import ThisInstance from './ThisInstance'; - -export const InfoBox = styled(View, 'rounded-md border gap-1 border-app bg-transparent p-2'); - -const CloudSettings = () => { - return ( - - - - ); -}; - -const AuthSensitiveChild = () => { - const authState = useAuthStateSnapshot(); - if (authState.status === 'loggedIn') return ; - if (authState.status === 'notLoggedIn' || authState.status === 'loggingIn') return ; - - return null; -}; - -const Authenticated = () => { - const { library } = useLibraryContext(); - const cloudLibrary = useLibraryQuery(['cloud.library.get'], { retry: false }); - const createLibrary = useLibraryMutation(['cloud.library.create']); - - const cloudInstances = useMemo( - () => - cloudLibrary.data?.instances.filter( - (instance) => instance.uuid !== library.instance_id - ), - [cloudLibrary.data, library.instance_id] - ); - - if (cloudLibrary.isLoading) { - return ( - - - - ); - } - - return ( - - {cloudLibrary.data ? ( - - - - - - - - {cloudInstances?.length} - - - Instances - - - - - } - contentContainerStyle={twStyle( - cloudInstances?.length === 0 && 'flex-row' - )} - showsHorizontalScrollIndicator={false} - ItemSeparatorComponent={() => } - renderItem={({ item }) => } - keyExtractor={(item) => item.id} - numColumns={1} - /> - - - - ) : ( - - - - - Uploading your library to the cloud will allow you to access your - library from other devices using your account & importing. - - - - - )} - - ); -}; - -export default CloudSettings; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/Instance.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/Instance.tsx deleted file mode 100644 index dbac4a60a..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/Instance.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import { Text, View } from 'react-native'; -import { CloudInstance, HardwareModel } from '@sd/client'; -import { Icon } from '~/components/icons/Icon'; -import { hardwareModelToIcon } from '~/components/overview/Devices'; -import { tw } from '~/lib/tailwind'; - -import { InfoBox } from './CloudSettings'; - -interface Props { - data: CloudInstance; -} - -const Instance = ({ data }: Props) => { - return ( - - - - - - - {data.metadata.name} - - - - Id: - - {data.id} - - - - - - - - UUID: - - {data.uuid} - - - - - - - - Public key: - - {data.identity} - - - - - - ); -}; - -export default Instance; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/Library.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/Library.tsx deleted file mode 100644 index 7e385bd31..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/Library.tsx +++ /dev/null @@ -1,66 +0,0 @@ -import { CheckCircle, XCircle } from 'phosphor-react-native'; -import { useMemo } from 'react'; -import { Text, View } from 'react-native'; -import { CloudLibrary, useLibraryContext, useLibraryMutation } from '@sd/client'; -import Card from '~/components/layout/Card'; -import { Button } from '~/components/primitive/Button'; -import { Divider } from '~/components/primitive/Divider'; -import { SettingsTitle } from '~/components/settings/SettingsContainer'; -import { tw } from '~/lib/tailwind'; -import { logout, useAuthStateSnapshot } from '~/stores/auth'; - -import { InfoBox } from './CloudSettings'; - -interface LibraryProps { - cloudLibrary?: CloudLibrary; -} - -const Library = ({ cloudLibrary }: LibraryProps) => { - const authState = useAuthStateSnapshot(); - const { library } = useLibraryContext(); - const syncLibrary = useLibraryMutation(['cloud.library.sync']); - const thisInstance = useMemo( - () => cloudLibrary?.instances.find((instance) => instance.uuid === library.instance_id), - [cloudLibrary, library.instance_id] - ); - - return ( - - - Library - {authState.status === 'loggedIn' && ( - - )} - - - Name - - {cloudLibrary?.name} - - - - ); -}; - -export default Library; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/Login.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/Login.tsx deleted file mode 100644 index 88738c329..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/Login.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import { Text, View } from 'react-native'; -import { Icon } from '~/components/icons/Icon'; -import Card from '~/components/layout/Card'; -import { Button } from '~/components/primitive/Button'; -import { tw } from '~/lib/tailwind'; -import { cancel, login, useAuthStateSnapshot } from '~/stores/auth'; - -const Login = () => { - const authState = useAuthStateSnapshot(); - const buttonText = { - notLoggedIn: 'Login', - loggingIn: 'Cancel' - }; - return ( - - - - - - Cloud Sync will upload your library to the cloud so you can access your - library from other devices by importing it from the cloud. - - - {(authState.status === 'notLoggedIn' || authState.status === 'loggingIn') && ( - - )} - - - ); -}; - -export default Login; diff --git a/apps/mobile/src/screens/settings/library/CloudSettings/ThisInstance.tsx b/apps/mobile/src/screens/settings/library/CloudSettings/ThisInstance.tsx deleted file mode 100644 index 041d6591c..000000000 --- a/apps/mobile/src/screens/settings/library/CloudSettings/ThisInstance.tsx +++ /dev/null @@ -1,76 +0,0 @@ -import { useMemo } from 'react'; -import { Text, View } from 'react-native'; -import { CloudLibrary, HardwareModel, useLibraryContext } from '@sd/client'; -import { Icon } from '~/components/icons/Icon'; -import Card from '~/components/layout/Card'; -import { hardwareModelToIcon } from '~/components/overview/Devices'; -import { Divider } from '~/components/primitive/Divider'; -import { tw } from '~/lib/tailwind'; - -import { InfoBox } from './CloudSettings'; - -interface ThisInstanceProps { - cloudLibrary?: CloudLibrary; -} - -const ThisInstance = ({ cloudLibrary }: ThisInstanceProps) => { - const { library } = useLibraryContext(); - const thisInstance = useMemo( - () => cloudLibrary?.instances.find((instance) => instance.uuid === library.instance_id), - [cloudLibrary, library.instance_id] - ); - - if (!thisInstance) return null; - - return ( - - - This Instance - - - - - - {thisInstance.metadata.name} - - - - - - Id: - {thisInstance.id} - - - - - - - UUID: - - {thisInstance.uuid} - - - - - - - - Publc Key: - - {thisInstance.identity} - - - - - - ); -}; - -export default ThisInstance; diff --git a/apps/mobile/src/screens/settings/library/EditLocationSettings.tsx b/apps/mobile/src/screens/settings/library/EditLocationSettings.tsx index ea617565d..5bc9935df 100644 --- a/apps/mobile/src/screens/settings/library/EditLocationSettings.tsx +++ b/apps/mobile/src/screens/settings/library/EditLocationSettings.tsx @@ -40,7 +40,7 @@ const EditLocationSettingsScreen = ({ onError: (e) => console.log({ e }), onSuccess: () => { form.reset(form.getValues()); - queryClient.invalidateQueries(['locations.list']); + queryClient.invalidateQueries({ queryKey: ['locations.list'] }); toast.success('Location updated!'); // TODO: navigate back & reset input focus! } @@ -90,19 +90,19 @@ const EditLocationSettingsScreen = ({ }); }, [form, navigation, onSubmit]); - useLibraryQuery(['locations.getWithRules', id], { - onSuccess: (data) => { - if (data && !form.formState.isDirty) - form.reset({ - displayName: data.name, - localPath: data.path, - indexer_rules_ids: data.indexer_rules.map((i) => i.id.toString()), - generatePreviewMedia: data.generate_preview_media, - syncPreviewMedia: data.sync_preview_media, - hidden: data.hidden - }); - } - }); + const query = useLibraryQuery(['locations.getWithRules', id]); + useEffect(() => { + const data = query.data; + if (data && !form.formState.isDirty) + form.reset({ + displayName: data.name, + localPath: data.path, + indexer_rules_ids: data.indexer_rules.map((i) => i.id.toString()), + generatePreviewMedia: data.generate_preview_media, + syncPreviewMedia: data.sync_preview_media, + hidden: data.hidden + }); + }, [form, query.data]); const fullRescan = useLibraryMutation('locations.fullRescan'); diff --git a/apps/mobile/src/screens/settings/library/SyncSettings.tsx b/apps/mobile/src/screens/settings/library/SyncSettings.tsx deleted file mode 100644 index d6cb460de..000000000 --- a/apps/mobile/src/screens/settings/library/SyncSettings.tsx +++ /dev/null @@ -1,158 +0,0 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; -import { useIsFocused } from '@react-navigation/native'; -import { MotiView } from 'moti'; -import { Circle } from 'phosphor-react-native'; -import React, { useEffect, useRef, useState } from 'react'; -import { Text, View } from 'react-native'; -import { - Procedures, - useLibraryMutation, - useLibraryQuery, - useLibrarySubscription -} from '@sd/client'; -import { Icon } from '~/components/icons/Icon'; -import Card from '~/components/layout/Card'; -import { ModalRef } from '~/components/layout/Modal'; -import ScreenContainer from '~/components/layout/ScreenContainer'; -import CloudModal from '~/components/modal/cloud/CloudModal'; -import { Button } from '~/components/primitive/Button'; -import { tw } from '~/lib/tailwind'; -import { SettingsStackScreenProps } from '~/navigation/tabs/SettingsStack'; - -const SyncSettingsScreen = ({ navigation }: SettingsStackScreenProps<'SyncSettings'>) => { - const syncEnabled = useLibraryQuery(['sync.enabled']); - const [data, setData] = useState>({}); - const modalRef = useRef(null); - - const [startBackfill, setStart] = useState(false); - const pageFocused = useIsFocused(); - const [showCloudModal, setShowCloudModal] = useState(false); - - useLibrarySubscription(['library.actors'], { onData: setData }); - - useEffect(() => { - if (startBackfill === true) { - navigation.navigate('BackfillWaitingStack', { - screen: 'BackfillWaiting' - }); - setTimeout(() => setShowCloudModal(true), 1000); - } - }, [startBackfill, navigation]); - - useEffect(() => { - if (pageFocused && showCloudModal) modalRef.current?.present(); - return () => { - if (showCloudModal) setShowCloudModal(false); - }; - }, [pageFocused, showCloudModal]); - - return ( - - {syncEnabled.data === false ? ( - - - - - - With Sync, you can share your library with other devices using P2P - technology. - - - Additionally, allowing you to enable Cloud services to upload your - library to the cloud, making it accessible on any of your devices. - - - - - - ) : ( - - {Object.keys(data).map((key) => { - return ( - - - - {key} - - {data[key] ? : } - - ); - })} - - )} - - - ); -}; - -export default SyncSettingsScreen; - -function OnlineIndicator({ online }: { online: boolean }) { - const size = 6; - return ( - - {online ? ( - - - - - ) : ( - - )} - - ); -} - -function StartButton({ name }: { name: string }) { - const startActor = useLibraryMutation(['library.startActor']); - return ( - - ); -} - -function StopButton({ name }: { name: string }) { - const stopActor = useLibraryMutation(['library.stopActor']); - return ( - - ); -} diff --git a/apps/mobile/src/stores/auth.ts b/apps/mobile/src/stores/auth.ts index c2851cf25..3f99024dd 100644 --- a/apps/mobile/src/stores/auth.ts +++ b/apps/mobile/src/stores/auth.ts @@ -1,4 +1,4 @@ -import { RSPCError } from '@oscartbeaumont-sd/rspc-client'; +import { RSPCError } from '@spacedrive/rspc-client'; import { Linking } from 'react-native'; import { createMutable } from 'solid-js/store'; import { nonLibraryClient, useSolidStore } from '@sd/client'; diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index f649ab23b..f4050942f 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -16,12 +16,13 @@ default = [] sd-core = { path = "../../core", features = ["ffmpeg", "heif"] } # Workspace dependencies -axum = { workspace = true, features = ["headers"] } -http = { workspace = true } -rspc = { workspace = true, features = ["axum"] } -tempfile = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "signal", "sync"] } -tracing = { workspace = true } +axum = { workspace = true } +axum-extra = { workspace = true, features = ["typed-header"] } +http = { workspace = true } +rspc = { workspace = true, features = ["axum"] } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "signal", "sync"] } +tracing = { workspace = true } # Specific Desktop dependencies include_dir = "0.7.3" diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index f5b2baec0..4a4246312 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -1,12 +1,15 @@ use std::{collections::HashMap, env, net::SocketAddr, path::Path}; use axum::{ + body::Body, extract::{FromRequestParts, State}, - headers::{authorization::Basic, Authorization}, http::Request, middleware::Next, response::{IntoResponse, Response}, routing::get, +}; +use axum_extra::{ + headers::{authorization::Basic, Authorization}, TypedHeader, }; use sd_core::{custom_uri, Node}; @@ -24,11 +27,7 @@ pub struct AppState { auth: HashMap, } -async fn basic_auth( - State(state): State, - request: Request, - next: Next, -) -> Response { +async fn basic_auth(State(state): State, request: Request, next: Next) -> Response { let request = if !state.auth.is_empty() { let (mut parts, body) = request.into_parts(); @@ -163,10 +162,7 @@ async fn main() { .route( "/", get(|| async move { - use axum::{ - body::{self, Full}, - response::Response, - }; + use axum::{body::Body, response::Response}; use http::{header, HeaderValue, StatusCode}; match ASSETS_DIR.get_file("index.html") { @@ -176,11 +172,11 @@ async fn main() { header::CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap(), ) - .body(body::boxed(Full::from(file.contents()))) + .body(Body::from(file.contents())) .unwrap(), None => Response::builder() .status(StatusCode::NOT_FOUND) - .body(body::boxed(axum::body::Empty::new())) + .body(Body::empty()) .unwrap(), } }), @@ -189,10 +185,7 @@ async fn main() { "/*id", get( |axum::extract::Path(path): axum::extract::Path| async move { - use axum::{ - body::{self, Empty, Full}, - response::Response, - }; + use axum::{body::Body, response::Response}; use http::{header, HeaderValue, StatusCode}; let path = path.trim_start_matches('/'); @@ -206,7 +199,7 @@ async fn main() { ) .unwrap(), ) - .body(body::boxed(Full::from(file.contents()))) + .body(Body::from(file.contents())) .unwrap(), None => match ASSETS_DIR.get_file("index.html") { Some(file) => Response::builder() @@ -215,11 +208,11 @@ async fn main() { header::CONTENT_TYPE, HeaderValue::from_str("text/html").unwrap(), ) - .body(body::boxed(Full::from(file.contents()))) + .body(Body::from(file.contents())) .unwrap(), None => Response::builder() .status(StatusCode::NOT_FOUND) - .body(body::boxed(Empty::new())) + .body(Body::empty()) .unwrap(), }, } @@ -242,8 +235,7 @@ async fn main() { let mut addr = "[::]:8080".parse::().unwrap(); // This listens on IPv6 and IPv4 addr.set_port(port); info!("Listening on http://localhost:{}", port); - axum::Server::bind(&addr) - .serve(app.into_make_service()) + axum::serve(tokio::net::TcpListener::bind(addr).await.unwrap(), app) .with_graceful_shutdown(signal) .await .expect("Error with HTTP server!"); diff --git a/apps/storybook/package.json b/apps/storybook/package.json index 0f5786227..5dfe9f15f 100644 --- a/apps/storybook/package.json +++ b/apps/storybook/package.json @@ -30,6 +30,6 @@ "storybook": "^8.0.1", "tailwindcss": "^3.4.10", "typescript": "^5.6.2", - "vite": "^5.2.0" + "vite": "^5.4.9" } } diff --git a/apps/web/package.json b/apps/web/package.json index b487de1d8..85ab05a77 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -17,10 +17,10 @@ "lint": "eslint src --cache" }, "dependencies": { - "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", "@sd/client": "workspace:*", "@sd/interface": "workspace:*", - "@tanstack/react-query": "^4.36.1", + "@tanstack/react-query": "^5.59", "html-to-image": "^1.11.11", "html2canvas": "^1.4.1", "react": "^18.2.0", @@ -41,7 +41,7 @@ "rollup-plugin-visualizer": "^5.12.0", "start-server-and-test": "^2.0.3", "typescript": "^5.6.2", - "vite": "^5.2.0", - "vite-tsconfig-paths": "^4.3.2" + "vite": "^5.4.9", + "vite-tsconfig-paths": "^5.0.1" } } diff --git a/apps/web/src/patches.ts b/apps/web/src/patches.ts index 2faac6997..a545a1217 100644 --- a/apps/web/src/patches.ts +++ b/apps/web/src/patches.ts @@ -1,4 +1,4 @@ -import { wsBatchLink } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { wsBatchLink } from '@spacedrive/rspc-client'; globalThis.isDev = import.meta.env.DEV; globalThis.rspcLinks = [ diff --git a/core/Cargo.toml b/core/Cargo.toml index 586e45550..c5f263529 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -68,7 +68,7 @@ rmp-serde = { workspace = true } rmpv = { workspace = true } rspc = { workspace = true, features = ["alpha", "axum", "chrono", "unstable", "uuid"] } sd-cloud-schema = { workspace = true } -serde = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive", "rc"] } serde_json = { workspace = true } specta = { workspace = true } strum = { workspace = true, features = ["derive"] } @@ -88,15 +88,16 @@ ctor = "0.2.8" directories = "5.0" flate2 = "1.0" hostname = "0.4.0" -http-body = "0.4.6" # Update blocked by http +http-body = "1.0" http-range = "0.1.5" -int-enum = "0.5" # Update blocked due to API breaking changes +hyper-util = { version = "0.1.9", features = ["tokio"] } +int-enum = "0.5" # Update blocked due to API breaking changes mini-moka = "0.10.3" serde-hashkey = "0.4.5" serde_repr = "0.1.19" serde_with = "3.8" slotmap = "1.0" -sysinfo = "0.29.11" # Update blocked due to API breaking changes +sysinfo = "0.29.11" # Update blocked due to API breaking changes tar = "0.4.41" tower-service = "0.3.2" tracing-appender = "0.2.3" diff --git a/core/crates/cloud-services/Cargo.toml b/core/crates/cloud-services/Cargo.toml index 359766e3d..baffe812d 100644 --- a/core/crates/cloud-services/Cargo.toml +++ b/core/crates/cloud-services/Cargo.toml @@ -39,7 +39,7 @@ zeroize = { workspace = true } # External dependencies anyhow = "1.0.86" dashmap = "6.1.0" -iroh-net = { version = "0.26", features = ["discovery-local-network", "iroh-relay"] } +iroh-net = { version = "0.27", features = ["discovery-local-network", "iroh-relay"] } paste = "=1.0.15" quic-rpc = { version = "0.12.1", features = ["quinn-transport"] } quinn = { package = "iroh-quinn", version = "0.11" } @@ -47,7 +47,7 @@ quinn = { package = "iroh-quinn", version = "0.11" } reqwest = { version = "0.12", features = ["json", "native-tls-vendored", "stream"] } reqwest-middleware = { version = "0.3", features = ["json"] } reqwest-retry = "0.6" -rustls = { version = "=0.23.13", default-features = false, features = ["brotli", "ring", "std"] } +rustls = { version = "=0.23.15", default-features = false, features = ["brotli", "ring", "std"] } rustls-platform-verifier = "0.3.3" diff --git a/core/crates/cloud-services/src/client.rs b/core/crates/cloud-services/src/client.rs index 71d4da975..d9ec361e1 100644 --- a/core/crates/cloud-services/src/client.rs +++ b/core/crates/cloud-services/src/client.rs @@ -2,11 +2,7 @@ use crate::p2p::{NotifyUser, UserResponse}; use sd_cloud_schema::{Client, Service, ServicesALPN}; -use std::{ - net::SocketAddr, - sync::{atomic::AtomicBool, Arc}, - time::Duration, -}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; use futures::Stream; use iroh_net::relay::RelayUrl; @@ -15,7 +11,7 @@ use quinn::{crypto::rustls::QuicClientConfig, ClientConfig, Endpoint}; use reqwest::{IntoUrl, Url}; use reqwest_middleware::{reqwest, ClientBuilder, ClientWithMiddleware}; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; -use tokio::sync::RwLock; +use tokio::sync::{Mutex, RwLock}; use tracing::warn; use super::{ @@ -53,7 +49,7 @@ pub struct CloudServices { notify_user_rx: flume::Receiver, user_response_tx: flume::Sender, pub(crate) user_response_rx: flume::Receiver, - pub has_bootstrapped: Arc, + pub has_bootstrapped: Arc>, } impl CloudServices { diff --git a/core/crates/cloud-services/src/error.rs b/core/crates/cloud-services/src/error.rs index 5fb4691be..f90ee028e 100644 --- a/core/crates/cloud-services/src/error.rs +++ b/core/crates/cloud-services/src/error.rs @@ -123,6 +123,8 @@ pub enum Error { EndUpdatePushSyncMessages(io::Error), #[error("Unexpected end of stream while encrypting sync messages")] UnexpectedEndOfStream, + #[error("Failed to create directory to store timestamp keeper files")] + FailedToCreateTimestampKeepersDirectory(io::Error), #[error("Failed to read last timestamp keeper for pulling sync messages: {0}")] FailedToReadLastTimestampKeeper(io::Error), #[error("Failed to handle last timestamp keeper serialization: {0}")] diff --git a/core/crates/cloud-services/src/p2p/mod.rs b/core/crates/cloud-services/src/p2p/mod.rs index 316b6f028..0f31f977c 100644 --- a/core/crates/cloud-services/src/p2p/mod.rs +++ b/core/crates/cloud-services/src/p2p/mod.rs @@ -9,20 +9,20 @@ use sd_cloud_schema::{ }; use sd_crypto::{CryptoRng, SeedableRng}; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use iroh_net::{ discovery::{ dns::DnsDiscovery, local_swarm_discovery::LocalSwarmDiscovery, pkarr::dht::DhtDiscovery, - ConcurrentDiscovery, + ConcurrentDiscovery, Discovery, }, relay::{RelayMap, RelayMode, RelayUrl}, Endpoint, NodeId, }; use reqwest::Url; use serde::{Deserialize, Serialize}; -use tokio::{spawn, sync::oneshot}; -use tracing::error; +use tokio::{spawn, sync::oneshot, time::sleep}; +use tracing::{debug, error, warn}; mod new_sync_messages_notifier; mod runner; @@ -110,6 +110,12 @@ impl CloudP2P { dns_pkarr_url: Url, relay_url: RelayUrl, ) -> Result { + let dht_discovery = DhtDiscovery::builder() + .secret_key(iroh_secret_key.clone()) + .pkarr_relay(dns_pkarr_url) + .build() + .map_err(Error::DhtDiscoveryInit)?; + let endpoint = Endpoint::builder() .alpns(vec![CloudP2PALPN::LATEST.to_vec()]) .discovery(Box::new(ConcurrentDiscovery::from_services(vec![ @@ -118,13 +124,7 @@ impl CloudP2P { LocalSwarmDiscovery::new(iroh_secret_key.public()) .map_err(Error::LocalSwarmDiscoveryInit)?, ), - Box::new( - DhtDiscovery::builder() - .secret_key(iroh_secret_key.clone()) - .pkarr_relay(dns_pkarr_url) - .build() - .map_err(Error::DhtDiscoveryInit)?, - ), + Box::new(dht_discovery.clone()), ]))) .secret_key(iroh_secret_key) .relay_mode(RelayMode::Custom(RelayMap::from_url(relay_url))) @@ -132,6 +132,23 @@ impl CloudP2P { .await .map_err(Error::CreateCloudP2PEndpoint)?; + spawn({ + let endpoint = endpoint.clone(); + async move { + loop { + let Ok(node_addr) = endpoint.node_addr().await.map_err(|e| { + warn!(?e, "Failed to get direct addresses to force publish on DHT"); + }) else { + sleep(Duration::from_secs(5)).await; + continue; + }; + + debug!("Force publishing peer on DHT"); + return dht_discovery.publish(&node_addr.info); + } + } + }); + let (msgs_tx, msgs_rx) = flume::bounded(16); spawn({ diff --git a/core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs b/core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs index aabac95f7..f4d0a3751 100644 --- a/core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs +++ b/core/crates/cloud-services/src/p2p/new_sync_messages_notifier.rs @@ -132,7 +132,7 @@ async fn connect_and_send_notification( ) -> Result<(), Error> { let client = Client::new(RpcClient::new(QuinnConnection::::from_connection( endpoint - .connect_by_node_id(*connection_id, CloudP2PALPN::LATEST) + .connect(*connection_id, CloudP2PALPN::LATEST) .await .map_err(Error::ConnectToCloudP2PNode)?, ))); diff --git a/core/crates/cloud-services/src/p2p/runner.rs b/core/crates/cloud-services/src/p2p/runner.rs index 298a6700d..3dfc33be2 100644 --- a/core/crates/cloud-services/src/p2p/runner.rs +++ b/core/crates/cloud-services/src/p2p/runner.rs @@ -601,7 +601,7 @@ async fn connect_to_first_available_client( ) -> Result, Service>, CloudP2PError> { for (device_pub_id, device_connection_id) in devices_in_group { if let Ok(connection) = endpoint - .connect_by_node_id(*device_connection_id, CloudP2PALPN::LATEST) + .connect(*device_connection_id, CloudP2PALPN::LATEST) .await .map_err( |e| error!(?e, %device_pub_id, "Failed to connect to authorizor device candidate"), diff --git a/core/crates/cloud-services/src/sync/ingest.rs b/core/crates/cloud-services/src/sync/ingest.rs index 065ceb964..a7dd65af3 100644 --- a/core/crates/cloud-services/src/sync/ingest.rs +++ b/core/crates/cloud-services/src/sync/ingest.rs @@ -1,10 +1,8 @@ use crate::Error; -use sd_core_sync::{from_cloud_crdt_ops, CompressedCRDTOperationsPerModelPerDevice, SyncManager}; +use sd_core_sync::SyncManager; use sd_actors::{Actor, Stopper}; -use sd_prisma::prisma::{cloud_crdt_operation, SortOrder}; -use sd_utils::timestamp_to_datetime; use std::{ future::IntoFuture, @@ -12,18 +10,18 @@ use std::{ atomic::{AtomicBool, Ordering}, Arc, }, - time::SystemTime, }; use futures::FutureExt; use futures_concurrency::future::Race; -use tokio::{sync::Notify, time::sleep}; +use tokio::{ + sync::Notify, + time::{sleep, Instant}, +}; use tracing::{debug, error}; use super::{ReceiveAndIngestNotifiers, SyncActors, ONE_MINUTE}; -const BATCH_SIZE: i64 = 1000; - /// Responsible for taking sync operations received from the cloud, /// and applying them to the local database via the sync system's ingest actor. @@ -43,20 +41,14 @@ impl Actor for Ingester { Stopped, } - 'outer: loop { + loop { self.active.store(true, Ordering::Relaxed); self.active_notify.notify_waiters(); - loop { - match self.run_loop_iteration().await { - Ok(IngestStatus::Completed) => break, - Ok(IngestStatus::InProgress) => {} - Err(e) => { - error!(?e, "Error during cloud sync ingester actor iteration"); - sleep(ONE_MINUTE).await; - continue 'outer; - } - } + if let Err(e) = self.run_loop_iteration().await { + error!(?e, "Error during cloud sync ingester actor iteration"); + sleep(ONE_MINUTE).await; + continue; } self.active.store(false, Ordering::Relaxed); @@ -79,11 +71,6 @@ impl Actor for Ingester { } } -enum IngestStatus { - Completed, - InProgress, -} - impl Ingester { pub const fn new( sync: SyncManager, @@ -99,48 +86,36 @@ impl Ingester { } } - async fn run_loop_iteration(&self) -> Result { - let (ops_ids, ops) = self + async fn run_loop_iteration(&self) -> Result<(), Error> { + let start = Instant::now(); + + let operations_to_ingest_count = self .sync .db .cloud_crdt_operation() - .find_many(vec![]) - .take(BATCH_SIZE) - .order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc)) - .exec() - .await - .map_err(sd_core_sync::Error::from)? - .into_iter() - .map(from_cloud_crdt_ops) - .collect::, Vec<_>), _>>()?; - - if ops_ids.is_empty() { - return Ok(IngestStatus::Completed); - } - - debug!( - messages_count = ops.len(), - first_message = ?ops - .first() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - last_message = ?ops - .last() - .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), - "Messages to ingest", - ); - - self.sync - .ingest_ops(CompressedCRDTOperationsPerModelPerDevice::new(ops)) - .await?; - - self.sync - .db - .cloud_crdt_operation() - .delete_many(vec![cloud_crdt_operation::id::in_vec(ops_ids)]) + .count(vec![]) .exec() .await .map_err(sd_core_sync::Error::from)?; - Ok(IngestStatus::InProgress) + if operations_to_ingest_count == 0 { + debug!("Nothing to ingest, early finishing ingester loop"); + return Ok(()); + } + + debug!( + operations_to_ingest_count, + "Starting sync messages cloud ingestion loop" + ); + + let ingested_count = self.sync.ingest_ops().await?; + + debug!( + ingested_count, + "Finished sync messages cloud ingestion loop in {:?}", + start.elapsed() + ); + + Ok(()) } } diff --git a/core/crates/cloud-services/src/sync/receive.rs b/core/crates/cloud-services/src/sync/receive.rs index c8e411107..f4db4b4c5 100644 --- a/core/crates/cloud-services/src/sync/receive.rs +++ b/core/crates/cloud-services/src/sync/receive.rs @@ -15,7 +15,7 @@ use sd_core_sync::{ use sd_actors::{Actor, Stopper}; use sd_crypto::{ cloud::{OneShotDecryption, SecretKey, StreamDecryption}, - primitives::{EncryptedBlock, OneShotNonce, StreamNonce}, + primitives::{EncryptedBlock, StreamNonce}, }; use sd_prisma::prisma::PrismaClient; @@ -23,34 +23,24 @@ use std::{ collections::{hash_map::Entry, HashMap}, future::IntoFuture, path::Path, - pin::Pin, sync::{ atomic::{AtomicBool, Ordering}, Arc, }, - task::{Context, Poll}, }; use chrono::{DateTime, Utc}; -use futures::{FutureExt, StreamExt, TryStreamExt}; +use futures::{FutureExt, StreamExt}; use futures_concurrency::future::{Race, TryJoin}; use quic_rpc::transport::quinn::QuinnConnection; -use reqwest::Response; -use reqwest_middleware::ClientWithMiddleware; use serde::{Deserialize, Serialize}; -use tokio::{ - fs, - io::{self, AsyncRead, AsyncReadExt, ReadBuf}, - sync::Notify, - time::sleep, -}; -use tokio_util::io::StreamReader; +use tokio::{fs, io, sync::Notify, time::sleep}; use tracing::{debug, error, instrument, warn}; use uuid::Uuid; use super::{ReceiveAndIngestNotifiers, SyncActors, ONE_MINUTE}; -const CLOUD_SYNC_DATA_KEEPER_FILE: &str = "cloud_sync_data_keeper.bin"; +const CLOUD_SYNC_DATA_KEEPER_DIRECTORY: &str = "cloud_sync_data_keeper"; /// Responsible for downloading sync operations from the cloud to be processed by the ingester @@ -121,7 +111,7 @@ impl Receiver { active_notify: Arc, ) -> Result { let (keeper, cloud_client, key_manager) = ( - LastTimestampKeeper::load(data_dir.as_ref()), + LastTimestampKeeper::load(data_dir.as_ref(), sync_group_pub_id), cloud_services.client(), cloud_services.key_manager(), ) @@ -209,7 +199,6 @@ impl Receiver { message, &self.key_manager, &self.sync, - self.cloud_services.http_client(), ) .await?; @@ -246,33 +235,23 @@ async fn handle_single_message( end_time, operations_count, key_hash, - signed_download_link, + encrypted_messages, .. }: MessagesCollection, key_manager: &KeyManager, sync: &SyncManager, - http_client: &ClientWithMiddleware, ) -> Result<(devices::PubId, DateTime), Error> { // FIXME(@fogodev): If we don't have the key hash, we need to fetch it from another device in the group if possible let Some(secret_key) = key_manager.get_key(sync_group_pub_id, &key_hash).await else { return Err(Error::MissingKeyHash); }; - let response = http_client - .get(signed_download_link) - .send() - .await - .map_err(Error::DownloadSyncMessages)? - .error_for_status() - .map_err(Error::ErrorResponseDownloadSyncMessages)?; + debug!( + size = encrypted_messages.len(), + "Received encrypted sync messages collection" + ); - let crdt_ops = if let Some(size) = response.content_length() { - debug!(size, "Received encrypted sync messages collection"); - extract_messages_known_size(response, size, secret_key, original_device_pub_id).await - } else { - debug!("Received encrypted sync messages collection of unknown size"); - extract_messages_unknown_size(response, secret_key, original_device_pub_id).await - }?; + let crdt_ops = decrypt_messages(encrypted_messages, secret_key, original_device_pub_id).await?; assert_eq!( crdt_ops.len(), @@ -285,44 +264,28 @@ async fn handle_single_message( Ok((original_device_pub_id, end_time)) } -#[instrument(skip(response, size, secret_key), err)] -async fn extract_messages_known_size( - response: Response, - size: u64, +#[instrument(skip(encrypted_messages, secret_key), fields(messages_size = %encrypted_messages.len()), err)] +async fn decrypt_messages( + encrypted_messages: Vec, secret_key: SecretKey, devices::PubId(device_pub_id): devices::PubId, ) -> Result, Error> { - let plain_text = if size <= EncryptedBlock::CIPHER_TEXT_SIZE as u64 { - OneShotDecryption::decrypt( - &secret_key, - response - .bytes() - .await - .map_err(Error::ErrorResponseDownloadReadBytesSyncMessages)? - .as_ref() - .into(), - ) - .map_err(Error::Decrypt)? + let plain_text = if encrypted_messages.len() <= EncryptedBlock::CIPHER_TEXT_SIZE { + OneShotDecryption::decrypt(&secret_key, encrypted_messages.as_slice().into()) + .map_err(Error::Decrypt)? } else { - let mut reader = StreamReader::new(response.bytes_stream().map_err(|e| { - error!(?e, "Failed to read sync messages bytes stream"); - io::Error::new(io::ErrorKind::Other, e) - })); + let (nonce, cipher_text) = encrypted_messages.split_at(size_of::()); - let mut nonce = StreamNonce::default(); + let mut plain_text = Vec::with_capacity(cipher_text.len()); - reader - .read_exact(&mut nonce) - .await - .map_err(Error::ReadNonceStreamDecryption)?; - - // TODO: Reimplement using async streaming with serde if it ever gets implemented - - let mut plain_text = vec![]; - - StreamDecryption::decrypt(&secret_key, &nonce, reader, &mut plain_text) - .await - .map_err(Error::Decrypt)?; + StreamDecryption::decrypt( + &secret_key, + nonce.try_into().expect("we split the correct amount"), + cipher_text, + &mut plain_text, + ) + .await + .map_err(Error::Decrypt)?; plain_text }; @@ -332,34 +295,6 @@ async fn extract_messages_known_size( .map_err(Error::DeserializationFailureToPullSyncMessages) } -#[instrument(skip_all, err)] -async fn extract_messages_unknown_size( - response: Response, - secret_key: SecretKey, - devices::PubId(device_pub_id): devices::PubId, -) -> Result, Error> { - let plain_text = match UnknownDownloadKind::new(response).await? { - UnknownDownloadKind::OneShot(buffer) => { - OneShotDecryption::decrypt(&secret_key, buffer.as_slice().into()) - .map_err(Error::Decrypt)? - } - - UnknownDownloadKind::Stream((nonce, reader)) => { - let mut plain_text = vec![]; - - StreamDecryption::decrypt(&secret_key, &nonce, reader, &mut plain_text) - .await - .map_err(Error::Decrypt)?; - - plain_text - } - }; - - rmp_serde::from_slice::(&plain_text) - .map(|compressed_ops| compressed_ops.into_ops(device_pub_id)) - .map_err(Error::DeserializationFailureToPullSyncMessages) -} - #[instrument(skip_all, err)] pub async fn write_cloud_ops_to_db( ops: Vec, @@ -382,8 +317,16 @@ struct LastTimestampKeeper { } impl LastTimestampKeeper { - async fn load(data_dir: &Path) -> Result { - let file_path = data_dir.join(CLOUD_SYNC_DATA_KEEPER_FILE).into_boxed_path(); + async fn load(data_dir: &Path, sync_group_pub_id: groups::PubId) -> Result { + let cloud_sync_data_directory = data_dir.join(CLOUD_SYNC_DATA_KEEPER_DIRECTORY); + + fs::create_dir_all(&cloud_sync_data_directory) + .await + .map_err(Error::FailedToCreateTimestampKeepersDirectory)?; + + let file_path = cloud_sync_data_directory + .join(format!("{sync_group_pub_id}.bin")) + .into_boxed_path(); match fs::read(&file_path).await { Ok(bytes) => Ok(Self { @@ -411,73 +354,3 @@ impl LastTimestampKeeper { .map_err(Error::FailedToWriteLastTimestampKeeper) } } - -struct UnknownDownloadSizeStreamer { - stream_reader: Box, - buffer: Vec, - was_read: usize, -} - -enum UnknownDownloadKind { - OneShot(Vec), - Stream((StreamNonce, UnknownDownloadSizeStreamer)), -} - -impl UnknownDownloadKind { - async fn new(response: Response) -> Result { - let mut buffer = Vec::with_capacity(EncryptedBlock::CIPHER_TEXT_SIZE * 2); - - let mut stream = response.bytes_stream(); - - while let Some(res) = stream.next().await { - buffer.extend(res.map_err(Error::ErrorResponseDownloadReadBytesSyncMessages)?); - if buffer.len() > EncryptedBlock::CIPHER_TEXT_SIZE { - break; - } - } - - if buffer.len() < size_of::() { - return Err(Error::IncompleteDownloadBytesSyncMessages); - } - - if buffer.len() <= EncryptedBlock::CIPHER_TEXT_SIZE { - Ok(Self::OneShot(buffer)) - } else { - let nonce_size = size_of::(); - - Ok(Self::Stream(( - StreamNonce::try_from(&buffer[..nonce_size]).expect("passing the right nonce size"), - UnknownDownloadSizeStreamer { - stream_reader: Box::new(StreamReader::new(stream.map_err(|e| { - error!(?e, "Failed to read sync messages bytes stream"); - io::Error::new(io::ErrorKind::Other, e) - }))), - buffer, - was_read: nonce_size, - }, - ))) - } - } -} - -impl AsyncRead for UnknownDownloadSizeStreamer { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - if buf.remaining() == 0 { - return Poll::Ready(Ok(())); - } - - if self.was_read == self.buffer.len() { - Pin::new(&mut self.stream_reader).poll_read(cx, buf) - } else { - let len = std::cmp::min(self.buffer.len() - self.was_read, buf.remaining()); - buf.put_slice(&self.buffer[self.was_read..(self.was_read + len)]); - self.was_read += len; - - Poll::Ready(Ok(())) - } - } -} diff --git a/core/crates/cloud-services/src/sync/send.rs b/core/crates/cloud-services/src/sync/send.rs index 2e36b8118..c0ab06e88 100644 --- a/core/crates/cloud-services/src/sync/send.rs +++ b/core/crates/cloud-services/src/sync/send.rs @@ -6,7 +6,7 @@ use sd_actors::{Actor, Stopper}; use sd_cloud_schema::{ devices, error::{ClientSideError, NotFoundError}, - sync::{self, groups, messages}, + sync::{groups, messages}, Client, Service, }; use sd_crypto::{ @@ -18,8 +18,7 @@ use sd_utils::{datetime_to_timestamp, timestamp_to_datetime}; use std::{ future::IntoFuture, - num::NonZero, - pin::{pin, Pin}, + pin::pin, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -27,15 +26,12 @@ use std::{ time::{Duration, UNIX_EPOCH}, }; -use async_stream::try_stream; use chrono::{DateTime, Utc}; -use futures::{FutureExt, SinkExt, Stream, StreamExt, TryStream, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use futures_concurrency::future::{Race, TryJoin}; -use quic_rpc::{client::UpdateSink, pattern::bidi_streaming, transport::quinn::QuinnConnection}; -use reqwest_middleware::reqwest::{header, Body}; +use quic_rpc::transport::quinn::QuinnConnection; use tokio::{ - spawn, - sync::{broadcast, oneshot, Notify, Semaphore}, + sync::{broadcast, Notify}, time::sleep, }; use tracing::{debug, error}; @@ -44,9 +40,8 @@ use uuid::Uuid; use super::{SyncActors, ONE_MINUTE}; const TEN_SECONDS: Duration = Duration::from_secs(10); -const THIRTY_SECONDS: Duration = Duration::from_secs(30); -const MESSAGES_COLLECTION_SIZE: u32 = 100_000; +const MESSAGES_COLLECTION_SIZE: u32 = 10_000; enum RaceNotifiedOrStopped { Notified, @@ -60,18 +55,6 @@ enum LoopStatus { type LatestTimestamp = NTP64; -type PushResponsesStream = Pin< - Box< - dyn Stream< - Item = Result< - Result, - bidi_streaming::ItemError>, - >, - > + Send - + Sync, - >, ->; - #[derive(Debug)] pub struct Sender { sync_group_pub_id: groups::PubId, @@ -164,6 +147,8 @@ impl Sender { } async fn run_loop_iteration(&mut self) -> Result { + debug!("Starting cloud sender actor loop iteration"); + let current_device_pub_id = devices::PubId(Uuid::from(&self.sync.device_pub_id)); let (key_hash, secret_key) = self @@ -183,6 +168,11 @@ impl Sender { let mut status = LoopStatus::Idle; let mut new_latest_timestamp = current_latest_timestamp; + + debug!( + chunk_size = MESSAGES_COLLECTION_SIZE, + "Trying to fetch chunk of sync messages from the database" + ); while let Some(ops_res) = crdt_ops_stream.next().await { let ops = ops_res?; @@ -190,9 +180,13 @@ impl Sender { break; }; + debug!("Got first and last sync messages"); + #[allow(clippy::cast_possible_truncation)] let operations_count = ops.len() as u32; + debug!(operations_count, "Got chunk of sync messages"); + new_latest_timestamp = last.timestamp; let start_time = timestamp_to_datetime(first.timestamp); @@ -205,17 +199,17 @@ impl Sender { let messages_bytes = rmp_serde::to_vec_named(&compressed_ops) .map_err(Error::SerializationFailureToPushSyncMessages)?; - let plain_text_size = messages_bytes.len(); - let expected_blob_size = if plain_text_size <= EncryptedBlock::PLAIN_TEXT_SIZE { - OneShotEncryption::cipher_text_size(&secret_key, plain_text_size) - } else { - StreamEncryption::cipher_text_size(&secret_key, plain_text_size) - } as u64; + let encrypted_messages = + encrypt_messages(&secret_key, &mut self.rng, messages_bytes).await?; - debug!(?expected_blob_size, ?key_hash, "Preparing sync message"); + let encrypted_messages_size = encrypted_messages.len(); - let (mut push_updates, mut push_responses) = self - .cloud_client + debug!( + operations_count, + encrypted_messages_size, "Sending sync messages to cloud", + ); + + self.cloud_client .sync() .messages() .push(messages::push::Request { @@ -228,60 +222,23 @@ impl Sender { device_pub_id: current_device_pub_id, key_hash: key_hash.clone(), operations_count, - start_time, - end_time, - expected_blob_size, + time_range: (start_time, end_time), + encrypted_messages, }) - .await?; + .await??; - let Some(response) = push_responses.next().await else { - return Err(Error::EmptyResponse("push initial response")); - }; - - let messages::push::Response(response_kind) = response??; - - match response_kind { - messages::push::ResponseKind::SinglePresignedUrl(url) => { - upload_to_single_url( - url, - secret_key.clone(), - self.cloud_services.http_client(), - messages_bytes, - &mut self.rng, - ) - .await?; - } - messages::push::ResponseKind::ManyPresignedUrls(urls) => { - upload_to_many_urls( - urls, - secret_key.clone(), - self.cloud_services.http_client().clone(), - messages_bytes, - &mut self.rng, - &mut push_updates, - &mut push_responses, - ) - .await?; - } - messages::push::ResponseKind::Pong => { - return Err(Error::UnexpectedResponse( - "Pong on first messages push request", - )) - } - messages::push::ResponseKind::End => { - return Err(Error::UnexpectedResponse( - "End on first messages push request", - )) - } - } - - finalize_protocol(&mut push_updates, &mut push_responses).await?; + debug!( + operations_count, + encrypted_messages_size, "Sent sync messages to cloud", + ); status = LoopStatus::SentMessages; } self.maybe_latest_timestamp = Some(new_latest_timestamp); + debug!("Finished cloud sender actor loop iteration"); + Ok(status) } @@ -303,8 +260,7 @@ impl Sender { .get_access_token() .await?, group_pub_id: self.sync_group_pub_id, - current_device_pub_id, - kind: messages::get_latest_time::Kind::ForCurrentDevice, + kind: messages::get_latest_time::Kind::ForCurrentDevice(current_device_pub_id), }) .await? { @@ -328,320 +284,44 @@ impl Sender { } } -async fn finalize_protocol( - push_updates: &mut UpdateSink< - Service, - QuinnConnection, - messages::push::RequestUpdate, - sync::Service, - >, - push_responses: &mut PushResponsesStream, -) -> Result<(), Error> { - push_updates - .send(messages::push::RequestUpdate( - messages::push::UpdateKind::End, - )) - .await - .map_err(Error::EndUpdatePushSyncMessages)?; - - let Some(response) = push_responses.next().await else { - return Err(Error::EmptyResponse("push initial response")); - }; - - let messages::push::Response(response_kind) = response??; - - match response_kind { - messages::push::ResponseKind::SinglePresignedUrl(_) - | messages::push::ResponseKind::ManyPresignedUrls(_) => { - return Err(Error::UnexpectedResponse( - "Urls responses on final messages push response", - )) - } - messages::push::ResponseKind::Pong => { - return Err(Error::UnexpectedResponse( - "Pong on final message push response", - )) - } - messages::push::ResponseKind::End => { - /* - Everything is awesome! - */ - } - } - - Ok(()) -} - -async fn upload_to_many_urls( - urls: Vec, - secret_key: SecretKey, - http_client: reqwest_middleware::ClientWithMiddleware, - messages_bytes: Vec, +async fn encrypt_messages( + secret_key: &SecretKey, rng: &mut CryptoRng, - push_updates: &mut UpdateSink< - Service, - QuinnConnection, - messages::push::RequestUpdate, - sync::Service, - >, - push_responses: &mut PushResponsesStream, -) -> Result<(), Error> { - let stop_ping_pong = Arc::new(AtomicBool::new(false)); - let (out_tx, mut out_rx) = oneshot::channel(); - let rng = CryptoRng::from_seed(rng.generate_fixed()); - - let handle = spawn(handle_multipart_upload( - urls, - secret_key, - http_client, - messages_bytes, - rng, - Arc::clone(&stop_ping_pong), - out_tx, - )); - - loop { - if stop_ping_pong.load(Ordering::Acquire) { - break; - } - - if let Err(e) = push_updates - .send(messages::push::RequestUpdate( - messages::push::UpdateKind::Ping, - )) - .await - { - error!(?e, "Failed to send push ping update"); - sleep(TEN_SECONDS).await; - continue; - } - - let Some(response) = push_responses.next().await else { - error!("Empty response from push ping response"); - continue; - }; - - match response { - Ok(Ok(messages::push::Response( - messages::push::ResponseKind::SinglePresignedUrl(_) - | messages::push::ResponseKind::ManyPresignedUrls(_), - ))) => { - unreachable!("can't receive url if we didn't send an initial request") - } - - Ok(Ok(messages::push::Response(messages::push::ResponseKind::Pong))) => { - /* - Everything is awesome! - */ - } - Ok(Ok(messages::push::Response(messages::push::ResponseKind::End))) => { - unreachable!("Can't receive an End if we didn't send an End first"); - } - - Ok(Err(e)) => { - error!(?e, "Error from push ping response"); - sleep(TEN_SECONDS).await; - continue; - } - - Err(e) => { - error!(?e, "Error from push ping response"); - sleep(TEN_SECONDS).await; - continue; - } - } - - if stop_ping_pong.load(Ordering::Acquire) { - break; - } - - sleep(THIRTY_SECONDS).await; - } - - let Ok(out) = out_rx.try_recv() else { - // SAFETY: This try_recv error can only happen if the upload task panicked - // so we're good to unwrap the error. - let e = handle.await.expect_err("upload task panicked"); - error!(?e, "Critical error while uploading sync messages"); - return Err(Error::CriticalErrorWhileUploadingSyncMessages); - }; - - out -} - -async fn handle_multipart_upload( - urls: Vec, - secret_key: SecretKey, - http_client: reqwest_middleware::ClientWithMiddleware, messages_bytes: Vec, - rng: CryptoRng, - stop_ping_pong: Arc, - out_tx: oneshot::Sender>, -) { - async fn inner( - urls: Vec, - secret_key: SecretKey, - http_client: reqwest_middleware::ClientWithMiddleware, - messages_bytes: Vec, - mut rng: CryptoRng, - ) -> Result<(), Error> { - let urls_count = urls.len(); - let message_size = messages_bytes.len(); - let blocks_per_url = message_size / urls_count / EncryptedBlock::PLAIN_TEXT_SIZE; - let cipher_text_size = StreamEncryption::cipher_text_size(&secret_key, message_size); - - let parallel_upload_semaphore = Arc::new(Semaphore::new( - std::thread::available_parallelism() - .map(NonZero::get) - .unwrap_or(1), +) -> Result, Error> { + if messages_bytes.len() <= EncryptedBlock::PLAIN_TEXT_SIZE { + let mut nonce_and_cipher_text = Vec::with_capacity(OneShotEncryption::cipher_text_size( + secret_key, + messages_bytes.len(), )); - // If we're uploading to many URLs, it implies that the message size is bigger than a single - // encryption block, so we always use stream encryption. - - let mut buffers = vec![Vec::with_capacity(cipher_text_size / urls_count); urls_count]; - let (nonce, cipher_stream) = - StreamEncryption::encrypt(&secret_key, messages_bytes.as_slice(), &mut rng); - - buffers[0].extend_from_slice(&nonce); - - let mut cipher_stream = pin!(cipher_stream); - - let mut handles = Vec::with_capacity(urls_count); - - for (idx, (mut buffer, url)) in buffers.into_iter().zip(urls).enumerate() { - for _ in 0..blocks_per_url { - if let Some(cipher_res) = cipher_stream.next().await { - buffer.extend(cipher_res.map_err(Error::Encrypt)?); - } else { - return Err(Error::UnexpectedEndOfStream); - } - } - - handles.push(spawn(upload_part( - idx, - url, - http_client.clone(), - buffer, - Arc::clone(¶llel_upload_semaphore), - ))); - } - - assert!( - cipher_stream.next().await.is_none(), - "Unexpected ciphered bytes still on stream" - ); - - handles.try_join().await.map_err(|e| { - error!(?e, "Error while uploading sync messages"); - Error::CriticalErrorWhileUploadingSyncMessages - })?; - - Ok(()) - } - - let res = inner(urls, secret_key, http_client, messages_bytes, rng).await; - stop_ping_pong.store(true, Ordering::Release); - out_tx - .send(res) - .expect("upload output channel never closes"); -} - -async fn upload_part( - idx: usize, - url: reqwest::Url, - http_client: reqwest_middleware::ClientWithMiddleware, - buffer: Vec, - parallel_upload_semaphore: Arc, -) -> Result<(), Error> { - let _permit = parallel_upload_semaphore - .acquire() - .await - .expect("Semaphore never closes"); - - let response = http_client - .put(url) - .header(header::CONTENT_LENGTH, buffer.len()) - .body(buffer) - .send() - .await - .map_err(Error::UploadSyncMessages)? - .error_for_status() - .map_err(Error::ErrorResponseUploadSyncMessages)?; - - debug!(?response, idx, "Uploaded sync messages part"); - - Ok(()) -} - -async fn upload_to_single_url( - url: reqwest::Url, - secret_key: SecretKey, - http_client: &reqwest_middleware::ClientWithMiddleware, - messages_bytes: Vec, - rng: &mut CryptoRng, -) -> Result<(), Error> { - let (cipher_text_size, body) = if messages_bytes.len() <= EncryptedBlock::PLAIN_TEXT_SIZE { let EncryptedBlock { nonce, cipher_text } = - OneShotEncryption::encrypt(&secret_key, messages_bytes.as_slice(), rng) + OneShotEncryption::encrypt(secret_key, messages_bytes.as_slice(), rng) .map_err(Error::Encrypt)?; - let cipher_text_size = nonce.len() + cipher_text.len(); + nonce_and_cipher_text.extend_from_slice(nonce.as_slice()); + nonce_and_cipher_text.extend(&cipher_text); - let mut body_bytes = Vec::with_capacity(cipher_text_size); - body_bytes.extend_from_slice(nonce.as_slice()); - body_bytes.extend(&cipher_text); - - (cipher_text_size, Body::from(body_bytes)) + Ok(nonce_and_cipher_text) } else { let mut rng = CryptoRng::from_seed(rng.generate_fixed()); - let cipher_text_size = - StreamEncryption::cipher_text_size(&secret_key, messages_bytes.len()); + let mut nonce_and_cipher_text = Vec::with_capacity(StreamEncryption::cipher_text_size( + secret_key, + messages_bytes.len(), + )); - let body_bytes = stream_encryption(secret_key, messages_bytes, &mut rng) - .try_fold( - Vec::with_capacity(cipher_text_size), - |mut body_bytes, ciphered_chunk| async move { - body_bytes.extend(ciphered_chunk); - Ok(body_bytes) - }, - ) - .await?; - - (cipher_text_size, Body::from(body_bytes)) - }; - - http_client - .put(url) - .header(header::CONTENT_LENGTH, cipher_text_size) - .body(body) - .send() - .await - .map_err(Error::UploadSyncMessages)? - .error_for_status() - .map_err(Error::ErrorResponseUploadSyncMessages)?; - - Ok(()) -} - -fn stream_encryption( - secret_key: SecretKey, - messages_bytes: Vec, - rng: &mut CryptoRng, -) -> impl TryStream, Error = Error> + Send + 'static { - let mut rng = CryptoRng::from_seed(rng.generate_fixed()); - - try_stream! { let (nonce, cipher_stream) = - StreamEncryption::encrypt(&secret_key, messages_bytes.as_slice(), &mut rng); + StreamEncryption::encrypt(secret_key, messages_bytes.as_slice(), &mut rng); + + nonce_and_cipher_text.extend_from_slice(nonce.as_slice()); let mut cipher_stream = pin!(cipher_stream); - yield nonce.to_vec(); - - while let Some(res) = cipher_stream.next().await { - yield res.map_err(Error::Encrypt)?; + while let Some(ciphered_chunk) = cipher_stream.try_next().await.map_err(Error::Encrypt)? { + nonce_and_cipher_text.extend(ciphered_chunk); } + + Ok(nonce_and_cipher_text) } } diff --git a/core/crates/file-path-helper/src/isolated_file_path_data.rs b/core/crates/file-path-helper/src/isolated_file_path_data.rs index 3e89cce0f..fe83bbee9 100644 --- a/core/crates/file-path-helper/src/isolated_file_path_data.rs +++ b/core/crates/file-path-helper/src/isolated_file_path_data.rs @@ -2,7 +2,7 @@ use sd_core_prisma_helpers::{ file_path_for_file_identifier, file_path_for_media_processor, file_path_for_object_validator, file_path_to_full_path, file_path_to_handle_custom_uri, file_path_to_handle_p2p_serve_file, file_path_to_isolate, file_path_to_isolate_with_id, file_path_to_isolate_with_pub_id, - file_path_walker, file_path_with_object, + file_path_walker, file_path_watcher_remove, file_path_with_object, }; use sd_prisma::prisma::{file_path, location}; @@ -506,7 +506,8 @@ impl_from_db!( file_path_to_isolate_with_pub_id, file_path_walker, file_path_to_isolate_with_id, - file_path_with_object + file_path_with_object, + file_path_watcher_remove ); impl_from_db_without_location_id!( diff --git a/core/crates/heavy-lifting/Cargo.toml b/core/crates/heavy-lifting/Cargo.toml index 75e99359c..20743e9fa 100644 --- a/core/crates/heavy-lifting/Cargo.toml +++ b/core/crates/heavy-lifting/Cargo.toml @@ -44,7 +44,7 @@ prisma-client-rust = { workspace = true } rmp-serde = { workspace = true } rmpv = { workspace = true } rspc = { workspace = true } -serde = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive", "rc"] } serde_json = { workspace = true } specta = { workspace = true } strum = { workspace = true, features = ["derive", "phf"] } diff --git a/core/crates/heavy-lifting/src/file_identifier/job.rs b/core/crates/heavy-lifting/src/file_identifier/job.rs index a75bbb2ca..dc2d6866c 100644 --- a/core/crates/heavy-lifting/src/file_identifier/job.rs +++ b/core/crates/heavy-lifting/src/file_identifier/job.rs @@ -14,7 +14,11 @@ use crate::{ use sd_core_file_path_helper::IsolatedFilePathData; use sd_core_prisma_helpers::{file_path_for_file_identifier, CasId}; -use sd_prisma::prisma::{device, file_path, location, SortOrder}; +use sd_prisma::{ + prisma::{device, file_path, location, SortOrder}, + prisma_sync, +}; +use sd_sync::{sync_db_not_null_entry, OperationFactory}; use sd_task_system::{ AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, TaskOutput, TaskStatus, @@ -267,15 +271,25 @@ impl Job for FileIdentifier { .. } = self; - ctx.db() - .location() - .update( - location::id::equals(location.id), - vec![location::scan_state::set( - LocationScanState::FilesIdentified as i32, - )], + let (sync_param, db_param) = sync_db_not_null_entry!( + LocationScanState::FilesIdentified as i32, + location::scan_state + ); + + ctx.sync() + .write_op( + ctx.db(), + ctx.sync().shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + [sync_param], + ), + ctx.db() + .location() + .update(location::id::equals(location.id), vec![db_param]) + .select(location::select!({ id })), ) - .exec() .await .map_err(file_identifier::Error::from)?; @@ -360,7 +374,7 @@ impl FileIdentifier { self.last_orphan_file_path_id = None; self.dispatch_deep_identifier_tasks( - &maybe_sub_iso_file_path, + maybe_sub_iso_file_path.as_ref(), ctx, device_id, dispatcher, @@ -405,7 +419,7 @@ impl FileIdentifier { self.last_orphan_file_path_id = None; self.dispatch_deep_identifier_tasks( - &maybe_sub_iso_file_path, + maybe_sub_iso_file_path.as_ref(), ctx, device_id, dispatcher, @@ -419,7 +433,7 @@ impl FileIdentifier { Phase::SearchingOrphans => { self.dispatch_deep_identifier_tasks( - &maybe_sub_iso_file_path, + maybe_sub_iso_file_path.as_ref(), ctx, device_id, dispatcher, @@ -738,7 +752,7 @@ impl FileIdentifier { async fn dispatch_deep_identifier_tasks( &mut self, - maybe_sub_iso_file_path: &Option>, + maybe_sub_iso_file_path: Option<&IsolatedFilePathData<'static>>, ctx: &impl JobContext, device_id: device::id::Type, dispatcher: &JobTaskDispatcher, diff --git a/core/crates/heavy-lifting/src/file_identifier/mod.rs b/core/crates/heavy-lifting/src/file_identifier/mod.rs index a44afbc40..f777c118d 100644 --- a/core/crates/heavy-lifting/src/file_identifier/mod.rs +++ b/core/crates/heavy-lifting/src/file_identifier/mod.rs @@ -176,7 +176,7 @@ fn orphan_path_filters_shallow( fn orphan_path_filters_deep( location_id: location::id::Type, file_path_id: Option, - maybe_sub_iso_file_path: &Option>, + maybe_sub_iso_file_path: Option<&IsolatedFilePathData<'_>>, ) -> Vec { sd_utils::chain_optional_iter( [ diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs b/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs index 22d244cac..125a72713 100644 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs +++ b/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs @@ -12,11 +12,11 @@ use sd_prisma::{ prisma::{device, file_path, location, PrismaClient}, prisma_sync, }; -use sd_sync::OperationFactory; +use sd_sync::{sync_db_entry, OperationFactory}; use sd_task_system::{ ExecStatus, Interrupter, InterruptionKind, IntoAnyTaskOutput, SerializableTask, Task, TaskId, }; -use sd_utils::{error::FileIOError, msgpack}; +use sd_utils::error::FileIOError; use std::{ collections::HashMap, convert::identity, future::IntoFuture, mem, path::PathBuf, pin::pin, @@ -403,19 +403,17 @@ async fn assign_cas_id_to_file_paths( let (ops, queries) = identified_files .iter() .map(|(pub_id, IdentifiedFile { cas_id, .. })| { + let (sync_param, db_param) = sync_db_entry!(cas_id, file_path::cas_id); + ( sync.shared_update( prisma_sync::file_path::SyncId { pub_id: pub_id.to_db(), }, - file_path::cas_id::NAME, - msgpack!(cas_id), + [sync_param], ), db.file_path() - .update( - file_path::pub_id::equals(pub_id.to_db()), - vec![file_path::cas_id::set(cas_id.into())], - ) + .update(file_path::pub_id::equals(pub_id.to_db()), vec![db_param]) // We don't need any data here, just the id avoids receiving the entire object // as we can't pass an empty select macro call .select(file_path::select!({ id })), diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs b/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs index 13e4d7d9f..59f75d0a9 100644 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs +++ b/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs @@ -9,7 +9,7 @@ use sd_prisma::{ prisma_sync, }; use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, CRDTOperation, OperationFactory}; -use sd_utils::{chain_optional_iter, msgpack}; +use sd_utils::chain_optional_iter; use std::collections::{HashMap, HashSet}; @@ -47,10 +47,12 @@ fn connect_file_path_to_object<'db>( prisma_sync::file_path::SyncId { pub_id: file_path_pub_id.to_db(), }, - file_path::object::NAME, - msgpack!(prisma_sync::object::SyncId { - pub_id: object_pub_id.to_db(), - }), + [sync_entry!( + prisma_sync::object::SyncId { + pub_id: object_pub_id.to_db(), + }, + file_path::object + )], ), db.file_path() .update( diff --git a/core/crates/heavy-lifting/src/indexer/job.rs b/core/crates/heavy-lifting/src/indexer/job.rs index d910fec81..cf19fbb90 100644 --- a/core/crates/heavy-lifting/src/indexer/job.rs +++ b/core/crates/heavy-lifting/src/indexer/job.rs @@ -16,7 +16,11 @@ use sd_core_file_path_helper::IsolatedFilePathData; use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; use sd_core_prisma_helpers::location_with_indexer_rules; -use sd_prisma::prisma::{device, location}; +use sd_prisma::{ + prisma::{device, location}, + prisma_sync, +}; +use sd_sync::{sync_db_not_null_entry, OperationFactory}; use sd_task_system::{ AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, TaskOutput, TaskStatus, @@ -269,7 +273,7 @@ impl Job for Indexer { .await?; } - update_location_size(location.id, ctx.db(), &ctx).await?; + update_location_size(location.id, location.pub_id.clone(), &ctx).await?; metadata.mean_db_write_time += start_size_update_time.elapsed(); } @@ -287,13 +291,23 @@ impl Job for Indexer { "all tasks must be completed here" ); - ctx.db() - .location() - .update( - location::id::equals(location.id), - vec![location::scan_state::set(LocationScanState::Indexed as i32)], + let (sync_param, db_param) = + sync_db_not_null_entry!(LocationScanState::Indexed as i32, location::scan_state); + + ctx.sync() + .write_op( + ctx.db(), + ctx.sync().shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + [sync_param], + ), + ctx.db() + .location() + .update(location::id::equals(location.id), vec![db_param]) + .select(location::select!({ id })), ) - .exec() .await .map_err(indexer::Error::from)?; diff --git a/core/crates/heavy-lifting/src/indexer/mod.rs b/core/crates/heavy-lifting/src/indexer/mod.rs index 1ad78902b..6880e6d91 100644 --- a/core/crates/heavy-lifting/src/indexer/mod.rs +++ b/core/crates/heavy-lifting/src/indexer/mod.rs @@ -10,11 +10,11 @@ use sd_prisma::{ prisma::{file_path, indexer_rule, location, PrismaClient, SortOrder}, prisma_sync, }; -use sd_sync::OperationFactory; +use sd_sync::{sync_db_entry, OperationFactory}; use sd_utils::{ db::{size_in_bytes_from_db, size_in_bytes_to_db, MissingFieldError}, error::{FileIOError, NonUtf8PathError}, - from_bytes_to_uuid, msgpack, + from_bytes_to_uuid, }; use std::{ @@ -146,22 +146,20 @@ async fn update_directory_sizes( .map(|file_path| { let size_bytes = iso_paths_and_sizes .get(&IsolatedFilePathData::try_from(&file_path)?) - .map(|size| size.to_be_bytes().to_vec()) + .map(|size| size_in_bytes_to_db(*size)) .expect("must be here"); + let (sync_param, db_param) = sync_db_entry!(size_bytes, file_path::size_in_bytes_bytes); + Ok(( sync.shared_update( prisma_sync::file_path::SyncId { pub_id: file_path.pub_id.clone(), }, - file_path::size_in_bytes_bytes::NAME, - msgpack!(size_bytes), + [sync_param], ), db.file_path() - .update( - file_path::pub_id::equals(file_path.pub_id), - vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))], - ) + .update(file_path::pub_id::equals(file_path.pub_id), vec![db_param]) .select(file_path::select!({ id })), )) }) @@ -178,35 +176,45 @@ async fn update_directory_sizes( async fn update_location_size( location_id: location::id::Type, - db: &PrismaClient, + location_pub_id: location::pub_id::Type, ctx: &impl OuterContext, ) -> Result<(), Error> { - let total_size = db - .file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some("/".to_string())), - ]) - .select(file_path::select!({ size_in_bytes_bytes })) - .exec() - .await? - .into_iter() - .filter_map(|file_path| { - file_path - .size_in_bytes_bytes - .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) - }) - .sum::(); + let db = ctx.db(); + let sync = ctx.sync(); - db.location() - .update( - location::id::equals(location_id), - vec![location::size_in_bytes::set(Some( - total_size.to_be_bytes().to_vec(), - ))], - ) - .exec() - .await?; + let total_size = size_in_bytes_to_db( + db.file_path() + .find_many(vec![ + file_path::location_id::equals(Some(location_id)), + file_path::materialized_path::equals(Some("/".to_string())), + ]) + .select(file_path::select!({ size_in_bytes_bytes })) + .exec() + .await? + .into_iter() + .filter_map(|file_path| { + file_path + .size_in_bytes_bytes + .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) + }) + .sum::(), + ); + + let (sync_param, db_param) = sync_db_entry!(total_size, location::size_in_bytes); + + sync.write_op( + db, + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: location_pub_id, + }, + [sync_param], + ), + db.location() + .update(location::id::equals(location_id), vec![db_param]) + .select(location::select!({ id })), + ) + .await?; ctx.invalidate_query("locations.list"); ctx.invalidate_query("locations.get"); @@ -334,18 +342,19 @@ pub async fn reverse_update_directories_sizes( { let size_bytes = size_in_bytes_to_db(size); + let (sync_param, db_param) = + sync_db_entry!(size_bytes, file_path::size_in_bytes_bytes); + Some(( sync.shared_update( prisma_sync::file_path::SyncId { pub_id: pub_id.clone(), }, - file_path::size_in_bytes_bytes::NAME, - msgpack!(size_bytes), - ), - db.file_path().update( - file_path::pub_id::equals(pub_id), - vec![file_path::size_in_bytes_bytes::set(Some(size_bytes))], + [sync_param], ), + db.file_path() + .update(file_path::pub_id::equals(pub_id), vec![db_param]) + .select(file_path::select!({ id })), )) } else { warn!("Got a missing ancestor for a file_path in the database, ignoring..."); diff --git a/core/crates/heavy-lifting/src/indexer/shallow.rs b/core/crates/heavy-lifting/src/indexer/shallow.rs index 90a22eead..1bc55b556 100644 --- a/core/crates/heavy-lifting/src/indexer/shallow.rs +++ b/core/crates/heavy-lifting/src/indexer/shallow.rs @@ -136,7 +136,7 @@ pub async fn shallow( .await?; } - update_location_size(location.id, db, ctx).await?; + update_location_size(location.id, location.pub_id, ctx).await?; } if indexed_count > 0 || removed_count > 0 { diff --git a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs b/core/crates/heavy-lifting/src/indexer/tasks/saver.rs index 9fbe24554..c5d0951d0 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs +++ b/core/crates/heavy-lifting/src/indexer/tasks/saver.rs @@ -9,10 +9,7 @@ use sd_prisma::{ }; use sd_sync::{sync_db_entry, sync_entry, OperationFactory}; use sd_task_system::{ExecStatus, Interrupter, IntoAnyTaskOutput, SerializableTask, Task, TaskId}; -use sd_utils::{ - db::{inode_to_db, size_in_bytes_to_db}, - msgpack, -}; +use sd_utils::db::{inode_to_db, size_in_bytes_to_db}; use std::{sync::Arc, time::Duration}; @@ -121,13 +118,13 @@ impl Task for Saver { new file_paths and they were not identified yet" ); - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ + let (sync_params, db_params) = [ ( - ( - location::NAME, - msgpack!(prisma_sync::location::SyncId { + sync_entry!( + prisma_sync::location::SyncId { pub_id: location_pub_id.clone() - }), + }, + location ), location_id::set(Some(*location_id)), ), @@ -152,7 +149,7 @@ impl Task for Saver { ), ] .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); ( sync.shared_create( diff --git a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs b/core/crates/heavy-lifting/src/indexer/tasks/updater.rs index 91eb72899..80cf3d6f4 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs +++ b/core/crates/heavy-lifting/src/indexer/tasks/updater.rs @@ -93,7 +93,7 @@ impl Task for Updater { check_interruption!(interrupter); - let (sync_stuff, paths_to_update) = walked_entries + let (crdt_ops, paths_to_update) = walked_entries .drain(..) .map( |WalkedEntry { @@ -138,18 +138,12 @@ impl Task for Updater { .unzip::<_, _, Vec<_>, Vec<_>>(); ( - sync_params - .into_iter() - .map(|(field, value)| { - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: pub_id.to_db(), - }, - field, - value, - ) - }) - .collect::>(), + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: pub_id.to_db(), + }, + sync_params, + ), db.file_path() .update(file_path::pub_id::equals(pub_id.into()), db_params) // selecting id to avoid fetching whole object from database @@ -159,9 +153,7 @@ impl Task for Updater { ) .unzip::<_, _, Vec<_>, Vec<_>>(); - let ops = sync_stuff.into_iter().flatten().collect::>(); - - if ops.is_empty() && paths_to_update.is_empty() { + if crdt_ops.is_empty() && paths_to_update.is_empty() { return Ok(ExecStatus::Done( Output { updated_count: 0, @@ -172,7 +164,7 @@ impl Task for Updater { } let updated = sync - .write_ops(db, (ops, paths_to_update)) + .write_ops(db, (crdt_ops, paths_to_update)) .await .map_err(indexer::Error::from)?; diff --git a/core/crates/heavy-lifting/src/job_system/report.rs b/core/crates/heavy-lifting/src/job_system/report.rs index 3d536e7dd..b747b8195 100644 --- a/core/crates/heavy-lifting/src/job_system/report.rs +++ b/core/crates/heavy-lifting/src/job_system/report.rs @@ -290,6 +290,7 @@ impl Report { .map(|id| job::parent::connect(job::id::equals(id.as_bytes().to_vec())))], ), ) + .select(job::select!({ id })) .exec() .await .map_err(ReportError::Create)?; @@ -318,6 +319,7 @@ impl Report { job::date_completed::set(self.completed_at.map(Into::into)), ], ) + .select(job::select!({ id })) .exec() .await .map_err(ReportError::Update)?; diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs b/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs index d1637ebb3..6d919fbae 100644 --- a/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs +++ b/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs @@ -25,7 +25,8 @@ use image::{imageops, DynamicImage, GenericImageView}; use serde::{Deserialize, Serialize}; use specta::Type; use tokio::{ - fs, io, + fs::{self, File}, + io::{self, AsyncWriteExt}, sync::{oneshot, Mutex}, task::spawn_blocking, time::{sleep, Instant}, @@ -450,15 +451,29 @@ async fn generate_image_thumbnail( trace!("Created shard directory and writing it to disk"); - let res = fs::write(output_path, &webp).await.map_err(|e| { + let mut file = File::create(output_path).await.map_err(|e| { + thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( + file_path.clone(), + FileIOError::from((output_path, e)).to_string(), + ) + })?; + + file.write_all(&webp).await.map_err(|e| { + thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( + file_path.clone(), + FileIOError::from((output_path, e)).to_string(), + ) + })?; + + file.sync_all().await.map_err(|e| { thumbnailer::NonCriticalThumbnailerError::SaveThumbnail( file_path, FileIOError::from((output_path, e)).to_string(), ) - }); + })?; trace!("Wrote thumbnail to disk"); - res + return Ok(()); } #[instrument( diff --git a/core/crates/heavy-lifting/src/media_processor/job.rs b/core/crates/heavy-lifting/src/media_processor/job.rs index cadeb5f03..fb622e162 100644 --- a/core/crates/heavy-lifting/src/media_processor/job.rs +++ b/core/crates/heavy-lifting/src/media_processor/job.rs @@ -14,7 +14,11 @@ use sd_core_file_path_helper::IsolatedFilePathData; use sd_core_prisma_helpers::file_path_for_media_processor; use sd_file_ext::extensions::Extension; -use sd_prisma::prisma::{location, PrismaClient}; +use sd_prisma::{ + prisma::{location, PrismaClient}, + prisma_sync, +}; +use sd_sync::{sync_db_not_null_entry, OperationFactory}; use sd_task_system::{ AnyTaskOutput, IntoTask, SerializableTask, Task, TaskDispatcher, TaskHandle, TaskId, TaskOutput, TaskStatus, TaskSystemError, @@ -214,15 +218,23 @@ impl Job for MediaProcessor { .. } = self; - ctx.db() - .location() - .update( - location::id::equals(location.id), - vec![location::scan_state::set( - LocationScanState::Completed as i32, - )], + let (sync_param, db_param) = + sync_db_not_null_entry!(LocationScanState::Completed as i32, location::scan_state); + + ctx.sync() + .write_op( + ctx.db(), + ctx.sync().shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + [sync_param], + ), + ctx.db() + .location() + .update(location::id::equals(location.id), vec![db_param]) + .select(location::select!({ id })), ) - .exec() .await .map_err(media_processor::Error::from)?; diff --git a/core/crates/heavy-lifting/src/media_processor/shallow.rs b/core/crates/heavy-lifting/src/media_processor/shallow.rs index fd7caac14..675dcd791 100644 --- a/core/crates/heavy-lifting/src/media_processor/shallow.rs +++ b/core/crates/heavy-lifting/src/media_processor/shallow.rs @@ -220,7 +220,7 @@ async fn dispatch_media_data_extractor_tasks( async fn dispatch_thumbnailer_tasks( parent_iso_file_path: &IsolatedFilePathData<'_>, should_regenerate: bool, - location_path: &PathBuf, + location_path: &Path, dispatcher: &BaseTaskDispatcher, ctx: &impl OuterContext, ) -> Result>, Error> { diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs b/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs index 0180014a9..1497e3cc4 100644 --- a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs +++ b/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs @@ -379,21 +379,20 @@ fn process_thumbnail_generation_output( match status { GenerationStatus::Generated => { *generated += 1; + // This if is REALLY needed, due to the sheer performance of the thumbnailer, + // I restricted to only send events notifying for thumbnails in the current + // opened directory, sending events for the entire location turns into a + // humongous bottleneck in the frontend lol, since it doesn't even knows + // what to do with thumbnails for inner directories lol + // - fogodev + if with_priority { + reporter.new_thumbnail(thumb_key); + } } GenerationStatus::Skipped => { *skipped += 1; } } - - // This if is REALLY needed, due to the sheer performance of the thumbnailer, - // I restricted to only send events notifying for thumbnails in the current - // opened directory, sending events for the entire location turns into a - // humongous bottleneck in the frontend lol, since it doesn't even knows - // what to do with thumbnails for inner directories lol - // - fogodev - if with_priority { - reporter.new_thumbnail(thumb_key); - } } Err(e) => { errors.push(media_processor::NonCriticalMediaProcessorError::from(e).into()); diff --git a/core/crates/indexer-rules/Cargo.toml b/core/crates/indexer-rules/Cargo.toml index 218f04d75..472bd2442 100644 --- a/core/crates/indexer-rules/Cargo.toml +++ b/core/crates/indexer-rules/Cargo.toml @@ -19,7 +19,7 @@ globset = { workspace = true, features = ["serde1"] } prisma-client-rust = { workspace = true } rmp-serde = { workspace = true } rspc = { workspace = true } -serde = { workspace = true, features = ["derive"] } +serde = { workspace = true, features = ["derive", "rc"] } specta = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs"] } diff --git a/core/crates/indexer-rules/src/serde_impl.rs b/core/crates/indexer-rules/src/serde_impl.rs index a0b24dd23..461630669 100644 --- a/core/crates/indexer-rules/src/serde_impl.rs +++ b/core/crates/indexer-rules/src/serde_impl.rs @@ -60,7 +60,7 @@ impl<'de> Deserialize<'de> for RulePerKind { struct FieldsVisitor; - impl<'de> de::Visitor<'de> for FieldsVisitor { + impl de::Visitor<'_> for FieldsVisitor { type Value = Fields; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/core/crates/prisma-helpers/src/lib.rs b/core/crates/prisma-helpers/src/lib.rs index 48a400e65..311d81947 100644 --- a/core/crates/prisma-helpers/src/lib.rs +++ b/core/crates/prisma-helpers/src/lib.rs @@ -74,6 +74,20 @@ file_path::select!(file_path_for_media_processor { pub_id } }); +file_path::select!(file_path_watcher_remove { + id + pub_id + location_id + materialized_path + is_dir + name + extension + object: select { + id + pub_id + } + +}); file_path::select!(file_path_to_isolate { location_id materialized_path @@ -324,7 +338,7 @@ impl Clone for CasId<'_> { } } -impl<'cas_id> CasId<'cas_id> { +impl CasId<'_> { #[must_use] pub fn as_str(&self) -> &str { self.0.as_ref() diff --git a/core/crates/sync/Cargo.toml b/core/crates/sync/Cargo.toml index 229a58ead..d2a7dfc1e 100644 --- a/core/crates/sync/Cargo.toml +++ b/core/crates/sync/Cargo.toml @@ -22,6 +22,7 @@ async-stream = { workspace = true } chrono = { workspace = true } futures = { workspace = true } futures-concurrency = { workspace = true } +itertools = { workspace = true } prisma-client-rust = { workspace = true, features = ["rspc"] } rmp-serde = { workspace = true } rmpv = { workspace = true } diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/sync/src/ingest_utils.rs index 3cc4c8a68..5f60ccfdf 100644 --- a/core/crates/sync/src/ingest_utils.rs +++ b/core/crates/sync/src/ingest_utils.rs @@ -1,47 +1,41 @@ use sd_core_prisma_helpers::DevicePubId; use sd_prisma::{ - prisma::{crdt_operation, PrismaClient, SortOrder}, + prisma::{crdt_operation, PrismaClient}, prisma_sync::ModelSyncData, }; use sd_sync::{ CRDTOperation, CRDTOperationData, CompressedCRDTOperation, ModelId, OperationKind, RecordId, }; -use std::{collections::BTreeMap, num::NonZeroU128}; +use std::{collections::BTreeMap, num::NonZeroU128, sync::Arc}; use futures_concurrency::future::TryJoin; +use tokio::sync::Mutex; use tracing::{debug, instrument, trace, warn}; use uhlc::{Timestamp, HLC, NTP64}; use uuid::Uuid; use super::{db_operation::write_crdt_op_to_db, Error, TimestampPerDevice}; +crdt_operation::select!(crdt_operation_id { id }); + // where the magic happens #[instrument(skip(clock, ops), fields(operations_count = %ops.len()), err)] pub async fn process_crdt_operations( clock: &HLC, timestamp_per_device: &TimestampPerDevice, + sync_lock: Arc>, db: &PrismaClient, device_pub_id: DevicePubId, - model: ModelId, - record_id: RecordId, - mut ops: Vec, + model_id: ModelId, + (record_id, mut ops): (RecordId, Vec), ) -> Result<(), Error> { ops.sort_by_key(|op| op.timestamp); let new_timestamp = ops.last().expect("Empty ops array").timestamp; - // first, we update the HLC's timestamp with the incoming one. - // this involves a drift check + sets the last time of the clock - clock - .update_with_timestamp(&Timestamp::new( - new_timestamp, - uhlc::ID::from( - NonZeroU128::new(Uuid::from(&device_pub_id).to_u128_le()).expect("Non zero id"), - ), - )) - .expect("timestamp has too much drift!"); + update_clock(clock, new_timestamp, &device_pub_id); // Delete - ignores all other messages if let Some(delete_op) = ops @@ -50,7 +44,15 @@ pub async fn process_crdt_operations( .find(|op| matches!(op.data, CRDTOperationData::Delete)) { trace!("Deleting operation"); - handle_crdt_deletion(db, &device_pub_id, model, record_id, delete_op).await?; + handle_crdt_deletion( + db, + &sync_lock, + &device_pub_id, + model_id, + record_id, + delete_op, + ) + .await?; } // Create + > 0 Update - overwrites the create's data with the updates else if let Some(timestamp) = ops @@ -61,24 +63,31 @@ pub async fn process_crdt_operations( trace!("Create + Updates operations"); // conflict resolution - let delete = db + let delete_count = db .crdt_operation() - .find_first(vec![ - crdt_operation::model::equals(i32::from(model)), + .count(vec![ + crdt_operation::model::equals(i32::from(model_id)), crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), crdt_operation::kind::equals(OperationKind::Delete.to_string()), ]) - .order_by(crdt_operation::timestamp::order(SortOrder::Desc)) .exec() .await?; - if delete.is_some() { + if delete_count > 0 { debug!("Found a previous delete operation with the same SyncId, will ignore these operations"); return Ok(()); } - handle_crdt_create_and_updates(db, &device_pub_id, model, record_id, ops, timestamp) - .await?; + handle_crdt_create_and_updates( + db, + &sync_lock, + &device_pub_id, + model_id, + record_id, + ops, + timestamp, + ) + .await?; } // > 0 Update - batches updates with a fake Create op else { @@ -87,94 +96,222 @@ pub async fn process_crdt_operations( let mut data = BTreeMap::new(); for op in ops.into_iter().rev() { - let CRDTOperationData::Update { field, value } = op.data else { + let CRDTOperationData::Update(fields_and_values) = op.data else { unreachable!("Create + Delete should be filtered out!"); }; - data.insert(field, (value, op.timestamp)); + for (field, value) in fields_and_values { + data.insert(field, (value, op.timestamp)); + } } + let earlier_time = data.values().fold( + NTP64(u64::from(u32::MAX)), + |earlier_time, (_, timestamp)| { + if timestamp.0 < earlier_time.0 { + *timestamp + } else { + earlier_time + } + }, + ); + // conflict resolution - let (create, updates) = db + let (create, possible_newer_updates_count) = db ._batch(( + db.crdt_operation().count(vec![ + crdt_operation::model::equals(i32::from(model_id)), + crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), + crdt_operation::kind::equals(OperationKind::Create.to_string()), + ]), + // Fetching all update operations newer than our current earlier timestamp db.crdt_operation() - .find_first(vec![ - crdt_operation::model::equals(i32::from(model)), + .find_many(vec![ + crdt_operation::timestamp::gt({ + #[allow(clippy::cast_possible_wrap)] + // SAFETY: we had to store using i64 due to SQLite limitations + { + earlier_time.as_u64() as i64 + } + }), + crdt_operation::model::equals(i32::from(model_id)), crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::equals(OperationKind::Create.to_string()), + crdt_operation::kind::starts_with("u".to_string()), ]) - .order_by(crdt_operation::timestamp::order(SortOrder::Desc)), - data.iter() - .map(|(k, (_, timestamp))| { - Ok(db - .crdt_operation() - .find_first(vec![ - crdt_operation::timestamp::gt({ - #[allow(clippy::cast_possible_wrap)] - // SAFETY: we had to store using i64 due to SQLite limitations - { - timestamp.as_u64() as i64 - } - }), - crdt_operation::model::equals(i32::from(model)), - crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), - crdt_operation::kind::equals(OperationKind::Update(k).to_string()), - ]) - .order_by(crdt_operation::timestamp::order(SortOrder::Desc))) - }) - .collect::, Error>>()?, + .select(crdt_operation::select!({ kind timestamp })), )) .await?; - if create.is_none() { + if create == 0 { warn!("Failed to find a previous create operation with the same SyncId"); return Ok(()); } - handle_crdt_updates(db, &device_pub_id, model, record_id, data, updates).await?; + for candidate in possible_newer_updates_count { + // The first element is "u" meaning that this is an update, so we skip it + for key in candidate + .kind + .split(':') + .filter(|field| !field.is_empty()) + .skip(1) + { + // remove entries if we possess locally more recent updates for this field + if data.get(key).is_some_and(|(_, new_timestamp)| { + #[allow(clippy::cast_sign_loss)] + { + // we need to store as i64 due to SQLite limitations + *new_timestamp < NTP64(candidate.timestamp as u64) + } + }) { + data.remove(key); + } + } + + if data.is_empty() { + break; + } + } + + handle_crdt_updates(db, &sync_lock, &device_pub_id, model_id, record_id, data).await?; } - // read the timestamp for the operation's device, or insert one if it doesn't exist - let current_last_timestamp = timestamp_per_device - .read() - .await - .get(&device_pub_id) - .copied(); - - // update the stored timestamp for this device - will be derived from the crdt operations table on restart - let new_ts = NTP64::max(current_last_timestamp.unwrap_or_default(), new_timestamp); - - timestamp_per_device - .write() - .await - .insert(device_pub_id, new_ts); + update_timestamp_per_device(timestamp_per_device, device_pub_id, new_timestamp).await; Ok(()) } +pub async fn bulk_ingest_create_only_ops( + clock: &HLC, + timestamp_per_device: &TimestampPerDevice, + db: &PrismaClient, + device_pub_id: DevicePubId, + model_id: ModelId, + ops: Vec<(RecordId, CompressedCRDTOperation)>, + sync_lock: Arc>, +) -> Result<(), Error> { + let latest_timestamp = ops.iter().fold(NTP64(0), |latest, (_, op)| { + if latest < op.timestamp { + op.timestamp + } else { + latest + } + }); + + update_clock(clock, latest_timestamp, &device_pub_id); + + let ops = ops + .into_iter() + .map(|(record_id, op)| { + rmp_serde::to_vec(&record_id) + .map(|serialized_record_id| (record_id, serialized_record_id, op)) + }) + .collect::, _>>()?; + + // conflict resolution + let delete_counts = db + ._batch( + ops.iter() + .map(|(_, serialized_record_id, _)| { + db.crdt_operation().count(vec![ + crdt_operation::model::equals(i32::from(model_id)), + crdt_operation::record_id::equals(serialized_record_id.clone()), + crdt_operation::kind::equals(OperationKind::Delete.to_string()), + ]) + }) + .collect::>(), + ) + .await?; + + let lock_guard = sync_lock.lock().await; + + db._transaction() + .with_timeout(30 * 10000) + .with_max_wait(30 * 10000) + .run(|db| { + let device_pub_id = device_pub_id.clone(); + + async move { + // complying with borrowck + let device_pub_id = &device_pub_id; + + let (crdt_creates, model_sync_data) = ops + .into_iter() + .zip(delete_counts) + .filter_map(|(data, delete_count)| (delete_count == 0).then_some(data)) + .map( + |( + record_id, + serialized_record_id, + CompressedCRDTOperation { timestamp, data }, + )| { + let crdt_create = crdt_operation::CreateUnchecked { + timestamp: { + #[allow(clippy::cast_possible_wrap)] + // SAFETY: we have to store using i64 due to SQLite limitations + { + timestamp.0 as i64 + } + }, + model: i32::from(model_id), + record_id: serialized_record_id, + kind: "c".to_string(), + data: rmp_serde::to_vec(&data)?, + device_pub_id: device_pub_id.to_db(), + _params: vec![], + }; + + // NOTE(@fogodev): I wish I could do a create many here instead of creating separately each + // entry, but it's not supported by PCR + let model_sync_data = ModelSyncData::from_op(CRDTOperation { + device_pub_id: Uuid::from(device_pub_id), + model_id, + record_id, + timestamp, + data, + })? + .exec(&db); + + Ok::<_, Error>((crdt_create, model_sync_data)) + }, + ) + .collect::, _>>()? + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + + model_sync_data.try_join().await?; + + db.crdt_operation().create_many(crdt_creates).exec().await?; + + Ok::<_, Error>(()) + } + }) + .await?; + + drop(lock_guard); + + update_timestamp_per_device(timestamp_per_device, device_pub_id, latest_timestamp).await; + + Ok(()) +} + +#[instrument(skip_all, err)] async fn handle_crdt_updates( db: &PrismaClient, + sync_lock: &Mutex<()>, device_pub_id: &DevicePubId, model_id: ModelId, record_id: rmpv::Value, - mut data: BTreeMap, - updates: Vec>, + data: BTreeMap, ) -> Result<(), Error> { - let keys = data.keys().cloned().collect::>(); let device_pub_id = sd_sync::DevicePubId::from(device_pub_id); - // does the same thing as processing ops one-by-one and returning early if a newer op was found - for (update, key) in updates.into_iter().zip(keys) { - if update.is_some() { - data.remove(&key); - } - } + let _lock_guard = sync_lock.lock().await; db._transaction() .with_timeout(30 * 10000) .with_max_wait(30 * 10000) .run(|db| async move { - // fake operation to batch them all at once + // fake operation to batch them all at once, inserting the latest data on appropriate table ModelSyncData::from_op(CRDTOperation { device_pub_id, model_id, @@ -185,41 +322,40 @@ async fn handle_crdt_updates( .map(|(k, (data, _))| (k.clone(), data.clone())) .collect(), ), - }) - .ok_or(Error::InvalidModelId(model_id))? + })? .exec(&db) .await?; - // need to only apply ops that haven't been filtered out - data.into_iter() - .map(|(field, (value, timestamp))| { - let record_id = record_id.clone(); - let db = &db; - - async move { - write_crdt_op_to_db( - &CRDTOperation { - device_pub_id, - model_id, - record_id, - timestamp, - data: CRDTOperationData::Update { field, value }, - }, - db, - ) - .await + let (fields_and_values, latest_timestamp) = data.into_iter().fold( + (BTreeMap::new(), NTP64::default()), + |(mut fields_and_values, mut latest_time_stamp), (field, (value, timestamp))| { + fields_and_values.insert(field, value); + if timestamp > latest_time_stamp { + latest_time_stamp = timestamp; } - }) - .collect::>() - .try_join() - .await - .map(|_| ()) + (fields_and_values, latest_time_stamp) + }, + ); + + write_crdt_op_to_db( + &CRDTOperation { + device_pub_id, + model_id, + record_id, + timestamp: latest_timestamp, + data: CRDTOperationData::Update(fields_and_values), + }, + &db, + ) + .await }) .await } +#[instrument(skip_all, err)] async fn handle_crdt_create_and_updates( db: &PrismaClient, + sync_lock: &Mutex<()>, device_pub_id: &DevicePubId, model_id: ModelId, record_id: rmpv::Value, @@ -244,13 +380,18 @@ async fn handle_crdt_create_and_updates( break; } - CRDTOperationData::Update { field, value } => { - data.insert(field.clone(), value.clone()); + CRDTOperationData::Update(fields_and_values) => { + for (field, value) in fields_and_values { + data.insert(field.clone(), value.clone()); + } + applied_ops.push(op); } } } + let _lock_guard = sync_lock.lock().await; + db._transaction() .with_timeout(30 * 10000) .with_max_wait(30 * 10000) @@ -262,8 +403,7 @@ async fn handle_crdt_create_and_updates( record_id: record_id.clone(), timestamp, data: CRDTOperationData::Create(data), - }) - .ok_or(Error::InvalidModelId(model_id))? + })? .exec(&db) .await?; @@ -294,14 +434,33 @@ async fn handle_crdt_create_and_updates( .await } +#[instrument(skip_all, err)] async fn handle_crdt_deletion( db: &PrismaClient, + sync_lock: &Mutex<()>, device_pub_id: &DevicePubId, model: u16, record_id: rmpv::Value, delete_op: &CompressedCRDTOperation, ) -> Result<(), Error> { - // deletes are the be all and end all, no need to check anything + // deletes are the be all and end all, except if we never created the object to begin with + // in this case we don't need to delete anything + + if db + .crdt_operation() + .count(vec![ + crdt_operation::model::equals(i32::from(model)), + crdt_operation::record_id::equals(rmp_serde::to_vec(&record_id)?), + ]) + .exec() + .await? + == 0 + { + // This means that in the other device this entry was created and deleted, before this + // device here could even take notice of it. So we don't need to do anything here. + return Ok(()); + } + let op = CRDTOperation { device_pub_id: device_pub_id.into(), model_id: model, @@ -310,16 +469,49 @@ async fn handle_crdt_deletion( data: CRDTOperationData::Delete, }; + let _lock_guard = sync_lock.lock().await; + db._transaction() .with_timeout(30 * 10000) .with_max_wait(30 * 10000) .run(|db| async move { - ModelSyncData::from_op(op.clone()) - .ok_or(Error::InvalidModelId(model))? - .exec(&db) - .await?; + ModelSyncData::from_op(op.clone())?.exec(&db).await?; write_crdt_op_to_db(&op, &db).await }) .await } + +fn update_clock(clock: &HLC, latest_timestamp: NTP64, device_pub_id: &DevicePubId) { + // first, we update the HLC's timestamp with the incoming one. + // this involves a drift check + sets the last time of the clock + clock + .update_with_timestamp(&Timestamp::new( + latest_timestamp, + uhlc::ID::from( + NonZeroU128::new(Uuid::from(device_pub_id).to_u128_le()).expect("Non zero id"), + ), + )) + .expect("timestamp has too much drift!"); +} + +async fn update_timestamp_per_device( + timestamp_per_device: &TimestampPerDevice, + device_pub_id: DevicePubId, + latest_timestamp: NTP64, +) { + // read the timestamp for the operation's device, or insert one if it doesn't exist + let current_last_timestamp = timestamp_per_device + .read() + .await + .get(&device_pub_id) + .copied(); + + // update the stored timestamp for this device - will be derived from the crdt operations table on restart + let new_ts = NTP64::max(current_last_timestamp.unwrap_or_default(), latest_timestamp); + + timestamp_per_device + .write() + .await + .insert(device_pub_id, new_ts); +} diff --git a/core/crates/sync/src/lib.rs b/core/crates/sync/src/lib.rs index 903fb812b..5b8d90efe 100644 --- a/core/crates/sync/src/lib.rs +++ b/core/crates/sync/src/lib.rs @@ -27,12 +27,15 @@ #![forbid(deprecated_in_future)] #![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] -use sd_prisma::prisma::{cloud_crdt_operation, crdt_operation}; +use sd_prisma::{ + prisma::{cloud_crdt_operation, crdt_operation}, + prisma_sync, +}; use sd_utils::uuid_to_bytes; use std::{collections::HashMap, sync::Arc}; -use tokio::sync::RwLock; +use tokio::{sync::RwLock, task::JoinError}; pub mod backfill; mod db_operation; @@ -66,12 +69,16 @@ pub enum Error { Deserialization(#[from] rmp_serde::decode::Error), #[error("database error: {0}")] Database(#[from] prisma_client_rust::QueryError), + #[error("PrismaSync error: {0}")] + PrismaSync(#[from] prisma_sync::Error), #[error("invalid model id: {0}")] InvalidModelId(ModelId), #[error("tried to write an empty operations list")] EmptyOperations, #[error("device not found: {0}")] DeviceNotFound(DevicePubId), + #[error("processes crdt task panicked")] + ProcessCrdtPanic(JoinError), } impl From for rspc::Error { diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs index 382261b9d..47460afac 100644 --- a/core/crates/sync/src/manager.rs +++ b/core/crates/sync/src/manager.rs @@ -1,38 +1,47 @@ use sd_core_prisma_helpers::DevicePubId; use sd_prisma::{ - prisma::{crdt_operation, device, PrismaClient, SortOrder}, + prisma::{cloud_crdt_operation, crdt_operation, device, PrismaClient, SortOrder}, prisma_sync, }; use sd_sync::{ - CRDTOperation, CompressedCRDTOperationsPerModel, CompressedCRDTOperationsPerModelPerDevice, - ModelId, OperationFactory, + CRDTOperation, CRDTOperationData, CompressedCRDTOperation, ModelId, OperationFactory, RecordId, }; use sd_utils::timestamp_to_datetime; use std::{ - collections::BTreeMap, - fmt, + collections::{hash_map::Entry, BTreeMap, HashMap}, + fmt, mem, num::NonZeroU128, sync::{ atomic::{self, AtomicBool}, Arc, }, + time::{Duration, SystemTime}, }; use async_stream::stream; -use futures::Stream; +use futures::{stream::FuturesUnordered, Stream, TryStreamExt}; use futures_concurrency::future::TryJoin; -use tokio::sync::{broadcast, Mutex, Notify, RwLock}; -use tracing::{debug, warn}; +use itertools::Itertools; +use tokio::{ + spawn, + sync::{broadcast, Mutex, Notify, RwLock}, + time::Instant, +}; +use tracing::{debug, instrument, warn}; use uhlc::{HLCBuilder, HLC}; use uuid::Uuid; use super::{ - crdt_op_db, db_operation::from_crdt_ops, ingest_utils::process_crdt_operations, Error, - SyncEvent, TimestampPerDevice, NTP64, + crdt_op_db, + db_operation::{from_cloud_crdt_ops, from_crdt_ops}, + ingest_utils::{bulk_ingest_create_only_ops, process_crdt_operations}, + Error, SyncEvent, TimestampPerDevice, NTP64, }; +const INGESTION_BATCH_SIZE: i64 = 10_000; + /// Wrapper that spawns the ingest actor and provides utilities for reading and writing sync operations. #[derive(Clone)] pub struct Manager { @@ -44,7 +53,8 @@ pub struct Manager { pub clock: Arc, pub active: Arc, pub active_notify: Arc, - pub sync_lock: Arc>, + pub(crate) sync_lock: Arc>, + pub(crate) available_parallelism: usize, } impl fmt::Debug for Manager { @@ -131,67 +141,264 @@ impl Manager { active: Arc::default(), active_notify: Arc::default(), sync_lock: Arc::new(Mutex::default()), + available_parallelism: std::thread::available_parallelism() + .map_or(1, std::num::NonZero::get), }, rx, )) } - pub async fn ingest_ops( + async fn fetch_cloud_crdt_ops( &self, - CompressedCRDTOperationsPerModelPerDevice(compressed_ops): CompressedCRDTOperationsPerModelPerDevice, - ) -> Result<(), Error> { - // WARN: this order here exists because sync messages MUST be processed in this exact order - // due to relationship dependencies between these tables. - const INGEST_ORDER: &[ModelId] = &[ - prisma_sync::device::MODEL_ID, - prisma_sync::storage_statistics::MODEL_ID, - prisma_sync::tag::MODEL_ID, - prisma_sync::location::MODEL_ID, - prisma_sync::object::MODEL_ID, - prisma_sync::exif_data::MODEL_ID, - prisma_sync::file_path::MODEL_ID, - prisma_sync::label::MODEL_ID, - prisma_sync::tag_on_object::MODEL_ID, - prisma_sync::label_on_object::MODEL_ID, - ]; + model_id: ModelId, + batch_size: i64, + ) -> Result<(Vec, Vec), Error> { + self.db + .cloud_crdt_operation() + .find_many(vec![cloud_crdt_operation::model::equals(i32::from( + model_id, + ))]) + .take(batch_size) + .order_by(cloud_crdt_operation::timestamp::order(SortOrder::Asc)) + .exec() + .await? + .into_iter() + .map(from_cloud_crdt_ops) + .collect::, Vec<_>), _>>() + } - let _lock_guard = self.sync_lock.lock().await; + #[instrument(skip(self))] + async fn ingest_by_model(&self, model_id: ModelId) -> Result { + let mut total_count = 0; - let mut ops_fut_by_model = INGEST_ORDER - .iter() - .map(|&model_id| (model_id, vec![])) - .collect::>(); + let mut buckets = (0..self.available_parallelism) + .map(|_| FuturesUnordered::new()) + .collect::>(); - for (device_pub_id, CompressedCRDTOperationsPerModel(ops_per_model)) in compressed_ops { - for (model_id, ops_per_record) in ops_per_model { - for (record_id, ops) in ops_per_record { - ops_fut_by_model - .get_mut(&model_id) - .ok_or(Error::InvalidModelId(model_id))? - .push(process_crdt_operations( - &self.clock, - &self.timestamp_per_device, - &self.db, - device_pub_id.into(), - model_id, - record_id, - ops, - )); + let mut total_fetch_time = Duration::ZERO; + let mut total_compression_time = Duration::ZERO; + let mut total_work_distribution_time = Duration::ZERO; + let mut total_process_time = Duration::ZERO; + + loop { + let fetching_start = Instant::now(); + + let (ops_ids, ops) = self + .fetch_cloud_crdt_ops(model_id, INGESTION_BATCH_SIZE) + .await?; + if ops_ids.is_empty() { + break; + } + + total_fetch_time += fetching_start.elapsed(); + + let messages_count = ops.len(); + + debug!( + messages_count, + first_message = ?ops + .first() + .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), + last_message = ?ops + .last() + .map_or_else(|| SystemTime::UNIX_EPOCH.into(), |op| timestamp_to_datetime(op.timestamp)), + "Messages by model to ingest", + ); + + let compression_start = Instant::now(); + + let mut compressed_map = + BTreeMap::, (RecordId, Vec)>>::new(); + + for CRDTOperation { + device_pub_id, + timestamp, + model_id: _, // Ignoring model_id as we know it already + record_id, + data, + } in ops + { + let records = compressed_map.entry(device_pub_id).or_default(); + + // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. + // So we use it's serialized bytes as a key. + let record_id_bytes = + rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); + + match records.entry(record_id_bytes) { + Entry::Occupied(mut entry) => { + entry + .get_mut() + .1 + .push(CompressedCRDTOperation { timestamp, data }); + } + Entry::Vacant(entry) => { + entry + .insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); + } } } + + // Now that we separated all operations by their record_ids, we can do an optimization + // to process all records that only posses a single create operation, batching them together + let mut create_only_ops: BTreeMap> = + BTreeMap::new(); + for (device_pub_id, records) in &mut compressed_map { + for (record_id, ops) in records.values_mut() { + if ops.len() == 1 && matches!(ops[0].data, CRDTOperationData::Create(_)) { + create_only_ops + .entry(*device_pub_id) + .or_default() + .push((mem::replace(record_id, rmpv::Value::Nil), ops.remove(0))); + } + } + } + + total_count += bulk_process_of_create_only_ops( + self.available_parallelism, + Arc::clone(&self.clock), + Arc::clone(&self.timestamp_per_device), + Arc::clone(&self.db), + Arc::clone(&self.sync_lock), + model_id, + create_only_ops, + ) + .await?; + + total_compression_time += compression_start.elapsed(); + + let work_distribution_start = Instant::now(); + + compressed_map + .into_iter() + .flat_map(|(device_pub_id, records)| { + records.into_values().filter_map(move |(record_id, ops)| { + if record_id.is_nil() { + return None; + } + + // We can process each record in parallel as they are independent + + let clock = Arc::clone(&self.clock); + let timestamp_per_device = Arc::clone(&self.timestamp_per_device); + let db = Arc::clone(&self.db); + let device_pub_id = device_pub_id.into(); + let sync_lock = Arc::clone(&self.sync_lock); + + Some(async move { + let count = ops.len(); + + process_crdt_operations( + &clock, + ×tamp_per_device, + sync_lock, + &db, + device_pub_id, + model_id, + (record_id, ops), + ) + .await + .map(|()| count) + }) + }) + }) + .enumerate() + .for_each(|(idx, fut)| buckets[idx % self.available_parallelism].push(fut)); + + total_work_distribution_time += work_distribution_start.elapsed(); + + let processing_start = Instant::now(); + + let handles = buckets + .iter_mut() + .enumerate() + .filter(|(_idx, bucket)| !bucket.is_empty()) + .map(|(idx, bucket)| { + let mut bucket = mem::take(bucket); + + spawn(async move { + let mut ops_count = 0; + let processing_start = Instant::now(); + while let Some(count) = bucket.try_next().await? { + ops_count += count; + } + + debug!( + "Ingested {ops_count} operations in {:?}", + processing_start.elapsed() + ); + + Ok::<_, Error>((ops_count, idx, bucket)) + }) + }) + .collect::>(); + + let results = handles.try_join().await.map_err(Error::ProcessCrdtPanic)?; + + total_process_time += processing_start.elapsed(); + + for res in results { + let (count, idx, bucket) = res?; + + buckets[idx] = bucket; + + total_count += count; + } + + self.db + .cloud_crdt_operation() + .delete_many(vec![cloud_crdt_operation::id::in_vec(ops_ids)]) + .exec() + .await?; } - for model_id in INGEST_ORDER { - if let Some(futs) = ops_fut_by_model.remove(model_id) { - futs.try_join().await?; - } - } + debug!( + total_count, + ?total_fetch_time, + ?total_compression_time, + ?total_work_distribution_time, + ?total_process_time, + "Ingested all operations of this model" + ); + + Ok(total_count) + } + + pub async fn ingest_ops(&self) -> Result { + let mut total_count = 0; + + // WARN: this order here exists because sync messages MUST be processed in this exact order + // due to relationship dependencies between these tables. + total_count += self.ingest_by_model(prisma_sync::device::MODEL_ID).await?; + + total_count += [ + self.ingest_by_model(prisma_sync::storage_statistics::MODEL_ID), + self.ingest_by_model(prisma_sync::tag::MODEL_ID), + self.ingest_by_model(prisma_sync::location::MODEL_ID), + self.ingest_by_model(prisma_sync::object::MODEL_ID), + self.ingest_by_model(prisma_sync::label::MODEL_ID), + ] + .try_join() + .await? + .into_iter() + .sum::(); + + total_count += [ + self.ingest_by_model(prisma_sync::exif_data::MODEL_ID), + self.ingest_by_model(prisma_sync::file_path::MODEL_ID), + self.ingest_by_model(prisma_sync::tag_on_object::MODEL_ID), + self.ingest_by_model(prisma_sync::label_on_object::MODEL_ID), + ] + .try_join() + .await? + .into_iter() + .sum::(); if self.tx.send(SyncEvent::Ingested).is_err() { warn!("failed to send ingested message on `ingest_ops`"); } - Ok(()) + Ok(total_count) } #[must_use] @@ -450,6 +657,88 @@ impl Manager { // } } +async fn bulk_process_of_create_only_ops( + available_parallelism: usize, + clock: Arc, + timestamp_per_device: TimestampPerDevice, + db: Arc, + sync_lock: Arc>, + model_id: ModelId, + create_only_ops: BTreeMap>, +) -> Result { + let buckets = (0..available_parallelism) + .map(|_| FuturesUnordered::new()) + .collect::>(); + + let mut bucket_idx = 0; + + for (device_pub_id, records) in create_only_ops { + records + .into_iter() + .chunks(100) + .into_iter() + .for_each(|chunk| { + let ops = chunk.collect::>(); + + buckets[bucket_idx % available_parallelism].push({ + let clock = Arc::clone(&clock); + let timestamp_per_device = Arc::clone(×tamp_per_device); + let db = Arc::clone(&db); + let device_pub_id = device_pub_id.into(); + let sync_lock = Arc::clone(&sync_lock); + + async move { + let count = ops.len(); + bulk_ingest_create_only_ops( + &clock, + ×tamp_per_device, + &db, + device_pub_id, + model_id, + ops, + sync_lock, + ) + .await + .map(|()| count) + } + }); + + bucket_idx += 1; + }); + } + + let handles = buckets + .into_iter() + .map(|mut bucket| { + spawn(async move { + let mut total_count = 0; + + let process_creates_batch_start = Instant::now(); + + while let Some(count) = bucket.try_next().await? { + total_count += count; + } + + debug!( + "Processed {total_count} creates in {:?}", + process_creates_batch_start.elapsed() + ); + + Ok::<_, Error>(total_count) + }) + }) + .collect::>(); + + Ok(handles + .try_join() + .await + .map_err(Error::ProcessCrdtPanic)? + .into_iter() + .collect::, _>>()? + .into_iter() + .sum()) +} + impl OperationFactory for Manager { fn get_clock(&self) -> &HLC { &self.clock diff --git a/core/crates/sync/tests/lib.rs b/core/crates/sync/tests/lib.rs deleted file mode 100644 index 5c9dbf584..000000000 --- a/core/crates/sync/tests/lib.rs +++ /dev/null @@ -1,234 +0,0 @@ -// mod mock_instance; - -// use sd_core_sync::*; - -// use sd_prisma::{prisma::location, prisma_sync}; -// use sd_sync::*; -// use sd_utils::{msgpack, uuid_to_bytes}; - -// use mock_instance::Device; -// use tracing::info; -// use tracing_test::traced_test; -// use uuid::Uuid; - -// const MOCK_LOCATION_NAME: &str = "Location 0"; -// const MOCK_LOCATION_PATH: &str = "/User/Anon/Documents"; - -// async fn write_test_location(instance: &Device) -> location::Data { -// let location_pub_id = Uuid::new_v4(); - -// let (sync_ops, db_ops): (Vec<_>, Vec<_>) = [ -// sync_db_entry!(MOCK_LOCATION_NAME, location::name), -// sync_db_entry!(MOCK_LOCATION_PATH, location::path), -// ] -// .into_iter() -// .unzip(); - -// let location = instance -// .sync -// .write_op( -// &instance.db, -// instance.sync.shared_create( -// prisma_sync::location::SyncId { -// pub_id: uuid_to_bytes(&location_pub_id), -// }, -// sync_ops, -// ), -// instance -// .db -// .location() -// .create(uuid_to_bytes(&location_pub_id), db_ops), -// ) -// .await -// .expect("failed to create mock location"); - -// instance -// .sync -// .write_ops(&instance.db, { -// let (sync_ops, db_ops): (Vec<_>, Vec<_>) = [ -// sync_db_entry!(1024, location::total_capacity), -// sync_db_entry!(512, location::available_capacity), -// ] -// .into_iter() -// .unzip(); - -// ( -// sync_ops -// .into_iter() -// .map(|(k, v)| { -// instance.sync.shared_update( -// prisma_sync::location::SyncId { -// pub_id: uuid_to_bytes(&location_pub_id), -// }, -// k, -// v, -// ) -// }) -// .collect::>(), -// instance -// .db -// .location() -// .update(location::id::equals(location.id), db_ops), -// ) -// }) -// .await -// .expect("failed to create mock location"); - -// location -// } - -// #[tokio::test] -// #[traced_test] -// async fn writes_operations_and_rows_together() -> Result<(), Box> { -// let instance = Device::new(Uuid::new_v4()).await; - -// write_test_location(&instance).await; - -// let operations = instance -// .db -// .crdt_operation() -// .find_many(vec![]) -// .exec() -// .await?; - -// // 1 create, 2 update -// assert_eq!(operations.len(), 3); -// assert_eq!(operations[0].model, prisma_sync::location::MODEL_ID as i32); - -// let out = instance.sync.get_ops(100, vec![]).await?; - -// assert_eq!(out.len(), 3); - -// let locations = instance.db.location().find_many(vec![]).exec().await?; - -// assert_eq!(locations.len(), 1); -// let location = locations.first().unwrap(); -// assert_eq!(location.name.as_deref(), Some(MOCK_LOCATION_NAME)); -// assert_eq!(location.path.as_deref(), Some(MOCK_LOCATION_PATH)); - -// Ok(()) -// } - -// #[tokio::test] -// #[traced_test] -// async fn operations_send_and_ingest() -> Result<(), Box> { -// let instance1 = Device::new(Uuid::new_v4()).await; -// let instance2 = Device::new(Uuid::new_v4()).await; - -// let mut instance2_sync_rx = instance2.sync_rx.resubscribe(); - -// info!("Created instances!"); - -// Device::pair(&instance1, &instance2).await; - -// info!("Paired instances!"); - -// write_test_location(&instance1).await; - -// info!("Created mock location!"); - -// assert!(matches!( -// instance2_sync_rx.recv().await?, -// SyncEvent::Ingested -// )); - -// let out = instance2.sync.get_ops(100, vec![]).await?; - -// assert_locations_equality( -// &instance1.db.location().find_many(vec![]).exec().await?[0], -// &instance2.db.location().find_many(vec![]).exec().await?[0], -// ); - -// assert_eq!(out.len(), 3); - -// instance1.teardown().await; -// instance2.teardown().await; - -// Ok(()) -// } - -// #[tokio::test] -// async fn no_update_after_delete() -> Result<(), Box> { -// let instance1 = Device::new(Uuid::new_v4()).await; -// let instance2 = Device::new(Uuid::new_v4()).await; - -// let mut instance2_sync_rx = instance2.sync_rx.resubscribe(); - -// Device::pair(&instance1, &instance2).await; - -// let location = write_test_location(&instance1).await; - -// assert!(matches!( -// instance2_sync_rx.recv().await?, -// SyncEvent::Ingested -// )); - -// instance2 -// .sync -// .write_op( -// &instance2.db, -// instance2.sync.shared_delete(prisma_sync::location::SyncId { -// pub_id: location.pub_id.clone(), -// }), -// instance2.db.location().delete_many(vec![]), -// ) -// .await?; - -// assert!(matches!( -// instance1.sync_rx.resubscribe().recv().await?, -// SyncEvent::Ingested -// )); - -// instance1 -// .sync -// .write_op( -// &instance1.db, -// instance1.sync.shared_update( -// prisma_sync::location::SyncId { -// pub_id: location.pub_id.clone(), -// }, -// "name", -// msgpack!("New Location"), -// ), -// instance1.db.location().find_many(vec![]), -// ) -// .await?; - -// // one spare update operation that actually gets ignored by instance 2 -// assert_eq!(instance1.db.crdt_operation().count(vec![]).exec().await?, 5); -// assert_eq!(instance2.db.crdt_operation().count(vec![]).exec().await?, 4); - -// assert_eq!(instance1.db.location().count(vec![]).exec().await?, 0); -// // the whole point of the test - the update (which is ingested as an upsert) should be ignored -// assert_eq!(instance2.db.location().count(vec![]).exec().await?, 0); - -// instance1.teardown().await; -// instance2.teardown().await; - -// Ok(()) -// } - -// fn assert_locations_equality(l1: &location::Data, l2: &location::Data) { -// assert_eq!(l1.pub_id, l2.pub_id, "pub id"); -// assert_eq!(l1.name, l2.name, "name"); -// assert_eq!(l1.path, l2.path, "path"); -// assert_eq!(l1.total_capacity, l2.total_capacity, "total capacity"); -// assert_eq!( -// l1.available_capacity, l2.available_capacity, -// "available capacity" -// ); -// assert_eq!(l1.size_in_bytes, l2.size_in_bytes, "size in bytes"); -// assert_eq!(l1.is_archived, l2.is_archived, "is archived"); -// assert_eq!( -// l1.generate_preview_media, l2.generate_preview_media, -// "generate preview media" -// ); -// assert_eq!( -// l1.sync_preview_media, l2.sync_preview_media, -// "sync preview media" -// ); -// assert_eq!(l1.hidden, l2.hidden, "hidden"); -// assert_eq!(l1.date_created, l2.date_created, "date created"); -// assert_eq!(l1.scan_state, l2.scan_state, "scan state"); -// assert_eq!(l1.instance_id, l2.instance_id, "instance id"); -// } diff --git a/core/crates/sync/tests/mock_instance.rs b/core/crates/sync/tests/mock_instance.rs deleted file mode 100644 index 9dd5f1aff..000000000 --- a/core/crates/sync/tests/mock_instance.rs +++ /dev/null @@ -1,143 +0,0 @@ -// use sd_core_sync::*; - -// use sd_prisma::prisma; -// use sd_sync::CompressedCRDTOperationsPerModelPerDevice; - -// use std::sync::{atomic::AtomicBool, Arc}; - -// use tokio::{fs, spawn, sync::broadcast}; -// use tracing::{info, instrument, warn, Instrument}; -// use uuid::Uuid; - -// fn db_path(id: Uuid) -> String { -// format!("/tmp/test-{id}.db") -// } - -// #[derive(Clone)] -// pub struct Device { -// pub pub_id: DevicePubId, -// pub db: Arc, -// pub sync: Arc, -// pub sync_rx: Arc>, -// } - -// impl Device { -// pub async fn new(id: Uuid) -> Arc { -// let url = format!("file:{}", db_path(id)); -// let device_pub_id = DevicePubId::from(id); - -// let db = Arc::new( -// prisma::PrismaClient::_builder() -// .with_url(url.to_string()) -// .build() -// .await -// .unwrap(), -// ); - -// db._db_push().await.unwrap(); - -// db.device() -// .create(device_pub_id.to_db(), vec![]) -// .exec() -// .await -// .unwrap(); - -// // let (sync, sync_rx) = sd_core_sync::SyncManager::new( -// // Arc::clone(&db), -// // &device_pub_id, -// // Arc::new(AtomicBool::new(true)), -// // Default::default(), -// // ) -// // .await -// // .expect("failed to create sync manager"); - -// // Arc::new(Self { -// // pub_id: device_pub_id, -// // db, -// // sync: Arc::new(sync), -// // sync_rx: Arc::new(sync_rx), -// // }) -// } - -// pub async fn teardown(&self) { -// fs::remove_file(db_path(Uuid::from(&self.pub_id))) -// .await -// .unwrap(); -// } - -// pub async fn pair(instance1: &Arc, instance2: &Arc) { -// #[instrument(skip(left, right))] -// async fn half(left: &Arc, right: &Arc, context: &'static str) { -// left.db -// .device() -// .create(right.pub_id.to_db(), vec![]) -// .exec() -// .await -// .unwrap(); - -// spawn({ -// let mut sync_rx_left = left.sync_rx.resubscribe(); -// let right = Arc::clone(right); - -// async move { -// while let Ok(msg) = sync_rx_left.recv().await { -// info!(?msg, "sync_rx_left received message"); -// if matches!(msg, SyncEvent::Created) { -// right -// .sync -// .ingest -// .event_tx -// .send(ingest::Event::Notification) -// .await -// .unwrap(); -// info!("sent notification to instance 2"); -// } -// } -// } -// .in_current_span() -// }); - -// spawn({ -// let left = Arc::clone(left); -// let right = Arc::clone(right); - -// async move { -// while let Ok(msg) = right.sync.ingest.req_rx.recv().await { -// info!(?msg, "right instance received request"); -// match msg { -// ingest::Request::Messages { timestamps, tx } => { -// let messages = left.sync.get_ops(100, timestamps).await.unwrap(); - -// let ingest = &right.sync.ingest; - -// ingest -// .event_tx -// .send(ingest::Event::Messages(ingest::MessagesEvent { -// messages: CompressedCRDTOperationsPerModelPerDevice::new( -// messages, -// ), -// has_more: false, -// device_pub_id: left.pub_id.clone(), -// wait_tx: None, -// })) -// .await -// .unwrap(); - -// if tx.send(()).is_err() { -// warn!("failed to send ack to instance 1"); -// } -// } -// ingest::Request::FinishedIngesting => { -// right.sync.tx.send(SyncEvent::Ingested).unwrap(); -// } -// } -// } -// } -// .in_current_span() -// }); -// } - -// half(instance1, instance2, "instance1 -> instance2").await; -// half(instance2, instance1, "instance2 -> instance1").await; -// } -// } diff --git a/core/src/api/cloud/mod.rs b/core/src/api/cloud/mod.rs index c07340c99..9a54c47a9 100644 --- a/core/src/api/cloud/mod.rs +++ b/core/src/api/cloud/mod.rs @@ -16,7 +16,7 @@ use sd_cloud_schema::{ use sd_crypto::{CryptoRng, SeedableRng}; use sd_utils::error::report_error; -use std::{pin::pin, sync::atomic::Ordering}; +use std::pin::pin; use async_stream::stream; use futures::{FutureExt, StreamExt}; @@ -51,7 +51,19 @@ pub(crate) fn mount() -> AlphaRouter { |node, (access_token, refresh_token): (auth::AccessToken, auth::RefreshToken)| async move { use sd_cloud_schema::devices; - if node.cloud_services.has_bootstrapped.load(Ordering::Acquire) { + // Only allow a single bootstrap request in flight at a time + let mut has_bootstrapped_lock = node + .cloud_services + .has_bootstrapped + .try_lock() + .map_err(|_| { + rspc::Error::new( + rspc::ErrorCode::Conflict, + String::from("Bootstrap in progress"), + ) + })?; + + if *has_bootstrapped_lock { return Err(rspc::Error::new( rspc::ErrorCode::Conflict, String::from("Already bootstrapped"), @@ -204,16 +216,22 @@ pub(crate) fn mount() -> AlphaRouter { }| { let node = &node; - async move { initialize_cloud_sync(pub_id, library, node).await } + async move { + match initialize_cloud_sync(pub_id, library, node).await { + // If we don't have this library locally, we didn't joined this group yet + Ok(()) | Err(LibraryManagerError::LibraryNotFound) => { + Ok(()) + } + Err(e) => Err(e), + } + } }, ) .collect::>() .try_join() .await?; - node.cloud_services - .has_bootstrapped - .store(true, Ordering::Release); + *has_bootstrapped_lock = true; Ok(()) }, @@ -243,7 +261,14 @@ pub(crate) fn mount() -> AlphaRouter { .procedure( "hasBootstrapped", R.query(|node, _: ()| async move { - Ok(node.cloud_services.has_bootstrapped.load(Ordering::Relaxed)) + // If we can't lock immediately, it means that there is a bootstrap in progress + // so we didn't bootstrapped yet + Ok(node + .cloud_services + .has_bootstrapped + .try_lock() + .map(|lock| *lock) + .unwrap_or(false)) }), ) } diff --git a/core/src/api/files.rs b/core/src/api/files.rs index 8e0f29992..155fd2884 100644 --- a/core/src/api/files.rs +++ b/core/src/api/files.rs @@ -28,8 +28,8 @@ use sd_prisma::{ prisma::{file_path, location, object}, prisma_sync, }; -use sd_sync::OperationFactory; -use sd_utils::{db::maybe_missing, error::FileIOError, msgpack}; +use sd_sync::{sync_db_entry, sync_db_nullable_entry, sync_entry, OperationFactory}; +use sd_utils::{db::maybe_missing, error::FileIOError}; use std::{ ffi::OsString, @@ -195,19 +195,19 @@ pub(crate) fn mount() -> AlphaRouter { ) })?; + let (sync_param, db_param) = sync_db_nullable_entry!(args.note, object::note); + sync.write_op( db, sync.shared_update( prisma_sync::object::SyncId { pub_id: object.pub_id, }, - object::note::NAME, - msgpack!(&args.note), - ), - db.object().update( - object::id::equals(args.id), - vec![object::note::set(args.note)], + [sync_param], ), + db.object() + .update(object::id::equals(args.id), vec![db_param]) + .select(object::select!({ id })), ) .await?; @@ -241,19 +241,19 @@ pub(crate) fn mount() -> AlphaRouter { ) })?; + let (sync_param, db_param) = sync_db_entry!(args.favorite, object::favorite); + sync.write_op( db, sync.shared_update( prisma_sync::object::SyncId { pub_id: object.pub_id, }, - object::favorite::NAME, - msgpack!(&args.favorite), - ), - db.object().update( - object::id::equals(args.id), - vec![object::favorite::set(Some(args.favorite))], + [sync_param], ), + db.object() + .update(object::id::equals(args.id), vec![db_param]) + .select(object::select!({ id })), ) .await?; @@ -346,19 +346,20 @@ pub(crate) fn mount() -> AlphaRouter { let date_accessed = Utc::now().into(); - let (ops, object_ids): (Vec<_>, Vec<_>) = objects + let (ops, object_ids) = objects .into_iter() - .map(|d| { + .map(|object| { ( sync.shared_update( - prisma_sync::object::SyncId { pub_id: d.pub_id }, - object::date_accessed::NAME, - msgpack!(date_accessed), + prisma_sync::object::SyncId { + pub_id: object.pub_id, + }, + [sync_entry!(date_accessed, object::date_accessed)], ), - d.id, + object.id, ) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !ops.is_empty() && !object_ids.is_empty() { sync.write_ops( @@ -392,19 +393,20 @@ pub(crate) fn mount() -> AlphaRouter { .exec() .await?; - let (ops, object_ids): (Vec<_>, Vec<_>) = objects + let (ops, object_ids) = objects .into_iter() - .map(|d| { + .map(|object| { ( sync.shared_update( - prisma_sync::object::SyncId { pub_id: d.pub_id }, - object::date_accessed::NAME, - msgpack!(nil), + prisma_sync::object::SyncId { + pub_id: object.pub_id, + }, + [sync_entry!(nil, object::date_accessed)], ), - d.id, + object.id, ) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !ops.is_empty() && !object_ids.is_empty() { sync.write_ops( @@ -487,11 +489,32 @@ pub(crate) fn mount() -> AlphaRouter { path = %full_path.display(), "File not found in the file system, will remove from database;", ); - library + + let file_path_pub_id = library .db .file_path() - .delete(file_path::id::equals(args.file_path_ids[0])) + .find_unique(file_path::id::equals(args.file_path_ids[0])) + .select(file_path::select!({ pub_id })) .exec() + .await? + .ok_or(LocationError::FilePath(FilePathError::IdNotFound( + args.file_path_ids[0], + )))? + .pub_id; + + library + .sync + .write_op( + &library.db, + library.sync.shared_delete( + prisma_sync::file_path::SyncId { + pub_id: file_path_pub_id, + }, + ), + library.db.file_path().delete(file_path::id::equals( + args.file_path_ids[0], + )), + ) .await .map_err(LocationError::from)?; diff --git a/core/src/api/labels.rs b/core/src/api/labels.rs index 9aaaf30e3..eed08d8d3 100644 --- a/core/src/api/labels.rs +++ b/core/src/api/labels.rs @@ -116,7 +116,7 @@ pub(crate) fn mount() -> AlphaRouter { .procedure( "delete", R.with2(library()) - .mutation(|(_, library), label_id: i32| async move { + .mutation(|(_, library), label_id: label::id::Type| async move { let Library { db, sync, .. } = library.as_ref(); let label = db @@ -131,6 +131,35 @@ pub(crate) fn mount() -> AlphaRouter { ) })?; + let delete_ops = db + .label_on_object() + .find_many(vec![label_on_object::label_id::equals(label_id)]) + .select(label_on_object::select!({ object: select { pub_id } })) + .exec() + .await? + .into_iter() + .map(|label_on_object| { + sync.relation_delete(prisma_sync::label_on_object::SyncId { + label: prisma_sync::label::SyncId { + name: label.name.clone(), + }, + object: prisma_sync::object::SyncId { + pub_id: label_on_object.object.pub_id, + }, + }) + }) + .collect::>(); + + sync.write_ops( + db, + ( + delete_ops, + db.label_on_object() + .delete_many(vec![label_on_object::label_id::equals(label_id)]), + ), + ) + .await?; + sync.write_op( db, sync.shared_delete(prisma_sync::label::SyncId { name: label.name }), diff --git a/core/src/api/search/saved.rs b/core/src/api/search/saved.rs index 37dec602e..957474c49 100644 --- a/core/src/api/search/saved.rs +++ b/core/src/api/search/saved.rs @@ -69,7 +69,7 @@ pub(crate) fn mount() -> AlphaRouter { let pub_id = Uuid::now_v7().as_bytes().to_vec(); let date_created: DateTime = Utc::now().into(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = chain_optional_iter( + let (sync_params, db_params) = chain_optional_iter( [ sync_db_entry!(date_created, saved_search::date_created), sync_db_entry!(args.name, saved_search::name), @@ -96,7 +96,7 @@ pub(crate) fn mount() -> AlphaRouter { ], ) .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); sync.write_op( db, @@ -106,7 +106,9 @@ pub(crate) fn mount() -> AlphaRouter { }, sync_params, ), - db.saved_search().create(pub_id, db_params), + db.saved_search() + .create(pub_id, db_params) + .select(saved_search::select!({ id })), ) .await?; @@ -162,7 +164,7 @@ pub(crate) fn mount() -> AlphaRouter { rspc::Error::new(rspc::ErrorCode::NotFound, "search not found".into()) })?; - let (ops, db_params): (Vec<_>, Vec<_>) = chain_optional_iter( + let (sync_params, db_params) = chain_optional_iter( [sync_db_entry!(updated_at, saved_search::date_modified)], [ option_sync_db_entry!(args.name.flatten(), saved_search::name), @@ -173,34 +175,23 @@ pub(crate) fn mount() -> AlphaRouter { ], ) .into_iter() - .map(|((k, v), p)| { - ( - sync.shared_update( - prisma_sync::saved_search::SyncId { - pub_id: search.pub_id.clone(), - }, - k, - v, - ), - p, - ) - }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); - if !ops.is_empty() && !db_params.is_empty() { - sync.write_ops( - db, - ( - ops, - db.saved_search() - .update_unchecked(saved_search::id::equals(id), db_params), - ), - ) - .await?; + sync.write_op( + db, + sync.shared_update( + prisma_sync::saved_search::SyncId { + pub_id: search.pub_id.clone(), + }, + sync_params, + ), + db.saved_search() + .update_unchecked(saved_search::id::equals(id), db_params), + ) + .await?; - invalidate_query!(library, "search.saved.list"); - invalidate_query!(library, "search.saved.get"); - } + invalidate_query!(library, "search.saved.list"); + invalidate_query!(library, "search.saved.get"); Ok(()) } diff --git a/core/src/api/tags.rs b/core/src/api/tags.rs index 451ac624e..0035ea592 100644 --- a/core/src/api/tags.rs +++ b/core/src/api/tags.rs @@ -4,8 +4,7 @@ use sd_prisma::{ prisma::{device, file_path, object, tag, tag_on_object}, prisma_sync, }; -use sd_sync::{option_sync_db_entry, sync_entry, OperationFactory}; -use sd_utils::{msgpack, uuid_to_bytes}; +use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, OperationFactory}; use std::collections::BTreeMap; @@ -14,7 +13,6 @@ use itertools::{Either, Itertools}; use rspc::{alpha::AlphaRouter, ErrorCode}; use serde::{Deserialize, Serialize}; use specta::Type; -use uuid::Uuid; use super::{utils::library, Ctx, R}; @@ -185,17 +183,6 @@ pub(crate) fn mount() -> AlphaRouter { }) .await?; - macro_rules! sync_id { - ($pub_id:expr) => { - prisma_sync::tag_on_object::SyncId { - tag: prisma_sync::tag::SyncId { - pub_id: tag.pub_id.clone(), - }, - object: prisma_sync::object::SyncId { pub_id: $pub_id }, - } - }; - } - if args.unassign { let query = db.tag_on_object().delete_many(vec![ tag_on_object::tag_id::equals(args.tag_id), @@ -220,63 +207,20 @@ pub(crate) fn mount() -> AlphaRouter { .into_iter() .filter_map(|fp| fp.object.map(|o| o.pub_id)), ) - .map(|pub_id| sync.relation_delete(sync_id!(pub_id))) + .map(|pub_id| { + sync.relation_delete(prisma_sync::tag_on_object::SyncId { + tag: prisma_sync::tag::SyncId { + pub_id: tag.pub_id.clone(), + }, + object: prisma_sync::object::SyncId { pub_id }, + }) + }) .collect::>(); if !ops.is_empty() { sync.write_ops(db, (ops, query)).await?; } } else { - let mut ops = vec![]; - - let db_params: (Vec<_>, Vec<_>) = file_paths - .iter() - .filter(|fp| fp.is_dir.unwrap_or_default() && fp.object.is_none()) - .map(|fp| { - let id = uuid_to_bytes(&Uuid::now_v7()); - let device_pub_id = sync.device_pub_id.to_db(); - - ops.push(sync.shared_create( - prisma_sync::object::SyncId { pub_id: id.clone() }, - [sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone(), - }, - object::device - )], - )); - - ops.push(sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: fp.pub_id.clone(), - }, - file_path::object::NAME, - msgpack!(id), - )); - - ( - db.object().create( - id.clone(), - vec![object::device::connect(device::pub_id::equals( - device_pub_id, - ))], - ), - db.file_path().update( - file_path::id::equals(fp.id), - vec![file_path::object::connect(object::pub_id::equals( - id, - ))], - ), - ) - }) - .unzip(); - - if ops.is_empty() { - return Ok(()); - } - - let (new_objects, _) = sync.write_ops(db, (ops, db_params)).await?; - let (sync_ops, db_creates) = objects .into_iter() .map(|o| (o.id, o.pub_id)) @@ -285,22 +229,23 @@ pub(crate) fn mount() -> AlphaRouter { .into_iter() .filter_map(|fp| fp.object.map(|o| (o.id, o.pub_id))), ) - .chain(new_objects.into_iter().map(|o| (o.id, o.pub_id))) - .fold( - (vec![], vec![]), - |(mut sync_ops, mut db_creates), (id, pub_id)| { - let device_pub_id = sync.device_pub_id.to_db(); - sync_ops.push(sync.relation_create( - sync_id!(pub_id), + .map(|(id, pub_id)| { + ( + sync.relation_create( + prisma_sync::tag_on_object::SyncId { + tag: prisma_sync::tag::SyncId { + pub_id: tag.pub_id.clone(), + }, + object: prisma_sync::object::SyncId { pub_id }, + }, [sync_entry!( prisma_sync::device::SyncId { - pub_id: device_pub_id.clone(), + pub_id: sync.device_pub_id.to_db(), }, tag_on_object::device )], - )); - - db_creates.push(tag_on_object::CreateUnchecked { + ), + tag_on_object::CreateUnchecked { tag_id: args.tag_id, object_id: id, _params: vec![ @@ -309,24 +254,21 @@ pub(crate) fn mount() -> AlphaRouter { )), tag_on_object::device_id::set(Some(device_id)), ], - }); + }, + ) + }) + .unzip::<_, _, Vec<_>, Vec<_>>(); - (sync_ops, db_creates) - }, - ); - - if sync_ops.is_empty() && db_creates.is_empty() { - return Ok(()); + if !sync_ops.is_empty() && !db_creates.is_empty() { + sync.write_ops( + db, + ( + sync_ops, + db.tag_on_object().create_many(db_creates).skip_duplicates(), + ), + ) + .await?; } - - sync.write_ops( - db, - ( - sync_ops, - db.tag_on_object().create_many(db_creates).skip_duplicates(), - ), - ) - .await?; } invalidate_query!(library, "tags.getForObject"); @@ -344,13 +286,17 @@ pub(crate) fn mount() -> AlphaRouter { pub color: Option, } - R.with2(library()) - .mutation(|(_, library), args: TagUpdateArgs| async move { + R.with2(library()).mutation( + |(_, library), TagUpdateArgs { id, name, color }: TagUpdateArgs| async move { + if name.is_none() && color.is_none() { + return Ok(()); + } + let Library { sync, db, .. } = library.as_ref(); let tag = db .tag() - .find_unique(tag::id::equals(args.id)) + .find_unique(tag::id::equals(id)) .select(tag::select!({ pub_id })) .exec() .await? @@ -359,68 +305,88 @@ pub(crate) fn mount() -> AlphaRouter { "Error finding tag in db".into(), ))?; - db.tag() - .update( - tag::id::equals(args.id), - vec![tag::date_modified::set(Some(Utc::now().into()))], - ) - .exec() - .await?; - - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - option_sync_db_entry!(args.name, tag::name), - option_sync_db_entry!(args.color, tag::color), + let (sync_params, db_params) = [ + option_sync_db_entry!(name, tag::name), + option_sync_db_entry!(color, tag::color), + Some(sync_db_entry!(Utc::now(), tag::date_modified)), ] .into_iter() .flatten() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); - if sync_params.is_empty() && db_params.is_empty() { - return Ok(()); - } - - sync.write_ops( + sync.write_op( db, - ( - sync_params - .into_iter() - .map(|(k, v)| { - sync.shared_update( - prisma_sync::tag::SyncId { - pub_id: tag.pub_id.clone(), - }, - k, - v, - ) - }) - .collect(), - db.tag().update(tag::id::equals(args.id), db_params), + sync.shared_update( + prisma_sync::tag::SyncId { + pub_id: tag.pub_id.clone(), + }, + sync_params, ), + db.tag() + .update(tag::id::equals(id), db_params) + .select(tag::select!({ id })), ) .await?; invalidate_query!(library, "tags.list"); Ok(()) - }) + }, + ) }) .procedure( "delete", R.with2(library()) - .mutation(|(_, library), tag_id: i32| async move { - library - .db - .tag_on_object() - .delete_many(vec![tag_on_object::tag_id::equals(tag_id)]) - .exec() - .await?; + .mutation(|(_, library), tag_id: tag::id::Type| async move { + let Library { sync, db, .. } = &*library; - library - .db + let tag_pub_id = db .tag() - .delete(tag::id::equals(tag_id)) + .find_unique(tag::id::equals(tag_id)) + .select(tag::select!({ pub_id })) .exec() - .await?; + .await? + .ok_or(rspc::Error::new( + rspc::ErrorCode::NotFound, + "Tag not found".to_string(), + ))? + .pub_id; + + let delete_ops = db + .tag_on_object() + .find_many(vec![tag_on_object::tag_id::equals(tag_id)]) + .select(tag_on_object::select!({ object: select { pub_id } })) + .exec() + .await? + .into_iter() + .map(|tag_on_object| { + sync.relation_delete(prisma_sync::tag_on_object::SyncId { + tag: prisma_sync::tag::SyncId { + pub_id: tag_pub_id.clone(), + }, + object: prisma_sync::object::SyncId { + pub_id: tag_on_object.object.pub_id, + }, + }) + }) + .collect::>(); + + sync.write_ops( + db, + ( + delete_ops, + db.tag_on_object() + .delete_many(vec![tag_on_object::tag_id::equals(tag_id)]), + ), + ) + .await?; + + sync.write_op( + db, + sync.shared_delete(prisma_sync::tag::SyncId { pub_id: tag_pub_id }), + db.tag().delete(tag::id::equals(tag_id)), + ) + .await?; invalidate_query!(library, "tags.list"); diff --git a/core/src/api/utils/invalidate.rs b/core/src/api/utils/invalidate.rs index 8df2eea6d..e888b08a2 100644 --- a/core/src/api/utils/invalidate.rs +++ b/core/src/api/utils/invalidate.rs @@ -121,6 +121,7 @@ impl InvalidRequests { } /// `invalidate_query` is a macro which stores a list of all of it's invocations so it can ensure all of the queries match the queries attached to the router. +/// /// This allows invalidate to be type-safe even when the router keys are stringly typed. /// ```ignore /// invalidate_query!( diff --git a/core/src/api/utils/library.rs b/core/src/api/utils/library.rs index effdb89ba..001943f55 100644 --- a/core/src/api/utils/library.rs +++ b/core/src/api/utils/library.rs @@ -22,7 +22,10 @@ pub(crate) struct LibraryArgs { pub(crate) struct LibraryArgsLike; impl MwArgMapper for LibraryArgsLike { - type Input = LibraryArgs where T: Type + DeserializeOwned + 'static; + type Input + = LibraryArgs + where + T: Type + DeserializeOwned + 'static; type State = Uuid; fn map( diff --git a/core/src/custom_uri/async_read_body.rs b/core/src/custom_uri/async_read_body.rs deleted file mode 100644 index 1a1cc523a..000000000 --- a/core/src/custom_uri/async_read_body.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::{ - io, - pin::Pin, - task::{Context, Poll}, -}; - -use axum::http::HeaderMap; -use bytes::Bytes; -use futures::Stream; -use http_body::Body; -use pin_project_lite::pin_project; -use tokio::io::{AsyncRead, AsyncReadExt, Take}; -use tokio_util::io::ReaderStream; - -// This code was taken from: https://github.com/tower-rs/tower-http/blob/e8eb54966604ea7fa574a2a25e55232f5cfe675b/tower-http/src/services/fs/mod.rs#L30 -pin_project! { - // NOTE: This could potentially be upstreamed to `http-body`. - /// Adapter that turns an [`impl AsyncRead`][tokio::io::AsyncRead] to an [`impl Body`][http_body::Body]. - #[derive(Debug)] - pub struct AsyncReadBody { - #[pin] - reader: ReaderStream, - } -} - -impl AsyncReadBody -where - T: AsyncRead, -{ - pub(crate) fn with_capacity_limited( - read: T, - capacity: usize, - max_read_bytes: u64, - ) -> AsyncReadBody> { - AsyncReadBody { - reader: ReaderStream::with_capacity(read.take(max_read_bytes), capacity), - } - } -} - -impl Body for AsyncReadBody -where - T: AsyncRead, -{ - type Data = Bytes; - type Error = io::Error; - - fn poll_data( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - self.project().reader.poll_next(cx) - } - - fn poll_trailers( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - ) -> Poll, Self::Error>> { - Poll::Ready(Ok(None)) - } -} diff --git a/core/src/custom_uri/mod.rs b/core/src/custom_uri/mod.rs index 33af7c727..89bd85122 100644 --- a/core/src/custom_uri/mod.rs +++ b/core/src/custom_uri/mod.rs @@ -29,7 +29,7 @@ use std::{ use async_stream::stream; use axum::{ - body::{self, Body, BoxBody, Full, StreamBody}, + body::Body, extract::{self, State}, http::{HeaderMap, HeaderValue, Request, Response, StatusCode}, middleware, @@ -38,8 +38,8 @@ use axum::{ Router, }; use bytes::Bytes; -use http_body::combinators::UnsyncBoxBody; use hyper::{header, upgrade::OnUpgrade}; +use hyper_util::rt::TokioIo; use mini_moka::sync::Cache; use tokio::{ fs::{self, File}, @@ -50,7 +50,6 @@ use uuid::Uuid; use self::{serve_file::serve_file, utils::*}; -mod async_read_body; mod mpsc_to_async_write; mod serve_file; mod utils; @@ -97,7 +96,7 @@ async fn request_to_remote_node( p2p: Arc, identity: RemoteIdentity, mut request: Request, -) -> Response> { +) -> Response { let request_upgrade_header = request.headers().get(header::UPGRADE).cloned(); let maybe_client_upgrade = request.extensions_mut().remove::(); @@ -121,17 +120,20 @@ async fn request_to_remote_node( }; tokio::spawn(async move { - let Ok(mut request_upgraded) = request_upgraded.await.map_err(|e| { + let Ok(request_upgraded) = request_upgraded.await.map_err(|e| { warn!(?e, "Error upgrading websocket request;"); }) else { return; }; - let Ok(mut response_upgraded) = response_upgraded.await.map_err(|e| { + let Ok(response_upgraded) = response_upgraded.await.map_err(|e| { warn!(?e, "Error upgrading websocket response;"); }) else { return; }; + let mut request_upgraded = TokioIo::new(request_upgraded); + let mut response_upgraded = TokioIo::new(response_upgraded); + copy_bidirectional(&mut request_upgraded, &mut response_upgraded) .await .map_err(|e| { @@ -147,7 +149,7 @@ async fn request_to_remote_node( async fn get_or_init_lru_entry( state: &LocalState, extract::Path((lib_id, loc_id, path_id)): ExtractedPath, -) -> Result<(CacheValue, Arc), Response> { +) -> Result<(CacheValue, Arc), Response> { let library_id = Uuid::from_str(&lib_id).map_err(bad_request)?; let location_id = loc_id.parse::().map_err(bad_request)?; let file_path_id = path_id @@ -245,7 +247,7 @@ pub fn base_router() -> Router { } else { StatusCode::INTERNAL_SERVER_ERROR }) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) })?; let metadata = file.metadata().await; serve_file( @@ -290,7 +292,7 @@ pub fn base_router() -> Router { } else { StatusCode::INTERNAL_SERVER_ERROR }) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) })?; let resp = InfallibleResponse::builder().header( @@ -335,11 +337,11 @@ pub fn base_router() -> Router { // TODO: Content Type Ok(InfallibleResponse::builder().status(StatusCode::OK).body( - body::boxed(StreamBody::new(stream! { + Body::from_stream(stream! { while let Some(item) = rx.recv().await { yield item; } - })), + }), )) } } @@ -364,7 +366,7 @@ pub fn base_router() -> Router { } else { StatusCode::INTERNAL_SERVER_ERROR }) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) })?; let resp = InfallibleResponse::builder().header( @@ -453,7 +455,7 @@ async fn infer_the_mime_type( ext: &str, file: &mut File, metadata: &Metadata, -) -> Result> { +) -> Result> { let ext = ext.to_lowercase(); let mime_type = match ext.as_str() { // AAC audio diff --git a/core/src/custom_uri/serve_file.rs b/core/src/custom_uri/serve_file.rs index fd80a69e5..460684cf5 100644 --- a/core/src/custom_uri/serve_file.rs +++ b/core/src/custom_uri/serve_file.rs @@ -3,18 +3,18 @@ use crate::util::InfallibleResponse; use std::{fs::Metadata, time::UNIX_EPOCH}; use axum::{ - body::{self, BoxBody, Full, StreamBody}, + body::Body, http::{header, request, HeaderValue, Method, Response, StatusCode}, }; use http_range::HttpRange; use tokio::{ fs::File, - io::{self, AsyncSeekExt, SeekFrom}, + io::{self, AsyncReadExt, AsyncSeekExt, SeekFrom}, }; use tokio_util::io::ReaderStream; use tracing::error; -use super::{async_read_body::AsyncReadBody, utils::*}; +use super::utils::*; // default capacity 64KiB const DEFAULT_CAPACITY: usize = 65536; @@ -31,7 +31,7 @@ pub(crate) async fn serve_file( metadata: io::Result, req: request::Parts, mut resp: InfallibleResponse, -) -> Result, Response> { +) -> Result, Response> { if let Ok(metadata) = metadata { // We only accept range queries if `files.metadata() == Ok(_)` // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges @@ -48,7 +48,7 @@ pub(crate) async fn serve_file( return Ok(resp .status(StatusCode::OK) .header("Content-Length", HeaderValue::from_static("0")) - .body(body::boxed(Full::from("")))); + .body(Body::from(""))); } // ETag @@ -73,9 +73,7 @@ pub(crate) async fn serve_file( // Used for normal requests if let Some(etag) = req.headers.get("If-None-Match") { if etag.as_bytes() == etag_header.as_bytes() { - return Ok(resp - .status(StatusCode::NOT_MODIFIED) - .body(body::boxed(Full::from("")))); + return Ok(resp.status(StatusCode::NOT_MODIFIED).body(Body::from(""))); } } @@ -104,7 +102,7 @@ pub(crate) async fn serve_file( .map_err(internal_server_error)?, ) .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(body::boxed(Full::from("")))); + .body(Body::from(""))); } let range = ranges.first().expect("checked above"); @@ -116,7 +114,7 @@ pub(crate) async fn serve_file( .map_err(internal_server_error)?, ) .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(body::boxed(Full::from("")))); + .body(Body::from(""))); } file.seek(SeekFrom::Start(range.start)) @@ -140,14 +138,13 @@ pub(crate) async fn serve_file( HeaderValue::from_str(&range.length.to_string()) .map_err(internal_server_error)?, ) - .body(body::boxed(AsyncReadBody::with_capacity_limited( - file, + .body(Body::from_stream(ReaderStream::with_capacity( + file.take(range.length), DEFAULT_CAPACITY, - range.length, )))); } } } - Ok(resp.body(body::boxed(StreamBody::new(ReaderStream::new(file))))) + Ok(resp.body(Body::from_stream(ReaderStream::new(file)))) } diff --git a/core/src/custom_uri/utils.rs b/core/src/custom_uri/utils.rs index 645da5106..cb54b815f 100644 --- a/core/src/custom_uri/utils.rs +++ b/core/src/custom_uri/utils.rs @@ -3,50 +3,49 @@ use crate::util::InfallibleResponse; use std::{fmt::Debug, panic::Location}; use axum::{ - body::{self, BoxBody}, + body::Body, http::{self, HeaderValue, Method, Request, Response, StatusCode}, middleware::Next, }; -use http_body::Full; use tracing::debug; #[track_caller] -pub(crate) fn bad_request(e: impl Debug) -> http::Response { +pub(crate) fn bad_request(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "400: Bad Request;"); InfallibleResponse::builder() .status(StatusCode::BAD_REQUEST) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } #[track_caller] -pub(crate) fn not_found(e: impl Debug) -> http::Response { +pub(crate) fn not_found(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "404: Not Found;"); InfallibleResponse::builder() .status(StatusCode::NOT_FOUND) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } #[track_caller] -pub(crate) fn internal_server_error(e: impl Debug) -> http::Response { +pub(crate) fn internal_server_error(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "500: Internal Server Error;"); InfallibleResponse::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } #[track_caller] -pub(crate) fn not_implemented(e: impl Debug) -> http::Response { +pub(crate) fn not_implemented(e: impl Debug) -> http::Response { debug!(caller = %Location::caller(), ?e, "501: Not Implemented;"); InfallibleResponse::builder() .status(StatusCode::NOT_IMPLEMENTED) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) } -pub(crate) async fn cors_middleware(req: Request, next: Next) -> Response { +pub(crate) async fn cors_middleware(req: Request, next: Next) -> Response { if req.method() == Method::OPTIONS { return Response::builder() .header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS") @@ -54,7 +53,7 @@ pub(crate) async fn cors_middleware(req: Request, next: Next) -> Respon .header("Access-Control-Allow-Headers", "*") .header("Access-Control-Max-Age", "86400") .status(StatusCode::OK) - .body(body::boxed(Full::from(""))) + .body(Body::from("")) .expect("Invalid static response!"); } diff --git a/core/src/library/manager/mod.rs b/core/src/library/manager/mod.rs index d98b2dea9..f31f7f3b0 100644 --- a/core/src/library/manager/mod.rs +++ b/core/src/library/manager/mod.rs @@ -10,8 +10,13 @@ use crate::{ }; use sd_core_sync::{SyncEvent, SyncManager}; + use sd_p2p::{Identity, RemoteIdentity}; -use sd_prisma::prisma::{device, instance, location}; +use sd_prisma::{ + prisma::{self, device, instance, location, PrismaClient}, + prisma_sync, +}; +use sd_sync::ModelId; use sd_utils::{ db, error::{FileIOError, NonUtf8PathError}, @@ -29,6 +34,7 @@ use std::{ use chrono::Utc; use futures_concurrency::future::{Join, TryJoin}; +use prisma_client_rust::Raw; use tokio::{ fs, io, spawn, sync::{broadcast, RwLock}, @@ -462,6 +468,10 @@ impl Libraries { ); let db = Arc::new(db::load_and_migrate(&db_url).await?); + // Configure database + configure_pragmas(&db).await?; + special_sync_indexes(&db).await?; + if let Some(create) = maybe_create_device { create.to_query(&db).exec().await?; } @@ -541,6 +551,7 @@ impl Libraries { )), ], ) + .select(instance::select!({ id })) .exec() .await?; } @@ -555,9 +566,6 @@ impl Libraries { ) .await?; - // Configure database - configure_pragmas(&db).await?; - let library = Library::new(id, config, instance_id, identity, db, node, sync).await; // This is an exception. Generally subscribe to this by `self.tx.subscribe`. @@ -638,3 +646,54 @@ async fn sync_rx_actor( } } } + +async fn special_sync_indexes(db: &PrismaClient) -> Result<(), LibraryManagerError> { + async fn create_index( + db: &PrismaClient, + model_id: ModelId, + model_name: &str, + ) -> Result<(), LibraryManagerError> { + db._execute_raw(Raw::new( + &format!( + "CREATE INDEX IF NOT EXISTS partial_index_model_{model_name} \ + ON crdt_operation(model,record_id,kind,timestamp) \ + WHERE model = {model_id} + " + ), + vec![], + )) + .exec() + .await?; + + debug!(model_name, "Created sync partial index"); + + Ok(()) + } + + for (model_id, model_name) in [ + (prisma_sync::device::MODEL_ID, prisma::device::NAME), + ( + prisma_sync::storage_statistics::MODEL_ID, + prisma::storage_statistics::NAME, + ), + (prisma_sync::tag::MODEL_ID, prisma::tag::NAME), + (prisma_sync::location::MODEL_ID, prisma::location::NAME), + (prisma_sync::object::MODEL_ID, prisma::object::NAME), + (prisma_sync::label::MODEL_ID, prisma::label::NAME), + (prisma_sync::exif_data::MODEL_ID, prisma::exif_data::NAME), + (prisma_sync::file_path::MODEL_ID, prisma::file_path::NAME), + ( + prisma_sync::tag_on_object::MODEL_ID, + prisma::tag_on_object::NAME, + ), + ( + prisma_sync::label_on_object::MODEL_ID, + prisma::label_on_object::NAME, + ), + ] { + // Creating indexes sequentially just in case + create_index(db, model_id, model_name).await?; + } + + Ok(()) +} diff --git a/core/src/location/manager/watcher/android.rs b/core/src/location/manager/watcher/android.rs index 01bd8a2a1..723f2e076 100644 --- a/core/src/location/manager/watcher/android.rs +++ b/core/src/location/manager/watcher/android.rs @@ -27,6 +27,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -40,9 +41,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self { + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self + where + Self: Sized, + { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -182,6 +192,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/ios.rs b/core/src/location/manager/watcher/ios.rs index 3a9c91500..25f0a49fd 100644 --- a/core/src/location/manager/watcher/ios.rs +++ b/core/src/location/manager/watcher/ios.rs @@ -33,6 +33,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -48,12 +49,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized, { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -183,6 +190,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/linux.rs b/core/src/location/manager/watcher/linux.rs index 0ec459a3c..34d37ed15 100644 --- a/core/src/location/manager/watcher/linux.rs +++ b/core/src/location/manager/watcher/linux.rs @@ -32,6 +32,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -45,9 +46,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self { + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self + where + Self: Sized, + { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -187,6 +197,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/macos.rs b/core/src/location/manager/watcher/macos.rs index 11486cd20..4d3b1ffec 100644 --- a/core/src/location/manager/watcher/macos.rs +++ b/core/src/location/manager/watcher/macos.rs @@ -42,6 +42,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -57,12 +58,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized, { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -206,6 +213,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/manager/watcher/mod.rs b/core/src/location/manager/watcher/mod.rs index 81b70ef87..d63709740 100644 --- a/core/src/location/manager/watcher/mod.rs +++ b/core/src/location/manager/watcher/mod.rs @@ -4,7 +4,7 @@ use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; use sd_core_prisma_helpers::{location_ids_and_path, location_with_indexer_rules}; use sd_prisma::prisma::{location, PrismaClient}; -use sd_utils::db::maybe_missing; +use sd_utils::{db::maybe_missing, uuid_to_bytes}; use std::{ collections::HashSet, @@ -76,7 +76,12 @@ const THIRTY_SECONDS: Duration = Duration::from_secs(30); const HUNDRED_MILLIS: Duration = Duration::from_millis(100); trait EventHandler: 'static { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized; @@ -200,7 +205,12 @@ impl LocationWatcher { Stop, } - let mut event_handler = Handler::new(location_id, Arc::clone(&library), Arc::clone(&node)); + let mut event_handler = Handler::new( + location_id, + uuid_to_bytes(&location_pub_id), + Arc::clone(&library), + Arc::clone(&node), + ); let mut last_event_at = Instant::now(); diff --git a/core/src/location/manager/watcher/utils.rs b/core/src/location/manager/watcher/utils.rs index 0adaf9f8c..88b065810 100644 --- a/core/src/location/manager/watcher/utils.rs +++ b/core/src/location/manager/watcher/utils.rs @@ -27,7 +27,9 @@ use sd_core_indexer_rules::{ seed::{GitIgnoreRules, GITIGNORE}, IndexerRuler, RulerDecision, }; -use sd_core_prisma_helpers::{file_path_with_object, object_ids, CasId, ObjectPubId}; +use sd_core_prisma_helpers::{ + file_path_watcher_remove, file_path_with_object, object_ids, CasId, ObjectPubId, +}; use sd_file_ext::{ extensions::{AudioExtension, ImageExtension, VideoExtension}, @@ -37,11 +39,11 @@ use sd_prisma::{ prisma::{device, file_path, location, object}, prisma_sync, }; -use sd_sync::{sync_entry, OperationFactory}; +use sd_sync::{option_sync_db_entry, sync_db_entry, sync_entry, OperationFactory}; use sd_utils::{ - db::{inode_from_db, inode_to_db, maybe_missing}, + chain_optional_iter, + db::{inode_from_db, inode_to_db, maybe_missing, size_in_bytes_to_db}, error::FileIOError, - msgpack, }; #[cfg(target_family = "unix")] @@ -354,32 +356,32 @@ async fn inner_create_file( let device_pub_id = sync.device_pub_id.to_db(); + let (sync_params, db_params) = [ + sync_db_entry!(date_created, object::date_created), + sync_db_entry!(int_kind, object::kind), + ( + sync_entry!( + prisma_sync::device::SyncId { + pub_id: device_pub_id.clone() + }, + object::device + ), + object::device::connect(device::pub_id::equals(device_pub_id)), + ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + sync.write_op( db, sync.shared_create( prisma_sync::object::SyncId { pub_id: pub_id.to_db(), }, - [ - sync_entry!(date_created, object::date_created), - sync_entry!(int_kind, object::kind), - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - object::device - ), - ], + sync_params, ), db.object() - .create( - pub_id.into(), - vec![ - object::date_created::set(Some(date_created)), - object::kind::set(Some(int_kind)), - object::device::connect(device::pub_id::equals(device_pub_id)), - ], - ) + .create(pub_id.into(), db_params) .select(object_ids::select()), ) .await? @@ -391,17 +393,21 @@ async fn inner_create_file( prisma_sync::location::SyncId { pub_id: created_file.pub_id.clone(), }, - file_path::object::NAME, - msgpack!(prisma_sync::object::SyncId { - pub_id: object_pub_id.clone() - }), - ), - db.file_path().update( - file_path::pub_id::equals(created_file.pub_id.clone()), - vec![file_path::object::connect(object::pub_id::equals( - object_pub_id.clone(), - ))], + [sync_entry!( + prisma_sync::object::SyncId { + pub_id: object_pub_id.clone() + }, + file_path::object + )], ), + db.file_path() + .update( + file_path::pub_id::equals(created_file.pub_id.clone()), + vec![file_path::object::connect(object::pub_id::equals( + object_pub_id.clone(), + ))], + ) + .select(file_path::select!({ id })), ) .await?; @@ -590,34 +596,22 @@ async fn inner_update_file( let is_hidden = path_is_hidden(full_path, &fs_metadata); if file_path.cas_id.as_deref() != cas_id.as_ref().map(CasId::as_str) { - let (sync_params, db_params): (Vec<_>, Vec<_>) = { - use file_path::*; - + let (sync_params, db_params) = chain_optional_iter( [ - ( - (cas_id::NAME, msgpack!(file_path.cas_id)), - Some(cas_id::set(file_path.cas_id.clone())), + sync_db_entry!( + size_in_bytes_to_db(fs_metadata.len()), + file_path::size_in_bytes_bytes ), - ( - ( - size_in_bytes_bytes::NAME, - msgpack!(fs_metadata.len().to_be_bytes().to_vec()), - ), - Some(size_in_bytes_bytes::set(Some( - fs_metadata.len().to_be_bytes().to_vec(), - ))), + sync_db_entry!( + DateTime::::from(fs_metadata.modified_or_now()), + file_path::date_modified ), - { - let date = DateTime::::from(fs_metadata.modified_or_now()).into(); - - ( - (date_modified::NAME, msgpack!(date)), - Some(date_modified::set(Some(date))), - ) - }, - { - // TODO: Should this be a skip rather than a null-set? - let checksum = if file_path.integrity_checksum.is_some() { + ], + [ + option_sync_db_entry!(file_path.cas_id.clone(), file_path::cas_id), + option_sync_db_entry!( + if file_path.integrity_checksum.is_some() { + // TODO: Should this be a skip rather than a null-set? // If a checksum was already computed, we need to recompute it Some( file_checksum(full_path) @@ -626,68 +620,39 @@ async fn inner_update_file( ) } else { None - }; - - ( - (integrity_checksum::NAME, msgpack!(checksum)), - Some(integrity_checksum::set(checksum)), - ) - }, - { - if current_inode != inode { - ( - (inode::NAME, msgpack!(inode)), - Some(inode::set(Some(inode_to_db(inode)))), - ) - } else { - ((inode::NAME, msgpack!(nil)), None) - } - }, - { - if is_hidden != file_path.hidden.unwrap_or_default() { - ( - (hidden::NAME, msgpack!(inode)), - Some(hidden::set(Some(is_hidden))), - ) - } else { - ((hidden::NAME, msgpack!(nil)), None) - } - }, - ] - .into_iter() - .filter_map(|(sync_param, maybe_db_param)| { - maybe_db_param.map(|db_param| (sync_param, db_param)) - }) - .unzip() - }; - - let ops = sync_params - .into_iter() - .map(|(field, value)| { - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), }, - field, - value, - ) - }) - .collect::>(); - - if !ops.is_empty() && !db_params.is_empty() { - // file content changed - sync.write_ops( - db, - ( - ops, - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - db_params, - ), + file_path::integrity_checksum ), - ) - .await?; - } + option_sync_db_entry!( + (current_inode != inode).then(|| inode_to_db(inode)), + file_path::inode + ), + option_sync_db_entry!( + (is_hidden != file_path.hidden.unwrap_or_default()).then_some(is_hidden), + file_path::hidden + ), + ], + ) + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + + // file content changed + sync.write_op( + db, + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: file_path.pub_id.clone(), + }, + sync_params, + ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + db_params, + ) + .select(file_path::select!({ id })), + ) + .await?; if let Some(ref object) = file_path.object { let int_kind = kind as i32; @@ -699,19 +664,18 @@ async fn inner_update_file( .await? == 1 { if object.kind.map(|k| k != int_kind).unwrap_or_default() { + let (sync_param, db_param) = sync_db_entry!(int_kind, object::kind); sync.write_op( db, sync.shared_update( prisma_sync::object::SyncId { pub_id: object.pub_id.clone(), }, - object::kind::NAME, - msgpack!(int_kind), - ), - db.object().update( - object::id::equals(object.id), - vec![object::kind::set(Some(int_kind))], + [sync_param], ), + db.object() + .update(object::id::equals(object.id), vec![db_param]) + .select(object::select!({ id })), ) .await?; } @@ -722,31 +686,31 @@ async fn inner_update_file( let device_pub_id = sync.device_pub_id.to_db(); + let (sync_params, db_params) = [ + sync_db_entry!(date_created, object::date_created), + sync_db_entry!(int_kind, object::kind), + ( + sync_entry!( + prisma_sync::device::SyncId { + pub_id: device_pub_id.clone() + }, + object::device + ), + object::device::connect(device::pub_id::equals(device_pub_id)), + ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); + sync.write_op( db, sync.shared_create( prisma_sync::object::SyncId { pub_id: pub_id.to_db(), }, - [ - sync_entry!(date_created, object::date_created), - sync_entry!(int_kind, object::kind), - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - object::device - ), - ], - ), - db.object().create( - pub_id.to_db(), - vec![ - object::date_created::set(Some(date_created)), - object::kind::set(Some(int_kind)), - object::device::connect(device::pub_id::equals(device_pub_id)), - ], + sync_params, ), + db.object().create(pub_id.to_db(), db_params), ) .await?; @@ -756,17 +720,21 @@ async fn inner_update_file( prisma_sync::location::SyncId { pub_id: file_path.pub_id.clone(), }, - file_path::object::NAME, - msgpack!(prisma_sync::object::SyncId { - pub_id: pub_id.to_db() - }), - ), - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![file_path::object::connect(object::pub_id::equals( - pub_id.into(), - ))], + [sync_entry!( + prisma_sync::object::SyncId { + pub_id: pub_id.to_db() + }, + file_path::object + )], ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + vec![file_path::object::connect(object::pub_id::equals( + pub_id.into(), + ))], + ) + .select(file_path::select!({ id })), ) .await?; } @@ -874,21 +842,22 @@ async fn inner_update_file( invalidate_query!(library, "search.paths"); invalidate_query!(library, "search.objects"); } else if is_hidden != file_path.hidden.unwrap_or_default() { - sync.write_ops( + let (sync_param, db_param) = sync_db_entry!(is_hidden, file_path::hidden); + + sync.write_op( db, - ( - vec![sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - file_path::hidden::NAME, - msgpack!(is_hidden), - )], - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![file_path::hidden::set(Some(is_hidden))], - ), + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: file_path.pub_id.clone(), + }, + [sync_param], ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + vec![db_param], + ) + .select(file_path::select!({ id })), ) .await?; @@ -972,7 +941,7 @@ pub(super) async fn rename( .await?; let total_paths_count = paths.len(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = paths + let (sync_params, db_params) = paths .into_iter() .filter_map(|path| path.materialized_path.map(|mp| (path.id, path.pub_id, mp))) .map(|(id, pub_id, mp)| { @@ -981,19 +950,20 @@ pub(super) async fn rename( &format!("{}/{}/", new_parts.materialized_path, new_parts.name), ); + let (sync_param, db_param) = + sync_db_entry!(new_path, file_path::materialized_path); + ( sync.shared_update( sd_prisma::prisma_sync::file_path::SyncId { pub_id }, - file_path::materialized_path::NAME, - msgpack!(&new_path), - ), - db.file_path().update( - file_path::id::equals(id), - vec![file_path::materialized_path::set(Some(new_path))], + [sync_param], ), + db.file_path() + .update(file_path::id::equals(id), vec![db_param]) + .select(file_path::select!({ id })), ) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !sync_params.is_empty() && !db_params.is_empty() { sync.write_ops(db, (sync_params, db_params)).await?; @@ -1002,65 +972,38 @@ pub(super) async fn rename( trace!(%total_paths_count, "Updated file_paths;"); } - let is_hidden = path_is_hidden(new_path, &new_path_metadata); - - let date_modified = DateTime::::from(new_path_metadata.modified_or_now()).into(); - - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - ( - ( - file_path::materialized_path::NAME, - msgpack!(new_path_materialized_str), - ), - file_path::materialized_path::set(Some(new_path_materialized_str)), + let (sync_params, db_params) = [ + sync_db_entry!(new_path_materialized_str, file_path::materialized_path), + sync_db_entry!(new_parts.name.to_string(), file_path::name), + sync_db_entry!(new_parts.extension.to_string(), file_path::extension), + sync_db_entry!( + DateTime::::from(new_path_metadata.modified_or_now()), + file_path::date_modified ), - ( - (file_path::name::NAME, msgpack!(new_parts.name)), - file_path::name::set(Some(new_parts.name.to_string())), - ), - ( - (file_path::extension::NAME, msgpack!(new_parts.extension)), - file_path::extension::set(Some(new_parts.extension.to_string())), - ), - ( - (file_path::date_modified::NAME, msgpack!(&date_modified)), - file_path::date_modified::set(Some(date_modified)), - ), - ( - (file_path::hidden::NAME, msgpack!(is_hidden)), - file_path::hidden::set(Some(is_hidden)), + sync_db_entry!( + path_is_hidden(new_path, &new_path_metadata), + file_path::hidden ), ] .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); - let ops = sync_params - .into_iter() - .map(|(k, v)| { - sync.shared_update( - prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), - }, - k, - v, - ) - }) - .collect::>(); + sync.write_op( + db, + sync.shared_update( + prisma_sync::file_path::SyncId { + pub_id: file_path.pub_id.clone(), + }, + sync_params, + ), + db.file_path() + .update(file_path::pub_id::equals(file_path.pub_id), db_params) + .select(file_path::select!({ id })), + ) + .await?; - if !ops.is_empty() && !db_params.is_empty() { - sync.write_ops( - db, - ( - ops, - db.file_path() - .update(file_path::pub_id::equals(file_path.pub_id), db_params), - ), - ) - .await?; - - invalidate_query!(library, "search.paths"); - invalidate_query!(library, "search.objects"); - } + invalidate_query!(library, "search.paths"); + invalidate_query!(library, "search.objects"); } Ok(()) @@ -1084,19 +1027,20 @@ pub(super) async fn remove( &location_path, full_path, )?) + .select(file_path_watcher_remove::select()) .exec() .await? else { return Ok(()); }; - remove_by_file_path(location_id, full_path, &file_path, library).await + remove_by_file_path(location_id, full_path, file_path, library).await } async fn remove_by_file_path( location_id: location::id::Type, path: impl AsRef + Send, - file_path: &file_path::Data, + file_path: file_path_watcher_remove::Data, library: &Library, ) -> Result<(), LocationManagerError> { // check file still exists on disk @@ -1120,28 +1064,42 @@ async fn remove_by_file_path( delete_directory( library, location_id, - Some(&IsolatedFilePathData::try_from(file_path)?), + Some(&IsolatedFilePathData::try_from(&file_path)?), ) .await?; } else { sync.write_op( db, sync.shared_delete(prisma_sync::file_path::SyncId { - pub_id: file_path.pub_id.clone(), + pub_id: file_path.pub_id, }), db.file_path().delete(file_path::id::equals(file_path.id)), ) .await?; - if let Some(object_id) = file_path.object_id { - db.object() - .delete_many(vec![ - object::id::equals(object_id), + if let Some(object) = file_path.object { + // If this object doesn't have any other file paths, delete it + if db + .object() + .count(vec![ + object::id::equals(object.id), // https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#none object::file_paths::none(vec![]), ]) .exec() + .await? == 1 + { + sync.write_op( + db, + sync.shared_delete(prisma_sync::object::SyncId { + pub_id: object.pub_id, + }), + db.object() + .delete(object::id::equals(object.id)) + .select(object::select!({ id })), + ) .await?; + } } } } @@ -1210,6 +1168,7 @@ pub(super) async fn recalculate_directories_size( candidates: &mut HashMap, buffer: &mut Vec<(PathBuf, Instant)>, location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: &Library, ) -> Result<(), LocationManagerError> { let mut location_path_cache = None; @@ -1268,7 +1227,7 @@ pub(super) async fn recalculate_directories_size( } if should_update_location_size { - update_location_size(location_id, library).await?; + update_location_size(location_id, location_pub_id, library).await?; } if should_invalidate { diff --git a/core/src/location/manager/watcher/windows.rs b/core/src/location/manager/watcher/windows.rs index a9b24c54c..bd85693e8 100644 --- a/core/src/location/manager/watcher/windows.rs +++ b/core/src/location/manager/watcher/windows.rs @@ -39,6 +39,7 @@ use super::{ #[derive(Debug)] pub(super) struct EventHandler { location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: Arc, node: Arc, last_events_eviction_check: Instant, @@ -54,12 +55,18 @@ pub(super) struct EventHandler { } impl super::EventHandler for EventHandler { - fn new(location_id: location::id::Type, library: Arc, node: Arc) -> Self + fn new( + location_id: location::id::Type, + location_pub_id: location::pub_id::Type, + library: Arc, + node: Arc, + ) -> Self where Self: Sized, { Self { location_id, + location_pub_id, library, node, last_events_eviction_check: Instant::now(), @@ -277,6 +284,7 @@ impl super::EventHandler for EventHandler { &mut self.to_recalculate_size, &mut self.path_and_instant_buffer, self.location_id, + self.location_pub_id.clone(), &self.library, ) .await diff --git a/core/src/location/mod.rs b/core/src/location/mod.rs index a1fd20073..d3baf8532 100644 --- a/core/src/location/mod.rs +++ b/core/src/location/mod.rs @@ -18,9 +18,9 @@ use sd_prisma::{ }; use sd_sync::*; use sd_utils::{ - db::{maybe_missing, size_in_bytes_to_db, MissingFieldError}, + db::{maybe_missing, size_in_bytes_from_db, size_in_bytes_to_db}, error::{FileIOError, NonUtf8PathError}, - msgpack, uuid_to_bytes, + uuid_to_bytes, }; use std::{ @@ -304,63 +304,36 @@ impl LocationUpdateArgs { let name = self.name.clone(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ - self.name - .filter(|name| location.name.as_ref() != Some(name)) - .map(|v| { - ( - (location::name::NAME, msgpack!(v)), - location::name::set(Some(v)), - ) - }), - self.generate_preview_media.map(|v| { - ( - (location::generate_preview_media::NAME, msgpack!(v)), - location::generate_preview_media::set(Some(v)), - ) - }), - self.sync_preview_media.map(|v| { - ( - (location::sync_preview_media::NAME, msgpack!(v)), - location::sync_preview_media::set(Some(v)), - ) - }), - self.hidden.map(|v| { - ( - (location::hidden::NAME, msgpack!(v)), - location::hidden::set(Some(v)), - ) - }), - self.path.clone().map(|v| { - ( - (location::path::NAME, msgpack!(v)), - location::path::set(Some(v)), - ) - }), + let (sync_params, db_params) = [ + option_sync_db_entry!( + self.name + .filter(|name| location.name.as_ref() != Some(name)), + location::name + ), + option_sync_db_entry!( + self.generate_preview_media, + location::generate_preview_media + ), + option_sync_db_entry!(self.sync_preview_media, location::sync_preview_media), + option_sync_db_entry!(self.hidden, location::hidden), + option_sync_db_entry!(self.path.clone(), location::path), ] .into_iter() .flatten() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); if !sync_params.is_empty() { - sync.write_ops( + sync.write_op( db, - ( - sync_params - .into_iter() - .map(|p| { - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: location.pub_id.clone(), - }, - p.0, - p.1, - ) - }) - .collect(), - db.location() - .update(location::id::equals(self.id), db_params), + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: location.pub_id.clone(), + }, + sync_params, ), + db.location() + .update(location::id::equals(self.id), db_params) + .select(location::select!({ id })), ) .await?; @@ -493,6 +466,7 @@ pub async fn scan_location( ) .await? } + ScanState::Indexed => { node.job_system .dispatch( @@ -505,6 +479,7 @@ pub async fn scan_location( ) .await? } + ScanState::FilesIdentified => { node.job_system .dispatch( @@ -651,33 +626,25 @@ pub async fn relink_location( .map(str::to_string) .ok_or_else(|| NonUtf8PathError(location_path.into()))?; - sync.write_op( - db, - sync.shared_update( - prisma_sync::location::SyncId { - pub_id: pub_id.clone(), - }, - location::path::NAME, - msgpack!(path), - ), - db.location().update( - location::pub_id::equals(pub_id.clone()), - vec![location::path::set(Some(path))], - ), - ) - .await?; + let (sync_param, db_param) = sync_db_entry!(path, location::path); - let location_id = db - .location() - .find_unique(location::pub_id::equals(pub_id)) - .select(location::select!({ id })) - .exec() + let location_id = sync + .write_op( + db, + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: pub_id.clone(), + }, + [sync_param], + ), + db.location() + .update(location::pub_id::equals(pub_id.clone()), vec![db_param]) + .select(location::select!({ id })), + ) .await? - .ok_or_else(|| { - LocationError::MissingField(MissingFieldError::new("missing id of location")) - })?; + .id; - Ok(location_id.id) + Ok(location_id) } #[derive(Debug)] @@ -1002,45 +969,44 @@ async fn check_nested_location( #[instrument(skip_all, err)] pub async fn update_location_size( location_id: location::id::Type, + location_pub_id: location::pub_id::Type, library: &Library, -) -> Result<(), QueryError> { - let Library { db, .. } = library; +) -> Result<(), sd_core_sync::Error> { + let Library { db, sync, .. } = library; - let total_size = db - .file_path() - .find_many(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some("/".to_string())), - ]) - .select(file_path::select!({ size_in_bytes_bytes })) - .exec() - .await? - .into_iter() - .filter_map(|file_path| { - file_path.size_in_bytes_bytes.map(|size_in_bytes_bytes| { - u64::from_be_bytes([ - size_in_bytes_bytes[0], - size_in_bytes_bytes[1], - size_in_bytes_bytes[2], - size_in_bytes_bytes[3], - size_in_bytes_bytes[4], - size_in_bytes_bytes[5], - size_in_bytes_bytes[6], - size_in_bytes_bytes[7], - ]) + let total_size = size_in_bytes_to_db( + db.file_path() + .find_many(vec![ + file_path::location_id::equals(Some(location_id)), + file_path::materialized_path::equals(Some("/".to_string())), + ]) + .select(file_path::select!({ size_in_bytes_bytes })) + .exec() + .await? + .into_iter() + .filter_map(|file_path| { + file_path + .size_in_bytes_bytes + .map(|size_in_bytes_bytes| size_in_bytes_from_db(&size_in_bytes_bytes)) }) - }) - .sum::(); + .sum::(), + ); - db.location() - .update( - location::id::equals(location_id), - vec![location::size_in_bytes::set(Some( - total_size.to_be_bytes().to_vec(), - ))], - ) - .exec() - .await?; + let (sync_param, db_param) = sync_db_entry!(total_size, location::size_in_bytes); + + sync.write_op( + db, + sync.shared_update( + prisma_sync::location::SyncId { + pub_id: location_pub_id, + }, + [sync_param], + ), + db.location() + .update(location::id::equals(location_id), vec![db_param]) + .select(location::select!({ id })), + ) + .await?; invalidate_query!(library, "locations.list"); invalidate_query!(library, "locations.get"); @@ -1100,69 +1066,60 @@ pub async fn create_file_path( location_id, ))?; - let (sync_params, db_params): (Vec<_>, Vec<_>) = { - use file_path::{ - cas_id, date_created, date_indexed, date_modified, device, extension, hidden, inode, - is_dir, location, materialized_path, name, size_in_bytes_bytes, - }; + let device_pub_id = sync.device_pub_id.to_db(); - let device_pub_id = sync.device_pub_id.to_db(); - - [ - ( - sync_entry!( - prisma_sync::location::SyncId { - pub_id: location.pub_id - }, - location - ), - location::connect(prisma::location::id::equals(location.id)), + let (sync_params, db_params) = [ + ( + sync_entry!( + prisma_sync::location::SyncId { + pub_id: location.pub_id + }, + file_path::location ), - ( - sync_entry!(cas_id, cas_id), - cas_id::set(cas_id.map(Into::into)), + file_path::location::connect(prisma::location::id::equals(location.id)), + ), + ( + sync_entry!(cas_id, file_path::cas_id), + file_path::cas_id::set(cas_id.map(Into::into)), + ), + sync_db_entry!(materialized_path, file_path::materialized_path), + sync_db_entry!(name, file_path::name), + sync_db_entry!(extension, file_path::extension), + sync_db_entry!( + size_in_bytes_to_db(metadata.size_in_bytes), + file_path::size_in_bytes_bytes + ), + sync_db_entry!(inode_to_db(metadata.inode), file_path::inode), + sync_db_entry!(is_dir, file_path::is_dir), + sync_db_entry!(metadata.created_at, file_path::date_created), + sync_db_entry!(metadata.modified_at, file_path::date_modified), + sync_db_entry!(indexed_at, file_path::date_indexed), + sync_db_entry!(metadata.hidden, file_path::hidden), + ( + sync_entry!( + prisma_sync::device::SyncId { + pub_id: device_pub_id.clone() + }, + file_path::device ), - sync_db_entry!(materialized_path, materialized_path), - sync_db_entry!(name, name), - sync_db_entry!(extension, extension), - sync_db_entry!( - size_in_bytes_to_db(metadata.size_in_bytes), - size_in_bytes_bytes - ), - sync_db_entry!(inode_to_db(metadata.inode), inode), - sync_db_entry!(is_dir, is_dir), - sync_db_entry!(metadata.created_at, date_created), - sync_db_entry!(metadata.modified_at, date_modified), - sync_db_entry!(indexed_at, date_indexed), - sync_db_entry!(metadata.hidden, hidden), - ( - sync_entry!( - prisma_sync::device::SyncId { - pub_id: device_pub_id.clone() - }, - device - ), - device::connect(prisma::device::pub_id::equals(device_pub_id)), - ), - ] - .into_iter() - .unzip() - }; + file_path::device::connect(prisma::device::pub_id::equals(device_pub_id)), + ), + ] + .into_iter() + .unzip::<_, _, Vec<_>, Vec<_>>(); let pub_id = sd_utils::uuid_to_bytes(&Uuid::now_v7()); - let created_path = sync - .write_op( - db, - sync.shared_create( - prisma_sync::file_path::SyncId { - pub_id: pub_id.clone(), - }, - sync_params, - ), - db.file_path().create(pub_id, db_params), - ) - .await?; - - Ok(created_path) + sync.write_op( + db, + sync.shared_create( + prisma_sync::file_path::SyncId { + pub_id: pub_id.clone(), + }, + sync_params, + ), + db.file_path().create(pub_id, db_params), + ) + .await + .map_err(Into::into) } diff --git a/core/src/object/fs/old_copy.rs b/core/src/object/fs/old_copy.rs index 8b760b920..2d7b0fb70 100644 --- a/core/src/object/fs/old_copy.rs +++ b/core/src/object/fs/old_copy.rs @@ -323,8 +323,8 @@ impl StatefulJob for OldFileCopierJobInit { .await?; dirs.extend(more_dirs); - let (dir_source_file_data, dir_target_full_path): (Vec<_>, Vec<_>) = - dirs.into_iter().unzip(); + let (dir_source_file_data, dir_target_full_path) = + dirs.into_iter().unzip::<_, _, Vec<_>, Vec<_>>(); let step_files = dir_source_file_data .into_iter() diff --git a/core/src/object/tag/mod.rs b/core/src/object/tag/mod.rs index 98238462b..34b609a83 100644 --- a/core/src/object/tag/mod.rs +++ b/core/src/object/tag/mod.rs @@ -23,14 +23,14 @@ impl TagCreateArgs { ) -> Result { let pub_id = Uuid::now_v7().as_bytes().to_vec(); - let (sync_params, db_params): (Vec<_>, Vec<_>) = [ + let (sync_params, db_params) = [ sync_db_entry!(self.name, tag::name), sync_db_entry!(self.color, tag::color), sync_db_entry!(false, tag::is_hidden), sync_db_entry!(Utc::now(), tag::date_created), ] .into_iter() - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); sync.write_op( db, diff --git a/core/src/object/validation/old_validator_job.rs b/core/src/object/validation/old_validator_job.rs index d90fc56cb..7ddd42938 100644 --- a/core/src/object/validation/old_validator_job.rs +++ b/core/src/object/validation/old_validator_job.rs @@ -15,8 +15,8 @@ use sd_prisma::{ prisma::{file_path, location}, prisma_sync, }; -use sd_sync::OperationFactory; -use sd_utils::{db::maybe_missing, error::FileIOError, msgpack}; +use sd_sync::{sync_db_entry, OperationFactory}; +use sd_utils::{db::maybe_missing, error::FileIOError}; use std::{ hash::{Hash, Hasher}, @@ -157,19 +157,22 @@ impl StatefulJob for OldObjectValidatorJobInit { .await .map_err(|e| ValidatorError::FileIO(FileIOError::from((full_path, e))))?; + let (sync_param, db_param) = sync_db_entry!(checksum, file_path::integrity_checksum); + sync.write_op( db, sync.shared_update( prisma_sync::file_path::SyncId { pub_id: file_path.pub_id.clone(), }, - file_path::integrity_checksum::NAME, - msgpack!(&checksum), - ), - db.file_path().update( - file_path::pub_id::equals(file_path.pub_id.clone()), - vec![file_path::integrity_checksum::set(Some(checksum))], + [sync_param], ), + db.file_path() + .update( + file_path::pub_id::equals(file_path.pub_id.clone()), + vec![db_param], + ) + .select(file_path::select!({ id })), ) .await?; } diff --git a/core/src/old_job/manager.rs b/core/src/old_job/manager.rs index f47164759..c9e5cc892 100644 --- a/core/src/old_job/manager.rs +++ b/core/src/old_job/manager.rs @@ -320,6 +320,7 @@ impl OldJobs { job::id::equals(job.id.as_bytes().to_vec()), vec![job::status::set(Some(JobStatus::Canceled as i32))], ) + .select(job::select!({ id })) .exec() .await?; } diff --git a/core/src/old_job/report.rs b/core/src/old_job/report.rs index ed40df23d..af7333267 100644 --- a/core/src/old_job/report.rs +++ b/core/src/old_job/report.rs @@ -395,6 +395,7 @@ impl OldJobReport { job::date_completed::set(self.completed_at.map(Into::into)), ], ) + .select(job::select!({ id })) .exec() .await?; Ok(()) diff --git a/core/src/p2p/operations/rspc.rs b/core/src/p2p/operations/rspc.rs index ed86c0912..e5e0b3cc8 100644 --- a/core/src/p2p/operations/rspc.rs +++ b/core/src/p2p/operations/rspc.rs @@ -1,9 +1,11 @@ use std::{error::Error, sync::Arc}; -use axum::{body::Body, http, Router}; -use hyper::{server::conn::Http, Response}; +use axum::{extract::Request, http, Router}; +use hyper::{body::Incoming, client::conn::http1::handshake, server::conn::http1, Response}; +use hyper_util::rt::TokioIo; use sd_p2p::{RemoteIdentity, UnicastStream, P2P}; use tokio::io::AsyncWriteExt; +use tower_service::Service; use tracing::debug; use crate::{p2p::Header, Node}; @@ -13,7 +15,7 @@ pub async fn remote_rspc( p2p: Arc, identity: RemoteIdentity, request: http::Request, -) -> Result, Box> { +) -> Result, Box> { let peer = p2p .peers() .get(&identity) @@ -23,7 +25,7 @@ pub async fn remote_rspc( stream.write_all(&Header::RspcRemote.to_bytes()).await?; - let (mut sender, conn) = hyper::client::conn::handshake(stream).await?; + let (mut sender, conn) = handshake(TokioIo::new(stream)).await?; tokio::task::spawn(async move { if let Err(e) = conn.await { println!("Connection error: {:?}", e); @@ -49,10 +51,12 @@ pub(crate) async fn receiver( todo!("No way buddy!"); } - Http::new() - .http1_only(true) - .http1_keep_alive(true) - .serve_connection(stream, service) + let hyper_service = + hyper::service::service_fn(move |request: Request| service.clone().call(request)); + + http1::Builder::new() + .keep_alive(true) + .serve_connection(TokioIo::new(stream), hyper_service) .with_upgrades() .await .map_err(Into::into) diff --git a/core/src/util/mpscrr.rs b/core/src/util/mpscrr.rs index 4c7826bea..72daf3441 100644 --- a/core/src/util/mpscrr.rs +++ b/core/src/util/mpscrr.rs @@ -230,7 +230,7 @@ impl<'a> Bomb<'a> { } } -impl<'a> Drop for Bomb<'a> { +impl Drop for Bomb<'_> { fn drop(&mut self) { self.0.store(false, Ordering::Relaxed); } diff --git a/crates/ai/Cargo.toml b/crates/ai/Cargo.toml index e24bd3c0c..35f9dc198 100644 --- a/crates/ai/Cargo.toml +++ b/crates/ai/Cargo.toml @@ -38,8 +38,9 @@ uuid = { workspace = true, features = ["serde", "v4"] } # Specific AI dependencies # Note: half and ndarray version must be the same as used in ort -half = { version = "2.1", features = ['num-traits'] } +half = { version = "2.4", features = ['num-traits'] } ndarray = "0.15" +ort-sys = '=2.0.0-rc.0' # lock sys crate to the same version as ort url = '2.5' # Microsoft does not provide a release for osx-gpu. See: https://github.com/microsoft/onnxruntime/releases diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index ef419cece..376769023 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -24,7 +24,7 @@ rand = { workspace = true } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "rt-multi-thread", "sync"] } -zeroize = { workspace = true, features = ["aarch64", "derive"] } +zeroize = { workspace = true, features = ["derive"] } # External dependencies aead = { version = "0.6.0-rc.0", default-features = false, features = ["stream"] } diff --git a/crates/crypto/src/cloud/decrypt.rs b/crates/crypto/src/cloud/decrypt.rs index c45a99110..94913f64b 100644 --- a/crates/crypto/src/cloud/decrypt.rs +++ b/crates/crypto/src/cloud/decrypt.rs @@ -31,7 +31,7 @@ impl OneShotDecryption for SecretKey { EncryptedBlockRef { nonce, cipher_text }: EncryptedBlockRef<'_>, ) -> Result, Error> { XChaCha20Poly1305::new(&self.0) - .decrypt(&nonce, cipher_text) + .decrypt(nonce, cipher_text) .map_err(|aead::Error| Error::Decrypt) } diff --git a/crates/crypto/src/cloud/secret_key.rs b/crates/crypto/src/cloud/secret_key.rs index c1df94f9f..2477684ad 100644 --- a/crates/crypto/src/cloud/secret_key.rs +++ b/crates/crypto/src/cloud/secret_key.rs @@ -191,7 +191,7 @@ mod tests { let EncryptedBlock { nonce, cipher_text } = key.encrypt(message, &mut rng).unwrap(); let mut bytes = Vec::with_capacity(nonce.len() + cipher_text.len()); - bytes.extend_from_slice(&nonce); + bytes.extend_from_slice(nonce.as_slice()); bytes.extend(cipher_text); assert_eq!( diff --git a/crates/crypto/src/ct.rs b/crates/crypto/src/ct.rs index e7edf6a89..8ce937ab9 100644 --- a/crates/crypto/src/ct.rs +++ b/crates/crypto/src/ct.rs @@ -87,7 +87,7 @@ impl ConstantTimeEq for String { } } -impl<'a> ConstantTimeEq for &'a str { +impl ConstantTimeEq for &str { fn ct_eq(&self, rhs: &Self) -> Choice { // Here we are just able to convert both values to bytes and use the // appropriate methods to compare the two in constant-time. diff --git a/crates/crypto/src/primitives.rs b/crates/crypto/src/primitives.rs index a37981a01..1d8335fc1 100644 --- a/crates/crypto/src/primitives.rs +++ b/crates/crypto/src/primitives.rs @@ -16,7 +16,7 @@ pub struct EncryptedBlock { } pub struct EncryptedBlockRef<'e> { - pub nonce: OneShotNonce, + pub nonce: &'e OneShotNonce, pub cipher_text: &'e [u8], } @@ -25,7 +25,7 @@ impl<'e> From<&'e [u8]> for EncryptedBlockRef<'e> { let (nonce, cipher_text) = cipher_text.split_at(size_of::()); Self { - nonce: OneShotNonce::try_from(nonce).expect("we split the correct amount"), + nonce: nonce.try_into().expect("we split the correct amount"), cipher_text, } } diff --git a/crates/ffmpeg/src/dict.rs b/crates/ffmpeg/src/dict.rs index 7d1d5726b..feb84184f 100644 --- a/crates/ffmpeg/src/dict.rs +++ b/crates/ffmpeg/src/dict.rs @@ -87,7 +87,7 @@ pub struct FFmpegDictIter<'a> { _lifetime: std::marker::PhantomData<&'a ()>, } -impl<'a> Iterator for FFmpegDictIter<'a> { +impl Iterator for FFmpegDictIter<'_> { type Item = (String, Option); fn next(&mut self) -> Option<(String, Option)> { diff --git a/crates/ffmpeg/src/thumbnailer.rs b/crates/ffmpeg/src/thumbnailer.rs index afd008813..6333e34c4 100644 --- a/crates/ffmpeg/src/thumbnailer.rs +++ b/crates/ffmpeg/src/thumbnailer.rs @@ -4,7 +4,7 @@ use std::{io, ops::Deref, path::Path}; use image::{imageops, DynamicImage, RgbImage}; use sd_utils::error::FileIOError; -use tokio::{fs, task::spawn_blocking}; +use tokio::{fs, io::AsyncWriteExt, task::spawn_blocking}; use tracing::error; use webp::Encoder; @@ -37,12 +37,18 @@ impl Thumbnailer { .await .map_err(|e| FileIOError::from((path, e)))?; - fs::write( - output_thumbnail_path, - &*self.process_to_webp_bytes(video_file_path).await?, - ) - .await - .map_err(|e| FileIOError::from((output_thumbnail_path, e)).into()) + let webp = self.process_to_webp_bytes(video_file_path).await?; + let mut file = fs::File::create(output_thumbnail_path) + .await + .map_err(|e: io::Error| FileIOError::from((output_thumbnail_path, e)))?; + + file.write_all(&webp) + .await + .map_err(|e| FileIOError::from((output_thumbnail_path, e)))?; + + file.sync_all() + .await + .map_err(|e| FileIOError::from((output_thumbnail_path, e)).into()) } /// Processes an video input file and returns a webp encoded thumbnail as bytes diff --git a/crates/images/src/consts.rs b/crates/images/src/consts.rs index cc844e19f..4afd8da84 100644 --- a/crates/images/src/consts.rs +++ b/crates/images/src/consts.rs @@ -159,7 +159,7 @@ impl serde::Serialize for ConvertibleExtension { struct ExtensionVisitor; #[cfg(feature = "serde")] -impl<'de> serde::de::Visitor<'de> for ExtensionVisitor { +impl serde::de::Visitor<'_> for ExtensionVisitor { type Value = ConvertibleExtension; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/crates/media-metadata/src/exif/datetime.rs b/crates/media-metadata/src/exif/datetime.rs index 39c6a40b6..b238dcd4f 100644 --- a/crates/media-metadata/src/exif/datetime.rs +++ b/crates/media-metadata/src/exif/datetime.rs @@ -77,7 +77,7 @@ impl serde::Serialize for MediaDate { struct MediaDateVisitor; -impl<'de> Visitor<'de> for MediaDateVisitor { +impl Visitor<'_> for MediaDateVisitor { type Value = MediaDate; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/crates/p2p/Cargo.toml b/crates/p2p/Cargo.toml index 19d36d2f1..d915e2024 100644 --- a/crates/p2p/Cargo.toml +++ b/crates/p2p/Cargo.toml @@ -35,15 +35,15 @@ zeroize = { workspace = true, features = ["derive"] } dns-lookup = "2.0" hash_map_diff = "0.2.0" if-watch = { version = "=3.2.0", features = ["tokio"] } # Override features used by libp2p-quic -libp2p-stream = "=0.1.0-alpha" # Update blocked due to custom patch -mdns-sd = "0.11.1" +libp2p-stream = "=0.2.0-alpha" # Update blocked due to custom patch +mdns-sd = "0.11.5" rand_core = "0.6.4" stable-vec = "0.4.1" sync_wrapper = "1.0" [dependencies.libp2p] features = ["autonat", "dcutr", "macros", "noise", "quic", "relay", "serde", "tokio", "yamux"] -version = "=0.53.2" # Update blocked due to custom patch +version = "=0.54.1" # Update blocked due to custom patch [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/p2p/src/smart_guards.rs b/crates/p2p/src/smart_guards.rs index 6177ed930..a920508aa 100644 --- a/crates/p2p/src/smart_guards.rs +++ b/crates/p2p/src/smart_guards.rs @@ -28,7 +28,7 @@ impl<'a, T: Clone> SmartWriteGuard<'a, T> { } } -impl<'a, T> Deref for SmartWriteGuard<'a, T> { +impl Deref for SmartWriteGuard<'_, T> { type Target = T; fn deref(&self) -> &Self::Target { @@ -36,13 +36,13 @@ impl<'a, T> Deref for SmartWriteGuard<'a, T> { } } -impl<'a, T> DerefMut for SmartWriteGuard<'a, T> { +impl DerefMut for SmartWriteGuard<'_, T> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.lock } } -impl<'a, T> Drop for SmartWriteGuard<'a, T> { +impl Drop for SmartWriteGuard<'_, T> { fn drop(&mut self) { (self.save)( self.p2p, diff --git a/crates/prisma-cli/Cargo.toml b/crates/prisma-cli/Cargo.toml index 37c764459..c5c0fb669 100644 --- a/crates/prisma-cli/Cargo.toml +++ b/crates/prisma-cli/Cargo.toml @@ -14,5 +14,5 @@ sd-sync-generator = { path = "../sync-generator" } [dependencies.prisma-client-rust-generator] default-features = false features = ["migrations", "specta", "sqlite", "sqlite-create-many"] -git = "https://github.com/brendonovich/prisma-client-rust" -rev = "4f9ef9d38c" +git = "https://github.com/spacedriveapp/prisma-client-rust" +rev = "b22ad7dc7d" diff --git a/crates/sync-generator/src/model.rs b/crates/sync-generator/src/model.rs index 767c1d820..e171634b8 100644 --- a/crates/sync-generator/src/model.rs +++ b/crates/sync-generator/src/model.rs @@ -46,7 +46,7 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { RefinedFieldWalker::Scalar(scalar_field) => { (!scalar_field.is_in_required_relation()).then(|| { quote! { - #model_name_snake::#field_name_snake::set(::rmpv::ext::from_value(val).unwrap()), + #model_name_snake::#field_name_snake::set(::rmpv::ext::from_value(val)?), } }) } @@ -59,11 +59,19 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { |i| { if i.count() == 1 { Some(quote! {{ - let val: std::collections::HashMap = ::rmpv::ext::from_value(val).unwrap(); - let val = val.into_iter().next().unwrap(); + + let (field, value) = ::rmpv + ::ext + ::from_value::>(val)? + .into_iter() + .next() + .ok_or(Error::MissingRelationData { + field: field.to_string(), + model: #relation_model_name_snake::NAME.to_string() + })?; #model_name_snake::#field_name_snake::connect( - #relation_model_name_snake::UniqueWhereParam::deserialize(&val.0, val.1).unwrap() + #relation_model_name_snake::UniqueWhereParam::deserialize(&field, value)? ) }}) } else { @@ -81,10 +89,13 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { } else { quote! { impl #model_name_snake::SetParam { - pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option { - Some(match field { + pub fn deserialize(field: &str, val: ::rmpv::Value) -> Result { + Ok(match field { #(#field_matches)* - _ => return None + _ => return Err(Error::FieldNotFound { + field: field.to_string(), + model: #model_name_snake::NAME.to_string(), + }), }) } } @@ -97,9 +108,12 @@ pub fn module((model, sync_type): ModelWithSyncType<'_>) -> Module { Module::new( model.name(), quote! { - use super::prisma::*; + use super::Error; + use prisma_client_rust::scalar_types::*; + use super::prisma::*; + #sync_id #set_param_impl @@ -172,7 +186,7 @@ fn process_unique_params(model: Walker<'_, ModelId>, model_name_snake: &Ident) - Some(quote!(#model_name_snake::#field_name_snake::NAME => #model_name_snake::#field_name_snake::equals( - ::rmpv::ext::from_value(val).unwrap() + ::rmpv::ext::from_value(val)? ), )) } @@ -185,10 +199,13 @@ fn process_unique_params(model: Walker<'_, ModelId>, model_name_snake: &Ident) - } else { quote! { impl #model_name_snake::UniqueWhereParam { - pub fn deserialize(field: &str, val: ::rmpv::Value) -> Option { - Some(match field { + pub fn deserialize(field: &str, val: ::rmpv::Value) -> Result { + Ok(match field { #(#field_matches)* - _ => return None + _ => return Err(Error::FieldNotFound { + field: field.to_string(), + model: #model_name_snake::NAME.to_string(), + }) }) } } diff --git a/crates/sync-generator/src/sync_data.rs b/crates/sync-generator/src/sync_data.rs index 9e9fdd937..e8ee713e6 100644 --- a/crates/sync-generator/src/sync_data.rs +++ b/crates/sync-generator/src/sync_data.rs @@ -7,7 +7,7 @@ use prisma_models::walkers::{FieldWalker, ScalarFieldWalker}; use crate::{ModelSyncType, ModelWithSyncType}; pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { - let (variants, matches): (Vec<_>, Vec<_>) = models + let (variants, matches) = models .iter() .filter_map(|(model, sync_type)| { let model_name_snake = snake_ident(model.name()); @@ -26,12 +26,12 @@ pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { quote!(#model_name_pascal(#model_name_snake::SyncId, sd_sync::CRDTOperationData)), quote! { #model_name_snake::MODEL_ID => - Self::#model_name_pascal(rmpv::ext::from_value(op.record_id).ok()?, op.data) + Self::#model_name_pascal(rmpv::ext::from_value(op.record_id)?, op.data) }, ) }) }) - .unzip(); + .unzip::<_, _, Vec<_>, Vec<_>>(); let exec_matches = models.iter().filter_map(|(model, sync_type)| { let model_name_pascal = pascal_ident(model.name()); @@ -54,20 +54,22 @@ pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { }) }); + let error_enum = declare_error_enum(); + quote! { pub enum ModelSyncData { #(#variants),* } impl ModelSyncData { - pub fn from_op(op: sd_sync::CRDTOperation) -> Option { - Some(match op.model_id { + pub fn from_op(op: sd_sync::CRDTOperation) -> Result { + Ok(match op.model_id { #(#matches),*, - _ => return None + _ => return Err(Error::InvalidModelId(op.model_id)), }) } - pub async fn exec(self, db: &prisma::PrismaClient) -> prisma_client_rust::Result<()> { + pub async fn exec(self, db: &prisma::PrismaClient) -> Result<(), Error> { match self { #(#exec_matches),* } @@ -75,6 +77,69 @@ pub fn enumerate(models: &[ModelWithSyncType<'_>]) -> TokenStream { Ok(()) } } + + #error_enum + } +} + +fn declare_error_enum() -> TokenStream { + quote! { + #[derive(Debug)] + pub enum Error { + Rmpv(rmpv::ext::Error), + RmpSerialize(rmp_serde::encode::Error), + Prisma(prisma_client_rust::QueryError), + InvalidModelId(sd_sync::ModelId), + FieldNotFound { field: String, model: String }, + MissingRelationData { field: String, model: String }, + RelatedEntryNotFound { field: String, model: String }, + } + + impl From for Error { + fn from(e: rmpv::ext::Error) -> Self { + Self::Rmpv(e) + } + } + + impl From for Error { + fn from(e: rmp_serde::encode::Error) -> Self { + Self::RmpSerialize(e) + } + } + + impl From for Error { + fn from(e: prisma_client_rust::QueryError) -> Self { + Self::Prisma(e) + } + } + + impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Rmpv(e) => write!(f, "Failed to serialize or deserialize rmpv data: {e}"), + Self::RmpSerialize(e) => write!(f, "Failed to serialize rmp data: {e}"), + Self::Prisma(e) => write!(f, "Prisma error: {e}"), + Self::InvalidModelId(id) => write!(f, "Invalid model id: {id}"), + Self::FieldNotFound { field, model } => { + write!(f, "Field '{field}' not found in model '{model}'") + } + Self::MissingRelationData { field, model } => { + write!( + f, + "Field '{field}' missing relation data in model '{model}'" + ) + } + Self::RelatedEntryNotFound { field, model } => { + write!( + f, + "Related entry for field '{field}' not found in table '{model}'" + ) + } + } + } + } + + impl std::error::Error for Error {} } } @@ -103,6 +168,7 @@ fn handle_crdt_ops_relation( .and_then(|(_m, sync)| sync.as_ref()) .map(|sync| snake_ident(sync.sync_id()[0].name())) .expect("missing sync id field name for relation"); + let item_model_name_snake = snake_ident(item.related_model().name()); let item_field_name_snake = snake_ident(item.name()); @@ -155,11 +221,15 @@ fn handle_crdt_ops_relation( vec![], ) .exec() - .await - .ok(); + .await?; }, - sd_sync::CRDTOperationData::Update { field, value } => { - let data = vec![prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap()]; + + sd_sync::CRDTOperationData::Update(data) => { + let data = data.into_iter() + .map(|(field, value)| { + prisma::#model_name_snake::SetParam::deserialize(&field, value) + }) + .collect::, _>>()?; db.#model_name_snake() .upsert( @@ -171,15 +241,14 @@ fn handle_crdt_ops_relation( data, ) .exec() - .await - .ok(); + .await?; }, + sd_sync::CRDTOperationData::Delete => { db.#model_name_snake() .delete(id) .exec() - .await - .ok(); + .await?; }, } } @@ -198,8 +267,10 @@ fn handle_crdt_ops_shared( .expect("missing fields") .next() .expect("empty fields"); + let id_name_snake = snake_ident(scalar_field.name()); let field_name_snake = snake_ident(rel.name()); + let opposite_model_name_snake = snake_ident( rel.opposite_relation_field() .expect("missing opposite relation field") @@ -211,12 +282,16 @@ fn handle_crdt_ops_shared( id.#field_name_snake.pub_id.clone() )); + let pub_id_field = format!("{field_name_snake}::pub_id"); + let rel_fetch = quote! { let rel = db.#opposite_model_name_snake() .find_unique(#relation_equals_condition) .exec() - .await? - .unwrap(); + .await?.ok_or_else(|| Error::RelatedEntryNotFound { + field: #pub_id_field.to_string(), + model: prisma::#opposite_model_name_snake::NAME.to_string(), + })?; }; ( @@ -226,6 +301,7 @@ fn handle_crdt_ops_shared( relation_equals_condition, ) } + RefinedFieldWalker::Scalar(s) => { let field_name_snake = snake_ident(s.name()); let thing = quote!(id.#field_name_snake.clone()); @@ -238,24 +314,12 @@ fn handle_crdt_ops_shared( #get_id match data { - sd_sync::CRDTOperationData::Create(data) => { - let data: Vec<_> = data.into_iter().map(|(field, value)| { - prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap() - }).collect(); - - db.#model_name_snake() - .upsert( - prisma::#model_name_snake::#id_name_snake::equals(#equals_value), - prisma::#model_name_snake::create(#create_id, data.clone()), - data - ) - .exec() - .await?; - }, - sd_sync::CRDTOperationData::Update { field, value } => { - let data = vec![ - prisma::#model_name_snake::SetParam::deserialize(&field, value).unwrap() - ]; + sd_sync::CRDTOperationData::Create(data) | sd_sync::CRDTOperationData::Update(data) => { + let data = data.into_iter() + .map(|(field, value)| { + prisma::#model_name_snake::SetParam::deserialize(&field, value) + }) + .collect::, _>>()?; db.#model_name_snake() .upsert( @@ -266,6 +330,7 @@ fn handle_crdt_ops_shared( .exec() .await?; }, + sd_sync::CRDTOperationData::Delete => { db.#model_name_snake() .delete(prisma::#model_name_snake::#id_name_snake::equals(#equals_value)) @@ -275,8 +340,8 @@ fn handle_crdt_ops_shared( db.crdt_operation() .delete_many(vec![ prisma::crdt_operation::model::equals(#model_id as i32), - prisma::crdt_operation::record_id::equals(rmp_serde::to_vec(&id).unwrap()), - prisma::crdt_operation::kind::equals(sd_sync::OperationKind::Create.to_string()) + prisma::crdt_operation::record_id::equals(rmp_serde::to_vec(&id)?), + prisma::crdt_operation::kind::equals(sd_sync::OperationKind::Create.to_string()), ]) .exec() .await?; diff --git a/crates/sync/example/Cargo.toml b/crates/sync/example/Cargo.toml deleted file mode 100644 index 6893daee8..000000000 --- a/crates/sync/example/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "sd-sync-example" -version = "0.1.0" - -edition.workspace = true -license.workspace = true -publish = false -repository.workspace = true -rust-version.workspace = true - -[dependencies] -# Spacedrive Sub-crates -sd-sync = { path = ".." } - -# Workspace dependencies -axum = { workspace = true } -http = { workspace = true } -prisma-client-rust = { workspace = true } -rspc = { workspace = true, features = ["axum"] } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } -tokio = { workspace = true, features = ["full"] } -uuid = { workspace = true, features = ["v4"] } - -# Specific Core dependencies -dotenv = "0.15.0" -tower-http = { version = "0.4.4", features = ["cors"] } # Update blocked by http diff --git a/crates/sync/example/README.md b/crates/sync/example/README.md deleted file mode 100644 index c3adc9cab..000000000 --- a/crates/sync/example/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Create rspc app - -This app was scaffolded using the [create-rspc-app](https://rspc.dev) CLI. - -## Usage - -```bash -# Terminal One -cd web -pnpm i -pnpm dev - -# Terminal Two -cd api/ -cargo prisma generate -cargo prisma db push -cargo run -``` diff --git a/crates/sync/example/prisma/migrations/.gitkeep b/crates/sync/example/prisma/migrations/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/crates/sync/example/prisma/schema.prisma b/crates/sync/example/prisma/schema.prisma deleted file mode 100644 index efc4ee911..000000000 --- a/crates/sync/example/prisma/schema.prisma +++ /dev/null @@ -1,34 +0,0 @@ -// This is your Prisma schema file, -// learn more about it in the docs: https://pris.ly/d/prisma-schema - -datasource db { - provider = "sqlite" - url = "file:dev.db" -} - -generator client { - provider = "cargo prisma" - output = "../src/prisma.rs" -} - -generator sync { - provider = "cargo run -p prisma-cli --bin sync --" - output = "../src/prisma_sync.rs" -} - -/// @owned -model FilePath { - id Bytes @id - path String - - object Object? @relation(fields: [object_id], references: [id]) - object_id Bytes? -} - -/// @shared -model Object { - id Bytes @id - name String - - paths FilePath[] @relation() -} diff --git a/crates/sync/example/src/api/mod.rs b/crates/sync/example/src/api/mod.rs deleted file mode 100644 index 14cc6d257..000000000 --- a/crates/sync/example/src/api/mod.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use rspc::*; -use sd_sync::*; -use serde_json::*; -use std::path::PathBuf; -use tokio::sync::Mutex; -use uuid::Uuid; - -use crate::prisma::{file_path, PrismaClient}; - -pub struct Ctx { - pub dbs: HashMap, - pub prisma: PrismaClient, -} - -type Router = rspc::Router>>; - -fn to_map(v: &impl serde::Serialize) -> serde_json::Map { - match to_value(v).unwrap() { - Value::Object(m) => m, - _ => unreachable!(), - } -} - -pub(crate) fn new() -> RouterBuilder>> { - Router::new() - .config(Config::new().export_ts_bindings( - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("web/src/utils/bindings.ts"), - )) - .mutation("testCreate", |r| { - r(|ctx, _: String| async move { - let prisma = &ctx.lock().await.prisma; - - let res = prisma - .file_path() - .create(vec![], String::new(), vec![]) - .exec_raw() - .await - .unwrap(); - - file_path::Create::operation_from_data(&res); - - Ok(()) - }) - }) - .mutation("createDatabase", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - let uuid = Uuid::new_v4(); - - dbs.insert(uuid, Db::new(uuid)); - - let ids = dbs.keys().copied().collect::>(); - - for db in dbs.values_mut() { - for id in &ids { - db.register_node(*id); - } - } - - Ok(uuid) - }) - }) - .mutation("removeDatabases", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - dbs.drain(); - - Ok(()) - }) - }) - .query("dbs", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - Ok(dbs.iter().map(|(id, _)| *id).collect::>()) - }) - }) - .query("db.tags", |r| { - r(|ctx, id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let id = id.parse().unwrap(); - - Ok(dbs.get(&id).unwrap().tags.clone()) - }) - }) - .query("file_path.list", |r| { - r(|ctx, id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db = dbs.get(&id.parse().unwrap()).unwrap(); - - let file_paths = db.file_paths.values().map(Clone::clone).collect::>(); - - Ok(file_paths) - }) - }) - .mutation("file_path.create", |r| { - r(|ctx, db: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db = dbs.get_mut(&db.parse().unwrap()).unwrap(); - - let id = Uuid::new_v4(); - - let file_path = FilePath { - id, - path: String::new(), - file: None, - }; - - let op = db.create_crdt_operation(CRDTOperationType::Owned(OwnedOperation { - model: "FilePath".to_string(), - items: vec![OwnedOperationItem { - id: serde_json::to_value(id).unwrap(), - data: OwnedOperationData::Create( - serde_json::from_value(serde_json::to_value(&file_path).unwrap()) - .unwrap(), - ), - }], - })); - - db.receive_crdt_operations(vec![op]); - - file_path - }) - }) - .query("message.list", |r| { - r(|ctx, id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db = dbs.get(&id.parse().unwrap()).unwrap(); - - Ok(db._operations.clone()) - }) - }) - .mutation("pullOperations", |r| { - r(|ctx, db_id: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let db_id = db_id.parse().unwrap(); - - let ops = dbs.values().flat_map(|db| db._operations.clone()).collect(); - - let db = dbs.get_mut(&db_id).unwrap(); - - db.receive_crdt_operations(ops); - - Ok(()) - }) - }) - .query("operations", |r| { - r(|ctx, _: String| async move { - let dbs = &mut ctx.lock().await.dbs; - - let mut hashmap = HashMap::new(); - - for db in dbs.values_mut() { - for op in &db._operations { - hashmap.insert(op.id, op.clone()); - } - } - - let mut array = hashmap.into_values().collect::>(); - - array.sort_by(|a, b| a.id.partial_cmp(&b.id).unwrap()); - - Ok(array) - }) - }) -} diff --git a/crates/sync/example/src/main.rs b/crates/sync/example/src/main.rs deleted file mode 100644 index a04eae216..000000000 --- a/crates/sync/example/src/main.rs +++ /dev/null @@ -1,45 +0,0 @@ -use api::Ctx; -use axum::{ - http::{HeaderValue, Method}, - routing::get, -}; -use std::{net::SocketAddr, sync::Arc}; -use tokio::sync::Mutex; -use tower_http::cors::CorsLayer; - -mod api; -mod prisma; -mod prisma_sync; -mod utils; - -async fn router() -> axum::Router { - let router = api::new().build().arced(); - - let ctx = Arc::new(Mutex::new(Ctx { - dbs: Default::default(), - prisma: prisma::new_client().await.unwrap(), - })); - - axum::Router::new() - .route("/", get(|| async { "Hello 'rspc'!" })) - .route("/rspc/:id", router.endpoint(move || ctx.clone()).axum()) - .layer( - CorsLayer::new() - .allow_origin("http://localhost:3000".parse::().unwrap()) - .allow_headers(vec![http::header::CONTENT_TYPE]) - .allow_methods([Method::GET, Method::POST]), - ) -} - -#[tokio::main] -async fn main() { - dotenv::dotenv().ok(); - - let addr = "[::]:9000".parse::().unwrap(); // This listens on IPv6 and IPv4 - println!("{} listening on http://{}", env!("CARGO_CRATE_NAME"), addr); - axum::Server::bind(&addr) - .serve(router().await.into_make_service()) - .with_graceful_shutdown(utils::axum_shutdown_signal()) - .await - .expect("Error with HTTP server!"); -} diff --git a/crates/sync/example/src/utils.rs b/crates/sync/example/src/utils.rs deleted file mode 100644 index f6437d365..000000000 --- a/crates/sync/example/src/utils.rs +++ /dev/null @@ -1,28 +0,0 @@ -use tokio::signal; - -/// shutdown_signal will inform axum to gracefully shutdown when the process is asked to shutdown. -pub async fn axum_shutdown_signal() { - let ctrl_c = async { - signal::ctrl_c() - .await - .expect("failed to install Ctrl+C handler"); - }; - - #[cfg(unix)] - let terminate = async { - signal::unix::signal(signal::unix::SignalKind::terminate()) - .expect("failed to install signal handler") - .recv() - .await; - }; - - #[cfg(not(unix))] - let terminate = std::future::pending::<()>(); - - tokio::select! { - _ = ctrl_c => {}, - _ = terminate => {}, - } - - println!("signal received, starting graceful shutdown"); -} diff --git a/crates/sync/example/web/.gitignore b/crates/sync/example/web/.gitignore deleted file mode 100644 index 76add878f..000000000 --- a/crates/sync/example/web/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -dist \ No newline at end of file diff --git a/crates/sync/example/web/README.md b/crates/sync/example/web/README.md deleted file mode 100644 index 434f7bb9d..000000000 --- a/crates/sync/example/web/README.md +++ /dev/null @@ -1,34 +0,0 @@ -## Usage - -Those templates dependencies are maintained via [pnpm](https://pnpm.io) via `pnpm up -Lri`. - -This is the reason you see a `pnpm-lock.yaml`. That being said, any package manager will work. This file can be safely be removed once you clone a template. - -```bash -$ npm install # or pnpm install or yarn install -``` - -### Learn more on the [Solid Website](https://solidjs.com) and come chat with us on our [Discord](https://discord.com/invite/solidjs) - -## Available Scripts - -In the project directory, you can run: - -### `npm dev` or `npm start` - -Runs the app in the development mode.
-Open [http://localhost:3000](http://localhost:3000) to view it in the browser. - -The page will reload if you make edits.
- -### `npm run build` - -Builds the app for production to the `dist` folder.
-It correctly bundles Solid in production mode and optimizes the build for the best performance. - -The build is minified and the filenames include the hashes.
-Your app is ready to be deployed! - -## Deployment - -You can deploy the `dist` folder to any static host provider (netlify, surge, now, etc.) diff --git a/crates/sync/example/web/index.html b/crates/sync/example/web/index.html deleted file mode 100644 index f22a9d4f1..000000000 --- a/crates/sync/example/web/index.html +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - Solid App - - - -
- - - - diff --git a/crates/sync/example/web/package.json b/crates/sync/example/web/package.json deleted file mode 100644 index 0c95d0ba5..000000000 --- a/crates/sync/example/web/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "example-2", - "version": "0.0.0", - "description": "", - "scripts": { - "dev": "vite", - "build": "vite build", - "serve": "vite preview", - "typecheck": "tsc --noEmit" - }, - "license": "MIT", - "dependencies": { - "clsx": "^2.0.0", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "solid-js": "^1.8.3" - }, - "devDependencies": { - "@tanstack/react-query": "^4.36.1", - "typescript": "^5.6.2", - "vite": "^5.2.0", - "tailwindcss": "^3.3.3" - } -} diff --git a/crates/sync/example/web/postcss.config.js b/crates/sync/example/web/postcss.config.js deleted file mode 100644 index 054c147cb..000000000 --- a/crates/sync/example/web/postcss.config.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - plugins: { - tailwindcss: {}, - autoprefixer: {} - } -}; diff --git a/crates/sync/example/web/src/App.tsx b/crates/sync/example/web/src/App.tsx deleted file mode 100644 index 6c90cbd4b..000000000 --- a/crates/sync/example/web/src/App.tsx +++ /dev/null @@ -1,172 +0,0 @@ -// import clsx from 'clsx'; -// import { Suspense, useState } from 'react'; -// import { tests } from './test'; -// import { CRDTOperationType, rspc } from './utils/rspc'; - -// export function App() { -// const dbs = rspc.useQuery(['dbs', 'cringe']); - -// const operations = rspc.useQuery(['operations', 'cringe']); - -// const createDb = rspc.useMutation('createDatabase'); -// const removeDbs = rspc.useMutation('removeDatabases'); -// const testCreate = rspc.useMutation('testCreate'); - -// return ( -//
-//
-//
-// -// -// -//
-//
    -// {Object.entries(tests).map(([key, test]) => ( -//
  • -// -//
    • -// ))} -//
-//
-//
-//
    -// {dbs.data?.map((id) => ( -// -// -// -// ))} -//
-//
-//
-//

All Operations

-//
    -// {operations.data?.map((op) => ( -//
  • -//

    ID: {op.id}

    -//

    Timestamp: {op.timestamp.toString()}

    -//

    Node: {op.node}

    -//
    • -// ))} -//
-//
-//
-// ); -// } - -// interface DatabaseViewProps { -// id: string; -// } -// const TABS = ['File Paths', 'Objects', 'Tags', 'Operations']; - -// function DatabaseView(props: DatabaseViewProps) { -// const [currentTab, setCurrentTab] = useState<(typeof TABS)[number]>('Operations'); - -// const pullOperations = rspc.useMutation('pullOperations'); - -// return ( -//
-//
-//

{props.id}

-// -//
-//
-// -// -// {currentTab === 'File Paths' && } -// {currentTab === 'Operations' && } -// -//
-//
-// ); -// } - -// function FilePathList(props: { db: string }) { -// const createFilePath = rspc.useMutation('file_path.create'); -// const filePaths = rspc.useQuery(['file_path.list', props.db]); - -// return ( -//
-// {filePaths.data && ( -//
    -// {filePaths.data -// .sort((a, b) => a.id.localeCompare(b.id)) -// .map((path) => ( -//
  • {JSON.stringify(path)}
    • -// ))} -//
-// )} -// -//
-// ); -// } - -// function messageType(msg: CRDTOperationType) { -// if ('items' in msg) { -// return 'Owned'; -// } else if ('record_id' in msg) { -// return 'Shared'; -// } -// } - -// function OperationList(props: { db: string }) { -// const messages = rspc.useQuery(['message.list', props.db]); - -// return ( -//
-// {messages.data && ( -// -// {messages.data -// .sort((a, b) => Number(a.timestamp - b.timestamp)) -// .map((message) => ( -// -// -// -// -// -// ))} -//
{message.id} -// {new Date( -// Number(message.timestamp) / 10000000 -// ).toLocaleTimeString()} -// -// {messageType(message.typ)} -//
-// )} -//
-// ); -// } - -// const ButtonStyles = 'bg-blue-500 text-white px-2 py-1 rounded-md'; - -export {}; diff --git a/crates/sync/example/web/src/index.css b/crates/sync/example/web/src/index.css deleted file mode 100644 index b5c61c956..000000000 --- a/crates/sync/example/web/src/index.css +++ /dev/null @@ -1,3 +0,0 @@ -@tailwind base; -@tailwind components; -@tailwind utilities; diff --git a/crates/sync/example/web/src/index.tsx b/crates/sync/example/web/src/index.tsx deleted file mode 100644 index 6c9a09bac..000000000 --- a/crates/sync/example/web/src/index.tsx +++ /dev/null @@ -1,16 +0,0 @@ -// /* @refresh reload */ -// import { Suspense } from 'react'; -// import { createRoot } from 'react-dom/client'; -// import { App } from './App'; -// import './index.css'; -// import { queryClient, rspc, rspcClient } from './utils/rspc'; - -// createRoot(document.getElementById('root') as HTMLElement).render( -// -// -// -// -// -// ); - -export {}; diff --git a/crates/sync/example/web/src/test.ts b/crates/sync/example/web/src/test.ts deleted file mode 100644 index e516517cc..000000000 --- a/crates/sync/example/web/src/test.ts +++ /dev/null @@ -1,47 +0,0 @@ -// import { queryClient, rspcClient } from './utils/rspc'; - -// function test(fn: () => Promise) { -// return async () => { -// await fn(); -// queryClient.invalidateQueries(); -// }; -// } - -// const wait = (ms: number) => new Promise((res) => setTimeout(res, ms)); - -// export const tests = { -// three: { -// name: 'Three', -// run: test(async () => { -// const [db1, db2, db3] = await Promise.all([ -// rspcClient.mutation(['createDatabase', ' ']), -// rspcClient.mutation(['createDatabase', ' ']), -// rspcClient.mutation(['createDatabase', ' ']) -// ]); - -// const dbs = await rspcClient.query(['dbs', 'cringe']); - -// for (const db of dbs) { -// await rspcClient.mutation(['file_path.create', db]); -// } - -// for (const db of dbs) { -// await rspcClient.mutation(['pullOperations', db]); -// } - -// await rspcClient.mutation(['file_path.create', dbs[0]]); -// await rspcClient.mutation(['file_path.create', dbs[0]]); - -// for (const db of dbs) { -// await rspcClient.mutation(['pullOperations', db]); -// } - -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// await rspcClient.mutation(['pullOperations', dbs[1]]); -// }) -// } -// }; - -export {}; diff --git a/crates/sync/example/web/src/utils/bindings.ts b/crates/sync/example/web/src/utils/bindings.ts deleted file mode 100644 index 3d31e95bb..000000000 --- a/crates/sync/example/web/src/utils/bindings.ts +++ /dev/null @@ -1,80 +0,0 @@ -// This file was generated by [rspc](https://github.com/oscartbeaumont/rspc). Do not edit this file manually. - -export type Procedures = { - queries: - | { key: 'db.tags'; input: string; result: Record } - | { key: 'dbs'; input: string; result: Array } - | { key: 'file_path.list'; input: string; result: Array } - | { key: 'message.list'; input: string; result: Array } - | { key: 'operations'; input: string; result: Array }; - mutations: - | { key: 'createDatabase'; input: string; result: string } - | { key: 'file_path.create'; input: string; result: FilePath } - | { key: 'pullOperations'; input: string; result: null } - | { key: 'removeDatabases'; input: string; result: null } - | { key: 'testCreate'; input: string; result: null }; - subscriptions: never; -}; - -export interface CRDTOperation { - node: string; - timestamp: bigint; - id: string; - typ: CRDTOperationType; -} - -export type CRDTOperationType = SharedOperation | RelationOperation | OwnedOperation; - -export interface Color { - red: number; - green: number; - blue: number; -} - -export interface FilePath { - id: string; - path: string; - file: string | null; -} - -export interface OwnedOperation { - model: string; - items: Array; -} - -export type OwnedOperationData = - | { Create: Record } - | { Update: Record } - | 'Delete'; - -export interface OwnedOperationItem { - id: any; - data: OwnedOperationData; -} - -export interface RelationOperation { - relation_item: string; - relation_group: string; - relation: string; - data: RelationOperationData; -} - -export type RelationOperationData = 'Create' | { Update: { field: string; value: any } } | 'Delete'; - -export interface SharedOperation { - record_id: string; - model: string; - data: SharedOperationData; -} - -export type SharedOperationCreateData = { Unique: Record } | 'Atomic'; - -export type SharedOperationData = - | { Create: SharedOperationCreateData } - | { Update: { field: string; value: any } } - | 'Delete'; - -export interface Tag { - color: Color; - name: string; -} diff --git a/crates/sync/example/web/src/utils/rspc.ts b/crates/sync/example/web/src/utils/rspc.ts deleted file mode 100644 index 9991bd5f0..000000000 --- a/crates/sync/example/web/src/utils/rspc.ts +++ /dev/null @@ -1,29 +0,0 @@ -// import { createClient, httpLink } from '@oscartbeaumont-sd/rspc-client'; -// import { createReactHooks } from '@oscartbeaumont-sd/rspc-react'; -// import { QueryClient } from '@tanstack/react-query'; -// import type { Procedures } from './bindings'; - -// export * from './bindings'; - -// // These are generated by rspc in Rust for you. - -// const rspc = createReactHooks(); - -// const rspcClient = rspc.createClient({ -// links: [httpLink({ url: 'http://localhost:9000/rspc' })] -// }); - -// const queryClient = new QueryClient({ -// defaultOptions: { -// queries: { -// suspense: true -// }, -// mutations: { -// onSuccess: () => queryClient.invalidateQueries() -// } -// } -// }); - -// export { rspc, rspcClient, queryClient }; - -export {}; diff --git a/crates/sync/example/web/tailwind.config.js b/crates/sync/example/web/tailwind.config.js deleted file mode 100644 index 7cf6cc57b..000000000 --- a/crates/sync/example/web/tailwind.config.js +++ /dev/null @@ -1,8 +0,0 @@ -/** @type {import('tailwindcss').Config} */ -module.exports = { - content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'], - theme: { - extend: {} - }, - plugins: [] -}; diff --git a/crates/sync/example/web/tsconfig.json b/crates/sync/example/web/tsconfig.json deleted file mode 100644 index 1d5d18140..000000000 --- a/crates/sync/example/web/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "node", - "allowSyntheticDefaultImports": true, - "esModuleInterop": true, - "jsx": "react-jsx", - "types": ["vite/client"], - "noEmit": true, - "isolatedModules": true - } -} diff --git a/crates/sync/example/web/vite.config.ts b/crates/sync/example/web/vite.config.ts deleted file mode 100644 index 29f022f92..000000000 --- a/crates/sync/example/web/vite.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import react from '@vitejs/plugin-react'; -import { defineConfig } from 'vite'; - -export default defineConfig({ - plugins: [react()], - server: { - port: 3000 - }, - build: { - target: 'esnext' - } -}); diff --git a/crates/sync/src/compressed.rs b/crates/sync/src/compressed.rs index aa084c4b7..a2e3a147d 100644 --- a/crates/sync/src/compressed.rs +++ b/crates/sync/src/compressed.rs @@ -1,6 +1,6 @@ use crate::{CRDTOperation, CRDTOperationData, DevicePubId, ModelId, RecordId}; -use std::collections::BTreeMap; +use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; use uhlc::NTP64; @@ -17,11 +17,16 @@ pub struct CompressedCRDTOperationsPerModelPerDevice( ); impl CompressedCRDTOperationsPerModelPerDevice { + /// Creates a new [`CompressedCRDTOperationsPerModelPerDevice`] from a vector of [`CRDTOperation`]s. + /// + /// # Panics + /// + /// Will panic if for some reason `rmp_serde::to_vec` fails to serialize a `rmpv::Value` to bytes. #[must_use] pub fn new(ops: Vec) -> Self { let mut compressed_map = BTreeMap::< DevicePubId, - BTreeMap)>>, + BTreeMap, (RecordId, Vec)>>, >::new(); for CRDTOperation { @@ -38,14 +43,21 @@ impl CompressedCRDTOperationsPerModelPerDevice { .entry(model_id) .or_default(); - // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq - if let Some((_, ops)) = records - .iter_mut() - .find(|(current_record_id, _)| *current_record_id == record_id) - { - ops.push(CompressedCRDTOperation { timestamp, data }); - } else { - records.push((record_id, vec![CompressedCRDTOperation { timestamp, data }])); + // Can't use RecordId as a key because rmpv::Value doesn't implement Hash + Eq. + // So we use it's serialized bytes as a key. + let record_id_bytes = + rmp_serde::to_vec_named(&record_id).expect("already serialized to Value"); + + match records.entry(record_id_bytes) { + Entry::Occupied(mut entry) => { + entry + .get_mut() + .1 + .push(CompressedCRDTOperation { timestamp, data }); + } + Entry::Vacant(entry) => { + entry.insert((record_id, vec![CompressedCRDTOperation { timestamp, data }])); + } } } @@ -55,7 +67,14 @@ impl CompressedCRDTOperationsPerModelPerDevice { .map(|(device_pub_id, model_map)| { ( device_pub_id, - CompressedCRDTOperationsPerModel(model_map.into_iter().collect()), + CompressedCRDTOperationsPerModel( + model_map + .into_iter() + .map(|(model_id, ops_per_record_map)| { + (model_id, ops_per_record_map.into_values().collect()) + }) + .collect(), + ), ) }) .collect(), diff --git a/crates/sync/src/crdt.rs b/crates/sync/src/crdt.rs index 13eda3ffa..3cbdf23d2 100644 --- a/crates/sync/src/crdt.rs +++ b/crates/sync/src/crdt.rs @@ -7,7 +7,7 @@ use uhlc::NTP64; pub enum OperationKind<'a> { Create, - Update(&'a str), + Update(Vec<&'a str>), Delete, } @@ -15,7 +15,7 @@ impl fmt::Display for OperationKind<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { OperationKind::Create => write!(f, "c"), - OperationKind::Update(field) => write!(f, "u:{field}"), + OperationKind::Update(fields) => write!(f, "u:{}:", fields.join(":")), OperationKind::Delete => write!(f, "d"), } } @@ -26,7 +26,7 @@ pub enum CRDTOperationData { #[serde(rename = "c")] Create(BTreeMap), #[serde(rename = "u")] - Update { field: String, value: rmpv::Value }, + Update(BTreeMap), #[serde(rename = "d")] Delete, } @@ -41,7 +41,9 @@ impl CRDTOperationData { pub fn as_kind(&self) -> OperationKind<'_> { match self { Self::Create(_) => OperationKind::Create, - Self::Update { field, .. } => OperationKind::Update(field), + Self::Update(fields_and_values) => { + OperationKind::Update(fields_and_values.keys().map(String::as_str).collect()) + } Self::Delete => OperationKind::Delete, } } diff --git a/crates/sync/src/factory.rs b/crates/sync/src/factory.rs index 9fed9a52f..7c73f8b5f 100644 --- a/crates/sync/src/factory.rs +++ b/crates/sync/src/factory.rs @@ -46,15 +46,16 @@ pub trait OperationFactory { fn shared_update( &self, id: impl SyncId, - field: impl Into, - value: rmpv::Value, + values: impl IntoIterator + 'static, ) -> CRDTOperation { self.new_op( &id, - CRDTOperationData::Update { - field: field.into(), - value, - }, + CRDTOperationData::Update( + values + .into_iter() + .map(|(name, value)| (name.to_string(), value)) + .collect(), + ), ) } @@ -77,20 +78,23 @@ pub trait OperationFactory { ), ) } + fn relation_update( &self, id: impl RelationSyncId, - field: impl Into, - value: rmpv::Value, + values: impl IntoIterator + 'static, ) -> CRDTOperation { self.new_op( &id, - CRDTOperationData::Update { - field: field.into(), - value, - }, + CRDTOperationData::Update( + values + .into_iter() + .map(|(name, value)| (name.to_string(), value)) + .collect(), + ), ) } + fn relation_delete( &self, id: impl RelationSyncId, @@ -101,9 +105,14 @@ pub trait OperationFactory { #[macro_export] macro_rules! sync_entry { + (nil, $($prisma_column_module:tt)+) => { + ($($prisma_column_module)+::NAME, ::sd_utils::msgpack!(nil)) + }; + ($value:expr, $($prisma_column_module:tt)+) => { ($($prisma_column_module)+::NAME, ::sd_utils::msgpack!($value)) - } + }; + } #[macro_export] @@ -124,6 +133,28 @@ macro_rules! sync_db_entry { }} } +#[macro_export] +macro_rules! sync_db_nullable_entry { + ($value:expr, $($prisma_column_module:tt)+) => {{ + let value = $value.into(); + ( + $crate::sync_entry!(&value, $($prisma_column_module)+), + $($prisma_column_module)+::set(value) + ) + }} +} + +#[macro_export] +macro_rules! sync_db_not_null_entry { + ($value:expr, $($prisma_column_module:tt)+) => {{ + let value = $value.into(); + ( + $crate::sync_entry!(&value, $($prisma_column_module)+), + $($prisma_column_module)+::set(value) + ) + }} +} + #[macro_export] macro_rules! option_sync_db_entry { ($value:expr, $($prisma_column_module:tt)+) => { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 45891b344..2960a21c4 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -14,6 +14,8 @@ sd-prisma = { path = "../prisma" } # Workspace dependencies chrono = { workspace = true } prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } rspc = { workspace = true, features = ["unstable"] } thiserror = { workspace = true } tracing = { workspace = true } diff --git a/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx b/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx index 381548a0a..d11ba36bc 100644 --- a/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx +++ b/interface/app/$libraryId/Explorer/ContextMenu/OpenWith.tsx @@ -81,9 +81,9 @@ const Items = ({ const paths = selectedEphemeralPaths.map((obj) => obj.path); const { t } = useLocale(); - const { data: apps } = useQuery( - ['openWith', ids, paths], - async () => { + const { data: apps } = useQuery({ + queryKey: ['openWith', ids, paths], + queryFn: async () => { const handleError = (res: Result) => { if (res?.status === 'error') { toast.error('Failed to get applications capable to open file'); @@ -104,8 +104,8 @@ const Items = ({ .then((res) => res.flat()) .then((res) => res.sort((a, b) => a.name.localeCompare(b.name))); }, - { initialData: [] } - ); + initialData: [] + }); return ( <> diff --git a/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx b/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx index 1c79d03ac..41a882e0e 100644 --- a/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx +++ b/interface/app/$libraryId/Explorer/ContextMenu/SharedItems.tsx @@ -224,7 +224,7 @@ const SpacedropNodes = () => { { spacedrop.mutateAsync({ identity: id, diff --git a/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx b/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx index aca3ec3f1..2dd8ef46e 100644 --- a/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx +++ b/interface/app/$libraryId/Explorer/ExplorerTagBar.tsx @@ -117,7 +117,7 @@ export const ExplorerTagBar = () => { const { data: allTags = [] } = useLibraryQuery(['tags.list']); const mutation = useLibraryMutation(['tags.assign'], { - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const { t } = useLocale(); diff --git a/interface/app/$libraryId/Explorer/FilePath/DecryptDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/DecryptDialog.tsx deleted file mode 100644 index 0f2cbbc2e..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/DecryptDialog.tsx +++ /dev/null @@ -1,179 +0,0 @@ -// import { RadioGroup } from '@headlessui/react'; -// import { Info } from '@phosphor-icons/react'; -// import { useLibraryMutation, useLibraryQuery } from '@sd/client'; -// import { Button, Dialog, Tooltip, UseDialogProps, useDialog } from '@sd/ui'; -// import { PasswordInput, Switch, useZodForm, z } from '@sd/ui/src/forms'; -// import { showAlertDialog } from '~/components'; -// import { usePlatform } from '~/util/Platform'; - -// const schema = z.object({ -// type: z.union([z.literal('password'), z.literal('key')]), -// outputPath: z.string(), -// mountAssociatedKey: z.boolean(), -// password: z.string(), -// saveToKeyManager: z.boolean() -// }); - -// interface Props extends UseDialogProps { -// location_id: number; -// path_id: number; -// } - -// export default (props: Props) => { -// const platform = usePlatform(); - -// const mountedUuids = useLibraryQuery(['keys.listMounted'], { -// onSuccess: (data) => { -// hasMountedKeys = data.length > 0 ? true : false; -// if (!hasMountedKeys) { -// form.setValue('type', 'password'); -// } else { -// form.setValue('type', 'key'); -// } -// } -// }); - -// let hasMountedKeys = -// mountedUuids.data !== undefined && mountedUuids.data.length > 0 ? true : false; - -// const decryptFile = useLibraryMutation('files.decryptFiles', { -// onSuccess: () => { -// showAlertDialog({ -// title: 'Success', -// value: 'The decryption job has started successfully. You may track the progress in the job overview panel.' -// }); -// }, -// onError: () => { -// showAlertDialog({ -// title: 'Error', -// value: 'The decryption job failed to start.' -// }); -// } -// }); - -// const form = useZodForm({ -// defaultValues: { -// type: hasMountedKeys ? 'key' : 'password', -// saveToKeyManager: true, -// outputPath: '', -// password: '', -// mountAssociatedKey: true -// }, -// schema -// }); - -// return ( -// -// decryptFile.mutateAsync({ -// location_id: props.location_id, -// file_path_ids: [props.path_id], -// output_path: data.outputPath !== '' ? data.outputPath : null, -// mount_associated_key: data.mountAssociatedKey, -// password: data.type === 'password' ? data.password : null, -// save_to_library: data.type === 'password' ? data.saveToKeyManager : null -// }) -// )} -// title="Decrypt a file" -// description="Leave the output file blank for the default." -// loading={decryptFile.isLoading} -// ctaLabel="Decrypt" -// > -//
-//

Key Type

-// form.setValue('type', e)} -// className="mt-2 flex flex-row gap-2" -// > -// -// {({ checked }) => ( -// -// )} -// -// -// {({ checked }) => ( -// -// )} -// -// - -// {form.watch('type') === 'key' && ( -//
-// form.setValue('mountAssociatedKey', e)} -// /> -// -// Automatically mount key -// -// -// -// -//
-// )} - -// {form.watch('type') === 'password' && ( -// <> -// - -//
-// -// -// Save to Key Manager -// -// -// -// -//
-// -// )} - -//

Output file

-// -//
-// -// ); -// }; diff --git a/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx index bd9d2fdb0..8ce5a5184 100644 --- a/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/DeleteDialog.tsx @@ -110,7 +110,7 @@ export default (props: Props) => { dialog={useDialog(props)} title={t('delete_dialog_title', { prefix, type: translatedType })} description={description} - loading={deleteFile.isLoading} + loading={deleteFile.isPending} ctaLabel={t('delete_forever')} ctaSecondLabel={t('move_to_trash')} closeLabel={t('close')} diff --git a/interface/app/$libraryId/Explorer/FilePath/EncryptDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/EncryptDialog.tsx deleted file mode 100644 index 9daf21a7a..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/EncryptDialog.tsx +++ /dev/null @@ -1,177 +0,0 @@ -// import { -// Algorithm, -// hashingAlgoSlugSchema, -// slugFromHashingAlgo, -// useLibraryMutation, -// useLibraryQuery -// } from '@sd/client'; -// import { Button, Dialog, Select, SelectOption, UseDialogProps, useDialog } from '@sd/ui'; -// import { CheckBox, useZodForm, z } from '@sd/ui/src/forms'; -// import { showAlertDialog } from '~/components'; -// import { usePlatform } from '~/util/Platform'; -// import { KeyListSelectOptions } from '../../KeyManager/List'; - -// interface Props extends UseDialogProps { -// location_id: number; -// path_id: number; -// } - -// const schema = z.object({ -// key: z.string(), -// encryptionAlgo: z.string(), -// hashingAlgo: hashingAlgoSlugSchema, -// metadata: z.boolean(), -// previewMedia: z.boolean(), -// outputPath: z.string() -// }); - -// export default (props: Props) => { -// const platform = usePlatform(); - -// const UpdateKey = (uuid: string) => { -// form.setValue('key', uuid); -// const hashAlg = keys.data?.find((key) => { -// return key.uuid === uuid; -// })?.hashing_algorithm; -// hashAlg && form.setValue('hashingAlgo', slugFromHashingAlgo(hashAlg)); -// }; - -// const keys = useLibraryQuery(['keys.list']); -// const mountedUuids = useLibraryQuery(['keys.listMounted'], { -// onSuccess: (data) => { -// UpdateKey(data[0] ?? ''); -// } -// }); - -// const encryptFile = useLibraryMutation('files.encryptFiles', { -// onSuccess: () => { -// showAlertDialog({ -// title: 'Success', -// value: 'The encryption job has started successfully. You may track the progress in the job overview panel.' -// }); -// }, -// onError: () => { -// showAlertDialog({ -// title: 'Error', -// value: 'The encryption job failed to start.' -// }); -// } -// }); - -// const form = useZodForm({ -// defaultValues: { encryptionAlgo: 'XChaCha20Poly1305', outputPath: '' }, -// schema -// }); - -// return ( -// -// encryptFile.mutateAsync({ -// algorithm: data.encryptionAlgo as Algorithm, -// key_uuid: data.key, -// location_id: props.location_id, -// file_path_ids: [props.path_id], -// metadata: data.metadata, -// preview_media: data.previewMedia -// }) -// )} -// dialog={useDialog(props)} -// title="Encrypt a file" -// description="Configure your encryption settings. Leave the output file blank for the default." -// loading={encryptFile.isLoading} -// ctaLabel="Encrypt" -// > -//
-//
-// Key -// -//
-//
-// Output file - -// -//
-//
- -//
-//
-// Encryption -// -//
-//
-// Hashing -// -//
-//
- -//
-//
-// Metadata -// -//
-//
-// Preview Media -// -//
-//
-// -// ); -// }; diff --git a/interface/app/$libraryId/Explorer/FilePath/EraseDialog.tsx b/interface/app/$libraryId/Explorer/FilePath/EraseDialog.tsx deleted file mode 100644 index 4bd84d28e..000000000 --- a/interface/app/$libraryId/Explorer/FilePath/EraseDialog.tsx +++ /dev/null @@ -1,68 +0,0 @@ -// import { useState } from 'react'; -// import { FilePath, useLibraryMutation, useZodForm } from '@sd/client'; -// import { Dialog, Slider, useDialog, UseDialogProps, z } from '@sd/ui'; -// import { useLocale } from '~/hooks'; - -// interface Props extends UseDialogProps { -// locationId: number; -// filePaths: FilePath[]; -// } - -// const schema = z.object({ -// passes: z.number() -// }); - -// export default (props: Props) => { -// const { t } = useLocale(); -// const eraseFile = useLibraryMutation('files.eraseFiles'); - -// const form = useZodForm({ -// schema, -// defaultValues: { -// passes: 4 -// } -// }); - -// const [passes, setPasses] = useState([4]); - -// return ( -// -// eraseFile.mutateAsync({ -// location_id: props.locationId, -// file_path_ids: props.filePaths.map((p) => p.id), -// passes: data.passes.toString() -// }) -// )} -// dialog={useDialog(props)} -// title={t('erase_a_file')} -// description={t('erase_a_file_description')} -// loading={eraseFile.isLoading} -// ctaLabel={t('erase')} -// > -//
-// {t('number_of_passes')} - -//
-//
-// { -// setPasses(val); -// form.setValue('passes', val[0] ?? 1); -// }} -// /> -//
-// {passes} -//
-//
- -// {/*

TODO: checkbox for "erase all matching files" (only if a file is selected)

*/} -// -// ); -// }; diff --git a/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx b/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx new file mode 100644 index 000000000..9290ca589 --- /dev/null +++ b/interface/app/$libraryId/Explorer/FilePath/ErrorBarrier.tsx @@ -0,0 +1,40 @@ +import React, { Component, ReactNode } from 'react'; + +interface ErrorBarrierProps { + onError: (error: Error, info: React.ErrorInfo) => void; + children: ReactNode; +} + +interface ErrorBarrierState { + hasError: boolean; +} + +export class ErrorBarrier extends Component { + constructor(props: ErrorBarrierProps) { + super(props); + this.state = { hasError: false }; + } + + static getDerivedStateFromError(error: Error) { + // Update state so the next render will show the fallback UI. + return { hasError: true }; + } + + componentDidCatch(error: Error, info: React.ErrorInfo) { + // Call the onError function passed as a prop + this.props.onError(error, info); + // Reset the error state after calling onError + Promise.resolve().then(() => this.setState({ hasError: false })); + } + + render() { + if (this.state.hasError) { + // Render nothing since the parent component will handle the error + return null; + } + + return this.props.children; + } +} + +export default ErrorBarrier; diff --git a/interface/app/$libraryId/Explorer/FilePath/Image.tsx b/interface/app/$libraryId/Explorer/FilePath/Image.tsx index 5be914315..e7036595b 100644 --- a/interface/app/$libraryId/Explorer/FilePath/Image.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/Image.tsx @@ -1,14 +1,7 @@ import { ComponentProps, forwardRef } from 'react'; -import { useSize } from './utils'; - -export interface ImageProps extends ComponentProps<'img'> { - extension?: string; - size: ReturnType; -} - -export const Image = forwardRef( - ({ crossOrigin, size, ...props }, ref) => ( +export const Image = forwardRef>( + ({ crossOrigin, ...props }, ref) => ( { +interface LayeredFileIconProps extends Omit, 'src'> { kind: ObjectKindKey; + isDir: boolean; extension: string | null; + customIcon: IconTypes | null; } const SUPPORTED_ICONS = ['Document', 'Code', 'Text', 'Config']; @@ -17,8 +20,18 @@ const positionConfig: Record = { }; const LayeredFileIcon = forwardRef( - ({ kind, extension, ...props }, ref) => { - const iconImg = ; + ({ kind, isDir, extension, customIcon, ...props }, ref) => { + const isDark = useIsDark(); + + const src = useMemo( + () => + customIcon + ? getIconByName(customIcon, isDark) + : getIcon(kind, isDark, extension, isDir), + [customIcon, isDark, kind, extension, isDir] + ); + + const iconImg = {`${kind}; if (SUPPORTED_ICONS.includes(kind) === false) { return iconImg; diff --git a/interface/app/$libraryId/Explorer/FilePath/Original.tsx b/interface/app/$libraryId/Explorer/FilePath/Original.tsx index 451b13d35..eb1496828 100644 --- a/interface/app/$libraryId/Explorer/FilePath/Original.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/Original.tsx @@ -8,27 +8,28 @@ import { useState, type VideoHTMLAttributes } from 'react'; -import { getItemFilePath, useLibraryContext } from '@sd/client'; +import { ObjectKindKey, useLibraryContext } from '@sd/client'; import i18n from '~/app/I18n'; import { PDFViewer, TextViewer } from '~/components'; -import { useLocale } from '~/hooks'; +import { useIsDark, useLocale } from '~/hooks'; import { pdfViewerEnabled } from '~/util/pdfViewer'; import { usePlatform } from '~/util/Platform'; import { useExplorerContext } from '../Context'; import { explorerStore } from '../store'; -import { ExplorerItemData } from '../useExplorerItemData'; import { Image } from './Image'; import { useBlackBars, useSize } from './utils'; interface OriginalRendererProps { src: string; - className: string; - frameClassName: string; - itemData: ExplorerItemData; - isDark: boolean; + fileId: number | null; + locationId: number | null; + path: string | null; + className?: string; + frameClassName?: string; + kind: ObjectKindKey; + extension: string | null; childClassName?: string; - size?: number; magnification?: number; mediaControls?: boolean; frame?: boolean; @@ -37,44 +38,53 @@ interface OriginalRendererProps { blackBars?: boolean; blackBarsSize?: number; onLoad?(): void; - onError?(e: ErrorEvent | SyntheticEvent): void; } export function Original({ - itemData, - filePath, + path, + fileId, + locationId, ...props -}: Omit & { - filePath: ReturnType; -}) { - const [error, setError] = useState(false); - if (error) throw new Error('onError'); +}: Omit) { + const [error, setError] = useState(null); + if (error != null) throw error; const Renderer = useMemo(() => { - const kind = originalRendererKind(itemData); + const kind = originalRendererKind(props.kind, props.extension); return ORIGINAL_RENDERERS[kind]; - }, [itemData]); + }, [props.kind, props.extension]); if (!Renderer) throw new Error('no renderer!'); const platform = usePlatform(); const { library } = useLibraryContext(); const { parent } = useExplorerContext(); + locationId = locationId ?? (parent?.type === 'Location' ? parent.location.id : null); const src = useMemo(() => { - const locationId = - itemData.locationId ?? (parent?.type === 'Location' ? parent.location.id : null); - - if (filePath && (itemData.extension !== 'pdf' || pdfViewerEnabled())) { - if ('id' in filePath && locationId) - return platform.getFileUrl(library.uuid, locationId, filePath.id); - else if ('path' in filePath) return platform.getFileUrlByPath(filePath.path); + if (props.extension !== 'pdf' || pdfViewerEnabled()) { + if (fileId != null && locationId) + return platform.getFileUrl(library.uuid, locationId, fileId); + else if (path) return platform.getFileUrlByPath(path); } - }, [itemData, filePath, library.uuid, parent, platform]); + }, [props.extension, fileId, locationId, platform, library.uuid, path]); if (src === undefined) throw new Error('no src!'); - return setError(true)} {...props} />; + return ( + + setError( + ('error' in event && event.error instanceof Error && event.error) || + new Error( + ('message' in event && event.message) || 'Filetype is not supported yet' + ) + ) + } + {...props} + /> + ); } const TEXT_RENDERER: OriginalRenderer = (props) => ( @@ -83,24 +93,26 @@ const TEXT_RENDERER: OriginalRenderer = (props) => ( onLoad={props.onLoad} onError={props.onError} className={clsx( - 'textviewer-scroll size-full overflow-y-auto whitespace-pre-wrap break-words px-4 font-mono', + 'textviewer-scroll font-mono size-full overflow-y-auto whitespace-pre-wrap break-words px-4', !props.mediaControls ? 'overflow-hidden' : 'overflow-auto', props.className, props.frame && [props.frameClassName, '!bg-none p-2'] )} codeExtension={ - ((props.itemData.kind === 'Code' || props.itemData.kind === 'Config') && - props.itemData.extension) || - '' + ((props.kind === 'Code' || props.kind === 'Config') && props.extension) || '' } isSidebarPreview={props.isSidebarPreview} /> ); -type OriginalRenderer = (props: OriginalRendererProps) => JSX.Element; +type OriginalRenderer = ( + props: Omit & { + onError?(e: ErrorEvent | SyntheticEvent): void; + } +) => JSX.Element; -function originalRendererKind(itemData: ExplorerItemData) { - return itemData.extension === 'pdf' ? 'PDF' : itemData.kind; +function originalRendererKind(kind: ObjectKindKey, extension: string | null) { + return extension === 'pdf' ? 'PDF' : kind; } type OriginalRendererKind = ReturnType; @@ -135,44 +147,45 @@ const ORIGINAL_RENDERERS: { )} /> ), - Audio: (props) => ( - <> - - {props.mediaControls && ( - - )} - - ), + Audio: (props) => { + const isDark = useIsDark(); + return ( + <> + + {props.mediaControls && ( + + )} + + ); + }, Image: (props) => { const ref = useRef(null); - const size = useSize(ref); return (
{ + cover?: boolean; + blackBars?: boolean; + blackBarsSize?: number; + videoExtension?: string; +} + +const Thumbnail = memo( + forwardRef( + ( + { + blackBars, + blackBarsSize, + videoExtension: extension, + cover, + className, + style, + ...props + }, + _ref + ) => { + const ref = useRef(null); + useImperativeHandle( + _ref, + () => ref.current + ); + + const size = useSize(ref); + + const { style: blackBarsStyle } = useBlackBars(ref, size, { + size: blackBarsSize, + disabled: !blackBars + }); + + return ( + <> + + + {(cover || size.width > 80) && extension && ( +
+ {extension} +
+ )} + + ); + } + ) +); + +interface ThumbProps extends ThumbnailProps { + src?: string; + kind: ObjectKindKey; + path: string | null; + isDir: boolean; + frame: boolean; + fileId: number | null; + onLoad: () => void; + onError: (error: Error | ErrorEvent | SyntheticEvent) => void; + thumbType: ThumbType; + extension: string | null; + customIcon: IconTypes | null; + locationId: number | null; + pauseVideo: boolean; + magnification: number; + mediaControls: boolean; + frameClassName: string; + isSidebarPreview: boolean; +} + +const Thumb = memo( + forwardRef( + ( + { + src, + kind, + path, + frame, + isDir, + cover, + fileId, + thumbType, + extension, + blackBars, + className, + pauseVideo, + locationId, + customIcon, + magnification, + mediaControls, + blackBarsSize, + videoExtension, + frameClassName, + isSidebarPreview, + onLoad, + ...props + }, + _ref + ) => { + const ref = useRef(null); + useImperativeHandle( + _ref, + () => ref.current + ); + const [isLoading, setIsLoading] = useState(true); + + const handleLoad = useCallback(() => { + const img = ref.current; + setIsLoading(!(img == null || (img.naturalHeight > 0 && img.naturalWidth > 0))); + onLoad?.(); + }, [onLoad]); + + let thumb: JSX.Element | null = null; + + switch (thumbType) { + case 'original': + thumb = ( + + ); + break; + case 'thumbnail': + thumb = ( + + ); + break; + } + + return ( + <> + {}} + decoding="sync" + draggable={false} + extension={extension} + className={clsx(ThumbClasses, className, !isLoading && 'hidden')} + customIcon={customIcon} + /> + {thumb ?? null} + + ); + } + ) +); + +export interface FileThumbProps { data: ExplorerItem; loadOriginal?: boolean; size?: number; cover?: boolean; frame?: boolean; - onLoad?: (state: ThumbType) => void; - onError?: (state: ThumbType, error: Error) => void; + onLoad?: (type: ThumbType) => void; + onError?: (state: LoadState, error: Error) => void; blackBars?: boolean; blackBarsSize?: number; extension?: boolean; @@ -43,253 +259,181 @@ export interface ThumbProps { magnification?: number; } -type ThumbType = { variant: 'original' } | { variant: 'thumbnail' } | { variant: 'icon' }; -type LoadState = { - [K in 'original' | 'thumbnail' | 'icon']: 'notLoaded' | 'loaded' | 'error'; -}; +/** + * This component is used to render a thumbnail of a file or folder. + * It will automatically choose the best thumbnail to display based on the item data. + * + * .. WARNING:: + * This Component is heavely used inside the explorer, and as such it is a performance critical component. + * Be careful with the performance of the code, make sure to always memoize any objects or functions to avoid unnecessary re-renders. + * + */ +export const FileThumb = memo( + forwardRef((props, ref) => { + const frame = useFrame(); + const platform = usePlatform(); + const itemData = useExplorerItemData(props.data); + const filePath = getItemFilePath(props.data); + const { library } = useLibraryContext(); + const [loadState, setLoadState] = useState({ + icon: 'normal', + original: 'normal', + thumbnail: 'normal' + }); -export const FileThumb = forwardRef((props, ref) => { - const isDark = useIsDark(); - const platform = usePlatform(); - const frame = useFrame(); + // WARNING: This is required so QuickPreview can work properly + useEffect(() => { + setLoadState({ + icon: 'normal', + original: 'normal', + thumbnail: 'normal' + }); + }, [props.data]); - const itemData = useExplorerItemData(props.data); - const filePath = getItemFilePath(props.data); + const thumbType = useMemo((): ThumbType => { + if (loadState.original !== 'error' && props.loadOriginal) return 'original'; + if (loadState.thumbnail !== 'error' && itemData.hasLocalThumbnail) return 'thumbnail'; + return 'icon'; + }, [itemData.hasLocalThumbnail, loadState, props.loadOriginal]); - const { library } = useLibraryContext(); - - const [loadState, setLoadState] = useState({ - original: 'notLoaded', - thumbnail: 'notLoaded', - icon: 'notLoaded' - }); - - const childClassName = 'max-h-full max-w-full object-contain'; - const frameClassName = clsx(frame.className, props.frameClassName); - - const thumbType = useMemo(() => { - const thumbType = 'thumbnail'; - - if (thumbType === 'thumbnail') - if ( - loadState.thumbnail !== 'error' && - itemData.hasLocalThumbnail && - itemData.thumbnailKey - ) - return { variant: 'thumbnail' }; - - return { variant: 'icon' }; - }, [itemData, loadState]); - - const src = useMemo(() => { - switch (thumbType.variant) { - case 'original': - if (filePath && (itemData.extension !== 'pdf' || pdfViewerEnabled())) { - if ('id' in filePath && itemData.locationId) - return platform.getFileUrl(library.uuid, itemData.locationId, filePath.id); - else if ('path' in filePath) return platform.getFileUrlByPath(filePath.path); + useEffect(() => { + let timeoutId = null; + // Reload thumbnail when it gets a notification from core that it has been generated + if (thumbType === 'icon' && loadState.thumbnail === 'error') { + for (const [, thumbId] of itemData.thumbnails) { + if (thumbId == null || !explorerStore.newThumbnails.has(thumbId)) continue; + // HACK: Delay removing the new thumbnail event from store + // to avoid some weird race condition with core that prevents + // us from accessing the new thumbnail immediately after it is created + timeoutId = setTimeout(() => explorerStore.removeThumbnail(thumbId), 0); + explorerStore.removeThumbnail(thumbId); + setLoadState((state) => ({ ...state, thumbnail: 'normal' })); + break; } - break; + } - case 'thumbnail': - if (itemData.thumbnailKey) - return platform.getThumbnailUrlByThumbKey(itemData.thumbnailKey); + return () => void (timeoutId && clearTimeout(timeoutId)); + }, [itemData.thumbnails, loadState.thumbnail, thumbType]); - break; - case 'icon': - if (itemData.customIcon) return getIconByName(itemData.customIcon as any, isDark); + const src = useMemo(() => { + switch (thumbType) { + case 'original': + if (filePath && (itemData.extension !== 'pdf' || pdfViewerEnabled())) { + if ('id' in filePath && itemData.locationId) + return platform.getFileUrl( + library.uuid, + itemData.locationId, + filePath.id + ); + else if ('path' in filePath) + return platform.getFileUrlByPath(filePath.path); + else setLoadState((state) => ({ ...state, [thumbType]: 'error' })); + } + break; - return getIcon( - // itemData.isDir || parent?.type === 'Node' ? 'Folder' : - itemData.kind, - isDark, - itemData.extension, - itemData.isDir - ); - } - }, [filePath, isDark, library.uuid, itemData, platform, thumbType]); + case 'thumbnail': { + const thumbnail = Array.from(itemData.thumbnails.keys()).find((key) => key); + if (thumbnail) return thumbnail; + else setLoadState((state) => ({ ...state, [thumbType]: 'error' })); - const onLoad = (s: 'original' | 'thumbnail' | 'icon') => { - setLoadState((state) => ({ ...state, [s]: 'loaded' })); - props.onLoad?.call(null, thumbType); - }; + break; + } + } + }, [ + filePath, + itemData.extension, + itemData.locationId, + itemData.thumbnails, + library.uuid, + platform, + thumbType + ]); - const onError = ( - s: 'original' | 'thumbnail' | 'icon', - event: ErrorEvent | SyntheticEvent - ) => { - setLoadState((state) => ({ ...state, [s]: 'error' })); + const onError = useCallback( + (event: Error | ErrorEvent | SyntheticEvent) => { + const rawError = + event instanceof Error + ? event + : ('error' in event && event.error) || + ('message' in event && event.message) || + 'Filetype is not supported yet'; - const rawError = - ('error' in event && event.error) || - ('message' in event && event.message) || - 'Filetype is not supported yet'; - - props.onError?.call( - null, - thumbType, - rawError instanceof Error ? rawError : new Error(rawError) + setLoadState((state) => { + state = { ...state, [thumbType]: 'error' }; + props.onError?.call( + null, + state, + rawError instanceof Error ? rawError : new Error(rawError) + ); + return state; + }); + }, + [props.onError, thumbType] ); - }; - const _childClassName = - typeof props.childClassName === 'function' - ? props.childClassName(thumbType) - : props.childClassName; + const onLoad = useCallback(() => { + props.onLoad?.call(null, thumbType); + }, [props.onLoad, thumbType]); - const className = clsx(childClassName, _childClassName); - - const thumbnail = (() => { - if (!src) return <>; - - switch (thumbType.variant) { - case 'thumbnail': - return ( - onLoad('thumbnail')} - onError={(e) => onError('thumbnail', e)} - decoding={props.size ? 'async' : 'sync'} - className={clsx( - props.cover - ? [ - 'min-h-full min-w-full object-cover object-center', - _childClassName - ] - : className, - props.frame && !(itemData.kind === 'Video' && props.blackBars) - ? frameClassName - : null - )} - crossOrigin="anonymous" // Here it is ok, because it is not a react attr - blackBars={props.blackBars && itemData.kind === 'Video' && !props.cover} - blackBarsSize={props.blackBarsSize} - extension={ - props.extension && itemData.extension && itemData.kind === 'Video' - ? itemData.extension - : undefined - } - /> - ); - - case 'icon': - return ( - + { + console.error('ErrorBoundary', error, info); + onError(error); + }, + [onError] + )} + > + onLoad('icon')} - onError={(e) => onError('icon', e)} - decoding={props.size ? 'async' : 'sync'} - className={className} - draggable={false} - /> - ); - default: - return <>; - } - })(); - - return ( -
- {props.loadOriginal ? ( - - onLoad('original')} - onError={(e) => onError('original', e)} - filePath={filePath} - className={className} - frameClassName={frameClassName} - itemData={itemData} - isDark={isDark} - childClassName={childClassName} - size={props.size} - magnification={props.magnification} - mediaControls={props.mediaControls} - frame={props.frame} - isSidebarPreview={props.isSidebarPreview} - pauseVideo={props.pauseVideo} blackBars={props.blackBars} + className={ + typeof props.childClassName === 'function' + ? props.childClassName(thumbType) + : props.childClassName + } + customIcon={itemData.customIcon as IconTypes | null} + locationId={itemData.locationId} + pauseVideo={props.pauseVideo ?? false} blackBarsSize={props.blackBarsSize} + mediaControls={props.mediaControls ?? false} + magnification={props.magnification ?? 1} + frameClassName={clsx(frame.className, props.frameClassName)} + videoExtension={ + props.extension && itemData.extension && itemData.kind === 'Video' + ? itemData.extension + : undefined + } + isSidebarPreview={props.isSidebarPreview ?? false} /> - - ) : ( - thumbnail - )} -
- ); -}); - -interface ThumbnailProps extends Omit { - cover?: boolean; - blackBars?: boolean; - blackBarsSize?: number; - extension?: string; -} - -const Thumbnail = forwardRef( - ({ blackBars, blackBarsSize, extension, cover, className, style, ...props }, _ref) => { - const ref = useRef(null); - useImperativeHandle( - _ref, - () => ref.current + +
); - - const size = useSize(ref); - - const { style: blackBarsStyle } = useBlackBars(ref, size, { - size: blackBarsSize, - disabled: !blackBars - }); - - return ( - <> - - - {(cover || size.width > 80) && extension && ( -
- {extension} -
- )} - - ); - } + }) ); diff --git a/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx b/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx index 97abc0d27..71eea2117 100644 --- a/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx +++ b/interface/app/$libraryId/Explorer/Inspector/FavoriteButton.tsx @@ -14,7 +14,7 @@ export default function FavoriteButton(props: Props) { setFavorite(!!props.data?.favorite); }, [props.data]); - const { mutate: fileToggleFavorite, isLoading: isFavoriteLoading } = useLibraryMutation( + const { mutate: fileToggleFavorite, isPending: isFavoriteLoading } = useLibraryMutation( 'files.setFavorite' // { // onError: () => setFavorite(!!props.data?.favorite) diff --git a/interface/app/$libraryId/Explorer/Inspector/index.tsx b/interface/app/$libraryId/Explorer/Inspector/index.tsx index 08b3255d0..7d0226396 100644 --- a/interface/app/$libraryId/Explorer/Inspector/index.tsx +++ b/interface/app/$libraryId/Explorer/Inspector/index.tsx @@ -154,7 +154,7 @@ const Thumbnails = ({ items }: { items: ExplorerItem[] }) => { i === 2 && 'z-10 !h-[84%] !w-[84%] rotate-[7deg]' )} childClassName={(type) => - type.variant !== 'icon' && thumbs.length > 1 + type !== 'icon' && thumbs.length > 1 ? 'shadow-md shadow-app-shade' : undefined } diff --git a/interface/app/$libraryId/Explorer/QuickPreview/index.tsx b/interface/app/$libraryId/Explorer/QuickPreview/index.tsx index ad1b0fdc5..918af62c9 100644 --- a/interface/app/$libraryId/Explorer/QuickPreview/index.tsx +++ b/interface/app/$libraryId/Explorer/QuickPreview/index.tsx @@ -50,7 +50,7 @@ import ExplorerContextMenu, { SharedItems } from '../ContextMenu'; import { Conditional } from '../ContextMenu/ConditionalItem'; -import { FileThumb } from '../FilePath/Thumb'; +import { FileThumb, ThumbType } from '../FilePath/Thumb'; import { SingleItemMetadata } from '../Inspector'; import { explorerStore } from '../store'; import { useExplorerViewContext } from '../View/Context'; @@ -84,19 +84,31 @@ export const QuickPreview = () => { const { open, itemIndex } = useQuickPreviewStore(); const thumb = createRef(); - const [thumbErrorToast, setThumbErrorToast] = useState(); const [showMetadata, setShowMetadata] = useState(false); const [magnification, setMagnification] = useState(1); const [isContextMenuOpen, setIsContextMenuOpen] = useState(false); const [isRenaming, setIsRenaming] = useState(false); const [newName, setNewName] = useState(null); - const [thumbnailLoading, setThumbnailLoading] = useState<'notLoaded' | 'loaded' | 'error'>( - 'notLoaded' - ); + const [thumbnailLoading, setThumbnailLoading] = useState({ + icon: 'notLoaded', + thumbnail: 'notLoaded', + original: 'notLoaded' + } as { + [K in ThumbType]: 'notLoaded' | 'loaded' | 'error'; + }); // the purpose of these refs is to prevent "jittering" when zooming with trackpads, as the deltaY value can be very high const deltaYRef = useRef(0); const lastZoomTimeRef = useRef(0); + const hasError = useMemo( + () => Object.values(thumbnailLoading).some((status) => status === 'error'), + [thumbnailLoading] + ); + const isLoaded = useMemo( + () => Object.values(thumbnailLoading).some((status) => status === 'loaded'), + [thumbnailLoading] + ); + const { t } = useLocale(); const items = useMemo(() => { @@ -122,50 +134,26 @@ export const QuickPreview = () => { const renameFile = useLibraryMutation(['files.renameFile'], { onError: () => setNewName(null), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const renameEphemeralFile = useLibraryMutation(['ephemeralFiles.renameFile'], { onError: () => setNewName(null), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const changeCurrentItem = (index: number) => { if (items[index]) getQuickPreviewStore().itemIndex = index; }; - // Error toast - useEffect(() => { - if (!thumbErrorToast) return; - - let id: string | number | undefined; - toast.error( - (_id) => { - id = _id; - return thumbErrorToast; - }, - { - ref: thumb, - duration: Infinity, - onClose() { - id = undefined; - setThumbErrorToast(undefined); - } - } - ); - - return () => void toast.dismiss(id); - }, [thumb, thumbErrorToast]); - // Reset state useEffect(() => { setNewName(null); - setThumbErrorToast(undefined); setMagnification(1); + setThumbnailLoading({ icon: 'notLoaded', thumbnail: 'notLoaded', original: 'notLoaded' }); if (open || item) return; - setThumbnailLoading('notLoaded'); getQuickPreviewStore().open = false; getQuickPreviewStore().itemIndex = 0; setShowMetadata(false); @@ -344,18 +332,12 @@ export const QuickPreview = () => { )} >
- {thumbnailLoading !== 'error' && - thumbnailLoading !== 'notLoaded' && - background && ( -
- -
-
- )} + {!hasError && isLoaded && background && ( +
+ +
+
+ )}
{ - {thumbnailLoading === 'error' && ( - -
- -

- {t('quickpreview_thumbnail_error_message')} -

-
- - )} + {thumbnailLoading.original === 'error' && + thumbnailLoading.thumbnail === 'loaded' && ( + +
+ +

+ {t( + 'quickpreview_thumbnail_error_message' + )} +

+
+ + )} {items.length > 1 && (
@@ -616,56 +603,42 @@ export const QuickPreview = () => {
- {thumbnailLoading === 'error' ? ( - <> - - - ) : ( - { - setThumbnailLoading('loaded'); - if (type.variant === 'original') - setThumbErrorToast(undefined); - }} - onError={(type, error) => { - setThumbnailLoading('error'); - if (type.variant === 'original') - setThumbErrorToast({ - title: t('error_loading_original_file'), - body: error.message - }); - }} - loadOriginal - frameClassName="!border-0" - mediaControls - className={clsx( - thumbnailLoading === 'notLoaded' && 'hidden', - 'm-3 !w-auto flex-1 !overflow-hidden rounded', - !background && !icon && 'bg-app-box shadow' - )} - childClassName={clsx( - 'rounded', - kind === 'Text' && 'p-3', - !icon && 'h-full', - textKinds.includes(kind) && 'select-text' - )} - magnification={magnification} - /> - )} + { + setThumbnailLoading((obj) => ({ + ...obj, + [type]: 'loaded' + })); + }} + onError={(state, error) => { + console.error(error); + setThumbnailLoading((obj) => { + const newState = { ...obj }; + for (const [type, loadState] of Object.entries( + state + ) as [ThumbType, string][]) + if (loadState === 'error') newState[type] = 'error'; + + return newState; + }); + }} + loadOriginal + frameClassName="!border-0" + mediaControls + className={clsx( + !isLoaded && 'hidden', + 'm-3 !w-auto flex-1 !overflow-hidden rounded', + !background && !icon && 'bg-app-box shadow' + )} + childClassName={clsx( + 'rounded', + kind === 'Text' && 'p-3', + !icon && 'h-full', + textKinds.includes(kind) && 'select-text' + )} + magnification={magnification} + /> {explorerLayoutStore.showImageSlider && activeItem && ( diff --git a/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx b/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx index e58a56787..e93f85355 100644 --- a/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx +++ b/interface/app/$libraryId/Explorer/View/Grid/DragSelect/index.tsx @@ -13,8 +13,16 @@ import { getElementIndex, SELECTABLE_DATA_ATTRIBUTE } from './util'; const CHROME_REGEX = /Chrome/; +type GridOpts = ReturnType>; + interface Props extends PropsWithChildren { - grid: ReturnType>; + columnCount: GridOpts['columnCount']; + gapY: GridOpts['gap']['y']; + getItem: GridOpts['getItem']; + totalColumnCount: GridOpts['totalColumnCount']; + totalCount: GridOpts['totalCount']; + totalRowCount: GridOpts['totalRowCount']; + virtualItemHeight: GridOpts['virtualItemHeight']; } export interface Drag { @@ -24,7 +32,7 @@ export interface Drag { endRow: number; } -export const DragSelect = ({ grid, children }: Props) => { +export const DragSelect = ({ children, ...props }: Props) => { const isChrome = CHROME_REGEX.test(navigator.userAgent); const { explorerOperatingSystem, matchingOperatingSystem } = useExplorerOperatingSystem(); @@ -62,7 +70,7 @@ export const DragSelect = ({ grid, children }: Props) => { function getGridItem(element: Element) { const index = getElementIndex(element); - return (index !== null && grid.getItem(index)) || undefined; + return (index !== null && props.getItem(index)) || undefined; } function handleScroll(e: SelectoEvents['scroll']) { @@ -176,9 +184,9 @@ export const DragSelect = ({ grid, children }: Props) => { // that are still in the DOM const elements: Element[] = []; - e.added.forEach((element) => { + for (const element of e.added) { const item = getGridItem(element); - if (!item?.data) return; + if (!item?.data) continue; // Add item to selected targets // Don't update selecto as it's already aware of it @@ -188,22 +196,22 @@ export const DragSelect = ({ grid, children }: Props) => { explorer.addSelectedItem(item.data); if (document.contains(element)) elements.push(element); - }); + } - e.removed.forEach((element) => { + for (const element of e.removed) { const item = getGridItem(element); - if (!item?.data) return; + if (!item?.data) continue; // Remove item from selected targets // Don't update selecto as it's already aware of it selectedTargets.removeSelectedTarget(String(item.id), { updateSelecto: false }); // Don't deselect item if element is unmounted by scroll - if (!document.contains(element)) return; + if (!document.contains(element)) continue; explorer.removeSelectedItem(item.data); elements.push(element); - }); + } const dragDirection = { x: inputEvent.x === e.rect.left ? 'left' : 'right', @@ -280,7 +288,7 @@ export const DragSelect = ({ grid, children }: Props) => { const addedRows = new Set(); const removedRows = new Set(); - columns.forEach((column) => { + for (const column of columns) { const { firstItem, lastItem } = columnItems[column]!; const { row: firstRow } = firstItem.item; @@ -353,7 +361,7 @@ export const DragSelect = ({ grid, children }: Props) => { // Remove row if dragged out of the last grid item // from a row that's above it - if (item.item.index === grid.totalCount - 1) { + if (item.item.index === props.totalCount - 1) { removedRows.add(item.item.row); } } @@ -372,21 +380,21 @@ export const DragSelect = ({ grid, children }: Props) => { // caches multiple rows at once, and the first one being removed if ( !isFirstRowInDrag && - firstRow === grid.totalRowCount - 2 && - firstItem.item.index + grid.totalColumnCount > grid.totalCount - 1 + firstRow === props.totalRowCount - 2 && + firstItem.item.index + props.totalColumnCount > props.totalCount - 1 ) { removedColumns.add(column); } // Return if first row equals the first/last row of the grid (depending on drag direction) // as there's no items to be selected beyond that point - if (!drag.current && (firstRow === 0 || firstRow === grid.totalRowCount - 1)) { - return; + if (!drag.current && (firstRow === 0 || firstRow === props.totalRowCount - 1)) { + continue; } // Return if column is already in drag range if (isColumnInDrag && isColumnInDragRange) { - return; + continue; } const viewTop = explorerView.ref.current?.getBoundingClientRect().top ?? 0; @@ -397,9 +405,9 @@ export const DragSelect = ({ grid, children }: Props) => { const hasEmptySpace = dragDirection.y === 'down' ? dragStart.y < itemTop : dragStart.y > itemBottom; - if (!hasEmptySpace) return; + if (!hasEmptySpace) continue; - // Get the heigh of the empty drag space between the start of the drag + // Get the height of the empty drag space between the start of the drag // and the first visible item const emptySpaceHeight = Math.abs( dragStart.y - (dragDirection.y === 'down' ? itemTop : itemBottom) @@ -407,8 +415,8 @@ export const DragSelect = ({ grid, children }: Props) => { // Check how many items we can fit into the empty space let itemsInEmptySpace = - (emptySpaceHeight - (grid.gap.y ?? 0)) / - (grid.virtualItemHeight + (grid.gap.y ?? 0)); + (emptySpaceHeight - (props.gapY ?? 0)) / + (props.virtualItemHeight + (props.gapY ?? 0)); if (itemsInEmptySpace > 1) { itemsInEmptySpace = Math.ceil(itemsInEmptySpace); @@ -416,15 +424,15 @@ export const DragSelect = ({ grid, children }: Props) => { itemsInEmptySpace = Math.round(itemsInEmptySpace); } - [...Array(itemsInEmptySpace)].forEach((_, i) => { + for (let i = 0; i < itemsInEmptySpace; i++) { i = dragDirection.y === 'down' ? itemsInEmptySpace - i : i + 1; const explorerItemIndex = firstItem.item.index + - (dragDirection.y === 'down' ? -i : i) * grid.columnCount; + (dragDirection.y === 'down' ? -i : i) * props.columnCount; - const item = grid.getItem(explorerItemIndex); - if (!item?.data) return; + const item = props.getItem(explorerItemIndex); + if (!item?.data) continue; // Set start row if not already set if (!drag.current && i === itemsInEmptySpace - 1) { @@ -438,13 +446,13 @@ export const DragSelect = ({ grid, children }: Props) => { explorer.addSelectedItem(item.data); } - return; + continue; } if (!isItemInDrag) explorer.removeSelectedItem(item.data); else explorer.addSelectedItem(item.data); - }); - }); + } + } const addedColumnsArray = [...addedColumns]; const removedColumnsArray = [...removedColumns]; diff --git a/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx b/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx index aeb40ee0f..a9d01994f 100644 --- a/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx +++ b/interface/app/$libraryId/Explorer/View/GridView/Item/index.tsx @@ -52,9 +52,10 @@ export const GridViewItem = memo((props: GridViewItemProps) => { ); }); -const InnerDroppable = () => { +const InnerDroppable = memo(() => { const item = useGridViewItemContext(); const { isDroppable } = useExplorerDroppableContext(); + return ( <>
{ (item.selected || isDroppable) && 'bg-app-selectedItem' )} > - +
); -}; +}); -const ItemFileThumb = () => { +const ItemFileThumb = memo((props: GridViewItemProps) => { const frame = useFrame(); - - const item = useGridViewItemContext(); - const isLabel = item.data.type === 'Label'; - const { attributes, listeners, style, setDraggableRef } = useExplorerDraggable({ - data: item.data + data: props.data }); + const isLabel = props.data.type === 'Label'; + return ( ({ + style, + ...attributes, + ...listeners + }), + [style, attributes, listeners] + )} /> ); -}; +}); -const ItemMetadata = () => { +const ItemMetadata = memo(() => { const item = useGridViewItemContext(); const { isDroppable } = useExplorerDroppableContext(); const explorerLayout = useExplorerLayoutStore(); @@ -123,9 +125,9 @@ const ItemMetadata = () => { {item.data.type === 'Label' && } ); -}; +}); -const ItemTags = () => { +const ItemTags = memo(() => { const item = useGridViewItemContext(); const object = getItemObject(item.data); const filePath = getItemFilePath(item.data); @@ -150,9 +152,9 @@ const ItemTags = () => { ))}
); -}; +}); -const ItemSize = () => { +const ItemSize = memo(() => { const item = useGridViewItemContext(); const { showBytesInGridView } = useExplorerContext().useSettingsSnapshot(); const isRenaming = useSelector(explorerStore, (s) => s.isRenaming); @@ -186,9 +188,9 @@ const ItemSize = () => { {`${bytes}`}
); -}; +}); -function LabelItemCount({ data }: { data: Extract }) { +const LabelItemCount = memo(({ data }: { data: Extract }) => { const { t } = useLocale(); const count = useLibraryQuery([ @@ -202,11 +204,11 @@ function LabelItemCount({ data }: { data: Extract {t('item_with_count', { count: count.data })}
); -} +}); diff --git a/interface/app/$libraryId/Explorer/View/GridView/index.tsx b/interface/app/$libraryId/Explorer/View/GridView/index.tsx index b5055ee49..507e5ee7d 100644 --- a/interface/app/$libraryId/Explorer/View/GridView/index.tsx +++ b/interface/app/$libraryId/Explorer/View/GridView/index.tsx @@ -1,5 +1,5 @@ import { Grid, useGrid } from '@virtual-grid/react'; -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback } from 'react'; import { useExplorerLayoutStore } from '@sd/client'; import { useExplorerContext } from '../../Context'; @@ -50,7 +50,15 @@ export const GridView = () => { useKeySelection(grid, { scrollToEnd: true }); return ( - + {(index) => { const item = explorer.items?.[index]; diff --git a/interface/app/$libraryId/Explorer/View/ListView/index.tsx b/interface/app/$libraryId/Explorer/View/ListView/index.tsx index fc6ac953d..e792a19ca 100644 --- a/interface/app/$libraryId/Explorer/View/ListView/index.tsx +++ b/interface/app/$libraryId/Explorer/View/ListView/index.tsx @@ -130,47 +130,49 @@ export const ListView = memo(() => { const [backRange, frontRange] = getRangesByRow(range.start); if (backRange && frontRange) { - [...Array(backRange.sorted.end.index - backRange.sorted.start.index + 1)].forEach( - (_, i) => { - const index = backRange.sorted.start.index + i; + for (let i = backRange.sorted.start.index; i <= backRange.sorted.end.index; i++) { + const index = backRange.sorted.start.index + i; - if (index === range.start.index) return; + if (index === range.start.index) continue; - const row = rows[index]; + const row = rows[index]; - if (row) explorer.removeSelectedItem(row.original); - } - ); + if (row) explorer.removeSelectedItem(row.original); + } _ranges = _ranges.filter((_, i) => i !== backRange.index); } - [...Array(Math.abs(range.end.index - rowIndex) + (changeDirection ? 1 : 0))].forEach( - (_, i) => { - if (!range.direction || direction === range.direction) i += 1; + for ( + let i = 0; + i < Math.abs(range.end.index - rowIndex) + (changeDirection ? 1 : 0); + i++ + ) { + if (!range.direction || direction === range.direction) i += 1; - const index = range.end.index + (direction === 'down' ? i : -i); + const index = range.end.index + (direction === 'down' ? i : -i); - const row = rows[index]; + const row = rows[index]; - if (!row) return; + if (!row) continue; - const item = row.original; + const item = row.original; - if (uniqueId(item) === uniqueId(range.start.original)) return; + if (uniqueId(item) === uniqueId(range.start.original)) continue; - if ( - !range.direction || - direction === range.direction || - (changeDirection && - (range.direction === 'down' - ? index < range.start.index - : index > range.start.index)) - ) { - explorer.addSelectedItem(item); - } else explorer.removeSelectedItem(item); + if ( + !range.direction || + direction === range.direction || + (changeDirection && + (range.direction === 'down' + ? index < range.start.index + : index > range.start.index)) + ) { + explorer.addSelectedItem(item); + } else { + explorer.removeSelectedItem(item); } - ); + } let newRangeEnd = item; let removeRangeIndex: number | null = null; @@ -186,15 +188,13 @@ export const ListView = memo(() => { rowIndex ); - [...Array(removableRowsCount)].forEach((_, i) => { - i += 1; - + for (let i = 1; i <= removableRowsCount; i++) { const index = rowIndex + (direction === 'down' ? i : -i); const row = rows[index]; if (row) explorer.removeSelectedItem(row.original); - }); + } removeRangeIndex = i; break; diff --git a/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx b/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx index c5b1652cf..dd71f2b31 100644 --- a/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx +++ b/interface/app/$libraryId/Explorer/View/MediaView/Item.tsx @@ -57,7 +57,7 @@ const ItemFileThumb = (props: Pick) => { filePath?.hidden && 'opacity-50' )} ref={setDraggableRef} - childClassName={({ variant }) => clsx(variant === 'icon' && 'size-2/4')} + childClassName={(type) => clsx(type === 'icon' && 'size-2/4')} childProps={{ style, ...attributes, diff --git a/interface/app/$libraryId/Explorer/View/MediaView/index.tsx b/interface/app/$libraryId/Explorer/View/MediaView/index.tsx index 6554d6a7d..cf8f6f866 100644 --- a/interface/app/$libraryId/Explorer/View/MediaView/index.tsx +++ b/interface/app/$libraryId/Explorer/View/MediaView/index.tsx @@ -82,11 +82,12 @@ export const MediaView = () => { let firstRowIndex: number | undefined = undefined; let lastRowIndex: number | undefined = undefined; + const scrollOffset = rowVirtualizer.scrollOffset ?? 0; // Find first row in viewport for (let i = 0; i < virtualRows.length; i++) { const row = virtualRows[i]!; - if (row.end >= rowVirtualizer.scrollOffset) { + if (row.end >= scrollOffset) { firstRowIndex = row.index; break; } @@ -95,7 +96,7 @@ export const MediaView = () => { // Find last row in viewport for (let i = virtualRows.length - 1; i >= 0; i--) { const row = virtualRows[i]!; - if (row.start <= rowVirtualizer.scrollOffset + rowVirtualizer.scrollRect.height) { + if (row.start <= scrollOffset + (rowVirtualizer.scrollRect?.height ?? 0)) { lastRowIndex = row.index; break; } @@ -163,15 +164,16 @@ export const MediaView = () => { ); } }, [ + isSortingByDate, + orderBy, + orderDirection, explorer.items, + rowVirtualizer.scrollOffset, + rowVirtualizer.scrollRect?.height, grid.columnCount, grid.options.count, - isSortingByDate, - rowVirtualizer.scrollOffset, - rowVirtualizer.scrollRect.height, - virtualRows, - orderBy, - orderDirection + dateFormat, + virtualRows ]); useKeySelection(grid); @@ -187,7 +189,15 @@ export const MediaView = () => { > {isSortingByDate && } - + {virtualRows.map((virtualRow) => ( {columnVirtualizer.getVirtualItems().map((virtualColumn) => { diff --git a/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx b/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx index b3852082e..2b53e43c8 100644 --- a/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx +++ b/interface/app/$libraryId/Explorer/View/RenamableItemText.tsx @@ -58,17 +58,17 @@ export const RenamableItemText = ({ const renameFile = useLibraryMutation(['files.renameFile'], { onError: () => reset(), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const renameEphemeralFile = useLibraryMutation(['ephemeralFiles.renameFile'], { onError: () => reset(), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const renameLocation = useLibraryMutation(['locations.update'], { onError: () => reset(), - onSuccess: () => rspc.queryClient.invalidateQueries(['search.paths']) + onSuccess: () => rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }) }); const reset = useCallback(() => { diff --git a/interface/app/$libraryId/Explorer/View/ViewItem.tsx b/interface/app/$libraryId/Explorer/View/ViewItem.tsx index 2537b854f..087c8b808 100644 --- a/interface/app/$libraryId/Explorer/View/ViewItem.tsx +++ b/interface/app/$libraryId/Explorer/View/ViewItem.tsx @@ -192,14 +192,15 @@ export const useViewItemDoubleClick = () => { } }, [ - searchParams, explorer.selectedItems, explorer.settingsStore.openOnDoubleClick, - library.uuid, - navigate, openFilePaths, - openEphemeralFiles, - updateAccessTime + updateAccessTime, + library.uuid, + t, + searchParams, + navigate, + openEphemeralFiles ] ); diff --git a/interface/app/$libraryId/Explorer/index.tsx b/interface/app/$libraryId/Explorer/index.tsx index 3dcb19261..2aa5971c0 100644 --- a/interface/app/$libraryId/Explorer/index.tsx +++ b/interface/app/$libraryId/Explorer/index.tsx @@ -61,7 +61,7 @@ export default function Explorer(props: PropsWithChildren) { // I had planned to somehow fetch the Object, but its a lot more work than its worth given // id have to fetch the file_path explicitly and patch the query // for now, it seems to work a treat just invalidating the whole query - rspc.queryClient.invalidateQueries(['search.paths']); + rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }); } } }); diff --git a/interface/app/$libraryId/Explorer/store.ts b/interface/app/$libraryId/Explorer/store.ts index 4814f972a..bb463ab11 100644 --- a/interface/app/$libraryId/Explorer/store.ts +++ b/interface/app/$libraryId/Explorer/store.ts @@ -123,8 +123,19 @@ export function flattenThumbnailKey(thumbKey: ThumbKey) { export const explorerStore = proxy({ ...state, reset: (_state?: typeof state) => resetStore(explorerStore, _state || state), - addNewThumbnail: (thumbKey: ThumbKey) => { - explorerStore.newThumbnails.add(flattenThumbnailKey(thumbKey)); + addNewThumbnail: (thumbKey: ThumbKey | string) => { + thumbKey = typeof thumbKey === 'string' ? thumbKey : flattenThumbnailKey(thumbKey); + // HACK: Ensure store propagates changes + const newThumbnails = new Set(explorerStore.newThumbnails); + newThumbnails.add(thumbKey); + explorerStore.newThumbnails = newThumbnails; + }, + removeThumbnail: (thumbKey: ThumbKey | string) => { + thumbKey = typeof thumbKey === 'string' ? thumbKey : flattenThumbnailKey(thumbKey); + // HACK: Ensure store propagates changes + const newThumbnails = new Set(explorerStore.newThumbnails); + newThumbnails.delete(thumbKey); + explorerStore.newThumbnails = newThumbnails; }, resetCache: () => { explorerStore.newThumbnails.clear(); diff --git a/interface/app/$libraryId/Explorer/useExplorer.ts b/interface/app/$libraryId/Explorer/useExplorer.ts index 97af144b8..c656362c7 100644 --- a/interface/app/$libraryId/Explorer/useExplorer.ts +++ b/interface/app/$libraryId/Explorer/useExplorer.ts @@ -105,7 +105,7 @@ export function useExplorerSettings({ >; data?: T | null; }) { - const [store] = useState(() => proxy(settings)); + const store = useMemo(() => proxy(settings), [settings]); const updateSettings = useDebouncedCallback((settings: ExplorerSettings, data: T) => { onSettingsChanged?.(settings, data); @@ -149,15 +149,8 @@ function useSelectedItems(items: ExplorerItem[] | null) { const itemHashesWeakMap = useRef(new WeakMap()); // Store hashes of items instead as objects are unique by reference but we - // still need to differentate between item variants - const [selectedItemHashes, setSelectedItemHashes] = useState(() => ({ - value: new Set() - })); - - const updateHashes = useCallback( - () => setSelectedItemHashes((h) => ({ ...h })), - [setSelectedItemHashes] - ); + // still need to differentiate between item variants + const [selectedItemHashes, setSelectedItemHashes] = useState(() => new Set()); const itemsMap = useMemo( () => @@ -172,7 +165,7 @@ function useSelectedItems(items: ExplorerItem[] | null) { const selectedItems = useMemo( () => - [...selectedItemHashes.value].reduce((items, hash) => { + [...selectedItemHashes].reduce((items, hash) => { const item = itemsMap.get(hash); if (item) items.add(item.data); return items; @@ -194,37 +187,37 @@ function useSelectedItems(items: ExplorerItem[] | null) { (item: ExplorerItem | ExplorerItem[]) => { const items = Array.isArray(item) ? item : [item]; - for (let i = 0; i < items.length; i++) { - selectedItemHashes.value.add(getItemUniqueId(items[i]!)); - } - - updateHashes(); + setSelectedItemHashes((oldHashes) => { + const newHashes = new Set(oldHashes); + for (const it of items) newHashes.add(getItemUniqueId(it)); + return newHashes; + }); }, - [getItemUniqueId, selectedItemHashes.value, updateHashes] + [getItemUniqueId] ), removeSelectedItem: useCallback( (item: ExplorerItem | ExplorerItem[]) => { const items = Array.isArray(item) ? item : [item]; - - for (let i = 0; i < items.length; i++) { - selectedItemHashes.value.delete(getItemUniqueId(items[i]!)); - } - - updateHashes(); + setSelectedItemHashes((oldHashes) => { + const newHashes = new Set(oldHashes); + for (const it of items) newHashes.delete(getItemUniqueId(it)); + return newHashes; + }); }, - [getItemUniqueId, selectedItemHashes.value, updateHashes] + [getItemUniqueId] ), resetSelectedItems: useCallback( (items?: ExplorerItem[]) => { - selectedItemHashes.value.clear(); - items?.forEach((item) => selectedItemHashes.value.add(getItemUniqueId(item))); - updateHashes(); + if (items) { + const newHashes = new Set(); + for (const it of items) newHashes.add(getItemUniqueId(it)); + setSelectedItemHashes(newHashes); + } else { + setSelectedItemHashes(new Set()); + } }, - [getItemUniqueId, selectedItemHashes.value, updateHashes] + [getItemUniqueId] ), - isItemSelected: useCallback( - (item: ExplorerItem) => selectedItems.has(item), - [selectedItems] - ) + isItemSelected: (item: ExplorerItem) => selectedItems.has(item) }; } diff --git a/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx b/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx index 39c23511f..b32105624 100644 --- a/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx +++ b/interface/app/$libraryId/Explorer/useExplorerDraggable.tsx @@ -1,5 +1,5 @@ import { useDraggable, UseDraggableArguments } from '@dnd-kit/core'; -import { CSSProperties, HTMLAttributes } from 'react'; +import { CSSProperties, HTMLAttributes, useCallback, useMemo } from 'react'; import { ExplorerItem } from '@sd/client'; import { explorerStore } from './store'; @@ -11,8 +11,26 @@ export interface UseExplorerDraggableProps extends Omit { - const disabled = props.disabled || !draggableTypes.includes(props.data.type); + const disabled = useMemo( + () => props.disabled || !draggableTypes.includes(props.data.type), + [props.disabled, props.data.type] + ); const { setNodeRef, ...draggable } = useDraggable({ ...props, @@ -20,30 +38,30 @@ export const useExplorerDraggable = (props: UseExplorerDraggableProps) => { disabled: disabled }); - const onMouseDown = () => { + const onMouseDown = useCallback(() => { if (!disabled) explorerStore.drag = { type: 'touched' }; - }; + }, [disabled]); - const onMouseLeave = () => { + const onMouseLeave = useCallback(() => { if (explorerStore.drag?.type !== 'dragging') explorerStore.drag = null; - }; + }, []); - const onMouseUp = () => (explorerStore.drag = null); - - const style = { - cursor: 'default', - outline: 'none' - } satisfies CSSProperties; + const onMouseUp = useCallback(() => { + explorerStore.drag = null; + }, []); return { ...draggable, setDraggableRef: setNodeRef, - listeners: { - ...draggable.listeners, - onMouseDown, - onMouseLeave, - onMouseUp - } satisfies HTMLAttributes, - style + listeners: useMemo( + () => ({ + ...draggable.listeners, + onMouseDown, + onMouseLeave, + onMouseUp + }), + [draggable.listeners, onMouseDown, onMouseLeave, onMouseUp] + ) satisfies HTMLAttributes, + style: DRAGGABLE_STYLE }; }; diff --git a/interface/app/$libraryId/Explorer/useExplorerItemData.tsx b/interface/app/$libraryId/Explorer/useExplorerItemData.tsx index 06d2f2af9..3aff87488 100644 --- a/interface/app/$libraryId/Explorer/useExplorerItemData.tsx +++ b/interface/app/$libraryId/Explorer/useExplorerItemData.tsx @@ -1,34 +1,91 @@ -import { useQueryClient } from '@tanstack/react-query'; -import { useEffect, useMemo } from 'react'; -import { getExplorerItemData, useSelector, type ExplorerItem } from '@sd/client'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { subscribe } from 'valtio'; +import { + compareHumanizedSizes, + getExplorerItemData, + humanizeSize, + ThumbKey, + type ExplorerItem +} from '@sd/client'; +import { usePlatform } from '~/util/Platform'; import { explorerStore, flattenThumbnailKey } from './store'; -// This is where we intercept the state of the explorer item to determine if we should rerender -// This hook is used inside every thumbnail in the explorer +/** + * This is where we intercept the state of the explorer item to determine if we should rerender + * + * .. WARNING:: + * This hook is used inside every thumbnail in the explorer. + * Be careful with the performance of the code, make sure to always memoize any objects or functions to avoid unnecessary re-renders. + * + * @param explorerItem - The explorer item to get data from + * @returns The extracted data from the explorer item + */ export function useExplorerItemData(explorerItem: ExplorerItem) { - const newThumbnail = useSelector(explorerStore, (s) => { - const thumbnailKey = - explorerItem.type === 'Label' - ? // labels have .thumbnails, plural - explorerItem.thumbnails?.[0] - : // all other explorer items have .thumbnail singular - 'thumbnail' in explorerItem && explorerItem.thumbnail; + const platform = usePlatform(); + const cachedSize = useRef | null>(null); + const [newThumbnails, setNewThumbnails] = useState>(new Map()); - return !!(thumbnailKey && s.newThumbnails.has(flattenThumbnailKey(thumbnailKey))); - }); + let thumbnails: ThumbKey | ThumbKey[] | null = null; + switch (explorerItem.type) { + case 'Label': + thumbnails = explorerItem.thumbnails; + break; + case 'Path': + case 'Object': + case 'NonIndexedPath': + thumbnails = explorerItem.thumbnail; + break; + } + + useEffect(() => { + const thumbnailKeys = thumbnails + ? Array.isArray(thumbnails) + ? thumbnails + : [thumbnails] + : []; + + const updateThumbnails = () => + setNewThumbnails((oldThumbs) => { + const thumbs = thumbnailKeys.reduce>((acc, thumbKey) => { + const url = platform.getThumbnailUrlByThumbKey(thumbKey); + const thumbId = flattenThumbnailKey(thumbKey); + acc.set(url, explorerStore.newThumbnails.has(thumbId) ? thumbId : null); + return acc; + }, new Map()); + + // Avoid unnecessary re-renders + return oldThumbs.size !== thumbs.size || + Array.from(oldThumbs.keys()).some( + (key) => oldThumbs.get(key) !== thumbs.get(key) + ) + ? thumbs + : oldThumbs; + }); + + updateThumbnails(); + + return subscribe(explorerStore, updateThumbnails); + }, [thumbnails, platform]); return useMemo(() => { - const itemData = getExplorerItemData(explorerItem); + const explorerItemData = getExplorerItemData(explorerItem); - if (!itemData.hasLocalThumbnail) { - itemData.hasLocalThumbnail = newThumbnail; + // Avoid unecessary re-renders + if ( + cachedSize.current == null || + !compareHumanizedSizes(cachedSize.current, explorerItemData.size) + ) { + cachedSize.current = explorerItemData.size; } - return itemData; - // whatever goes here, is what can cause an atomic re-render of an explorer item - // this is used for when new thumbnails are generated, and files identified - }, [explorerItem, newThumbnail]); + return { + ...explorerItemData, + size: cachedSize.current, + thumbnails: newThumbnails, + hasLocalThumbnail: explorerItemData.hasLocalThumbnail || newThumbnails.size > 0 + }; + }, [explorerItem, newThumbnails]); } export type ExplorerItemData = ReturnType; diff --git a/interface/app/$libraryId/Explorer/useExplorerPreferences.ts b/interface/app/$libraryId/Explorer/useExplorerPreferences.ts index 6af94ec7b..60d4bc3d5 100644 --- a/interface/app/$libraryId/Explorer/useExplorerPreferences.ts +++ b/interface/app/$libraryId/Explorer/useExplorerPreferences.ts @@ -53,7 +53,7 @@ export function useExplorerPreferences({ try { await updatePreferences.mutateAsync(writeSettings(settings)); - rspc.queryClient.invalidateQueries(['preferences.get']); + rspc.queryClient.invalidateQueries({ queryKey: ['preferences.get'] }); } catch (e) { alert('An error has occurred while updating your preferences.'); } diff --git a/interface/app/$libraryId/Layout/CMDK/index.tsx b/interface/app/$libraryId/Layout/CMDK/index.tsx index 1055e3708..7cd205911 100644 --- a/interface/app/$libraryId/Layout/CMDK/index.tsx +++ b/interface/app/$libraryId/Layout/CMDK/index.tsx @@ -1,6 +1,7 @@ import './CMDK.css'; import './CMDK.scss'; +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import { useEffect, useState } from 'react'; import CommandPalette, { filterItems, getItemIndex } from 'react-cmdk'; @@ -50,7 +51,9 @@ const CMDK = () => { const [page, setPage] = useState<'root' | 'locations' | 'tags'>('root'); const [search, setSearch] = useState(''); - const locationsQuery = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const locationsQuery = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = locationsQuery.data; const onlineLocations = useOnlineLocations(); diff --git a/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx b/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx index f878dea0a..b0696c50b 100644 --- a/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx +++ b/interface/app/$libraryId/Layout/CMDK/pages/CMDKLocations.tsx @@ -1,3 +1,4 @@ +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import CommandPalette from 'react-cmdk'; import { useNavigate } from 'react-router'; @@ -5,7 +6,9 @@ import { arraysEqual, useLibraryQuery, useOnlineLocations } from '@sd/client'; import { Icon } from '~/components'; export default function CMDKLocations() { - const locationsQuery = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const locationsQuery = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = locationsQuery.data; const onlineLocations = useOnlineLocations(); diff --git a/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx b/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx index a3cc2c836..7ca898691 100644 --- a/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx +++ b/interface/app/$libraryId/Layout/CMDK/pages/CMDKTags.tsx @@ -1,9 +1,10 @@ +import { keepPreviousData } from '@tanstack/react-query'; import CommandPalette from 'react-cmdk'; import { useNavigate } from 'react-router'; import { useLibraryQuery, type Tag } from '@sd/client'; export default function CMDKTags() { - const result = useLibraryQuery(['tags.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['tags.list'], { placeholderData: keepPreviousData }); const tags = result.data || []; const navigate = useNavigate(); diff --git a/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx b/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx index 5bc76063c..5fd40631f 100644 --- a/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/DebugPopover.tsx @@ -154,15 +154,12 @@ export default () => { title="React Query Devtools" description="Configure the React Query devtools." > - + + (debugState.reactQueryDevtools = !debugState.reactQueryDevtools) + } + /> { - queryClient.invalidateQueries(['jobs.reports']); + queryClient.invalidateQueries({ queryKey: ['jobs.reports'] }); }) ); diff --git a/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx b/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx index 84f3deb39..383e0273f 100644 --- a/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/JobManager/index.tsx @@ -97,7 +97,7 @@ export function JobManager() { title: t('success'), body: t('all_jobs_have_been_cleared') }); - queryClient.invalidateQueries(['jobs.reports']); + queryClient.invalidateQueries({ queryKey: ['jobs.reports'] }); } catch (error) { toast.error({ title: t('error'), diff --git a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx index ba5f967d3..6876bee9d 100644 --- a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/Footer.tsx @@ -1,5 +1,5 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; import { Gear } from '@phosphor-icons/react'; +import { inferSubscriptionResult } from '@spacedrive/rspc-client'; import { useState } from 'react'; import { useNavigate } from 'react-router'; import { diff --git a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx index 0511cf33b..44515a41c 100644 --- a/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/SidebarLayout/LibrariesDropdown.tsx @@ -1,9 +1,7 @@ -import { CloudArrowDown, Gear, Lock, Plus } from '@phosphor-icons/react'; +import { Gear, Plus } from '@phosphor-icons/react'; import clsx from 'clsx'; import { useClientContext } from '@sd/client'; import { dialogManager, Dropdown, DropdownMenu } from '@sd/ui'; -import JoinDialog from '~/app/$libraryId/settings/node/libraries/JoinDialog'; -import RequestAddDialog from '~/components/RequestAddDialog'; import { useLocale } from '~/hooks'; import CreateDialog from '../../../settings/node/libraries/CreateDialog'; @@ -64,17 +62,6 @@ export default () => { onClick={() => dialogManager.create((dp) => )} className="font-medium" /> - - dialogManager.create((dp) => ( - - )) - } - className="font-medium" - /> { to="settings/library/general" className="font-medium" /> - {/* alert('TODO: Not implemented yet!')} - className="font-medium" - /> */} ); }; diff --git a/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx b/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx index ef80dc378..703eddae0 100644 --- a/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/sections/Locations/index.tsx @@ -1,3 +1,4 @@ +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import { Link, useMatch } from 'react-router-dom'; import { @@ -18,7 +19,9 @@ import { SeeMore } from '../../SidebarLayout/SeeMore'; import { ContextMenu } from './ContextMenu'; export default function Locations() { - const locationsQuery = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const locationsQuery = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = locationsQuery.data; const onlineLocations = useOnlineLocations(); diff --git a/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx b/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx index f251fc042..4131bbfe6 100644 --- a/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx +++ b/interface/app/$libraryId/Layout/Sidebar/sections/Tags/index.tsx @@ -1,3 +1,4 @@ +import { keepPreviousData } from '@tanstack/react-query'; import clsx from 'clsx'; import { NavLink, useMatch } from 'react-router-dom'; import { useLibraryQuery, type Tag } from '@sd/client'; @@ -11,7 +12,7 @@ import { SeeMore } from '../../SidebarLayout/SeeMore'; import { ContextMenu } from './ContextMenu'; export default function TagsSection() { - const result = useLibraryQuery(['tags.list'], { keepPreviousData: true }); + const result = useLibraryQuery(['tags.list'], { placeholderData: keepPreviousData }); const tags = result.data; const { t } = useLocale(); diff --git a/interface/app/$libraryId/Spacedrop/index.tsx b/interface/app/$libraryId/Spacedrop/index.tsx index 4c9a81342..6803a5283 100644 --- a/interface/app/$libraryId/Spacedrop/index.tsx +++ b/interface/app/$libraryId/Spacedrop/index.tsx @@ -93,7 +93,7 @@ export function Spacedrop({ triggerClose }: { triggerClose: () => void }) { }); const onDropped = (id: string, files: string[]) => { - if (doSpacedrop.isLoading) { + if (doSpacedrop.isPending) { toast.warning(t('spacedrop_already_progress')); return; } diff --git a/interface/app/$libraryId/debug/actors.tsx b/interface/app/$libraryId/debug/actors.tsx deleted file mode 100644 index a744583a6..000000000 --- a/interface/app/$libraryId/debug/actors.tsx +++ /dev/null @@ -1,69 +0,0 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; -import { useMemo, useState } from 'react'; -import { Procedures, useLibraryMutation, useLibrarySubscription } from '@sd/client'; -import { Button } from '@sd/ui'; -import { useRouteTitle } from '~/hooks/useRouteTitle'; - -// @million-ignore -export const Component = () => { - useRouteTitle('Actors'); - - const [data, setData] = useState>({}); - - useLibrarySubscription(['library.actors'], { onData: setData }); - - const sortedData = useMemo(() => { - const sorted = Object.entries(data).sort(([a], [b]) => a.localeCompare(b)); - return sorted; - }, [data]); - - return ( -
- - - - - - {sortedData.map(([name, running]) => ( - - - - - - ))} -
NameRunning
{name} - {running ? 'Running' : 'Not Running'} - - {running ? : } -
-
- ); -}; - -function StartButton({ name }: { name: string }) { - const startActor = useLibraryMutation(['library.startActor']); - - return ( - - ); -} - -function StopButton({ name }: { name: string }) { - const stopActor = useLibraryMutation(['library.stopActor']); - - return ( - - ); -} diff --git a/interface/app/$libraryId/debug/cloud.tsx b/interface/app/$libraryId/debug/cloud.tsx deleted file mode 100644 index dfc60cbea..000000000 --- a/interface/app/$libraryId/debug/cloud.tsx +++ /dev/null @@ -1,262 +0,0 @@ -import { CheckCircle, XCircle } from '@phosphor-icons/react'; -import { Suspense, useMemo } from 'react'; -import { - auth, - HardwareModel, - useBridgeQuery, - useLibraryContext, - useLibraryMutation, - useLibraryQuery -} from '@sd/client'; -import { Button, Card, Loader, tw } from '@sd/ui'; -import { Icon } from '~/components'; -import { AuthRequiredOverlay } from '~/components/AuthRequiredOverlay'; -import { LoginButton } from '~/components/LoginButton'; -import { useLocale, useRouteTitle } from '~/hooks'; -import { hardwareModelToIcon } from '~/util/hardware'; - -const DataBox = tw.div`max-w-[300px] rounded-md border border-app-line/50 bg-app-lightBox/20 p-2`; -const Count = tw.div`min-w-[20px] flex h-[20px] px-1 items-center justify-center rounded-full border border-app-button/40 text-[9px]`; - -export const Component = () => { - useRouteTitle('Cloud'); - - // const authState = auth.useStateSnapshot(); - - // const authSensitiveChild = () => { - // if (authState.status === 'loggedIn') return ; - // if (authState.status === 'notLoggedIn' || authState.status === 'loggingIn') - // return ( - //
- // - //
- // - //

- // To access cloud related features, please login - //

- //
- // - // - //
- // ); - - // return null; - // }; - - // return
{Authenticated()}
; - return
; -}; - -// million-ignore -// function Authenticated() { -// const { library } = useLibraryContext(); -// const cloudLibrary: any = useLibraryQuery(['cloud.library.get'], { -// suspense: true, -// retry: false -// }); -// const getCloudDevice = useBridgeQuery(['cloud.devices.get'], { -// suspense: true, -// retry: false -// }); -// const cloudDevicesList = useBridgeQuery(['cloud.devices.list'], { -// suspense: true, -// retry: false -// }); -// console.log('[DEBUG] fetch cloud device:', getCloudDevice.data); -// console.log('[DEBUG] cloudDevicesList', cloudDevicesList.data); -// const createLibrary = useLibraryMutation(['cloud.library.create']); -// const { t } = useLocale(); - -// const thisInstance = useMemo(() => { -// if (!cloudLibrary.data) return undefined; -// return cloudLibrary.data.instances.find( -// (instance: any) => instance.uuid === library.instance_id -// ); -// }, [cloudLibrary.data, library.instance_id]); - -// return ( -// -// -// -// } -// > -// {cloudLibrary.data ? ( -//
-// -// {thisInstance && } -// -//
-// ) : ( -//
-// -// -//
-// -//

-// {t('cloud_connect_description')} -//

-//
-// -// -//
-// )} -// -// ); -// } - -// // million-ignore -// const Instances = ({ instances }: { instances: any[] }) => { -// const { library } = useLibraryContext(); -// const filteredInstances = instances.filter((instance) => instance.uuid !== library.instance_id); -// return ( -//
-//
-//

Instances

-// {filteredInstances.length} -//
-//
-// {filteredInstances.map((instance) => ( -// -//
-// -//

-// {instance.metadata.name} -//

-//
-//
-// -//

-// Id:{' '} -// {instance.id} -//

-// -// -//

-// UUID:{' '} -// -// {instance.uuid} -// -//

-// -// -//

-// Public Key:{' '} -// -// {instance.identity} -// -//

-// -//
-// -// ))} -//
-//
-// ); -// }; - -// interface LibraryProps { -// cloudLibrary: any; -// thisInstance: any | undefined; -// } - -// // million-ignore -// const Library = ({ thisInstance, cloudLibrary }: LibraryProps) => { -// const syncLibrary = useLibraryMutation(['cloud.library.sync']); -// return ( -//
-//

Library

-// -//

-// Name: {cloudLibrary.name} -//

-// -// -//
-// ); -// }; - -// interface ThisInstanceProps { -// instance: any; -// } - -// // million-ignore -// const ThisInstance = ({ instance }: ThisInstanceProps) => { -// return ( -//
-//

This Instance

-// -//
-// -//

-// {instance.metadata.name} -//

-//
-//
-// -//

-// Id: {instance.id} -//

-// -// -//

-// UUID: {instance.uuid} -//

-// -// -//

-// Public Key:{' '} -// {instance.identity} -//

-// -//
-// -//
-// ); -// }; diff --git a/interface/app/$libraryId/debug/index.ts b/interface/app/$libraryId/debug/index.ts deleted file mode 100644 index 4cf60b56c..000000000 --- a/interface/app/$libraryId/debug/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { RouteObject } from 'react-router'; - -export const debugRoutes = [ - { path: 'cloud', lazy: () => import('./cloud') }, - { path: 'actors', lazy: () => import('./actors') } -] satisfies RouteObject[]; diff --git a/interface/app/$libraryId/ephemeral.tsx b/interface/app/$libraryId/ephemeral.tsx index 4aacac2b1..a96f5cbb1 100644 --- a/interface/app/$libraryId/ephemeral.tsx +++ b/interface/app/$libraryId/ephemeral.tsx @@ -1,15 +1,12 @@ -import { type AlphaClient } from '@oscartbeaumont-sd/rspc-client/src/v2'; import { ArrowLeft, ArrowRight, Info } from '@phosphor-icons/react'; import * as Dialog from '@radix-ui/react-dialog'; import { iconNames } from '@sd/assets/util'; import clsx from 'clsx'; -import { memo, Suspense, useDeferredValue, useMemo } from 'react'; +import { memo, Suspense, useCallback, useDeferredValue, useEffect, useMemo } from 'react'; import { ExplorerItem, getExplorerItemData, - ItemData, nonIndexedPathOrderingSchema, - SortOrder, useLibraryContext, useUnsafeStreamedQuery, type EphemeralPathOrder @@ -58,15 +55,13 @@ const NOTICE_ITEMS: { icon: keyof typeof iconNames; name: string }[] = [ } ]; -const EphemeralNotice = ({ path }: { path: string }) => { +const EphemeralNotice = memo(({ path }: { path: string }) => { const { t } = useLocale(); - const isDark = useIsDark(); const { ephemeral: dismissed } = useDismissibleNoticeStore(); - const topbar = useTopBarContext(); - const dismiss = () => (getDismissibleNoticeStore().ephemeral = true); + const dismiss = useCallback(() => (getDismissibleNoticeStore().ephemeral = true), []); return ( @@ -155,11 +150,9 @@ const EphemeralNotice = ({ path }: { path: string }) => { ); -}; - -const EphemeralExplorer = memo((props: { args: PathParams }) => { - const { path } = props.args; +}); +const EphemeralExplorer = memo(({ args: path }: { args: PathParams['path'] }) => { const os = useOperatingSystem(); const explorerSettings = useExplorerSettings({ @@ -193,12 +186,12 @@ const EphemeralExplorer = memo((props: { args: PathParams }) => { ], { enabled: path != null, - suspense: true, - onSuccess: () => explorerStore.resetCache(), - onBatch: (item) => {} + onBatch: () => {} } ); + useEffect(() => explorerStore.resetCache(), [query]); + const entries = useMemo(() => { return ( query.data?.flatMap((item) => item.entries) || @@ -258,15 +251,15 @@ const EphemeralExplorer = memo((props: { args: PathParams }) => { }); export const Component = () => { - const [pathParams] = useZodSearchParams(PathParamsSchema); + let [{ path }] = useZodSearchParams(PathParamsSchema); - const path = useDeferredValue(pathParams); + path = useDeferredValue(path); - useRouteTitle(path.path ?? ''); + useRouteTitle(path ?? ''); return ( - + ); diff --git a/interface/app/$libraryId/index.tsx b/interface/app/$libraryId/index.tsx index 6be3b0dd3..6ff9f4b3e 100644 --- a/interface/app/$libraryId/index.tsx +++ b/interface/app/$libraryId/index.tsx @@ -3,16 +3,12 @@ import { type RouteObject } from 'react-router-dom'; import { guessOperatingSystem } from '~/hooks'; import { Platform } from '~/util/Platform'; -import { debugRoutes } from './debug'; import settingsRoutes from './settings'; // Routes that should be contained within the standard Page layout const pageRoutes: RouteObject = { lazy: () => import('./PageLayout'), - children: [ - { path: 'overview', lazy: () => import('./overview') }, - { path: 'debug', children: debugRoutes } - ] + children: [{ path: 'overview', lazy: () => import('./overview') }] }; // Routes that render the explorer and don't need padding and stuff @@ -37,8 +33,7 @@ function loadTopBarRoutes() { return [ ...explorerRoutes, pageRoutes, - { path: 'settings', lazy: () => import('./settings/Layout'), children: settingsRoutes }, - { path: 'debug', children: debugRoutes } + { path: 'settings', lazy: () => import('./settings/Layout'), children: settingsRoutes } ]; } else return [...explorerRoutes, pageRoutes]; } diff --git a/interface/app/$libraryId/location/$id.tsx b/interface/app/$libraryId/location/$id.tsx index cb2ab1304..57f8191d2 100644 --- a/interface/app/$libraryId/location/$id.tsx +++ b/interface/app/$libraryId/location/$id.tsx @@ -1,4 +1,5 @@ import { ArrowClockwise, Info } from '@phosphor-icons/react'; +import { keepPreviousData } from '@tanstack/react-query'; import { useCallback, useEffect, useMemo } from 'react'; import { stringify } from 'uuid'; import { @@ -42,7 +43,7 @@ export const Component = () => { const { id: locationId } = useZodRouteParams(LocationIdParamsSchema); const [{ path }] = useExplorerSearchParams(); const result = useLibraryQuery(['locations.get', locationId], { - keepPreviousData: true, + placeholderData: keepPreviousData, suspense: true }); const location = result.data; diff --git a/interface/app/$libraryId/overview/index.tsx b/interface/app/$libraryId/overview/index.tsx index a017ea6bb..2f4c6143d 100644 --- a/interface/app/$libraryId/overview/index.tsx +++ b/interface/app/$libraryId/overview/index.tsx @@ -1,3 +1,4 @@ +import { keepPreviousData } from '@tanstack/react-query'; import { Key, useEffect } from 'react'; import { Link } from 'react-router-dom'; import { HardwareModel, useBridgeQuery, useLibraryQuery } from '@sd/client'; @@ -30,7 +31,9 @@ export const Component = () => { const { t } = useLocale(); const accessToken = useAccessToken(); - const locationsQuery = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const locationsQuery = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = locationsQuery.data ?? []; // not sure if we'll need the node state in the future, as it should be returned with the cloud.devices.list query diff --git a/interface/app/$libraryId/search/Filters.tsx b/interface/app/$libraryId/search/Filters.tsx index f9e12a55b..f71f28877 100644 --- a/interface/app/$libraryId/search/Filters.tsx +++ b/interface/app/$libraryId/search/Filters.tsx @@ -5,9 +5,9 @@ import { Heart, Icon, SelectionSlash, - Tag, Textbox } from '@phosphor-icons/react'; +import { keepPreviousData } from '@tanstack/react-query'; import { useState } from 'react'; import { InOrNotIn, ObjectKind, SearchFilterArgs, TextMatch, useLibraryQuery } from '@sd/client'; import { Button, Input } from '@sd/ui'; @@ -436,7 +436,9 @@ export const filterRegistry = [ .filter(Boolean) as any; }, useOptions: () => { - const query = useLibraryQuery(['locations.list'], { keepPreviousData: true }); + const query = useLibraryQuery(['locations.list'], { + placeholderData: keepPreviousData + }); const locations = query.data; return (locations ?? []).map((location) => ({ diff --git a/interface/app/$libraryId/settings/client/backups.tsx b/interface/app/$libraryId/settings/client/backups.tsx index 69689b16f..0ebfe10e7 100644 --- a/interface/app/$libraryId/settings/client/backups.tsx +++ b/interface/app/$libraryId/settings/client/backups.tsx @@ -27,7 +27,7 @@ export const Component = () => { rightArea={
diff --git a/interface/app/$libraryId/settings/library/general.tsx b/interface/app/$libraryId/settings/library/general.tsx index d79b29363..a8b850923 100644 --- a/interface/app/$libraryId/settings/library/general.tsx +++ b/interface/app/$libraryId/settings/library/general.tsx @@ -109,7 +109,7 @@ export const Component = () => {
-// } -// {...form.register('masterPassword', { required: true })} -// /> - -// setShow((old) => ({ ...old, secretKey: !old.secretKey }))} -// size="icon" -// > -// -// -// } -// {...form.register('secretKey')} -// /> - -//
-// -//
-// -// ); -// }; diff --git a/interface/app/$libraryId/settings/library/keys/KeyViewerDialog.tsx b/interface/app/$libraryId/settings/library/keys/KeyViewerDialog.tsx deleted file mode 100644 index 9fe44cbac..000000000 --- a/interface/app/$libraryId/settings/library/keys/KeyViewerDialog.tsx +++ /dev/null @@ -1,160 +0,0 @@ -// import { Buffer } from 'buffer'; -// import { Clipboard } from '@phosphor-icons/react'; -// import { useState } from 'react'; -// import { slugFromHashingAlgo, useLibraryQuery } from '@sd/client'; -// import { Button, Dialog, Input, Select, SelectOption, UseDialogProps, useDialog } from '@sd/ui'; -// import { useZodForm } from '@sd/ui/src/forms'; -// import { KeyListSelectOptions } from '~/app/$libraryId/KeyManager/List'; - -// export const KeyUpdater = (props: { -// uuid: string; -// setKey: (value: string) => void; -// setEncryptionAlgo: (value: string) => void; -// setHashingAlgo: (value: string) => void; -// setContentSalt: (value: string) => void; -// }) => { -// useLibraryQuery(['keys.getKey', props.uuid], { -// onSuccess: (data) => { -// props.setKey(data); -// } -// }); - -// const keys = useLibraryQuery(['keys.list']); - -// const key = keys.data?.find((key) => key.uuid == props.uuid); - -// if (key) { -// props.setEncryptionAlgo(key?.algorithm); -// props.setHashingAlgo(slugFromHashingAlgo(key?.hashing_algorithm)); -// props.setContentSalt(Buffer.from(key.content_salt).toString('hex')); -// } - -// return <>; -// }; - -// export default (props: UseDialogProps) => { -// const keys = useLibraryQuery(['keys.list'], { -// onSuccess: (data) => { -// if (key === '' && data.length !== 0) { -// setKey(data[0]?.uuid ?? ''); -// } -// } -// }); - -// const [key, setKey] = useState(''); -// const [keyValue, setKeyValue] = useState(''); -// const [contentSalt, setContentSalt] = useState(''); -// const [encryptionAlgo, setEncryptionAlgo] = useState(''); -// const [hashingAlgo, setHashingAlgo] = useState(''); - -// return ( -// -// - -//
-//
-// Key -// -//
-//
-//
-//
-// Encryption -// -//
-//
-// Hashing -// -//
-//
-//
-//
-// Content Salt (hex) -// { -// navigator.clipboard.writeText(contentSalt); -// }} -// size="icon" -// > -// -// -// } -// /> -//
-//
-//
-//
-// Key Value -// { -// navigator.clipboard.writeText(keyValue); -// }} -// size="icon" -// > -// -// -// } -// /> -//
-//
-// -// ); -// }; diff --git a/interface/app/$libraryId/settings/library/keys/MasterPasswordDialog.tsx b/interface/app/$libraryId/settings/library/keys/MasterPasswordDialog.tsx deleted file mode 100644 index 7b07a3135..000000000 --- a/interface/app/$libraryId/settings/library/keys/MasterPasswordDialog.tsx +++ /dev/null @@ -1,187 +0,0 @@ -// import { ArrowsClockwise, Clipboard, Eye, EyeSlash } from '@phosphor-icons/react'; -// import { useState } from 'react'; -// import { -// Algorithm, -// HASHING_ALGOS, -// HashingAlgoSlug, -// hashingAlgoSlugSchema, -// useLibraryMutation -// } from '@sd/client'; -// import { Button, Dialog, Input, Select, SelectOption, UseDialogProps, useDialog } from '@sd/ui'; -// import { useZodForm, z } from '@sd/ui/src/forms'; -// import { PasswordMeter, showAlertDialog } from '~/components'; -// import { generatePassword } from '~/util'; - -// const schema = z.object({ -// masterPassword: z.string(), -// masterPassword2: z.string(), -// encryptionAlgo: z.string(), -// hashingAlgo: hashingAlgoSlugSchema -// }); - -// export default (props: UseDialogProps) => { -// const changeMasterPassword = useLibraryMutation('keys.changeMasterPassword', { -// onSuccess: () => { -// showAlertDialog({ -// title: 'Success', -// value: 'Your master password was changed successfully' -// }); -// }, -// onError: () => { -// // this should never really happen -// showAlertDialog({ -// title: 'Master Password Change Error', -// value: 'There was an error while changing your master password.' -// }); -// } -// }); - -// const [show, setShow] = useState({ -// masterPassword: false, -// masterPassword2: false -// }); - -// const MP1CurrentEyeIcon = show.masterPassword ? EyeSlash : Eye; -// const MP2CurrentEyeIcon = show.masterPassword2 ? EyeSlash : Eye; - -// const form = useZodForm({ -// schema, -// defaultValues: { -// encryptionAlgo: 'XChaCha20Poly1305', -// hashingAlgo: 'Argon2id-s', -// masterPassword: '', -// masterPassword2: '' -// } -// }); - -// const onSubmit = form.handleSubmit((data) => { -// if (data.masterPassword !== data.masterPassword2) { -// showAlertDialog({ -// title: 'Error', -// value: 'Passwords are not the same, please try again.' -// }); -// } else { -// const hashing_algorithm = HASHING_ALGOS[data.hashingAlgo]; - -// return changeMasterPassword.mutateAsync({ -// algorithm: data.encryptionAlgo as Algorithm, -// hashing_algorithm, -// password: data.masterPassword -// }); -// } -// }); - -// return ( -// -// -// -// -// -//
-// } -// /> - -// -// setShow((old) => ({ ...old, masterPassword2: !old.masterPassword2 })) -// } -// size="icon" -// type="button" -// > -// -// -// } -// /> - -// - -//
-//
-// Encryption -// -//
-//
-// Hashing -// -//
-//
-// -// ); -// }; diff --git a/interface/app/$libraryId/settings/library/keys/index.tsx b/interface/app/$libraryId/settings/library/keys/index.tsx deleted file mode 100644 index e0c3bc314..000000000 --- a/interface/app/$libraryId/settings/library/keys/index.tsx +++ /dev/null @@ -1,293 +0,0 @@ -// import * as DropdownMenu from '@radix-ui/react-dropdown-menu'; -// import { animated, useTransition } from '@react-spring/web'; -// import clsx from 'clsx'; -// import { Lock, Plus } from '@phosphor-icons/react'; -// import { PropsWithChildren, ReactNode, useState } from 'react'; -// import QRCode from 'react-qr-code'; -// import { useLibraryMutation, useLibraryQuery } from '@sd/client'; -// import { Button, PasswordInput, dialogManager } from '@sd/ui'; -// import { showAlertDialog } from '~/components/AlertDialog'; -// import { usePlatform } from '~/util/Platform'; -// import KeyList from '../../../KeyManager/List'; -// import KeyMounter from '../../../KeyManager/Mounter'; -// import { Heading } from '../../Layout'; -// import BackupRestoreDialog from './BackupRestoreDialog'; -// import KeyViewerDialog from './KeyViewerDialog'; -// import MasterPasswordDialog from './MasterPasswordDialog'; - -// interface Props extends DropdownMenu.MenuContentProps { -// trigger: React.ReactNode; -// transformOrigin?: string; -// disabled?: boolean; -// } - -// export const KeyMounterDropdown = ({ -// trigger, -// children, -// transformOrigin, -// className -// }: PropsWithChildren) => { -// const [open, setOpen] = useState(false); - -// const transitions = useTransition(open, { -// from: { -// opacity: 0, -// transform: `scale(0.9)`, -// transformOrigin: transformOrigin || 'top' -// }, -// enter: { opacity: 1, transform: 'scale(1)' }, -// leave: { opacity: -0.5, transform: 'scale(0.95)' }, -// config: { mass: 0.4, tension: 200, friction: 10 } -// }); - -// return ( -// -// {trigger} -// {transitions( -// (styles, show) => -// show && ( -// -// -// -// {children} -// -// -// -// ) -// )} -// -// ); -// }; - -// export const Component = () => { -// const platform = usePlatform(); -// const isUnlocked = useLibraryQuery(['keys.isUnlocked']); -// const keyringSk = useLibraryQuery(['keys.getSecretKey'], { initialData: '' }); // assume true by default, as it will often be the case. need to fix this with an rspc subscription+such -// const unlockKeyManager = useLibraryMutation('keys.unlockKeyManager', { -// onError: () => { -// showAlertDialog({ -// title: 'Unlock Error', -// value: 'The information provided to the key manager was incorrect' -// }); -// } -// }); - -// const unmountAll = useLibraryMutation('keys.unmountAll'); -// const clearMasterPassword = useLibraryMutation('keys.clearMasterPassword'); -// const backupKeystore = useLibraryMutation('keys.backupKeystore'); -// const isKeyManagerUnlocking = useLibraryQuery(['keys.isKeyManagerUnlocking']); - -// const [masterPassword, setMasterPassword] = useState(''); -// const [secretKey, setSecretKey] = useState(''); // for the unlock form -// const [viewSecretKey, setViewSecretKey] = useState(false); // for the settings page - -// const keys = useLibraryQuery(['keys.list']); - -// const [enterSkManually, setEnterSkManually] = useState(keyringSk?.data === null); - -// if (!isUnlocked?.data) { -// return ( -//
-// setMasterPassword(e.target.value)} -// autoFocus -// placeholder="Master Password" -// className="mb-2" -// /> - -// {enterSkManually && ( -// setSecretKey(e.target.value)} -// placeholder="Secret Key" -// className="mb-2" -// /> -// )} - -// -// {!enterSkManually && ( -//
-//

setEnterSkManually(true)}> -// or enter secret key manually -//

-//
-// )} -//
-// ); -// } else { -// return ( -// <> -// -// -// -// -// -// } -// > -// -// -//
-// } -// /> - -// {isUnlocked && ( -//
-// -//
-// )} - -// {keyringSk?.data && ( -// <> -// -// {!viewSecretKey && ( -//
-// -//
-// )} -// {viewSecretKey && ( -//
{ -// keyringSk.data && navigator.clipboard.writeText(keyringSk.data); -// }} -// > -// <> -// -//

{keyringSk.data}

-// -//
-// )} -// -// )} - -// -//
-// -// -//
- -// -//
-// -// -//
-// -// ); -// } -// }; - -// interface SubheadingProps { -// title: string; -// rightArea?: ReactNode; -// } - -// const Subheading = (props: SubheadingProps) => ( -//
-//
-//

{props.title}

-//
-// {props.rightArea} -//
-// ); diff --git a/interface/app/$libraryId/settings/library/locations/$id.tsx b/interface/app/$libraryId/settings/library/locations/$id.tsx index 41fd59571..3bcec7b99 100644 --- a/interface/app/$libraryId/settings/library/locations/$id.tsx +++ b/interface/app/$libraryId/settings/library/locations/$id.tsx @@ -78,7 +78,7 @@ const EditLocationForm = () => { }, onSuccess: () => { form.reset(form.getValues()); - queryClient.invalidateQueries(['locations.list']); + queryClient.invalidateQueries({ queryKey: ['locations.list'] }); } }); diff --git a/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx b/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx index 23b70e901..e9cc0fd75 100644 --- a/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx +++ b/interface/app/$libraryId/settings/library/locations/AddLocationButton.tsx @@ -71,7 +71,7 @@ export const AddLocationButton = ({ {...props} > {path ? ( -
+
{ await deleteSavedSearch.mutateAsync(savedSearch.id); onDelete(); diff --git a/interface/app/$libraryId/settings/library/sync.tsx b/interface/app/$libraryId/settings/library/sync.tsx deleted file mode 100644 index fe6124525..000000000 --- a/interface/app/$libraryId/settings/library/sync.tsx +++ /dev/null @@ -1,228 +0,0 @@ -import { inferSubscriptionResult } from '@oscartbeaumont-sd/rspc-client'; -import clsx from 'clsx'; -import { useEffect, useState } from 'react'; -import { - Procedures, - useFeatureFlag, - useLibraryMutation, - useLibraryQuery, - useLibrarySubscription, - useZodForm -} from '@sd/client'; -import { Button, Dialog, dialogManager, useDialog, UseDialogProps, z } from '@sd/ui'; -import { useLocale } from '~/hooks'; - -import { Heading } from '../Layout'; -import Setting from '../Setting'; - -const ACTORS = { - Ingest: 'Sync Ingest', - CloudSend: 'Cloud Sync Sender', - CloudReceive: 'Cloud Sync Receiver', - CloudIngest: 'Cloud Sync Ingest' -}; - -export const Component = () => { - const { t } = useLocale(); - - const syncEnabled = useLibraryQuery(['sync.enabled']); - - const backfillSync = useLibraryMutation(['sync.backfill'], { - onSuccess: async () => { - await syncEnabled.refetch(); - } - }); - - const [data, setData] = useState>({}); - - useLibrarySubscription(['library.actors'], { onData: setData }); - - const cloudSync = useFeatureFlag('cloudSync'); - - return ( - <> - - {syncEnabled.data === false ? ( - -
- -
- - ) : ( - <> - - {t('ingester')} - - - } - description={t('injester_description')} - > -
- {data[ACTORS.Ingest] ? ( - - ) : ( - - )} -
- - - {cloudSync && } - - )} - - ); -}; - -function SyncBackfillDialog(props: UseDialogProps & { onEnabled: () => void }) { - const form = useZodForm({ schema: z.object({}) }); - const dialog = useDialog(props); - const { t } = useLocale(); - - const enableSync = useLibraryMutation(['sync.backfill'], {}); - - // dialog is in charge of enabling sync - useEffect(() => { - form.handleSubmit( - async () => { - await enableSync.mutateAsync(null).then(() => (dialog.state.open = false)); - await props.onEnabled(); - }, - () => {} - )(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - return ( - - ); -} - -function CloudSync({ data }: { data: inferSubscriptionResult }) { - const { t } = useLocale(); - return ( - <> -
-

{t('cloud_sync')}

-

{t('cloud_sync_description')}

-
- - {t('sender')} - - } - description={t('sender_description')} - > -
- {data[ACTORS.CloudSend] ? ( - - ) : ( - - )} -
- - - {t('receiver')} - - - } - description={t('receiver_description')} - > -
- {data[ACTORS.CloudReceive] ? ( - - ) : ( - - )} -
- - - {t('ingester')} - - - } - description={t('ingester_description')} - > -
- {data[ACTORS.CloudIngest] ? ( - - ) : ( - - )} -
- - - ); -} - -function StartButton({ name }: { name: string }) { - const startActor = useLibraryMutation(['library.startActor']); - const { t } = useLocale(); - - return ( - - ); -} - -function StopButton({ name }: { name: string }) { - const stopActor = useLibraryMutation(['library.stopActor']); - const { t } = useLocale(); - - return ( - - ); -} - -function OnlineIndicator({ online }: { online: boolean }) { - return ( -
- ); -} diff --git a/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx b/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx index 93032ecf4..60a54a93f 100644 --- a/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx +++ b/interface/app/$libraryId/settings/library/tags/CreateDialog.tsx @@ -28,7 +28,7 @@ export function useAssignItemsToTag() { const mutation = useLibraryMutation(['tags.assign'], { onSuccess: () => { submitPlausibleEvent({ event: { type: 'tagAssign' } }); - rspc.queryClient.invalidateQueries(['search.paths']); + rspc.queryClient.invalidateQueries({ queryKey: ['search.paths'] }); } }); diff --git a/interface/app/$libraryId/settings/node/libraries/DeleteDeviceDialog.tsx b/interface/app/$libraryId/settings/node/libraries/DeleteDeviceDialog.tsx index 60f06670f..0a6f1e51e 100644 --- a/interface/app/$libraryId/settings/node/libraries/DeleteDeviceDialog.tsx +++ b/interface/app/$libraryId/settings/node/libraries/DeleteDeviceDialog.tsx @@ -54,7 +54,7 @@ export default function DeleteLibraryDialog(props: Props) { } await deleteDevice.mutateAsync(props.pubId); - queryClient.invalidateQueries(['library.list']); + queryClient.invalidateQueries({ queryKey: ['library.list'] }); // eslint-disable-next-line @typescript-eslint/no-unused-expressions platform.refreshMenuBar && platform.refreshMenuBar(); diff --git a/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx b/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx index 7425a0df0..075fb5280 100644 --- a/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx +++ b/interface/app/$libraryId/settings/node/libraries/DeleteDialog.tsx @@ -25,7 +25,7 @@ export default function DeleteLibraryDialog(props: Props) { try { await deleteLib.mutateAsync(props.libraryUuid); - queryClient.invalidateQueries(['library.list']); + queryClient.invalidateQueries({ queryKey: ['library.list'] }); if (platform.refreshMenuBar) platform.refreshMenuBar(); diff --git a/interface/app/$libraryId/settings/node/libraries/JoinDialog.tsx b/interface/app/$libraryId/settings/node/libraries/JoinDialog.tsx deleted file mode 100644 index e57f5340c..000000000 --- a/interface/app/$libraryId/settings/node/libraries/JoinDialog.tsx +++ /dev/null @@ -1,99 +0,0 @@ -import { useQueryClient } from '@tanstack/react-query'; -import { useNavigate } from 'react-router'; -import { LibraryConfigWrapped, useBridgeMutation, useBridgeQuery, useZodForm } from '@sd/client'; -import { Dialog, Loader, Select, SelectOption, toast, useDialog, UseDialogProps, z } from '@sd/ui'; -import { useLocale } from '~/hooks'; -import { usePlatform } from '~/util/Platform'; - -const schema = z.object({ - libraryId: z.string().refine((value) => value !== 'select_library', { - message: 'Please select a library' - }) -}); - -export default (props: UseDialogProps & { librariesCtx: LibraryConfigWrapped[] | undefined }) => { - const cloudLibraries = useBridgeQuery(['cloud.library.list']); - const joinLibrary = useBridgeMutation(['cloud.library.join']); - - const { t } = useLocale(); - const navigate = useNavigate(); - const platform = usePlatform(); - const queryClient = useQueryClient(); - - const form = useZodForm({ schema, defaultValues: { libraryId: 'select_library' } }); - - // const queryClient = useQueryClient(); - // const submitPlausibleEvent = usePlausibleEvent(); - // const platform = usePlatform(); - - const onSubmit = form.handleSubmit(async (data) => { - try { - const library = await joinLibrary.mutateAsync(data.libraryId); - - queryClient.setQueryData(['library.list'], (libraries: any) => { - // The invalidation system beat us to it - if ((libraries || []).find((l: any) => l.uuid === library.uuid)) return libraries; - - return [...(libraries || []), library]; - }); - - if (platform.refreshMenuBar) platform.refreshMenuBar(); - - navigate(`/${library.uuid}`, { replace: true }); - } catch (e: any) { - console.error(e); - toast.error(e); - } - }); - - return ( - -
- {cloudLibraries.isLoading && ( -
- - {t('loading')}... -
- )} - {cloudLibraries.data && ( - - )} -
- - ); -}; diff --git a/interface/app/$libraryId/settings/node/libraries/ListItem.tsx b/interface/app/$libraryId/settings/node/libraries/ListItem.tsx index 78bc8beec..0d1bc8550 100644 --- a/interface/app/$libraryId/settings/node/libraries/ListItem.tsx +++ b/interface/app/$libraryId/settings/node/libraries/ListItem.tsx @@ -1,13 +1,12 @@ import { CaretRight, Pencil, Trash } from '@phosphor-icons/react'; import { AnimatePresence, motion } from 'framer-motion'; -import { Key, useState } from 'react'; -import { LibraryConfigWrapped, useBridgeQuery } from '@sd/client'; +import { useState } from 'react'; +import { LibraryConfigWrapped } from '@sd/client'; import { Button, ButtonLink, Card, dialogManager, Tooltip } from '@sd/ui'; import { Icon } from '~/components'; import { useAccessToken, useLocale } from '~/hooks'; import DeleteDialog from './DeleteDialog'; -import DeviceItem from './DeviceItem'; interface Props { library: LibraryConfigWrapped; @@ -19,7 +18,6 @@ export default (props: Props) => { const [isExpanded, setIsExpanded] = useState(false); const accessToken = useAccessToken(); - const cloudDevicesList = useBridgeQuery(['cloud.devices.list', { access_token: accessToken }]); const toggleExpansion = () => { setIsExpanded((prev) => !prev); }; @@ -86,43 +84,6 @@ export default (props: Props) => { className="relative mt-2 flex origin-top flex-col gap-1 pl-8" >
- - {cloudDevicesList.data?.map( - ( - device: { - pub_id: Key | null | undefined; - name: string; - os: string; - storage_size: bigint; - used_storage: bigint; - created_at: string; - device_model: string; - }, - index: number - ) => ( -
- -
-
- -
- -
- ) - )} )} diff --git a/interface/app/$libraryId/settings/node/libraries/index.tsx b/interface/app/$libraryId/settings/node/libraries/index.tsx index dc8ff541b..db7d6f5d8 100644 --- a/interface/app/$libraryId/settings/node/libraries/index.tsx +++ b/interface/app/$libraryId/settings/node/libraries/index.tsx @@ -1,18 +1,15 @@ -import { useBridgeQuery, useClientContext, useFeatureFlag, useLibraryContext } from '@sd/client'; +import { useBridgeQuery, useClientContext, useLibraryContext } from '@sd/client'; import { Button, dialogManager } from '@sd/ui'; import { useLocale } from '~/hooks'; import { Heading } from '../../Layout'; import CreateDialog from './CreateDialog'; -import JoinDialog from './JoinDialog'; import ListItem from './ListItem'; export const Component = () => { const librariesQuery = useBridgeQuery(['library.list']); const libraries = librariesQuery.data; - const cloudEnabled = useFeatureFlag('cloudSync'); - const { library } = useLibraryContext(); const { libraries: librariesCtx } = useClientContext(); const librariesCtxData = librariesCtx.data; @@ -35,19 +32,6 @@ export const Component = () => { > {t('add_library')} - {cloudEnabled && ( - - )}
} /> diff --git a/interface/app/$libraryId/settings/resources/changelog.tsx b/interface/app/$libraryId/settings/resources/changelog.tsx index 011ef1178..ca012ac74 100644 --- a/interface/app/$libraryId/settings/resources/changelog.tsx +++ b/interface/app/$libraryId/settings/resources/changelog.tsx @@ -9,9 +9,10 @@ import { Heading } from '../Layout'; export const Component = () => { const platform = usePlatform(); const isDark = useIsDark(); - const changelog = useQuery(['changelog'], () => - fetch(`${platform.landingApiOrigin}/api/releases`).then((r) => r.json()) - ); + const changelog = useQuery({ + queryKey: ['changelog'], + queryFn: () => fetch(`${platform.landingApiOrigin}/api/releases`).then((r) => r.json()) + }); const { t } = useLocale(); diff --git a/interface/app/$libraryId/settings/resources/dependencies.tsx b/interface/app/$libraryId/settings/resources/dependencies.tsx deleted file mode 100644 index 2024f9252..000000000 --- a/interface/app/$libraryId/settings/resources/dependencies.tsx +++ /dev/null @@ -1,51 +0,0 @@ -// import { useQuery } from '@tanstack/react-query'; -// import { ScreenHeading } from '@sd/ui'; -// import { usePlatform } from '~/util/Platform'; - -// export const Component = () => { -// const frontEnd = useQuery( -// ['frontend-deps'], -// () => import('@sd/assets/deps/frontend-deps.json') -// ); -// const backEnd = useQuery(['backend-deps'], () => import('@sd/assets/deps/backend-deps.json')); -// const platform = usePlatform(); - -// return ( -//
-// Dependencies - -// {/* item has a LOT more data that we can display, i just went with the basics */} - -// Frontend Dependencies -//
-// {frontEnd.data && -// frontEnd.data?.default.map((item) => { -// return ( -// platform.openLink(item.url ?? '')}> -//
-//

-// {item.title.trimEnd().substring(0, 24) + -// (item.title.length > 24 ? '...' : '')} -//

-//
-// -// ); -// })} -//
- -// Backend Dependencies -//
-// {backEnd.data && -// backEnd.data?.default.map((item) => { -// return ( -// platform.openLink(item.url ?? '')}> -//
-//

{item.title.trimEnd()}

-//
-// -// ); -// })} -//
-//
-// ); -// }; diff --git a/interface/app/index.tsx b/interface/app/index.tsx index 3216925a1..2f70f199a 100644 --- a/interface/app/index.tsx +++ b/interface/app/index.tsx @@ -1,4 +1,4 @@ -import { initRspc, wsBatchLink, type AlphaClient } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { initRspc, wsBatchLink, type AlphaClient } from '@spacedrive/rspc-client'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { useEffect, useMemo, useState } from 'react'; import { diff --git a/interface/app/onboarding/index.tsx b/interface/app/onboarding/index.tsx index f57a4e379..22ba946c3 100644 --- a/interface/app/onboarding/index.tsx +++ b/interface/app/onboarding/index.tsx @@ -4,7 +4,6 @@ import { onboardingStore } from '@sd/client'; import { useOnboardingContext } from './context'; import CreatingLibrary from './creating-library'; import { FullDisk } from './full-disk'; -import { JoinLibrary } from './join-library'; import Locations from './locations'; import NewLibrary from './new-library'; import PreRelease from './prerelease'; @@ -38,7 +37,6 @@ export default [ // path: 'login' // }, { Component: NewLibrary, path: 'new-library' }, - { Component: JoinLibrary, path: 'join-library' }, { Component: FullDisk, path: 'full-disk' }, { Component: Locations, path: 'locations' }, { Component: Privacy, path: 'privacy' }, diff --git a/interface/app/onboarding/join-library.tsx b/interface/app/onboarding/join-library.tsx deleted file mode 100644 index d4bbed977..000000000 --- a/interface/app/onboarding/join-library.tsx +++ /dev/null @@ -1,84 +0,0 @@ -import { useQueryClient } from '@tanstack/react-query'; -import { useNavigate } from 'react-router'; -import { - resetOnboardingStore, - useBridgeMutation, - useBridgeQuery, - useLibraryMutation -} from '@sd/client'; -import { Button } from '@sd/ui'; -import { Icon } from '~/components'; -import { AuthRequiredOverlay } from '~/components/AuthRequiredOverlay'; -import { useLocale, useRouteTitle } from '~/hooks'; -import { usePlatform } from '~/util/Platform'; - -import { OnboardingContainer, OnboardingDescription, OnboardingTitle } from './components'; - -export function JoinLibrary() { - const { t } = useLocale(); - - useRouteTitle('Join Library'); - - return ( - - - {t('join_library')} - {t('join_library_description')} - -
- Cloud Libraries -
    - - -
-
- - ); -} - -function CloudLibraries() { - const { t } = useLocale(); - - const cloudLibraries = useBridgeQuery(['cloud.library.list']); - const joinLibrary = useBridgeMutation(['cloud.library.join']); - - const navigate = useNavigate(); - const queryClient = useQueryClient(); - const platform = usePlatform(); - - if (cloudLibraries.isLoading) return {t('loading')}...; - - return ( - <> - {cloudLibraries.data?.map((cloudLibrary) => ( -
  • - {cloudLibrary.name} - -
    • - ))} - - ); -} diff --git a/interface/app/onboarding/new-library.tsx b/interface/app/onboarding/new-library.tsx index 6242a2d1e..36d6ea453 100644 --- a/interface/app/onboarding/new-library.tsx +++ b/interface/app/onboarding/new-library.tsx @@ -1,6 +1,5 @@ import { useState } from 'react'; import { useNavigate } from 'react-router'; -import { useFeatureFlag } from '@sd/client'; import { Button, Form, InputField } from '@sd/ui'; import { Icon } from '~/components'; import { useLocale, useOperatingSystem } from '~/hooks'; @@ -21,8 +20,6 @@ export default function OnboardingNewLibrary() { // TODO }; - const cloudFeatureFlag = useFeatureFlag('cloudSync'); - return (
    */}
    - {cloudFeatureFlag && ( - <> - {t('or')} - - - )} )} diff --git a/interface/app/onboarding/prerelease.tsx b/interface/app/onboarding/prerelease.tsx index 156d6e5a5..5432ebec5 100644 --- a/interface/app/onboarding/prerelease.tsx +++ b/interface/app/onboarding/prerelease.tsx @@ -17,7 +17,7 @@ export default function OnboardingPreRelease() {
    diff --git a/interface/components/Authentication.tsx b/interface/components/Authentication.tsx index 4d530bb32..22c4c7269 100644 --- a/interface/components/Authentication.tsx +++ b/interface/components/Authentication.tsx @@ -1,13 +1,13 @@ -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/v2'; import { GoogleLogo, Icon } from '@phosphor-icons/react'; import { Apple, Github } from '@sd/assets/svgs/brands'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseMutationResult } from '@tanstack/react-query'; import { open } from '@tauri-apps/plugin-shell'; import clsx from 'clsx'; import { motion } from 'framer-motion'; import { Dispatch, SetStateAction, useState } from 'react'; import { getAuthorisationURLWithQueryParamsAndSetState } from 'supertokens-web-js/recipe/thirdparty'; -import { Card, Divider, toast } from '@sd/ui'; +import { Card, toast } from '@sd/ui'; import { Icon as Logo } from '~/components'; import { useIsDark } from '~/hooks'; @@ -32,7 +32,7 @@ export const Authentication = ({ cloudBootstrap }: { reload: Dispatch>; - cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation + cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation }) => { const [activeTab, setActiveTab] = useState<'Login' | 'Register'>('Login'); const isDark = useIsDark(); diff --git a/interface/components/Devtools.tsx b/interface/components/Devtools.tsx index acb72e803..952019859 100644 --- a/interface/components/Devtools.tsx +++ b/interface/components/Devtools.tsx @@ -1,4 +1,3 @@ -import { defaultContext } from '@tanstack/react-query'; import { ReactQueryDevtools } from '@tanstack/react-query-devtools'; import { useDebugState } from '@sd/client'; @@ -7,18 +6,9 @@ export const Devtools = () => { return ( <> - {debugState.reactQueryDevtools !== 'disabled' ? ( - - ) : null} + {debugState.reactQueryDevtools && ( + + )} ); }; diff --git a/interface/components/Login.tsx b/interface/components/Login.tsx index 463029c57..a4ac8bf63 100644 --- a/interface/components/Login.tsx +++ b/interface/components/Login.tsx @@ -1,5 +1,5 @@ -import { AlphaRSPCError } from '@oscartbeaumont-sd/rspc-client/v2'; import { ArrowLeft } from '@phosphor-icons/react'; +import { RSPCError } from '@spacedrive/rspc-client'; import { UseMutationResult } from '@tanstack/react-query'; import clsx from 'clsx'; import { Dispatch, SetStateAction, useState } from 'react'; @@ -17,7 +17,7 @@ async function signInClicked( email: string, password: string, reload: Dispatch>, - cloudBootstrap: UseMutationResult // Cloud bootstrap mutation + cloudBootstrap: UseMutationResult // Cloud bootstrap mutation ) { try { const response = await signIn({ @@ -80,7 +80,7 @@ const Login = ({ cloudBootstrap }: { reload: Dispatch>; - cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation + cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation }) => { const [continueWithEmail, setContinueWithEmail] = useState(false); @@ -105,7 +105,7 @@ const Login = ({ interface LoginProps { reload: Dispatch>; - cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation + cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation setContinueWithEmail: Dispatch>; } @@ -226,7 +226,7 @@ const LoginForm = ({ reload, cloudBootstrap, setContinueWithEmail }: LoginProps) interface Props { setContinueWithEmail: Dispatch>; reload: Dispatch>; - cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation + cloudBootstrap: UseMutationResult; // Cloud bootstrap mutation } const ContinueWithEmail = ({ setContinueWithEmail, reload, cloudBootstrap }: Props) => { diff --git a/interface/components/Sparkles.tsx b/interface/components/Sparkles.tsx index ec6b73ffa..2b1dc612d 100644 --- a/interface/components/Sparkles.tsx +++ b/interface/components/Sparkles.tsx @@ -38,7 +38,8 @@ type SparklesProps = { children: React.ReactNode; }; -const Sparkles = ({ color = DEFAULT_COLOR, children, ...props }: SparklesProps) => { +// million-ignore +const Sparkles = ({ color = DEFAULT_COLOR, children }: SparklesProps) => { const [sparkles, setSparkles] = useState(() => { return range(3).map(() => generateSparkle(color)); }); @@ -60,7 +61,7 @@ const Sparkles = ({ color = DEFAULT_COLOR, children, ...props }: SparklesProps) ); return ( - + {sparkles.map((sparkle) => ( { + return useSuspenseQuery({ + queryKey: ['userDirs', 'home'], + queryFn: () => { if (platform.userHomeDir) return platform.userHomeDir(); else return null; - }, - { suspense: true } - ); + } + }); } diff --git a/interface/hooks/useOperatingSystem.ts b/interface/hooks/useOperatingSystem.ts index c5797661e..acba5a8c6 100644 --- a/interface/hooks/useOperatingSystem.ts +++ b/interface/hooks/useOperatingSystem.ts @@ -17,17 +17,15 @@ export function guessOperatingSystem(): OperatingSystem { // Setting `realOs` to true will return a best guess of the underlying operating system instead of 'browser'. export function useOperatingSystem(realOs?: boolean): OperatingSystem { const platform = usePlatform(); - const { data } = useQuery( - ['_tauri', 'platform'], - async () => { + const { data } = useQuery({ + queryKey: ['_tauri', 'platform'], + queryFn: async () => { return platform.getOs ? await platform.getOs() : guessOperatingSystem(); }, - { - // Here we guess the users operating system from the user agent for the first render. - initialData: guessOperatingSystem, - enabled: platform.getOs !== undefined - } - ); + // Here we guess the users operating system from the user agent for the first render. + initialData: guessOperatingSystem, + enabled: platform.getOs !== undefined + }); return platform.platform === 'web' && !realOs ? 'browser' : data; } diff --git a/interface/locales/ja/common.json b/interface/locales/ja/common.json index 218106803..f5e840084 100644 --- a/interface/locales/ja/common.json +++ b/interface/locales/ja/common.json @@ -3,7 +3,7 @@ "about_vision_text": "私達は通常、複数のクラウドのアカウントを持ち、バックアップのないドライブを利用し、データを失う危険にさらされています。またGoogle PhotosやiCloudのようなクラウドサービスに依存していますが、それらは容量に制限があり、サービスやOS間に互換性はほとんどありません。フォトアルバムは、デバイスのエコシステムに縛られたり広告データとして利用されたりすべきではなく、OSにとらわれず、永続的で、個人所有のものであるべきです。私達が作成したデータは私達の遺産であり、私達よりもずっと長生きします。オープンソース・テクノロジーは、無制限のスケールで、私達の生活を定義するデータの絶対的なコントロールを確実に保持する唯一の方法なのです。", "about_vision_title": "ビジョン", "accept": "Accept", - "accept_files": "Accept files", + "accept_files": "ファイルを含める", "accessed": "最終アクセス", "account": "アカウント", "actions": "操作", @@ -20,7 +20,7 @@ "add_tag": "タグを追加", "added_location": "ロケーション {{name}} を追加しました", "adding_location": "ロケーション {{name}} を追加中", - "advanced": "高度な", + "advanced": "高度", "advanced_settings": "高度な設定", "album": "アルバム", "alias": "エイリアス", @@ -46,6 +46,7 @@ "backfill_sync_description": "バックフィルが完了するまでライブラリは一時停止されます", "backups": "バックアップ", "backups_description": "Spacedriveデータベースのバックアップの設定を行います。", + "bar_graph_info": "各バーにカーソルを合わせると、ファイルの種類が表示されます。 ダブルクリックで移動します。", "bitrate": "ビットレート", "blur_effects": "ぼかし効果", "blur_effects_description": "いくつかのUI要素にぼかし効果を適用します。", @@ -55,23 +56,23 @@ "canceled": "キャンセル", "celcius": "摂氏", "change": "変更", - "change_view_setting_description": "デフォルトのエクスプローラー ビューを変更する", + "change_view_setting_description": "エクスプローラーのデフォルトの表示形式を変更します。", "changelog": "変更履歴", "changelog_page_description": "Spacedriveの魅力ある新機能をご確認ください。", "changelog_page_title": "変更履歴", "checksum": "チェックサム", "clear_finished_jobs": "完了ジョブを削除", - "click_to_hide": "非表示にするにはクリック", - "click_to_lock": "ロックするにはクリック", + "click_to_hide": "クリックで非表示", + "click_to_lock": "クリックでロック", "client": "クライアント", "close": "閉じる", "close_command_palette": "コマンドパレットを閉じる", "close_current_tab": "タブを閉じる", "cloud": "クラウド", - "cloud_connect_description": "ライブラリをクラウドに接続しますか?", + "cloud_connect_description": "ライブラリをクラウドに接続しますか？", "cloud_drives": "クラウドドライブ", "cloud_sync": "クラウド同期", - "cloud_sync_description": "ライブラリを Spacedrive Cloud と同期するプロセスを管理する", + "cloud_sync_description": "ライブラリをSpacedrive Cloudと同期するプロセスを管理します。", "clouds": "クラウド", "code": "コード", "collection": "コレクション", @@ -87,13 +88,13 @@ "confirm": "Confirm", "connect": "接続する", "connect_cloud": "クラウドに接続する", - "connect_cloud_description": "クラウドアカウントをSpacedriveに接続する。", + "connect_cloud_description": "クラウドアカウントをSpacedriveに接続", "connect_device": "デバイスを接続する", "connect_device_description": "Spacedriveはすべてのデバイスで最適に機能します。", "connect_library_to_cloud": "ライブラリをSpacedrive Cloudに接続する", "connected": "接続中", "connecting": "接続中", - "connecting_library_to_cloud": "ライブラリを Spacedrive Cloud に接続しています...", + "connecting_library_to_cloud": "ライブラリをSpacedrive Cloudに接続しています……", "contacts": "連絡先", "contacts_description": "Spacedriveで連絡先を管理。", "contains": "が次を含む", @@ -140,7 +141,7 @@ "date_modified": "更新日時", "date_taken": "取得日時", "date_time_format": "日付と時刻のフォーマット", - "date_time_format_description": "Spacedriveに表示される日付フォーマットを選択します", + "date_time_format_description": "Spacedriveに表示される日付フォーマットを選択します。", "debug_mode": "デバッグモード", "debug_mode_description": "アプリ内で追加のデバッグ機能を有効にします。", "default": "デフォルト", @@ -196,7 +197,7 @@ "enable_networking_description": "あなたのノードが周囲の他のSpacedriveノードと通信できるようにします。", "enable_networking_description_required": "ライブラリの同期やSpacedropに必要です。", "enable_relay": "リレーを有効にする", - "enable_relay_description": "リレー サーバーを有効にして、デバイスがパブリック インターネット経由で通信できるようにします。", + "enable_relay_description": "リレー・サーバーを有効にして、デバイスがパブリック・インターネット経由で通信できるようにします。", "enable_sync": "同期を有効にする", "enable_sync_description": "このライブラリ内のすべての既存データに対して同期操作を生成し、将来何かが発生したときに同期操作を生成するように Spacedrive を構成します。", "enabled": "有効", @@ -220,7 +221,7 @@ "explorer": "エクスプローラー", "explorer_settings": "エクスプローラーの設定", "explorer_shortcut_description": "ファイルシステムの移動・操作を設定します。", - "explorer_view": "エクスプローラービュー", + "explorer_view": "エクスプローラーの表示形式", "export": "エクスポート", "export_library": "ライブラリのエクスポート", "export_library_coming_soon": "ライブラリのエクスポート機能は今後実装予定です", @@ -229,7 +230,7 @@ "extensions": "拡張機能", "extensions_description": "このクライアントの機能を拡張するための拡張機能をインストールします。", "fahrenheit": "華氏", - "failed": "失敗した", + "failed": "失敗しました", "failed_to_add_location": "ロケーションの追加に失敗", "failed_to_cancel_job": "ジョブの中止に失敗", "failed_to_clear_all_jobs": "全てのジョブの削除に失敗", @@ -262,8 +263,8 @@ "feedback_placeholder": "フィードバックを入力...", "feedback_toast_error_message": "フィードバックの送信中にエラーが発生しました。もう一度お試しください。", "file_already_exist_in_this_location": "このファイルは既にこのロケーションに存在します", - "file_directory_name": "ファイル/ディレクトリ名", - "file_extension_description": "ファイル拡張子 (例: .mp4、.jpg、.txt)", + "file_directory_name": "ファイル／ディレクトリ名", + "file_extension_description": "拡張子（例: .mp4、.jpg、.txt）", "file_from": "File {{file}} from {{name}}", "file_indexing_rules": "ファイルのインデックス化ルール", "file_other": "ファイル", @@ -288,8 +289,8 @@ "general_shortcut_description": "一般に使用されるショートカットキー。", "generate_checksums": "チェックサムを作成", "generate_preview_media_label": "このロケーションのプレビューメディアを作成する", - "gitignore": "Git 無視", - "glob_description": "グロブ (例: **/.git)", + "gitignore": "Git Ignore", + "glob_description": "グロブ（例: **/.git）", "go_back": "戻る", "go_to_labels": "ラベルに移動", "go_to_location": "ロケーションに移動", @@ -323,11 +324,11 @@ "indexer_rule_reject_allow_label": "デフォルトでは、インデックス化ルールはブラックリストとして機能し、その基準に一致する全てのファイルを除外します。このオプションを有効にすると、ホワイトリストに変換され、指定されたルールに一致するファイルのみをインデックス化するようになります。", "indexer_rules": "インデックス化のルール", "indexer_rules_error": "インデクス化ルールの取得エラー", - "indexer_rules_info": "globを使用して無視するパスを指定できます。", + "indexer_rules_info": "グロブを使用して無視するパスを指定できます。", "indexer_rules_not_available": "利用可能なインデックス化ルールがありません", "ingester": "インジェスター", "ingester_description": "このプロセスは、受信したクラウド操作を取得し、メインの同期インジェスターに送信します。", - "injester_description": "このプロセスは、P2P 接続と Spacedrive Cloud から同期操作を取得し、ライブラリに適用します。", + "injester_description": "このプロセスは、P2P接続とSpacedrive Cloudから同期操作を取得し、ライブラリに適用します。", "install": "インストール", "install_update": "アップデートをインストールする", "installed": "インストール完了", @@ -338,7 +339,7 @@ "ipv4_ipv6_listeners_error": "IPv4・IPv6リスナーの作成エラー。ファイアウォールの設定を確認してください！", "ipv4_listeners_error": "IPv4リスナーの作成エラー。ファイアウォールの設定を確認してください！", "ipv6": "IPv6ネットワーキング", - "ipv6_description": "IPv6 ネットワークを使用したピアツーピア通信を許可する", + "ipv6_description": "IPv6ネットワークを使用したピアツーピア通信を許可します。", "ipv6_listeners_error": "IPv6リスナーの作成エラー。ファイアウォールの設定を確認してください！", "is": "が", "is_not": "が次と異なる", @@ -346,7 +347,7 @@ "item_other": "項目", "item_size": "アイテムの表示サイズ", "items": "項目", - "job_error_description": "ジョブはエラーで完了しました。\n詳細については、以下のエラー ログを参照してください。\nサポートが必要な場合は、サポートに連絡してこのエラーを伝えてください。", + "job_error_description": "ジョブはエラーで完了しました。\n詳細については、以下のエラーログを参照してください。\nサポートが必要な場合は、サポートに連絡してこのエラーを伝えてください。", "job_has_been_canceled": "ジョブが中止されました。", "job_has_been_paused": "ジョブが一時停止されました。", "job_has_been_removed": "ジョブが削除されました。", @@ -375,7 +376,7 @@ "libraries_description": "データベースには、すべてのライブラリデータとファイルのメタデータが含まれています。", "library": "ライブラリ", "library_bytes": "ライブラリのサイズ", - "library_bytes_description": "ライブラリ内のすべての場所の合計サイズ。", + "library_bytes_description": "ライブラリ内のすべてのロケーションの合計サイズ。", "library_db_size": "インデックスサイズ", "library_db_size_description": "ライブラリのデータベースのサイズ。", "library_name": "ライブラリの名前", @@ -391,9 +392,9 @@ "local_locations": "ローカルロケーション", "local_node": "ローカルノード", "location": "ロケーション", - "location_added_successfully": "場所が正常に追加されました。", + "location_added_successfully": "ロケーションが正常に追加されました。", "location_connected_tooltip": "ロケーションの変化が監視されています", - "location_deleted_successfully": "場所は正常に削除されました。", + "location_deleted_successfully": "ロケーションは正常に削除されました。", "location_disconnected_tooltip": "ロケーションの変更は監視されていません", "location_display_name_info": "サイドバーに表示されるロケーションの名前を設定します。ディスク上の実際のフォルダの名前は変更されません。", "location_empty_notice_message": "ファイルが見つかりません", @@ -407,7 +408,7 @@ "locations": "ロケーション", "locations_description": "ローケーションを管理します。", "lock": "ロック", - "lock_sidebar": "サイドバーをロックする", + "lock_sidebar": "サイドバーをロック", "log_in": "ログイン", "log_in_with_browser": "ブラウザでログイン", "log_out": "ログアウト", @@ -416,9 +417,9 @@ "login": "ログイン", "logout": "ログアウト", "manage_library": "ライブラリの設定", - "managed": "マネージド", + "managed": "Managed", "manual_peers": "ピアを手動で追加する", - "manual_peers_description": "IP アドレスとポートを入力してピアを手動で追加します。\nこれは、自動検出が不可能な場合に役立ちます。", + "manual_peers_description": "IPアドレスとポートを入力してピアを手動で追加します。\nこれは、自動検出が不可能な場合に役立ちます。", "media": "メディア", "media_view": "メディアビュー", "media_view_context": "メディア ビュー", @@ -453,12 +454,12 @@ "network_settings": "ネットワーク設定", "network_settings_advanced": "高度なネットワークの概要", "network_settings_advanced_description": "現在のネットワーク設定に関する詳細情報。", - "network_settings_description": "ネットワークと接続に関する設定。", + "network_settings_description": "ネットワークと接続に関する設定を行います。", "networking": "ネットワーク", "networking_error": "ネットワークの起動エラー！", "networking_port": "ネットワークポート", "networking_port_description": "SpacedriveのP2Pネットワークが使用するポートを設定します。ファイアウォールによる制限がない限り、無効のままにしておくことを推奨します。インターネット上に公開しないでください！", - "new": "新しい", + "new": "新規", "new_folder": "新しいフォルダー", "new_library": "新しいライブラリ", "new_location": "新しいロケーション", @@ -468,14 +469,14 @@ "new_update_available": "アップデートが利用可能です！", "no_apps_available": "利用可能なアプリはありません", "no_favorite_items": "お気に入りのアイテムはありません", - "no_git_files": "Git ファイルがありません", - "no_hidden_files": "隠しファイルはありません", + "no_git_files": "Gitファイルを除外", + "no_hidden_files": "隠しファイルを除外", "no_items_found": "アイテムが見つかりませんでした", "no_jobs": "ジョブがありません。", "no_labels": "ラベルなし", - "no_nodes_found": "Spacedriveノードが見つかりませんでした。", + "no_nodes_found": "Spacedriveノードが見つかりません。", "no_search_selected": "検索が選択されていません", - "no_system_files": "システムファイルがありません", + "no_system_files": "システムファイルを除外", "no_tag_selected": "タグが選択されていません。", "no_tags": "タグがありません。", "no_tags_description": "タグが作成されていません", @@ -486,7 +487,7 @@ "normal": "Normal", "not_you": "あなたではありませんか？", "note": "Note", - "nothing_selected": "何も選択されていない", + "nothing_selected": "何も選択されていません", "number_of_passes": "# パス数", "object": "対象", "object_id": "オブジェクトID", @@ -507,12 +508,13 @@ "open_with": "プログラムから開く", "opening_trash": "ごみ箱を開く", "or": "OR", + "other": "その他", "overview": "概要", "p2p_visibility": "P2Pの可視性", "p2p_visibility_contacts_only": "連絡先のみ", "p2p_visibility_description": "Spacedrive インストールを閲覧できるユーザーを設定します。", "p2p_visibility_disabled": "無効", - "p2p_visibility_everyone": "みんな", + "p2p_visibility_everyone": "全員", "package": "パッケージ", "page": "ページ", "page_shortcut_description": "アプリ内の各ページへの移動のショートカット", @@ -544,10 +546,10 @@ "quick_rescan_started": "簡易再スキャンを開始", "quick_view": "クイック プレビュー", "quickpreview_thumbnail_error_message": "フル解像度の画像を読み込めません", - "quickpreview_thumbnail_error_tip": "画像が見つかりませんでした。そのため、サムネイルが表示されます。", + "quickpreview_thumbnail_error_tip": "画像が見つかりませんでした。代わりにサムネイルを表示します。", "random": "ランダム", "receiver": "受信機", - "receiver_description": "このプロセスは、Spacedrive Cloud から操作を受信して​​保存します。", + "receiver_description": "このプロセスは、Spacedrive Cloudから操作を受信して保存します。", "recent_jobs": "最近のジョブ", "recents": "最近のアクセス", "recents_notice_message": "ファイルを開くと最近のアクセスが表示されます。", @@ -556,8 +558,8 @@ "regenerate_thumbs": "サムネイルを再作成", "reindex": "再インデックス化", "reject": "拒否", - "reject_files": "Reject files", - "relay_listeners_error": "リレー リスナーの作成中にエラーが発生しました。ファイアウォールの設定を確認してください。", + "reject_files": "ファイルを除外", + "relay_listeners_error": "リレー・リスナーの作成中にエラーが発生しました。ファイアウォールの設定を確認してください。", "reload": "更新", "remote_access": "リモートアクセスを有効にする", "remote_access_description": "他のノードがこのノードに直接接続できるようにします。", @@ -602,7 +604,7 @@ "send": "送信", "send_report": "レポートを送信", "sender": "送信者", - "sender_description": "このプロセスでは、同期操作が Spacedrive Cloud に送信されます。", + "sender_description": "このプロセスでは、同期操作がSpacedrive Cloudに送信されます。", "settings": "設定", "setup": "セットアップ", "share": "共有", @@ -632,7 +634,7 @@ "spacedrive_account": "Spacedriveアカウント", "spacedrive_cloud": "Spacedriveクラウド", "spacedrive_cloud_description": "Spacedriveは常にローカルでの利用を優先しますが、将来的には独自オプションのクラウドサービスを提供する予定です。現在、アカウント認証はフィードバック機能のみに使用されており、それ以外では必要ありません。", - "spacedrop": "Spacedropの対照", + "spacedrop": "Spacedropの可視性", "spacedrop_a_file": "ファイルをSpacedropへ", "spacedrop_already_progress": "Spacedropは既に実行中です", "spacedrop_contacts_only": "連絡先のみ", @@ -643,10 +645,10 @@ "square_thumbnails": "正方形のサムネイル", "star_on_github": "Star on GitHub", "start": "始める", - "starting": "起動...", + "starting": "起動中……", "starts_with": "が次で始まる", "stop": "中止", - "stopping": "停止中...", + "stopping": "停止中……", "success": "成功", "support": "サポート", "switch_to_grid_view": "グリッド ビューに切り替え", @@ -665,17 +667,17 @@ "tags": "タグ", "tags_description": "タグを管理します。", "tags_notice_message": "このタグに割り当てられたアイテムはありません。", - "task": "task", - "task_other": "tasks", + "task": "タスク", + "task_other": "タスク", "telemetry_description": "有効にすると、アプリを改善するための詳細なテレメトリ・利用状況データが開発者に提供されます。無効にすると、基本的なデータ（実行状況、アプリバージョン、コアバージョン、プラットフォーム［モバイル／ウェブ／デスクトップなど］）のみが送信されます。", - "telemetry_share_anonymous": "利用状況を送信する", - "telemetry_share_anonymous_description": "アプリの改善のために、完全に匿名のテレメトリデータを送信します", - "telemetry_share_anonymous_short": "匿名での使用", - "telemetry_share_minimal": "最小限のデータのみを送信する", - "telemetry_share_minimal_description": "自分がSpacedriveのアクティブユーザーであることと、多少の技術的データのみを送信します", + "telemetry_share_anonymous": "匿名で利用状況を送信", + "telemetry_share_anonymous_description": "アプリの改善のために、完全に匿名のテレメトリデータを送信します。", + "telemetry_share_anonymous_short": "匿名で送信", + "telemetry_share_minimal": "最小限のデータのみを送信", + "telemetry_share_minimal_description": "自分がSpacedriveのアクティブユーザーであることと、多少の技術的データのみを送信します。", "telemetry_share_minimal_short": "最小限", - "telemetry_share_none": "共有しないでください", - "telemetry_share_none_description": "Spacedrive アプリから分析データをまったく送信しません。", + "telemetry_share_none": "送信しない", + "telemetry_share_none_description": "Spacedriveアプリから分析データをまったく送信しません。", "telemetry_share_none_short": "なし", "telemetry_title": "テレメトリ・利用状況データを送信する", "temperature": "温度", @@ -703,10 +705,13 @@ "total_bytes_free_description": "ライブラリに接続されているすべてのノードで利用可能な空き容量。", "total_bytes_used": "総使用量", "total_bytes_used_description": "ライブラリに接続されているすべてのノードで使用されているスペースの合計。", + "total_files": "総ファイル数", "trash": "ごみ箱", "type": "種類", "ui_animations": "UIアニメーション", "ui_animations_description": "ダイアログやその他のUI要素を開いたり閉じたりするときにアニメーションを有効にします。", + "unidentified_files": "不明ファイル数", + "unidentified_files_info": "Spacedriveが識別できなかったファイル。", "unknown": "不明", "unnamed_location": "名前の無いロケーション", "update": "アップデート", @@ -716,10 +721,10 @@ "usage": "利用状況", "usage_description": "ライブラリの利用状況とハードウェア情報", "vacuum": "バキューム", - "vacuum_library": "バキュームライブラリ", + "vacuum_library": "ライブラリをバキューム", "vacuum_library_description": "データベースを再パックして、不要なスペースを解放します。", "value": "値", - "value_required": "必要な値", + "value_required": "必須項目", "version": "バージョン {{version}}", "video": "ビデオ", "video_preview_not_supported": "ビデオのプレビューには対応していません。", @@ -736,4 +741,4 @@ "zoom": "ズーム", "zoom_in": "拡大する", "zoom_out": "縮小する" -} +} \ No newline at end of file diff --git a/interface/package.json b/interface/package.json index dda41dbc4..cbcb35c67 100644 --- a/interface/package.json +++ b/interface/package.json @@ -13,6 +13,7 @@ "@dnd-kit/utilities": "^3.2.2", "@headlessui/react": "^1.7.17", "@icons-pack/react-simple-icons": "^9.1.0", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", "@phosphor-icons/react": "^2.0.13", "@radix-ui/react-dialog": "^1.0.5", "@radix-ui/react-dropdown-menu": "^2.0.6", @@ -26,10 +27,10 @@ "@sd/client": "workspace:*", "@sd/ui": "workspace:*", "@sentry/browser": "^7.74.1", - "@tanstack/react-query": "^4.36.1", - "@tanstack/react-query-devtools": "^4.36.1", - "@tanstack/react-table": "^8.10.7", - "@tanstack/react-virtual": "3.0.0-beta.66", + "@tanstack/react-query": "^5.59", + "@tanstack/react-query-devtools": "^5.59", + "@tanstack/react-table": "^8.20.5", + "@tanstack/react-virtual": "3.10.8", "@total-typescript/ts-reset": "^0.5.1", "@virtual-grid/react": "^2.0.2", "class-variance-authority": "^0.7.0", @@ -70,7 +71,7 @@ "use-debounce": "^9.0.4", "use-resize-observer": "^9.1.0", "uuid": "^10.0.0", - "valtio": "^1.11.2" + "valtio": "^2.0" }, "devDependencies": { "@sd/config": "workspace:*", @@ -82,7 +83,7 @@ "tailwindcss": "^3.4.10", "type-fest": "^4.13.0", "typescript": "^5.6.2", - "vite": "^5.2.0", + "vite": "^5.4.9", "vite-plugin-svgr": "^3.3.0" } } diff --git a/interface/util/useTraceUpdate.tsx b/interface/util/useTraceUpdate.tsx new file mode 100644 index 000000000..c40ebc326 --- /dev/null +++ b/interface/util/useTraceUpdate.tsx @@ -0,0 +1,28 @@ +import { useEffect, useRef } from 'react'; + +/** + * DO NOT DELETE THIS HOOK + * It probably isn't used in the codebase, but it's a useful debugging tool. + */ +export function useTraceUpdate(name: string, props: object | null) { + const prev = useRef<{ [key: string]: any } | null>(props); + useEffect(() => { + const { current } = prev; + if (props == null) { + console.log(`Change ${name} to null`); + } else if (current == null) { + console.log(`Change ${name} from null to`, props); + } else { + const changedProps = Object.entries(props).reduce((ps: any, [k, v]) => { + if (current[k] !== v) { + ps[k] = [current[k], v]; + } + return ps; + }, {}); + if (Object.keys(changedProps).length > 0) { + console.log(`Changed ${name}:`, changedProps); + } + } + prev.current = props; + }); +} diff --git a/package.json b/package.json index 80924fac7..ea61bc8d9 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "turbo": "^1.12.5", "turbo-ignore": "^1.12.5", "typescript": "^5.6.2", - "vite": "^5.2.0" + "vite": "^5.4.9" }, "engines": { "pnpm": ">=9.0.0", @@ -73,5 +73,5 @@ "eslintConfig": { "root": true }, - "packageManager": "pnpm@9.9.0" + "packageManager": "pnpm@9.12.2" } diff --git a/packages/client/package.json b/packages/client/package.json index 68f7f1a07..f2db671f6 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -11,11 +11,11 @@ "typecheck": "tsc -b" }, "dependencies": { - "@oscartbeaumont-sd/rspc-client": "github:spacedriveapp/rspc#path:packages/client&bc882f4724", - "@oscartbeaumont-sd/rspc-react": "github:spacedriveapp/rspc#path:packages/react&bc882f4724", + "@spacedrive/rspc-client": "github:spacedriveapp/rspc#path:packages/client&6a77167495", + "@spacedrive/rspc-react": "github:spacedriveapp/rspc#path:packages/react&6a77167495", "@solid-primitives/deep": "^0.2.4", - "@tanstack/react-query": "^4.36.1", - "@tanstack/solid-query": "^5.17.9", + "@tanstack/react-query": "^5.59", + "@tanstack/solid-query": "^5.59", "@zxcvbn-ts/core": "^3.0.4", "@zxcvbn-ts/language-common": "^3.0.4", "@zxcvbn-ts/language-en": "^3.0.2", @@ -23,7 +23,7 @@ "plausible-tracker": "^0.3.8", "react-hook-form": "^7.47.0", "solid-js": "^1.8.8", - "zod": "~3.22.4" + "zod": "^3.23" }, "devDependencies": { "@sd/config": "workspace:*", diff --git a/packages/client/src/core.ts b/packages/client/src/core.ts index ac070c58e..7593d5e98 100644 --- a/packages/client/src/core.ts +++ b/packages/client/src/core.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// This file was generated by [rspc](https://github.com/oscartbeaumont/rspc). Do not edit this file manually. +// This file was generated by [rspc](https://github.com/spacedriveapp/rspc). Do not edit this file manually. export type Procedures = { queries: diff --git a/packages/client/src/explorer/index.ts b/packages/client/src/explorer/index.ts index 34dcf4dce..9703b13af 100644 --- a/packages/client/src/explorer/index.ts +++ b/packages/client/src/explorer/index.ts @@ -1,5 +1,4 @@ export * from './useExplorerInfiniteQuery'; -export * from './usePathsInfiniteQuery'; export * from './usePathsOffsetInfiniteQuery'; export * from './usePathsExplorerQuery'; export * from './useObjectsInfiniteQuery'; diff --git a/packages/client/src/explorer/useExplorerInfiniteQuery.ts b/packages/client/src/explorer/useExplorerInfiniteQuery.ts index 1b3c25ce5..80b4c5767 100644 --- a/packages/client/src/explorer/useExplorerInfiniteQuery.ts +++ b/packages/client/src/explorer/useExplorerInfiniteQuery.ts @@ -6,5 +6,4 @@ import { Ordering } from './index'; export type UseExplorerInfiniteQueryArgs = { arg: TArg; order: TOrder | null; - onSuccess?: () => void; -} & Pick>, 'enabled' | 'suspense'>; +}; diff --git a/packages/client/src/explorer/useExplorerQuery.ts b/packages/client/src/explorer/useExplorerQuery.ts index b41c9d247..ad7630b5d 100644 --- a/packages/client/src/explorer/useExplorerQuery.ts +++ b/packages/client/src/explorer/useExplorerQuery.ts @@ -1,13 +1,16 @@ -import { UseInfiniteQueryResult, UseQueryResult } from '@tanstack/react-query'; +import { InfiniteData, UseInfiniteQueryResult, UseQueryResult } from '@tanstack/react-query'; import { useCallback, useMemo } from 'react'; import { SearchData } from '../core'; export function useExplorerQuery( - query: UseInfiniteQueryResult>, + query: UseInfiniteQueryResult>>, count: UseQueryResult ) { - const items = useMemo(() => query.data?.pages.flatMap((d) => d.items) ?? null, [query.data]); + const items = useMemo( + () => query.data?.pages.flatMap((data) => data.items) ?? null, + [query.data] + ); const loadMore = useCallback(() => { if (query.hasNextPage && !query.isFetchingNextPage) { diff --git a/packages/client/src/explorer/useObjectsInfiniteQuery.ts b/packages/client/src/explorer/useObjectsInfiniteQuery.ts index 6e758e1ba..c8680a2b8 100644 --- a/packages/client/src/explorer/useObjectsInfiniteQuery.ts +++ b/packages/client/src/explorer/useObjectsInfiniteQuery.ts @@ -8,8 +8,7 @@ import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; export function useObjectsInfiniteQuery({ arg, - order, - ...args + order }: UseExplorerInfiniteQueryArgs) { const { library } = useLibraryContext(); const ctx = useRspcLibraryContext(); @@ -21,25 +20,23 @@ export function useObjectsInfiniteQuery({ const query = useInfiniteQuery({ queryKey: ['search.objects', { library_id: library.uuid, arg }] as const, queryFn: ({ pageParam, queryKey: [_, { arg }] }) => { - const cItem: Extract = pageParam; - let orderAndPagination: (typeof arg)['orderAndPagination']; - if (!cItem) { + if (!pageParam || pageParam.type !== 'Object') { if (order) orderAndPagination = { orderOnly: order }; } else { let cursor: ObjectCursor | undefined; if (!order) cursor = 'none'; - else if (cItem) { + else if (pageParam) { switch (order.field) { case 'kind': { - const data = cItem.item.kind; + const data = pageParam.item.kind; if (data !== null) cursor = { kind: { order: order.value, data } }; break; } case 'dateAccessed': { - const data = cItem.item.date_accessed; + const data = pageParam.item.date_accessed; if (data !== null) cursor = { dateAccessed: { order: order.value, data } }; break; @@ -47,18 +44,18 @@ export function useObjectsInfiniteQuery({ } } - if (cursor) orderAndPagination = { cursor: { cursor, id: cItem.item.id } }; + if (cursor) orderAndPagination = { cursor: { cursor, id: pageParam.item.id } }; } arg.orderAndPagination = orderAndPagination; return ctx.client.query(['search.objects', arg]); }, + initialPageParam: undefined as ExplorerItem | undefined, getNextPageParam: (lastPage) => { if (lastPage.items.length < arg.take) return undefined; else return lastPage.items[arg.take - 1]; - }, - ...args + } }); return query; diff --git a/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts b/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts index 7bb20c9e4..3580bcf67 100644 --- a/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts +++ b/packages/client/src/explorer/useObjectsOffsetInfiniteQuery.ts @@ -8,8 +8,7 @@ import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; export function useObjectsOffsetInfiniteQuery({ arg, - order, - ...args + order }: UseExplorerInfiniteQueryArgs) { const { library } = useLibraryContext(); const ctx = useRspcLibraryContext(); @@ -40,10 +39,10 @@ export function useObjectsOffsetInfiniteQuery({ return { ...result, offset: pageParam, arg }; }, + initialPageParam: 0, getNextPageParam: ({ items, offset, arg }) => { if (items.length >= arg.take) return (offset ?? 0) + 1; - }, - ...args + } }); return query; diff --git a/packages/client/src/explorer/usePathsExplorerQuery.ts b/packages/client/src/explorer/usePathsExplorerQuery.ts index f64f56504..4894fbc74 100644 --- a/packages/client/src/explorer/usePathsExplorerQuery.ts +++ b/packages/client/src/explorer/usePathsExplorerQuery.ts @@ -8,8 +8,6 @@ export function usePathsExplorerQuery(props: { order: FilePathOrder | null; enabled?: boolean; suspense?: boolean; - /** This callback will fire any time the query successfully fetches new data. (NOTE: This will be removed on the next major version (react-query)) */ - onSuccess?: () => void; }) { const query = usePathsOffsetInfiniteQuery(props); diff --git a/packages/client/src/explorer/usePathsInfiniteQuery.ts b/packages/client/src/explorer/usePathsInfiniteQuery.ts deleted file mode 100644 index c28a44b8a..000000000 --- a/packages/client/src/explorer/usePathsInfiniteQuery.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { useInfiniteQuery } from '@tanstack/react-query'; - -import { - ExplorerItem, - FilePathCursorVariant, - FilePathObjectCursor, - FilePathOrder, - FilePathSearchArgs -} from '../core'; -import { useLibraryContext } from '../hooks'; -import { useRspcLibraryContext } from '../rspc'; -import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; - -export function usePathsInfiniteQuery({ - arg, - order, - onSuccess, - ...args -}: UseExplorerInfiniteQueryArgs) { - const { library } = useLibraryContext(); - const ctx = useRspcLibraryContext(); - - if (order) { - arg.orderAndPagination = { orderOnly: order }; - if (arg.orderAndPagination.orderOnly.field === 'sizeInBytes') delete arg.take; - } - - const query = useInfiniteQuery({ - queryKey: ['search.paths', { library_id: library.uuid, arg }] as const, - queryFn: async ({ pageParam, queryKey: [_, { arg }] }) => { - const cItem: Extract = pageParam; - - let orderAndPagination: (typeof arg)['orderAndPagination']; - - if (!cItem) { - if (order) orderAndPagination = { orderOnly: order }; - } else { - let variant: FilePathCursorVariant | undefined; - - if (!order) variant = 'none'; - else if (cItem) { - switch (order.field) { - case 'name': { - const data = cItem.item.name; - if (data !== null) - variant = { - name: { order: order.value, data } - }; - break; - } - case 'sizeInBytes': { - variant = { sizeInBytes: order.value }; - break; - } - case 'dateCreated': { - const data = cItem.item.date_created; - if (data !== null) - variant = { - dateCreated: { order: order.value, data } - }; - break; - } - case 'dateModified': { - const data = cItem.item.date_modified; - if (data !== null) - variant = { - dateModified: { order: order.value, data } - }; - break; - } - case 'dateIndexed': { - const data = cItem.item.date_indexed; - if (data !== null) - variant = { - dateIndexed: { order: order.value, data } - }; - break; - } - case 'object': { - const object = cItem.item.object; - if (!object) break; - - let objectCursor: FilePathObjectCursor | undefined; - - switch (order.value.field) { - case 'dateAccessed': { - const data = object.date_accessed; - if (data !== null) - objectCursor = { - dateAccessed: { order: order.value.value, data } - }; - break; - } - case 'kind': { - const data = object.kind; - if (data !== null) - objectCursor = { - kind: { order: order.value.value, data } - }; - break; - } - } - - if (objectCursor) variant = { object: objectCursor }; - - break; - } - } - } - - if (cItem.item.is_dir === null) throw new Error(); - - if (variant) - orderAndPagination = { - cursor: { cursor: { variant, isDir: cItem.item.is_dir }, id: cItem.item.id } - }; - } - - arg.orderAndPagination = orderAndPagination; - - const result = await ctx.client.query(['search.paths', arg]); - return result; - }, - getNextPageParam: (lastPage) => { - if (arg.take === null || arg.take === undefined) return undefined; - if (lastPage.items.length < arg.take) return undefined; - else return lastPage.items[arg.take - 1]; - }, - onSuccess, - ...args - }); - - return query; -} diff --git a/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts b/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts index 52880b489..95d6fba3b 100644 --- a/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts +++ b/packages/client/src/explorer/usePathsOffsetInfiniteQuery.ts @@ -7,9 +7,7 @@ import { UseExplorerInfiniteQueryArgs } from './useExplorerInfiniteQuery'; export function usePathsOffsetInfiniteQuery({ arg, - order, - onSuccess, - ...args + order }: UseExplorerInfiniteQueryArgs) { const take = arg.take ?? 100; @@ -49,11 +47,10 @@ export function usePathsOffsetInfiniteQuery({ return { ...result, offset: pageParam, arg }; }, + initialPageParam: 0, getNextPageParam: ({ items, offset, arg }) => { if (items.length >= arg.take) return (offset ?? 0) + 1; - }, - onSuccess, - ...args + } }); return query; diff --git a/packages/client/src/hooks/useClientContext.tsx b/packages/client/src/hooks/useClientContext.tsx index 26c870c76..f66825a82 100644 --- a/packages/client/src/hooks/useClientContext.tsx +++ b/packages/client/src/hooks/useClientContext.tsx @@ -1,5 +1,6 @@ -import { AlphaClient } from '@oscartbeaumont-sd/rspc-client/src/v2'; -import { createContext, PropsWithChildren, useContext, useMemo } from 'react'; +import { AlphaClient } from '@spacedrive/rspc-client'; +import { keepPreviousData } from '@tanstack/react-query'; +import { createContext, PropsWithChildren, useContext, useEffect, useMemo } from 'react'; import { LibraryConfigWrapped, Procedures } from '../core'; import { valtioPersist } from '../lib'; @@ -9,8 +10,8 @@ import { useBridgeQuery } from '../rspc'; const libraryCacheLocalStorageKey = 'sd-library-list3'; // number is because the format of this underwent breaking changes export const useCachedLibraries = () => { - const result = useBridgeQuery(['library.list'], { - keepPreviousData: true, + const query = useBridgeQuery(['library.list'], { + placeholderData: keepPreviousData, initialData: () => { const cachedData = localStorage.getItem(libraryCacheLocalStorageKey); @@ -24,14 +25,15 @@ export const useCachedLibraries = () => { } return undefined; - }, - onSuccess: (data) => { - if (data.length > 0) - localStorage.setItem(libraryCacheLocalStorageKey, JSON.stringify(data)); } }); - return result; + useEffect(() => { + if ((query.data?.length ?? 0) > 0) + localStorage.setItem(libraryCacheLocalStorageKey, JSON.stringify(query.data)); + }, [query.data]); + + return query; }; export async function getCachedLibraries(client: AlphaClient) { diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 4a96e699f..dc0c5d41a 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,4 +1,4 @@ -import { Link } from '@oscartbeaumont-sd/rspc-client/src/v2'; +import { Link } from '@spacedrive/rspc-client'; declare global { // eslint-disable-next-line diff --git a/packages/client/src/lib/humanizeSize.ts b/packages/client/src/lib/humanizeSize.ts index 45a2cd167..654fe000f 100644 --- a/packages/client/src/lib/humanizeSize.ts +++ b/packages/client/src/lib/humanizeSize.ts @@ -146,3 +146,10 @@ export const humanizeSize = ( } }; }; + +export const compareHumanizedSizes = ( + size1: ReturnType, + size2: ReturnType +): boolean => { + return size1.bytes === size2.bytes && size1.unit === size2.unit && size1.value === size2.value; +}; diff --git a/packages/client/src/rspc-cursed.ts b/packages/client/src/rspc-cursed.ts index 774dcfcfb..ff1c026ef 100644 --- a/packages/client/src/rspc-cursed.ts +++ b/packages/client/src/rspc-cursed.ts @@ -1,5 +1,5 @@ -import { _inferProcedureHandlerInput, inferProcedureResult } from '@oscartbeaumont-sd/rspc-client'; -import { useQuery, UseQueryOptions, UseQueryResult } from '@tanstack/react-query'; +import { _inferProcedureHandlerInput, inferProcedureResult } from '@spacedrive/rspc-client'; +import { UseQueryOptions, useSuspenseQuery, UseSuspenseQueryResult } from '@tanstack/react-query'; import { useRef } from 'react'; import { Procedures } from './core'; @@ -26,19 +26,20 @@ export function useUnsafeStreamedQuery< TData = inferProcedureResult >( keyAndInput: [K, ..._inferProcedureHandlerInput], - opts: UseQueryOptions & { + opts: Omit, 'queryKey'> & { onBatch(item: TData): void; } -): UseQueryResult & { streaming: TData[] } { +): UseSuspenseQueryResult & { streaming: TData[] } { const data = useRef([]); const rspc = useRspcContext(); // TODO: The normalised cache might cleanup nodes for this query before it's finished streaming. We need a global mutex on the cleanup routine. - const query = useQuery({ + const query = useSuspenseQuery({ + ...opts, queryKey: keyAndInput, queryFn: ({ signal }) => - new Promise((resolve) => { + new Promise((resolve) => { permits += 1; try { @@ -48,7 +49,7 @@ export function useUnsafeStreamedQuery< if (item === null || item === undefined) return; if (typeof item === 'object' && '__stream_complete' in item) { - resolve(data.current as any); + resolve(data.current); return; } @@ -60,8 +61,7 @@ export function useUnsafeStreamedQuery< } finally { permits -= 1; } - }), - ...opts + }) }); return { diff --git a/packages/client/src/rspc.tsx b/packages/client/src/rspc.tsx index 40ec57782..579eacd7f 100644 --- a/packages/client/src/rspc.tsx +++ b/packages/client/src/rspc.tsx @@ -1,6 +1,5 @@ -import { ProcedureDef } from '@oscartbeaumont-sd/rspc-client'; -import { AlphaRSPCError, initRspc } from '@oscartbeaumont-sd/rspc-client/src/v2'; -import { Context, createReactQueryHooks } from '@oscartbeaumont-sd/rspc-react/src/v2'; +import { initRspc, ProcedureDef, RSPCError } from '@spacedrive/rspc-client'; +import { Context, createReactQueryHooks } from '@spacedrive/rspc-react/src/v2'; import { QueryClient } from '@tanstack/react-query'; import { createContext, PropsWithChildren, useContext } from 'react'; import { match, P } from 'ts-pattern'; @@ -102,7 +101,7 @@ export function useInvalidateQuery() { for (const op of ops) { match(op) .with({ type: 'single', data: P.select() }, (op) => { - let key: any[] = [op.key]; + let key: unknown[] = [op.key]; if (op.arg !== null) { key = key.concat(op.arg); } @@ -110,7 +109,7 @@ export function useInvalidateQuery() { if (op.result !== null) { context.queryClient.setQueryData(key, op.result); } else { - context.queryClient.invalidateQueries(key); + context.queryClient.invalidateQueries({ queryKey: key }); } }) .with({ type: 'all' }, (op) => { @@ -124,6 +123,6 @@ export function useInvalidateQuery() { // TODO: Remove/fix this when rspc typesafe errors are working export function extractInfoRSPCError(error: unknown) { - if (!(error instanceof AlphaRSPCError)) return null; + if (!(error instanceof RSPCError)) return null; return error; } diff --git a/packages/client/src/solid/index.ts b/packages/client/src/solid/index.ts index dad153583..3a7aaf513 100644 --- a/packages/client/src/solid/index.ts +++ b/packages/client/src/solid/index.ts @@ -2,7 +2,6 @@ export * from './createPersistedMutable'; export * from './react'; export * from './solid.solid'; export * from './useObserver'; -export * from './useUniversalQuery'; export * from './useSolidStore'; export { InteropProviderReact } from './portals'; export { createSharedContext } from './context'; diff --git a/packages/client/src/solid/solid.solid.tsx b/packages/client/src/solid/solid.solid.tsx index 8036b6e07..dfe24b907 100644 --- a/packages/client/src/solid/solid.solid.tsx +++ b/packages/client/src/solid/solid.solid.tsx @@ -1,5 +1,4 @@ /** @jsxImportSource solid-js */ - import { trackDeep } from '@solid-primitives/deep'; import { createElement, StrictMode, type FunctionComponent } from 'react'; import { createPortal } from 'react-dom'; diff --git a/packages/client/src/solid/useUniversalQuery.ts b/packages/client/src/solid/useUniversalQuery.ts deleted file mode 100644 index 2e9435448..000000000 --- a/packages/client/src/solid/useUniversalQuery.ts +++ /dev/null @@ -1,14 +0,0 @@ -// import { useQuery } from '@tanstack/react-query'; -// import { createQuery } from '@tanstack/solid-query'; - -// import { insideReactRender } from './internal'; - -// export function useUniversalQuery() { -// if (insideReactRender()) { -// useQuery(); -// } else { -// createQuery(); -// } -// } - -export {}; // TODO diff --git a/packages/client/src/stores/auth.ts b/packages/client/src/stores/auth.ts index 1d3b6719d..936988544 100644 --- a/packages/client/src/stores/auth.ts +++ b/packages/client/src/stores/auth.ts @@ -1,4 +1,4 @@ -import { RSPCError } from '@oscartbeaumont-sd/rspc-client'; +import { RSPCError } from '@spacedrive/rspc-client'; import { createMutable } from 'solid-js/store'; import { nonLibraryClient } from '../rspc'; diff --git a/packages/client/src/stores/debugState.ts b/packages/client/src/stores/debugState.ts index 23fd751f4..7bfc80855 100644 --- a/packages/client/src/stores/debugState.ts +++ b/packages/client/src/stores/debugState.ts @@ -6,7 +6,7 @@ import { createPersistedMutable, useSolidStore } from '../solid'; export interface DebugState { enabled: boolean; rspcLogger: boolean; - reactQueryDevtools: 'enabled' | 'disabled' | 'invisible'; + reactQueryDevtools: boolean; shareFullTelemetry: boolean; // used for sending telemetry even if the app is in debug mode telemetryLogging: boolean; } @@ -16,7 +16,7 @@ export const debugState = createPersistedMutable( createMutable({ enabled: globalThis.isDev, rspcLogger: false, - reactQueryDevtools: globalThis.isDev ? 'invisible' : 'enabled', + reactQueryDevtools: false, shareFullTelemetry: false, telemetryLogging: false }) diff --git a/packages/config/package.json b/packages/config/package.json index 90ee1a5fc..fab5fce96 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -11,6 +11,7 @@ "lint": "eslint . --cache" }, "devDependencies": { + "@babel/preset-typescript": "^7.24.0", "@typescript-eslint/eslint-plugin": "^8.8.0", "@typescript-eslint/parser": "^8.8.0", "@vitejs/plugin-react-swc": "^3.6.0", @@ -26,8 +27,8 @@ "eslint-utils": "^3.0.0", "regexpp": "^3.2.0", "vite-plugin-html": "^3.2.2", - "vite-plugin-i18next-loader": "^2.0.12", - "vite-plugin-inspect": "^0.8.3", + "vite-plugin-i18next-loader": "^2.0.14", + "vite-plugin-inspect": "^0.8.7", "vite-plugin-solid": "^2.10.2", "vite-plugin-svgr": "^3.3.0" }, diff --git a/packages/config/vite/narrowSolidPlugin.ts b/packages/config/vite/narrowSolidPlugin.ts index 425970ff5..1791ff0fd 100644 --- a/packages/config/vite/narrowSolidPlugin.ts +++ b/packages/config/vite/narrowSolidPlugin.ts @@ -13,10 +13,14 @@ export interface NarrowSolidPluginOptions extends Partial { export function narrowSolidPlugin({ include, exclude, ...rest }: NarrowSolidPluginOptions = {}) { const plugin = solidPlugin(rest); - const originalConfig = plugin.config!.bind(plugin); + const originalConfig = + typeof plugin.config == 'function' + ? (plugin.config.bind(plugin) as typeof plugin.config) + : plugin.config; const filter = createFilter(include, exclude); plugin.config = (...args) => { - const baseConfig = originalConfig(...args); + const baseConfig = + typeof originalConfig == 'function' ? originalConfig?.(...args) : originalConfig; return { ...baseConfig, esbuild: { diff --git a/packages/ui/package.json b/packages/ui/package.json index 89dee0358..60f785072 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -45,7 +45,7 @@ "react-router-dom": "=6.20.1", "sonner": "^1.0.3", "use-debounce": "^9.0.4", - "zod": "~3.22.4" + "zod": "^3.23" }, "devDependencies": { "@babel/core": "^7.24.0", diff --git a/patches/@react-navigation__drawer@6.6.15.patch b/patches/@react-navigation__drawer@6.6.15.patch index fa4c115a9..067d678a9 100644 --- a/patches/@react-navigation__drawer@6.6.15.patch +++ b/patches/@react-navigation__drawer@6.6.15.patch @@ -1,59 +1,21 @@ diff --git a/src/views/modern/Drawer.tsx b/src/views/modern/Drawer.tsx -index 9909e9698e51379de6469eb2053a1432636d0c7d..220fa07f6784c5da13e6949e9c4893e015a5d1f8 100644 +index 9909e96..a7dd9b7 100644 --- a/src/views/modern/Drawer.tsx +++ b/src/views/modern/Drawer.tsx -@@ -1,26 +1,27 @@ +@@ -1,5 +1,6 @@ import * as React from 'react'; import { -- I18nManager, -- InteractionManager, -- Keyboard, -- Platform, -- StatusBar, -- StyleSheet, -- View, -+ Dimensions, -+ I18nManager, -+ InteractionManager, -+ Keyboard, -+ Platform, -+ StatusBar, -+ StyleSheet, -+ View, - } from 'react-native'; - import { -- PanGestureHandler, -- PanGestureHandlerGestureEvent, -- State as GestureState, -+ PanGestureHandler, -+ PanGestureHandlerGestureEvent, -+ State as GestureState, - } from 'react-native-gesture-handler'; - import Animated, { -- interpolate, -- runOnJS, -- useAnimatedGestureHandler, -- useAnimatedStyle, -- useDerivedValue, -- useSharedValue, -- withSpring, -+ interpolate, -+ runOnJS, -+ useAnimatedGestureHandler, -+ useAnimatedStyle, -+ useDerivedValue, -+ useSharedValue, -+ withSpring, - } from 'react-native-reanimated'; - - import type { DrawerProps } from '../../types'; ++ Dimensions, + I18nManager, + InteractionManager, + Keyboard, @@ -72,7 +73,8 @@ export default function Drawer({ const percentage = Number(width.replace(/%$/, '')); if (Number.isFinite(percentage)) { - return dimensions.width * (percentage / 100); -+ const dimensionsWidth = Dimensions.get("screen").width -+ return dimensionsWidth * (percentage / 100); ++ const dimensionsWidth = Dimensions.get("screen").width ++ return dimensionsWidth * (percentage / 100); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 634052ff9..8fc70a91e 100644 Binary files a/pnpm-lock.yaml and b/pnpm-lock.yaml differ diff --git a/scripts/list-dup-deps.sh b/scripts/list-dup-deps.sh deleted file mode 100755 index 97c4023ff..000000000 --- a/scripts/list-dup-deps.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -_root="$(CDPATH='' cd "$(dirname "$0")" && pwd -P)" - -grep -Po '^\s+"[\w-]+\s+\d(\.\d+)*[^"]*"' "${_root}/../Cargo.lock" \ - | xargs printf '%s\n' \ - | sort -u -k 1b,2V diff --git a/scripts/preprep.mjs b/scripts/preprep.mjs index 7ffa7a06a..7e0799e03 100755 --- a/scripts/preprep.mjs +++ b/scripts/preprep.mjs @@ -1,5 +1,4 @@ #!/usr/bin/env node - import * as fs from 'node:fs/promises' import * as path from 'node:path' import { env, exit, umask } from 'node:process' @@ -9,11 +8,11 @@ import { extractTo } from 'archive-wasm/src/fs.mjs' import * as _mustache from 'mustache' import { parse as parseTOML } from 'smol-toml' -import { getConst, NATIVE_DEPS_URL, NATIVE_DEPS_ASSETS } from './utils/consts.mjs' +import { getConst, NATIVE_DEPS_ASSETS, NATIVE_DEPS_URL } from './utils/consts.mjs' import { get } from './utils/fetch.mjs' import { getMachineId } from './utils/machineId.mjs' import { getRustTargetList } from './utils/rustup.mjs' -import { symlinkSharedLibsMacOS, symlinkSharedLibsLinux } from './utils/shared.mjs' +import { symlinkSharedLibsLinux, symlinkSharedLibsMacOS } from './utils/shared.mjs' import { spinTask } from './utils/spinner.mjs' import { which } from './utils/which.mjs' diff --git a/scripts/tauri.mjs b/scripts/tauri.mjs index 0eed0d73f..46e5f5640 100755 --- a/scripts/tauri.mjs +++ b/scripts/tauri.mjs @@ -1,8 +1,7 @@ #!/usr/bin/env node - import * as fs from 'node:fs/promises' import * as path from 'node:path' -import { env, exit, umask, platform } from 'node:process' +import { env, exit, platform, umask } from 'node:process' import { setTimeout } from 'node:timers/promises' import { fileURLToPath } from 'node:url' diff --git a/scripts/utils/fetch.mjs b/scripts/utils/fetch.mjs index cf8cd6c41..5907e418e 100644 --- a/scripts/utils/fetch.mjs +++ b/scripts/utils/fetch.mjs @@ -4,7 +4,7 @@ import { env } from 'node:process' import { fileURLToPath } from 'node:url' import { getSystemProxy } from 'os-proxy-config' -import { fetch, Headers, Agent, ProxyAgent } from 'undici' +import { Agent, fetch, Headers, ProxyAgent } from 'undici' const CONNECT_TIMEOUT = 5 * 60 * 1000 const __debug = env.NODE_ENV === 'debug'