From 02f03f5351bf3476d1f5bca262ab3576dd52d44d Mon Sep 17 00:00:00 2001 From: Oscar Beaumont Date: Fri, 29 Sep 2023 14:58:29 +1000 Subject: [PATCH] Improve QuickPreview (#1350) * Handle large text files * wip * nit * Fix syntax highlighting * Requiring an API call for my dev builds, no shot * backend for line counting + wip frontend * 600 lines is too much for this file, ngl * wip: `LimitedByLinesBody` & some more restructuring * Virtualised list for QuickPreview * yeet bad ideas * general cleanup + hack to fix broken toml * fix --------- Co-authored-by: Utku <74243531+utkubakir@users.noreply.github.com> --- apps/desktop/package.json | 1 - apps/desktop/src/vite-env.d.ts | 1 - apps/desktop/vite.config.ts | 11 +- core/src/custom_uri.rs | 645 ------------------ core/src/custom_uri/async_read_body.rs | 61 ++ core/src/custom_uri/mod.rs | 422 ++++++++++++ core/src/custom_uri/mpsc_to_async_write.rs | 44 ++ core/src/custom_uri/serve_file.rs | 153 +++++ core/src/custom_uri/utils.rs | 74 ++ .../$libraryId/Explorer/FilePath/Thumb.tsx | 9 +- .../$libraryId/Explorer/Inspector/index.tsx | 1 + interface/components/TextViewer/index.tsx | 183 +++-- interface/components/TextViewer/prism.ts | 33 + interface/components/TextViewer/worker.ts | 45 -- interface/package.json | 5 +- interface/tsconfig.json | 2 +- packages/config/package.json | 1 - packages/config/vite/index.ts | 7 +- pnpm-lock.yaml | Bin 895962 -> 895387 bytes 19 files changed, 922 insertions(+), 776 deletions(-) delete mode 100644 core/src/custom_uri.rs create mode 100644 core/src/custom_uri/async_read_body.rs create mode 100644 core/src/custom_uri/mod.rs create mode 100644 core/src/custom_uri/mpsc_to_async_write.rs create mode 100644 core/src/custom_uri/serve_file.rs create mode 100644 core/src/custom_uri/utils.rs delete mode 100644 interface/components/TextViewer/worker.ts diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 90c0352a2..a74da72f4 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -22,7 +22,6 @@ "@sentry/vite-plugin": "^2.7.0", "@tanstack/react-query": "^4.24.4", "@tauri-apps/api": "1.3.0", - "comlink": "^4.4.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-router-dom": "6.9.0" diff --git a/apps/desktop/src/vite-env.d.ts b/apps/desktop/src/vite-env.d.ts index 7f26f947a..16334d7f0 100644 --- a/apps/desktop/src/vite-env.d.ts +++ b/apps/desktop/src/vite-env.d.ts @@ -1,5 +1,4 @@ /// -/// declare interface ImportMetaEnv { VITE_OS: string; diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index ded92f6a5..ed4884870 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -27,11 +27,12 @@ export default defineConfig(({ mode }) => { }, plugins: [ devtoolsPlugin, - sentryVitePlugin({ - authToken: process.env.SENTRY_AUTH_TOKEN, - org: 'spacedriveapp', - project: 'desktop' - }) + process.env.SENTRY_AUTH_TOKEN && + sentryVitePlugin({ + authToken: process.env.SENTRY_AUTH_TOKEN, + org: 'spacedriveapp', + project: 'desktop' + }) ] }); }); diff --git a/core/src/custom_uri.rs b/core/src/custom_uri.rs deleted file mode 100644 index eeefb14bf..000000000 --- a/core/src/custom_uri.rs +++ /dev/null @@ -1,645 +0,0 @@ -use crate::{ - location::file_path_helper::{file_path_to_handle_custom_uri, IsolatedFilePathData}, - p2p::{sync::InstanceState, IdentityOrRemoteIdentity}, - prisma::{file_path, location}, - util::{db::*, InfallibleResponse}, - Node, -}; - -use std::{ - cmp::min, - ffi::OsStr, - fmt::Debug, - fs::Metadata, - io::{self, SeekFrom}, - panic::Location, - path::{Path, PathBuf}, - pin::Pin, - str::FromStr, - sync::{atomic::Ordering, Arc}, - task::{Context, Poll}, - time::UNIX_EPOCH, -}; - -use async_stream::stream; -use axum::{ - body::{self, Body, BoxBody, Full, HttpBody, StreamBody}, - extract::{self, State}, - http::{self, header, request, HeaderMap, HeaderValue, Method, Request, Response, StatusCode}, - middleware::{self, Next}, - routing::get, - Router, -}; -use bytes::Bytes; -use futures::Stream; -use http_range::HttpRange; -use mini_moka::sync::Cache; -use pin_project_lite::pin_project; -use sd_file_ext::text::is_text; -use sd_p2p::{spaceblock::Range, spacetunnel::RemoteIdentity}; -use tokio::{ - fs::File, - io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, Take}, -}; -use tokio_util::{io::ReaderStream, sync::PollSender}; -use tracing::{debug, error}; -use uuid::Uuid; - -type CacheKey = (Uuid, file_path::id::Type); - -#[derive(Debug, Clone)] -struct CacheValue { - name: PathBuf, - ext: String, - file_path_pub_id: Uuid, - serve_from: ServeFrom, -} - -const MAX_TEXT_READ_LENGTH: usize = 10 * 1024; // 10KB - -// default capacity 64KiB -const DEFAULT_CAPACITY: usize = 65536; - -#[derive(Debug, Clone)] -pub enum ServeFrom { - /// Serve from the local filesystem - Local, - /// Serve from a specific instance - Remote(RemoteIdentity), -} - -#[derive(Clone)] -struct LocalState { - node: Arc, - - // This LRU cache allows us to avoid doing a DB lookup on every request. - // The main advantage of this LRU Cache is for video files. Video files are fetch in multiple chunks and the cache prevents a DB lookup on every chunk reducing the request time from 15-25ms to 1-10ms. - // TODO: We should listen to events when deleting or moving a location and evict the cache accordingly. - file_metadata_cache: Cache, -} - -// We are using Axum on all platforms because Tauri's custom URI protocols can't be async! -// TODO(@Oscar): Long-term hopefully this can be moved into rspc but streaming files is a hard thing for rspc to solve (Eg. how does batching work, dyn-safe handler, etc). -pub fn router(node: Arc) -> Router<()> { - Router::new() - .route( - "/thumbnail/*path", - get( - |State(state): State, - extract::Path(path): extract::Path, - request: Request| async move { - let thumbnail_path = state.node.config.data_directory().join("thumbnails"); - let path = thumbnail_path.join(path); - - // Prevent directory traversal attacks (Eg. requesting `../../../etc/passwd`) - // For now we only support `webp` thumbnails. - (path.starts_with(&thumbnail_path) && path.extension() == Some(OsStr::new("webp"))).then_some(()).ok_or_else(|| not_found(()))?; - - let file = File::open(&path).await.map_err(|err| { - InfallibleResponse::builder() - .status(if err.kind() == io::ErrorKind::NotFound { - StatusCode::NOT_FOUND - } else { - StatusCode::INTERNAL_SERVER_ERROR - }) - .body(body::boxed(Full::from(""))) - })?; - let metadata = file.metadata().await; - serve_file( - file, - metadata, - request.into_parts().0, - InfallibleResponse::builder().header("Content-Type", HeaderValue::from_static("image/webp")), - ) - .await - }, - ), - ) - .route( - "/file/:lib_id/:loc_id/:path_id", - get( - |State(state): State, - extract::Path((lib_id, loc_id, path_id)): extract::Path<( - String, - String, - String, - )>, - request: Request| async move { - let library_id = Uuid::from_str(&lib_id).map_err(bad_request)?; - let location_id = loc_id.parse::().map_err(bad_request)?; - let file_path_id = path_id.parse::().map_err(bad_request)?; - - let lru_cache_key = (library_id, file_path_id); - let library = state.node.libraries.get_library(&library_id).await.ok_or_else(|| internal_server_error(()))?; - - let CacheValue { name: file_path_full_path, ext: extension, file_path_pub_id, serve_from } = if let Some(entry) = - state.file_metadata_cache.get(&lru_cache_key) - { - entry - } else { - let file_path = library - .db - .file_path() - .find_unique(file_path::id::equals(file_path_id)) - // TODO: This query could be seen as a security issue as it could load the private key (`identity`) when we 100% don't need it. We are gonna wanna fix that! - .select(file_path_to_handle_custom_uri::select()) - .exec() - .await - .map_err(internal_server_error)? - .ok_or_else(|| not_found(()))?; - - let location = - maybe_missing(&file_path.location, "file_path.location").map_err(internal_server_error)?; - let path = - maybe_missing(&location.path, "file_path.location.path").map_err(internal_server_error)?; - let instance = - maybe_missing(&location.instance, "file_path.location.instance").map_err(internal_server_error)?; - - let path = Path::new(path).join( - IsolatedFilePathData::try_from((location_id, &file_path)).map_err(not_found)? - ); - - let identity = IdentityOrRemoteIdentity::from_bytes(&instance.identity).map_err(internal_server_error)?.remote_identity(); - let lru_entry = CacheValue { - name: path, - ext: maybe_missing(file_path.extension, "extension").map_err(not_found)?, - file_path_pub_id: Uuid::from_slice(&file_path.pub_id).map_err(internal_server_error)?, - serve_from: if identity == library.identity.to_remote_identity() { - ServeFrom::Local - } else { - ServeFrom::Remote(identity) - }, - }; - - state - .file_metadata_cache - .insert(lru_cache_key, lru_entry.clone()); - - lru_entry - }; - - match serve_from { - ServeFrom::Local => { - let metadata = file_path_full_path.metadata().map_err(internal_server_error)?; - (!metadata.is_dir()).then_some(()).ok_or_else(|| not_found(()))?; - - let mut file = File::open(&file_path_full_path).await.map_err(|err| { - InfallibleResponse::builder() - .status(if err.kind() == io::ErrorKind::NotFound { - StatusCode::NOT_FOUND - } else { - StatusCode::INTERNAL_SERVER_ERROR - }) - .body(body::boxed(Full::from(""))) - })?; - - let resp = InfallibleResponse::builder().header("Content-Type", HeaderValue::from_str(&plz_for_the_love_of_all_that_is_good_replace_this_with_the_db_instead_of_adding_variants_to_it(&extension, &mut file, &metadata).await?).map_err(|err| { - error!("Error converting mime-type into header value: {}", err); - internal_server_error(()) - })?); - - serve_file(file, Ok(metadata), request.into_parts().0, resp).await - } - ServeFrom::Remote(identity) => { - if !state.node.files_over_p2p_flag.load(Ordering::Relaxed) { - return Ok(not_found(())) - } - - // TODO: Support `Range` requests and `ETag` headers - #[allow(clippy::unwrap_used)] - match *state.node.nlm.state().await.get(&library_id).unwrap().instances.get(&identity).unwrap() { - InstanceState::Discovered(_) | InstanceState::Unavailable => Ok(not_found(())), - InstanceState::Connected(peer_id) => { - let (tx, mut rx) = tokio::sync::mpsc::channel::>(150); - // TODO: We only start a thread because of stupid `ManagerStreamAction2` and libp2p's `!Send/!Sync` bounds on a stream. - let node = state.node.clone(); - tokio::spawn(async move { - node.p2p - .request_file( - peer_id, - &library, - file_path_pub_id, - Range::Full, - MpscToAsyncWrite(PollSender::new(tx)), - ) - .await; - }); - - // TODO: Content Type - Ok(InfallibleResponse::builder() - .status(StatusCode::OK) - .body(body::boxed(StreamBody::new(stream! { - while let Some(item) = rx.recv().await { - yield item; - } - })))) - } - } - }, - } - }, - ), - ) - .route_layer(middleware::from_fn(cors_middleware)) - .with_state(LocalState { - node, - file_metadata_cache: Cache::new(100), - }) -} - -#[track_caller] -fn bad_request(err: impl Debug) -> http::Response { - debug!("400: Bad Request at {}: {err:?}", Location::caller()); - - InfallibleResponse::builder() - .status(StatusCode::BAD_REQUEST) - .body(body::boxed(Full::from(""))) -} - -#[track_caller] -fn not_found(err: impl Debug) -> http::Response { - debug!("404: Not Found at {}: {err:?}", Location::caller()); - - InfallibleResponse::builder() - .status(StatusCode::NOT_FOUND) - .body(body::boxed(Full::from(""))) -} - -#[track_caller] -fn internal_server_error(err: impl Debug) -> http::Response { - debug!( - "500 - Internal Server Error at {}: {err:?}", - Location::caller() - ); - - InfallibleResponse::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(body::boxed(Full::from(""))) -} - -async fn cors_middleware(req: Request, next: Next) -> Response { - if req.method() == Method::OPTIONS { - return Response::builder() - .header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS") - .header("Access-Control-Allow-Headers", "*") - .header("Access-Control-Max-Age", "86400") - .status(StatusCode::OK) - .body(body::boxed(Full::from(""))) - .expect("Invalid static response!"); - } - - let mut response = next.run(req).await; - - response - .headers_mut() - .insert("Access-Control-Allow-Origin", HeaderValue::from_static("*")); - - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection - response - .headers_mut() - .insert("Connection", HeaderValue::from_static("Keep-Alive")); - - response - .headers_mut() - .insert("Server", HeaderValue::from_static("Spacedrive")); - - response -} - -/// Serve a Tokio file as a HTTP response. -/// -/// This function takes care of: -/// - 304 Not Modified using ETag's -/// - Range requests for partial content -/// -/// BE AWARE this function does not do any path traversal protection so that's up to the caller! -async fn serve_file( - mut file: File, - metadata: io::Result, - req: request::Parts, - mut resp: InfallibleResponse, -) -> Result, Response> { - if let Ok(metadata) = metadata { - // We only accept range queries if `files.metadata() == Ok(_)` - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges - resp = resp - .header("Accept-Ranges", HeaderValue::from_static("bytes")) - .header( - "Content-Length", - HeaderValue::from_str(&metadata.len().to_string()) - .expect("number won't fail conversion"), - ); - - // Empty files - if metadata.len() == 0 { - return Ok(resp - .status(StatusCode::OK) - .header("Content-Length", HeaderValue::from_static("0")) - .body(body::boxed(Full::from("")))); - } - - // ETag - let mut status_code = StatusCode::PARTIAL_CONTENT; - if let Ok(time) = metadata.modified() { - let etag_header = format!( - r#""{}""#, - // The ETag's can be any value so we just use the modified time to make it easy. - time.duration_since(UNIX_EPOCH) - .expect("are you a time traveller? cause that's the only explanation for this error") - .as_millis() - ); - - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag - if let Ok(etag_header) = HeaderValue::from_str(&etag_header) { - resp = resp.header("etag", etag_header); - } else { - error!("Failed to convert ETag into header value!"); - } - - // Used for normal requests - if let Some(etag) = req.headers.get("If-None-Match") { - if etag.as_bytes() == etag_header.as_bytes() { - return Ok(resp - .status(StatusCode::NOT_MODIFIED) - .body(body::boxed(Full::from("")))); - } - } - - // Used checking if the resource has been modified since starting the download - if let Some(if_range) = req.headers.get("If-Range") { - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Range - if if_range.as_bytes() != etag_header.as_bytes() { - status_code = StatusCode::OK - } - } - }; - - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests - if req.method == Method::GET { - if let Some(range) = req.headers.get("range") { - // TODO: Error handling - let ranges = HttpRange::parse(range.to_str().map_err(bad_request)?, metadata.len()) - .map_err(bad_request)?; - - // TODO: Multipart requests are not support, yet - if ranges.len() != 1 { - return Ok(resp - .header( - header::CONTENT_RANGE, - HeaderValue::from_str(&format!("bytes */{}", metadata.len())) - .map_err(internal_server_error)?, - ) - .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(body::boxed(Full::from("")))); - } - let range = ranges.first().expect("checked above"); - - if (range.start + range.length) > metadata.len() { - return Ok(resp - .header( - header::CONTENT_RANGE, - HeaderValue::from_str(&format!("bytes */{}", metadata.len())) - .map_err(internal_server_error)?, - ) - .status(StatusCode::RANGE_NOT_SATISFIABLE) - .body(body::boxed(Full::from("")))); - } - - file.seek(SeekFrom::Start(range.start)) - .await - .map_err(internal_server_error)?; - - return Ok(resp - .status(status_code) - .header( - "Content-Range", - HeaderValue::from_str(&format!( - "bytes {}-{}/{}", - range.start, - range.start + range.length - 1, - metadata.len() - )) - .map_err(internal_server_error)?, - ) - .header( - "Content-Length", - HeaderValue::from_str(&range.length.to_string()) - .map_err(internal_server_error)?, - ) - .body(body::boxed(AsyncReadBody::with_capacity_limited( - file, - DEFAULT_CAPACITY, - range.length, - )))); - } - } - } - - Ok(resp.body(body::boxed(StreamBody::new(ReaderStream::new(file))))) -} - -// TODO: This should be determined from magic bytes when the file is indexed and stored it in the DB on the file path -async fn plz_for_the_love_of_all_that_is_good_replace_this_with_the_db_instead_of_adding_variants_to_it( - ext: &str, - file: &mut File, - metadata: &Metadata, -) -> Result> { - let mime_type = match ext { - // AAC audio - "aac" => "audio/aac", - // Musical Instrument Digital Interface (MIDI) - "mid" | "midi" => "audio/midi, audio/x-midi", - // MP3 audio - "mp3" => "audio/mpeg", - // MP4 audio - "m4a" => "audio/mp4", - // OGG audio - "oga" => "audio/ogg", - // Opus audio - "opus" => "audio/opus", - // Waveform Audio Format - "wav" => "audio/wav", - // WEBM audio - "weba" => "audio/webm", - // AVI: Audio Video Interleave - "avi" => "video/x-msvideo", - // MP4 video - "mp4" | "m4v" => "video/mp4", - // TODO: Bruh - #[cfg(not(target_os = "macos"))] - // TODO: Bruh - // FIX-ME: This media types break macOS video rendering - // MPEG transport stream - "ts" => "video/mp2t", - // TODO: Bruh - #[cfg(not(target_os = "macos"))] - // FIX-ME: This media types break macOS video rendering - // MPEG Video - "mpeg" => "video/mpeg", - // OGG video - "ogv" => "video/ogg", - // WEBM video - "webm" => "video/webm", - // 3GPP audio/video container (TODO: audio/3gpp if it doesn't contain video) - "3gp" => "video/3gpp", - // 3GPP2 audio/video container (TODO: audio/3gpp2 if it doesn't contain video) - "3g2" => "video/3gpp2", - // Quicktime movies - "mov" => "video/quicktime", - // Windows OS/2 Bitmap Graphics - "bmp" => "image/bmp", - // Graphics Interchange Format (GIF) - "gif" => "image/gif", - // Icon format - "ico" => "image/vnd.microsoft.icon", - // JPEG images - "jpeg" | "jpg" => "image/jpeg", - // Portable Network Graphics - "png" => "image/png", - // Scalable Vector Graphics (SVG) - "svg" => "image/svg+xml", - // Tagged Image File Format (TIFF) - "tif" | "tiff" => "image/tiff", - // WEBP image - "webp" => "image/webp", - // PDF document - "pdf" => "application/pdf", - // HEIF/HEIC images - "heif" | "heifs" => "image/heif,image/heif-sequence", - "heic" | "heics" => "image/heic,image/heic-sequence", - // AVIF images - "avif" | "avci" | "avcs" => "image/avif", - _ => "text/plain", - }; - - Ok(if mime_type == "text/plain" { - let mut text_buf = vec![ - 0; - min( - metadata.len().try_into().unwrap_or(usize::MAX), - MAX_TEXT_READ_LENGTH - ) - ]; - if !text_buf.is_empty() { - file.read_exact(&mut text_buf) - .await - .map_err(internal_server_error)?; - file.seek(SeekFrom::Start(0)) - .await - .map_err(internal_server_error)?; - } - - let charset = is_text(&text_buf, text_buf.len() == (metadata.len() as usize)).unwrap_or(""); - - // Only browser recognized types, everything else should be text/plain - // https://www.iana.org/assignments/media-types/media-types.xhtml#table-text - let mime_type = match ext { - // HyperText Markup Language - "html" | "htm" => "text/html", - // Cascading Style Sheets - "css" => "text/css", - // Javascript - "js" | "mjs" => "text/javascript", - // Comma-separated values - "csv" => "text/csv", - // Markdown - "md" | "markdown" => "text/markdown", - // Rich text format - "rtf" => "text/rtf", - // Web Video Text Tracks - "vtt" => "text/vtt", - // Extensible Markup Language - "xml" => "text/xml", - // Text - "txt" => "text/plain", - _ => { - if charset.is_empty() { - todo!(); - // "TODO: This filetype is not supported because of the missing mime type!", - }; - mime_type - } - }; - - format!("{mime_type}; charset={charset}") - } else { - mime_type.to_string() - }) -} - -// This code was taken from: https://github.com/tower-rs/tower-http/blob/e8eb54966604ea7fa574a2a25e55232f5cfe675b/tower-http/src/services/fs/mod.rs#L30 -pin_project! { - // NOTE: This could potentially be upstreamed to `http-body`. - /// Adapter that turns an [`impl AsyncRead`][tokio::io::AsyncRead] to an [`impl Body`][http_body::Body]. - #[derive(Debug)] - pub struct AsyncReadBody { - #[pin] - reader: ReaderStream, - } -} - -impl AsyncReadBody -where - T: AsyncRead, -{ - fn with_capacity_limited( - read: T, - capacity: usize, - max_read_bytes: u64, - ) -> AsyncReadBody> { - AsyncReadBody { - reader: ReaderStream::with_capacity(read.take(max_read_bytes), capacity), - } - } -} - -impl HttpBody for AsyncReadBody -where - T: AsyncRead, -{ - type Data = Bytes; - type Error = io::Error; - - fn poll_data( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - self.project().reader.poll_next(cx) - } - - fn poll_trailers( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - ) -> Poll, Self::Error>> { - Poll::Ready(Ok(None)) - } -} - -/// Allowing wrapping an `mpsc::Sender` into an `AsyncWrite` -pub struct MpscToAsyncWrite(PollSender>); - -impl AsyncWrite for MpscToAsyncWrite { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - #[allow(clippy::unwrap_used)] - match self.0.poll_reserve(cx) { - Poll::Ready(Ok(())) => { - self.0.send_item(Ok(Bytes::from(buf.to_vec()))).unwrap(); - Poll::Ready(Ok(buf.len())) - } - Poll::Ready(Err(_)) => todo!(), - Poll::Pending => Poll::Pending, - } - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } -} diff --git a/core/src/custom_uri/async_read_body.rs b/core/src/custom_uri/async_read_body.rs new file mode 100644 index 000000000..1a1cc523a --- /dev/null +++ b/core/src/custom_uri/async_read_body.rs @@ -0,0 +1,61 @@ +use std::{ + io, + pin::Pin, + task::{Context, Poll}, +}; + +use axum::http::HeaderMap; +use bytes::Bytes; +use futures::Stream; +use http_body::Body; +use pin_project_lite::pin_project; +use tokio::io::{AsyncRead, AsyncReadExt, Take}; +use tokio_util::io::ReaderStream; + +// This code was taken from: https://github.com/tower-rs/tower-http/blob/e8eb54966604ea7fa574a2a25e55232f5cfe675b/tower-http/src/services/fs/mod.rs#L30 +pin_project! { + // NOTE: This could potentially be upstreamed to `http-body`. + /// Adapter that turns an [`impl AsyncRead`][tokio::io::AsyncRead] to an [`impl Body`][http_body::Body]. + #[derive(Debug)] + pub struct AsyncReadBody { + #[pin] + reader: ReaderStream, + } +} + +impl AsyncReadBody +where + T: AsyncRead, +{ + pub(crate) fn with_capacity_limited( + read: T, + capacity: usize, + max_read_bytes: u64, + ) -> AsyncReadBody> { + AsyncReadBody { + reader: ReaderStream::with_capacity(read.take(max_read_bytes), capacity), + } + } +} + +impl Body for AsyncReadBody +where + T: AsyncRead, +{ + type Data = Bytes; + type Error = io::Error; + + fn poll_data( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + self.project().reader.poll_next(cx) + } + + fn poll_trailers( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll, Self::Error>> { + Poll::Ready(Ok(None)) + } +} diff --git a/core/src/custom_uri/mod.rs b/core/src/custom_uri/mod.rs new file mode 100644 index 000000000..cd9921543 --- /dev/null +++ b/core/src/custom_uri/mod.rs @@ -0,0 +1,422 @@ +use crate::{ + library::Library, + location::file_path_helper::{file_path_to_handle_custom_uri, IsolatedFilePathData}, + p2p::{sync::InstanceState, IdentityOrRemoteIdentity}, + prisma::{file_path, location}, + util::{db::*, InfallibleResponse}, + Node, +}; + +use std::{ + cmp::min, + ffi::OsStr, + fmt::Debug, + fs::Metadata, + io::{self, SeekFrom}, + path::{Path, PathBuf}, + str::FromStr, + sync::{atomic::Ordering, Arc}, +}; + +use async_stream::stream; +use axum::{ + body::{self, Body, BoxBody, Full, StreamBody}, + extract::{self, State}, + http::{HeaderValue, Request, Response, StatusCode}, + middleware, + routing::get, + Router, +}; +use bytes::Bytes; + +use mini_moka::sync::Cache; +use sd_file_ext::text::is_text; +use sd_p2p::{spaceblock::Range, spacetunnel::RemoteIdentity}; +use tokio::{ + fs::File, + io::{AsyncReadExt, AsyncSeekExt}, +}; +use tokio_util::sync::PollSender; +use tracing::error; +use uuid::Uuid; + +use self::{mpsc_to_async_write::MpscToAsyncWrite, serve_file::serve_file, utils::*}; + +mod async_read_body; +mod mpsc_to_async_write; +mod serve_file; +mod utils; + +type CacheKey = (Uuid, file_path::id::Type); + +#[derive(Debug, Clone)] +struct CacheValue { + name: PathBuf, + ext: String, + file_path_pub_id: Uuid, + serve_from: ServeFrom, +} + +const MAX_TEXT_READ_LENGTH: usize = 10 * 1024; // 10KB + +#[derive(Debug, Clone)] +pub enum ServeFrom { + /// Serve from the local filesystem + Local, + /// Serve from a specific instance + Remote(RemoteIdentity), +} + +#[derive(Clone)] +struct LocalState { + node: Arc, + + // This LRU cache allows us to avoid doing a DB lookup on every request. + // The main advantage of this LRU Cache is for video files. Video files are fetch in multiple chunks and the cache prevents a DB lookup on every chunk reducing the request time from 15-25ms to 1-10ms. + // TODO: We should listen to events when deleting or moving a location and evict the cache accordingly. + file_metadata_cache: Cache, +} + +type ExtractedPath = extract::Path<(String, String, String)>; + +async fn get_or_init_lru_entry( + state: &LocalState, + extract::Path((lib_id, loc_id, path_id)): ExtractedPath, +) -> Result<(CacheValue, Arc), Response> { + let library_id = Uuid::from_str(&lib_id).map_err(bad_request)?; + let location_id = loc_id.parse::().map_err(bad_request)?; + let file_path_id = path_id + .parse::() + .map_err(bad_request)?; + + let lru_cache_key = (library_id, file_path_id); + let library = state + .node + .libraries + .get_library(&library_id) + .await + .ok_or_else(|| internal_server_error(()))?; + + if let Some(entry) = state.file_metadata_cache.get(&lru_cache_key) { + Ok((entry, library)) + } else { + let file_path = library + .db + .file_path() + .find_unique(file_path::id::equals(file_path_id)) + // TODO: This query could be seen as a security issue as it could load the private key (`identity`) when we 100% don't need it. We are gonna wanna fix that! + .select(file_path_to_handle_custom_uri::select()) + .exec() + .await + .map_err(internal_server_error)? + .ok_or_else(|| not_found(()))?; + + let location = maybe_missing(&file_path.location, "file_path.location") + .map_err(internal_server_error)?; + let path = maybe_missing(&location.path, "file_path.location.path") + .map_err(internal_server_error)?; + let instance = maybe_missing(&location.instance, "file_path.location.instance") + .map_err(internal_server_error)?; + + let path = Path::new(path) + .join(IsolatedFilePathData::try_from((location_id, &file_path)).map_err(not_found)?); + + let identity = IdentityOrRemoteIdentity::from_bytes(&instance.identity) + .map_err(internal_server_error)? + .remote_identity(); + + let lru_entry = CacheValue { + name: path, + ext: maybe_missing(file_path.extension, "extension").map_err(not_found)?, + file_path_pub_id: Uuid::from_slice(&file_path.pub_id).map_err(internal_server_error)?, + serve_from: if identity == library.identity.to_remote_identity() { + ServeFrom::Local + } else { + ServeFrom::Remote(identity) + }, + }; + + state + .file_metadata_cache + .insert(lru_cache_key, lru_entry.clone()); + + Ok((lru_entry, library)) + } +} + +// We are using Axum on all platforms because Tauri's custom URI protocols can't be async! +pub fn router(node: Arc) -> Router<()> { + Router::new() + .route( + "/thumbnail/*path", + get( + |State(state): State, + extract::Path(path): extract::Path, + request: Request| async move { + let thumbnail_path = state.node.config.data_directory().join("thumbnails"); + let path = thumbnail_path.join(path); + + // Prevent directory traversal attacks (Eg. requesting `../../../etc/passwd`) + // For now we only support `webp` thumbnails. + (path.starts_with(&thumbnail_path) + && path.extension() == Some(OsStr::new("webp"))) + .then_some(()) + .ok_or_else(|| not_found(()))?; + + let file = File::open(&path).await.map_err(|err| { + InfallibleResponse::builder() + .status(if err.kind() == io::ErrorKind::NotFound { + StatusCode::NOT_FOUND + } else { + StatusCode::INTERNAL_SERVER_ERROR + }) + .body(body::boxed(Full::from(""))) + })?; + let metadata = file.metadata().await; + serve_file( + file, + metadata, + request.into_parts().0, + InfallibleResponse::builder() + .header("Content-Type", HeaderValue::from_static("image/webp")), + ) + .await + }, + ), + ) + .route( + "/file/:lib_id/:loc_id/:path_id", + get( + |State(state): State, path: ExtractedPath, request: Request| async move { + let ( + CacheValue { + name: file_path_full_path, + ext: extension, + file_path_pub_id, + serve_from, + .. + }, + library, + ) = get_or_init_lru_entry(&state, path).await?; + + match serve_from { + ServeFrom::Local => { + let metadata = file_path_full_path + .metadata() + .map_err(internal_server_error)?; + (!metadata.is_dir()) + .then_some(()) + .ok_or_else(|| not_found(()))?; + + let mut file = + File::open(&file_path_full_path).await.map_err(|err| { + InfallibleResponse::builder() + .status(if err.kind() == io::ErrorKind::NotFound { + StatusCode::NOT_FOUND + } else { + StatusCode::INTERNAL_SERVER_ERROR + }) + .body(body::boxed(Full::from(""))) + })?; + + let resp = InfallibleResponse::builder().header( + "Content-Type", + HeaderValue::from_str( + &infer_the_mime_type(&extension, &mut file, &metadata).await?, + ) + .map_err(|err| { + error!("Error converting mime-type into header value: {}", err); + internal_server_error(()) + })?, + ); + + serve_file(file, Ok(metadata), request.into_parts().0, resp).await + } + ServeFrom::Remote(identity) => { + if !state.node.files_over_p2p_flag.load(Ordering::Relaxed) { + return Ok(not_found(())); + } + + // TODO: Support `Range` requests and `ETag` headers + #[allow(clippy::unwrap_used)] + match *state + .node + .nlm + .state() + .await + .get(&library.id) + .unwrap() + .instances + .get(&identity) + .unwrap() + { + InstanceState::Discovered(_) | InstanceState::Unavailable => { + Ok(not_found(())) + } + InstanceState::Connected(peer_id) => { + let (tx, mut rx) = + tokio::sync::mpsc::channel::>(150); + // TODO: We only start a thread because of stupid `ManagerStreamAction2` and libp2p's `!Send/!Sync` bounds on a stream. + let node = state.node.clone(); + tokio::spawn(async move { + node.p2p + .request_file( + peer_id, + &library, + file_path_pub_id, + Range::Full, + MpscToAsyncWrite::new(PollSender::new(tx)), + ) + .await; + }); + + // TODO: Content Type + Ok(InfallibleResponse::builder().status(StatusCode::OK).body( + body::boxed(StreamBody::new(stream! { + while let Some(item) = rx.recv().await { + yield item; + } + })), + )) + } + } + } + } + }, + ), + ) + .route_layer(middleware::from_fn(cors_middleware)) + .with_state(LocalState { + node, + file_metadata_cache: Cache::new(150), + }) +} + +// TODO: This should possibly be determined from magic bytes when the file is indexed and stored it in the DB on the file path +async fn infer_the_mime_type( + ext: &str, + file: &mut File, + metadata: &Metadata, +) -> Result> { + let mime_type = match ext { + // AAC audio + "aac" => "audio/aac", + // Musical Instrument Digital Interface (MIDI) + "mid" | "midi" => "audio/midi, audio/x-midi", + // MP3 audio + "mp3" => "audio/mpeg", + // MP4 audio + "m4a" => "audio/mp4", + // OGG audio + "oga" => "audio/ogg", + // Opus audio + "opus" => "audio/opus", + // Waveform Audio Format + "wav" => "audio/wav", + // WEBM audio + "weba" => "audio/webm", + // AVI: Audio Video Interleave + "avi" => "video/x-msvideo", + // MP4 video + "mp4" | "m4v" => "video/mp4", + // TODO: Bruh + #[cfg(not(target_os = "macos"))] + // TODO: Bruh + // FIX-ME: This media types break macOS video rendering + // MPEG transport stream + "ts" => "video/mp2t", + // TODO: Bruh + #[cfg(not(target_os = "macos"))] + // FIX-ME: This media types break macOS video rendering + // MPEG Video + "mpeg" => "video/mpeg", + // OGG video + "ogv" => "video/ogg", + // WEBM video + "webm" => "video/webm", + // 3GPP audio/video container (TODO: audio/3gpp if it doesn't contain video) + "3gp" => "video/3gpp", + // 3GPP2 audio/video container (TODO: audio/3gpp2 if it doesn't contain video) + "3g2" => "video/3gpp2", + // Quicktime movies + "mov" => "video/quicktime", + // Windows OS/2 Bitmap Graphics + "bmp" => "image/bmp", + // Graphics Interchange Format (GIF) + "gif" => "image/gif", + // Icon format + "ico" => "image/vnd.microsoft.icon", + // JPEG images + "jpeg" | "jpg" => "image/jpeg", + // Portable Network Graphics + "png" => "image/png", + // Scalable Vector Graphics (SVG) + "svg" => "image/svg+xml", + // Tagged Image File Format (TIFF) + "tif" | "tiff" => "image/tiff", + // WEBP image + "webp" => "image/webp", + // PDF document + "pdf" => "application/pdf", + // HEIF/HEIC images + "heif" | "heifs" => "image/heif,image/heif-sequence", + "heic" | "heics" => "image/heic,image/heic-sequence", + // AVIF images + "avif" | "avci" | "avcs" => "image/avif", + _ => "text/plain", + }; + + Ok(if mime_type == "text/plain" { + let mut text_buf = vec![ + 0; + min( + metadata.len().try_into().unwrap_or(usize::MAX), + MAX_TEXT_READ_LENGTH + ) + ]; + if !text_buf.is_empty() { + file.read_exact(&mut text_buf) + .await + .map_err(internal_server_error)?; + file.seek(SeekFrom::Start(0)) + .await + .map_err(internal_server_error)?; + } + + let charset = is_text(&text_buf, text_buf.len() == (metadata.len() as usize)).unwrap_or(""); + + // Only browser recognized types, everything else should be text/plain + // https://www.iana.org/assignments/media-types/media-types.xhtml#table-text + let mime_type = match ext { + // HyperText Markup Language + "html" | "htm" => "text/html", + // Cascading Style Sheets + "css" => "text/css", + // Javascript + "js" | "mjs" => "text/javascript", + // Comma-separated values + "csv" => "text/csv", + // Markdown + "md" | "markdown" => "text/markdown", + // Rich text format + "rtf" => "text/rtf", + // Web Video Text Tracks + "vtt" => "text/vtt", + // Extensible Markup Language + "xml" => "text/xml", + // Text + "txt" => "text/plain", + _ => { + if charset.is_empty() { + todo!(); + // "TODO: This filetype is not supported because of the missing mime type!", + }; + mime_type + } + }; + + format!("{mime_type}; charset={charset}") + } else { + mime_type.to_string() + }) +} diff --git a/core/src/custom_uri/mpsc_to_async_write.rs b/core/src/custom_uri/mpsc_to_async_write.rs new file mode 100644 index 000000000..889ba185e --- /dev/null +++ b/core/src/custom_uri/mpsc_to_async_write.rs @@ -0,0 +1,44 @@ +use std::{ + io, + pin::Pin, + task::{Context, Poll}, +}; + +use bytes::Bytes; +use tokio::io::AsyncWrite; +use tokio_util::sync::PollSender; + +/// Allowing wrapping an `mpsc::Sender` into an `AsyncWrite` +pub struct MpscToAsyncWrite(PollSender>); + +impl MpscToAsyncWrite { + pub fn new(sender: PollSender>) -> Self { + Self(sender) + } +} + +impl AsyncWrite for MpscToAsyncWrite { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + #[allow(clippy::unwrap_used)] + match self.0.poll_reserve(cx) { + Poll::Ready(Ok(())) => { + self.0.send_item(Ok(Bytes::from(buf.to_vec()))).unwrap(); + Poll::Ready(Ok(buf.len())) + } + Poll::Ready(Err(_)) => todo!(), + Poll::Pending => Poll::Pending, + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} diff --git a/core/src/custom_uri/serve_file.rs b/core/src/custom_uri/serve_file.rs new file mode 100644 index 000000000..0b86a5f20 --- /dev/null +++ b/core/src/custom_uri/serve_file.rs @@ -0,0 +1,153 @@ +use crate::util::InfallibleResponse; + +use std::{ + fs::Metadata, + io::{self, SeekFrom}, + time::UNIX_EPOCH, +}; + +use axum::{ + body::{self, BoxBody, Full, StreamBody}, + http::{header, request, HeaderValue, Method, Response, StatusCode}, +}; +use http_range::HttpRange; +use tokio::{fs::File, io::AsyncSeekExt}; +use tokio_util::io::ReaderStream; +use tracing::error; + +use super::{async_read_body::AsyncReadBody, utils::*}; + +// default capacity 64KiB +const DEFAULT_CAPACITY: usize = 65536; + +/// Serve a Tokio file as a HTTP response. +/// +/// This function takes care of: +/// - 304 Not Modified using ETag's +/// - Range requests for partial content +/// +/// BE AWARE this function does not do any path traversal protection so that's up to the caller! +pub(crate) async fn serve_file( + mut file: File, + metadata: io::Result, + req: request::Parts, + mut resp: InfallibleResponse, +) -> Result, Response> { + if let Ok(metadata) = metadata { + // We only accept range queries if `files.metadata() == Ok(_)` + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Ranges + resp = resp + .header("Accept-Ranges", HeaderValue::from_static("bytes")) + .header( + "Content-Length", + HeaderValue::from_str(&metadata.len().to_string()) + .expect("number won't fail conversion"), + ); + + // Empty files + if metadata.len() == 0 { + return Ok(resp + .status(StatusCode::OK) + .header("Content-Length", HeaderValue::from_static("0")) + .body(body::boxed(Full::from("")))); + } + + // ETag + let mut status_code = StatusCode::PARTIAL_CONTENT; + if let Ok(time) = metadata.modified() { + let etag_header = format!( + r#""{}""#, + // The ETag's can be any value so we just use the modified time to make it easy. + time.duration_since(UNIX_EPOCH) + .expect("are you a time traveller? cause that's the only explanation for this error") + .as_millis() + ); + + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag + if let Ok(etag_header) = HeaderValue::from_str(&etag_header) { + resp = resp.header("etag", etag_header); + } else { + error!("Failed to convert ETag into header value!"); + } + + // Used for normal requests + if let Some(etag) = req.headers.get("If-None-Match") { + if etag.as_bytes() == etag_header.as_bytes() { + return Ok(resp + .status(StatusCode::NOT_MODIFIED) + .body(body::boxed(Full::from("")))); + } + } + + // Used checking if the resource has been modified since starting the download + if let Some(if_range) = req.headers.get("If-Range") { + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Range + if if_range.as_bytes() != etag_header.as_bytes() { + status_code = StatusCode::OK + } + } + }; + + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests + if req.method == Method::GET { + if let Some(range) = req.headers.get("range") { + // TODO: Error handling + let ranges = HttpRange::parse(range.to_str().map_err(bad_request)?, metadata.len()) + .map_err(bad_request)?; + + // TODO: Multipart requests are not support, yet + if ranges.len() != 1 { + return Ok(resp + .header( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes */{}", metadata.len())) + .map_err(internal_server_error)?, + ) + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .body(body::boxed(Full::from("")))); + } + let range = ranges.first().expect("checked above"); + + if (range.start + range.length) > metadata.len() { + return Ok(resp + .header( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes */{}", metadata.len())) + .map_err(internal_server_error)?, + ) + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .body(body::boxed(Full::from("")))); + } + + file.seek(SeekFrom::Start(range.start)) + .await + .map_err(internal_server_error)?; + + return Ok(resp + .status(status_code) + .header( + "Content-Range", + HeaderValue::from_str(&format!( + "bytes {}-{}/{}", + range.start, + range.start + range.length - 1, + metadata.len() + )) + .map_err(internal_server_error)?, + ) + .header( + "Content-Length", + HeaderValue::from_str(&range.length.to_string()) + .map_err(internal_server_error)?, + ) + .body(body::boxed(AsyncReadBody::with_capacity_limited( + file, + DEFAULT_CAPACITY, + range.length, + )))); + } + } + } + + Ok(resp.body(body::boxed(StreamBody::new(ReaderStream::new(file))))) +} diff --git a/core/src/custom_uri/utils.rs b/core/src/custom_uri/utils.rs new file mode 100644 index 000000000..6b5401c11 --- /dev/null +++ b/core/src/custom_uri/utils.rs @@ -0,0 +1,74 @@ +use std::{fmt::Debug, panic::Location}; + +use axum::{ + body::{self, BoxBody}, + http::{self, HeaderValue, Method, Request, Response, StatusCode}, + middleware::Next, +}; +use http_body::Full; +use tracing::debug; + +use crate::util::InfallibleResponse; + +#[track_caller] +pub(crate) fn bad_request(err: impl Debug) -> http::Response { + debug!("400: Bad Request at {}: {err:?}", Location::caller()); + + InfallibleResponse::builder() + .status(StatusCode::BAD_REQUEST) + .body(body::boxed(Full::from(""))) +} + +#[track_caller] +pub(crate) fn not_found(err: impl Debug) -> http::Response { + debug!("404: Not Found at {}: {err:?}", Location::caller()); + + InfallibleResponse::builder() + .status(StatusCode::NOT_FOUND) + .body(body::boxed(Full::from(""))) +} + +#[track_caller] +pub(crate) fn internal_server_error(err: impl Debug) -> http::Response { + debug!( + "500 - Internal Server Error at {}: {err:?}", + Location::caller() + ); + + InfallibleResponse::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(body::boxed(Full::from(""))) +} + +pub(crate) async fn cors_middleware(req: Request, next: Next) -> Response { + if req.method() == Method::OPTIONS { + return Response::builder() + .header("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS") + .header("Access-Control-Allow-Origin", "*") + .header("Access-Control-Allow-Headers", "*") + .header("Access-Control-Max-Age", "86400") + .status(StatusCode::OK) + .body(body::boxed(Full::from(""))) + .expect("Invalid static response!"); + } + + let mut response = next.run(req).await; + + { + let headers = response.headers_mut(); + + headers.insert("Access-Control-Allow-Origin", HeaderValue::from_static("*")); + + headers.insert( + "Access-Control-Allow-Headers", + HeaderValue::from_static("*"), + ); + + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection + headers.insert("Connection", HeaderValue::from_static("Keep-Alive")); + + headers.insert("Server", HeaderValue::from_static("Spacedrive")); + } + + response +} diff --git a/interface/app/$libraryId/Explorer/FilePath/Thumb.tsx b/interface/app/$libraryId/Explorer/FilePath/Thumb.tsx index 8e49083db..9e6c5214d 100644 --- a/interface/app/$libraryId/Explorer/FilePath/Thumb.tsx +++ b/interface/app/$libraryId/Explorer/FilePath/Thumb.tsx @@ -2,6 +2,7 @@ import { getIcon, iconNames } from '@sd/assets/util'; import clsx from 'clsx'; import { memo, + useCallback, useEffect, useLayoutEffect, useMemo, @@ -44,6 +45,7 @@ export interface ThumbProps { className?: string; frameClassName?: string; childClassName?: string | ((type: ThumbType | `${ThumbType}`) => string | undefined); + isSidebarPreview?: boolean; } export const FileThumb = memo((props: ThumbProps) => { @@ -67,16 +69,16 @@ export const FileThumb = memo((props: ThumbProps) => { isDark ? classes.checkers : classes.checkersLight ); - const onLoad = () => setLoaded(true); + const onLoad = useCallback(() => setLoaded(true), []); - const onError = () => { + const onError = useCallback(() => { setLoaded(false); setThumbType((prevThumbType) => prevThumbType === ThumbType.Original && itemData.hasLocalThumbnail ? ThumbType.Thumbnail : ThumbType.Icon ); - }; + }, [itemData.hasLocalThumbnail]); // useLayoutEffect is required to ensure the thumbType is always updated before the onError listener can execute, // thus avoiding improper thumb types changes @@ -197,6 +199,7 @@ export const FileThumb = memo((props: ThumbProps) => { itemData.extension) || '' } + isSidebarPreview={props.isSidebarPreview} /> ); diff --git a/interface/app/$libraryId/Explorer/Inspector/index.tsx b/interface/app/$libraryId/Explorer/Inspector/index.tsx index 2e65f3169..8e92b5424 100644 --- a/interface/app/$libraryId/Explorer/Inspector/index.tsx +++ b/interface/app/$libraryId/Explorer/Inspector/index.tsx @@ -148,6 +148,7 @@ const Thumbnails = ({ items }: { items: ExplorerItem[] }) => { ? 'shadow-md shadow-app-shade' : undefined } + isSidebarPreview={true} /> ))} diff --git a/interface/components/TextViewer/index.tsx b/interface/components/TextViewer/index.tsx index ef239a285..414d6fbfb 100644 --- a/interface/components/TextViewer/index.tsx +++ b/interface/components/TextViewer/index.tsx @@ -1,62 +1,59 @@ +import { useVirtualizer, VirtualItem } from '@tanstack/react-virtual'; import clsx from 'clsx'; +import Prism from 'prismjs'; import { memo, useEffect, useRef, useState } from 'react'; -import './prism.css'; +import * as prism from './prism'; export interface TextViewerProps { src: string; + className?: string; onLoad?: (event: HTMLElementEventMap['load']) => void; onError?: (event: HTMLElementEventMap['error']) => void; - className?: string; codeExtension?: string; + isSidebarPreview?: boolean; } -// prettier-ignore -type Worker = typeof import('./worker') -export const worker = new ComlinkWorker(new URL('./worker', import.meta.url)); - -const NEW_LINE_EXP = /\n(?!$)/g; +// TODO: ANSI support export const TextViewer = memo( - ({ src, onLoad, onError, className, codeExtension }: TextViewerProps) => { - const ref = useRef(null); - const [highlight, setHighlight] = useState<{ - code: string; - length: number; - language: string; - }>(); - const [textContent, setTextContent] = useState(''); + ({ src, className, onLoad, onError, codeExtension, isSidebarPreview }: TextViewerProps) => { + const [lines, setLines] = useState([]); + + const parentRef = useRef(null); + const rowVirtualizer = useVirtualizer({ + count: lines.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 25 + }); useEffect(() => { // Ignore empty urls if (!src || src === '#') return; const controller = new AbortController(); - - fetch(src, { mode: 'cors', signal: controller.signal }) + fetch(src, { + mode: 'cors', + signal: controller.signal + }) .then(async (response) => { if (!response.ok) throw new Error(`Invalid response: ${response.statusText}`); - const text = await response.text(); - - if (controller.signal.aborted) return; - + if (!response.body) return; onLoad?.(new UIEvent('load', {})); - setTextContent(text); - if (codeExtension) { - try { - const env = await worker.highlight(text, codeExtension); - if (env && !controller.signal.aborted) { - const match = text.match(NEW_LINE_EXP); - setHighlight({ - ...env, - length: (match ? match.length + 1 : 1) + 1 - }); - } - } catch (error) { - console.error(error); - } - } + const reader = response.body.pipeThrough(new TextDecoderStream()).getReader(); + const ingestLines = async () => { + const { done, value } = await reader.read(); + if (done) return; + + const chunks = value.split('\n'); + setLines((lines) => [...lines, ...chunks]); + + if (isSidebarPreview) return; + + await ingestLines(); + }; + ingestLines(); }) .catch((error) => { if (!controller.signal.aborted) @@ -64,40 +61,94 @@ export const TextViewer = memo( }); return () => controller.abort(); - }, [src, onError, onLoad, codeExtension]); + }, [src, onError, onLoad, codeExtension, isSidebarPreview]); return ( - 
-				{highlight ? (
-					<>
-						
-							{Array.from(highlight, (_, i) => (
-								
-									{i + 1}
-								
-							))}
-						
-						
+				

+					{rowVirtualizer.getVirtualItems().map((row) => (
+						
-					
-				) : (
-					textContent
-				)}
+					))}
+
); } ); + +function TextRow({ + codeExtension, + row, + content +}: { + codeExtension?: string; + row: VirtualItem; + content: string; +}) { + const contentRef = useRef(null); + + useEffect(() => { + if (contentRef.current) { + const cb: IntersectionObserverCallback = (events) => { + for (const event of events) { + if ( + !event.isIntersecting || + contentRef.current?.getAttribute('data-highlighted') === 'true' + ) + continue; + contentRef.current?.setAttribute('data-highlighted', 'true'); + Prism.highlightElement(event.target, false); // Prism's async seems to be broken + + // With this class present TOML headers are broken Eg. `[dependencies]` will format over multiple lines + const children = contentRef.current?.children; + if (children) { + for (const elem of children) { + elem.classList.remove('table'); + } + } + } + }; + + new IntersectionObserver(cb).observe(contentRef.current); + } + }, []); + + return ( +
+ {codeExtension && ( +
+ {row.index + 1} +
+ )} + + {content} + +
+ ); +} diff --git a/interface/components/TextViewer/prism.ts b/interface/components/TextViewer/prism.ts index d364a31e2..489eb466d 100644 --- a/interface/components/TextViewer/prism.ts +++ b/interface/components/TextViewer/prism.ts @@ -2,6 +2,12 @@ // WARNING: Import order matters +window.Prism = window.Prism || {}; +Prism.manual = true; + +import "prismjs"; +import './prism.css'; + // Languages // Do not include default ones: markup, html, xml, svg, mathml, ssml, atom, rss, css, clike, javascript, js import 'prismjs/components/prism-applescript.js'; @@ -52,3 +58,30 @@ import 'prismjs/components/prism-typoscript.js'; import 'prismjs/components/prism-vala.js'; import 'prismjs/components/prism-yaml.js'; import 'prismjs/components/prism-zig.js'; + +// Mapping between extensions and prismjs language identifier +// Only for those that are not already internally resolved by prismjs +// https://prismjs.com/#supported-languages +export const languageMapping = Object.entries({ + applescript: ['scpt', 'scptd'], + // This is not entirely correct, but better than nothing: + // https://github.com/PrismJS/prism/issues/3656 + // https://github.com/PrismJS/prism/issues/3660 + sh: ['zsh', 'fish'], + c: ['h'], + cpp: ['hpp'], + js: ['mjs'], + crystal: ['cr'], + cs: ['csx'], + makefile: ['make'], + nim: ['nims'], + objc: ['m', 'mm'], + ocaml: ['ml', 'mli', 'mll', 'mly'], + perl: ['pl'], + php: ['php', 'php1', 'php2', 'php3', 'php4', 'php5', 'php6', 'phps', 'phpt', 'phtml'], + powershell: ['ps1', 'psd1', 'psm1'], + rust: ['rs'] +}).reduce>((mapping, [id, exts]) => { + for (const ext of exts) mapping.set(ext, id); + return mapping; +}, new Map()); diff --git a/interface/components/TextViewer/worker.ts b/interface/components/TextViewer/worker.ts deleted file mode 100644 index 128ef66c9..000000000 --- a/interface/components/TextViewer/worker.ts +++ /dev/null @@ -1,45 +0,0 @@ -import Prism from 'prismjs'; - -import './prism'; - -// if you are intending to use Prism functions manually, you will need to set: -Prism.manual = true; - -// Mapping between extensions and prismjs language identifier -// Only for those that are not already internally resolved by prismjs -// https://prismjs.com/#supported-languages -const languageMapping = Object.entries({ - applescript: ['scpt', 'scptd'], - // This is not entirely correct, but better than nothing: - // https://github.com/PrismJS/prism/issues/3656 - // https://github.com/PrismJS/prism/issues/3660 - sh: ['zsh', 'fish'], - c: ['h'], - cpp: ['hpp'], - js: ['mjs'], - crystal: ['cr'], - cs: ['csx'], - makefile: ['make'], - nim: ['nims'], - objc: ['m', 'mm'], - ocaml: ['ml', 'mli', 'mll', 'mly'], - perl: ['pl'], - php: ['php', 'php1', 'php2', 'php3', 'php4', 'php5', 'php6', 'phps', 'phpt', 'phtml'], - powershell: ['ps1', 'psd1', 'psm1'], - rust: ['rs'] -}).reduce>((mapping, [id, exts]) => { - for (const ext of exts) mapping.set(ext, id); - return mapping; -}, new Map()); - -export const highlight = (code: string, ext: string) => { - const language = languageMapping.get(ext) ?? ext; - const grammar = Prism.languages[language]; - - return grammar - ? { - code: Prism.highlight(code, grammar, language), - language - } - : null; -}; diff --git a/interface/package.json b/interface/package.json index 84e617937..3ce14dc9c 100644 --- a/interface/package.json +++ b/interface/package.json @@ -34,8 +34,9 @@ "@tailwindcss/forms": "^0.5.3", "@tanstack/react-query": "^4.12.0", "@tanstack/react-query-devtools": "^4.22.0", - "@tanstack/react-table": "^8.10.0", - "@tanstack/react-virtual": "3.0.0-beta.54", + "@tanstack/react-table": "^8.8.5", + "@tanstack/react-virtual": "3.0.0-beta.61", + "@types/react-scroll-sync": "^0.8.4", "@types/uuid": "^9.0.2", "@vitejs/plugin-react": "^2.1.0", "autoprefixer": "^10.4.12", diff --git a/interface/tsconfig.json b/interface/tsconfig.json index bbfbb211e..3ad612626 100644 --- a/interface/tsconfig.json +++ b/interface/tsconfig.json @@ -5,7 +5,7 @@ "paths": { "~/*": ["./*"] }, - "types": ["vite-plugin-comlink/client", "vite-plugin-svgr/client", "vite/client", "node"] + "types": ["vite-plugin-svgr/client", "vite/client", "node"] }, "include": ["./**/*"], "exclude": ["dist"], diff --git a/packages/config/package.json b/packages/config/package.json index 83ced7b05..a1f9a78d2 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -23,7 +23,6 @@ "eslint-plugin-tailwindcss": "^3.12.0", "eslint-utils": "^3.0.0", "regexpp": "^3.2.0", - "vite-plugin-comlink": "^3.0.5", "vite-plugin-html": "^3.2.0", "vite-plugin-svgr": "^2.2.1" } diff --git a/packages/config/vite/index.ts b/packages/config/vite/index.ts index e68cb1dc2..a9306a6a7 100644 --- a/packages/config/vite/index.ts +++ b/packages/config/vite/index.ts @@ -1,6 +1,5 @@ import react from '@vitejs/plugin-react'; import { defineConfig } from 'vite'; -import { comlink } from 'vite-plugin-comlink'; import { createHtmlPlugin } from 'vite-plugin-html'; import svg from 'vite-plugin-svgr'; import tsconfigPaths from 'vite-tsconfig-paths'; @@ -12,8 +11,7 @@ export default defineConfig({ svg({ svgrOptions: { icon: true } }), createHtmlPlugin({ minify: true - }), - comlink() + }) ], css: { modules: { @@ -25,8 +23,5 @@ export default defineConfig({ sourcemap: true, outDir: '../dist', assetsDir: '.' - }, - worker: { - plugins: [comlink()] } }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f60a9f239bc876f618d3054ce3b0194eff34eb0c..11ad442bdd18c1389c1cae0368a46909d34fc902 100644 GIT binary patch delta 803 zcmYk)O^6eB7{_t4o7rsExJDG+SS+?^1w%4<*?9>R*}QCKlFVk3*}Q|`f09XNlF4Rf zlFTH7;ELcLiXhD4U+`*sEG~NpdWv@`v{G7%Ac6-^dd@1_PU*$N=luIV@cjKJ{`eLC z^mx^B=|5T@>bI>X?K{e?b*W`Ch*|s61;}4SZde}+XW;rV;<0 z@WKcLaTwloZorVwdCC(QHno8)2ZqzVruBB}bCA|y={yn&lEKZR+uXu;e=Z-)WruCa zwrq3G<}&i^2L}n&D+pEUprc558_gtX9VFQ-D-Kh64J(2Kii1>fn@;l+v@T1AQAU&5 zlDMN%{?=}U)b~n*OrnBfJBf}W)CLqMMGTP2RAd0LkpH92M?mX%_g=kXd9l}8HMSK4W&2w&u#0l5DJxeNWgCr~h*Y$F3&j4X{qIC4#Dlsq_NtpC= zN~DmfH<^S~M>CSiw2V{*15VU58Vj4s9q`7R~YXcz0|trLHP*v39c^lWScr z!O$sl$e=rTlOs@?*)7L50l(X-Xy#Z_IlVxP2_cjg)Iu)AMk~Btox-;-mwdCoJ|jO- z-Y?%|%YF`c_Njd*N#-MZr_rjYvY}yMN~0K+kH@?1mTZb6p@pi2(zMnop((Ry49A&j z&qync(SQ}FRFdmY>}~@y;{Jx%Q+N|}GPPT_2M^u0HQTA#gNL5?zs_FYI(@S9FOpyxbpQYW delta 1163 zcmZ9L&u`mw7{-+*Zu6sIo0vvB7_>v%S_Pcgu@k$Hi6lKDgs>X%;wUbt!+tNPlb%7k#34QC6&yqVi|3tAdoEA?~vk1Rl(0cLIJS-hI1C&{oL{#IKJ<&gSu+I1l*4xE3h9y zcECM`AQJ-iZaYH2^R?wHI6j1IVDB+BKZRpJ@teIog~-%e4dc0zsxj$8K^4Q&5HD?H zLuPj+J5z35`)hO0iQd8wREjNigMY5G#h|bmMOpaS^3+Mo0yj6cya0zv8Lj$IJ zjwP_?gf4@<73X>I+9SvTzIg&&Jh46Zz`Xo#kV!@-&_11B=Bs`WG7Pc^QKTu(Cy&`W~y*W)8;4)p{t7wStV>yf>JQa4p*P2X4i_ol(R)@is6b>h{)rP-XWW~J9 z3Bx=vN#<*yGi`N#c5;kgj-WXc4i*ogt9(Oi>Pm$})jTSO!_;~(+AAhPjJGC6M3gLM zTWDI)V(sRTpt4ypTq$EjN9r()T8-;+AK_wQ77MaGAITII@1RK33Jl=_-*4IG#_#rS?!X#u2;KytnKEKDuyGfzf{n5uMjP9!PQo_Bg&(=Tb8j@AaHB*I50E8IzA5A zzDj`TmhCE!2ueJ zdYwX0Rw%u|%j*R+RYDuBA(zaRDTa`ViUht~H(de;QS(Ire=ubQe|%)KZtmW(O|VaJ zjCb$MKK$vW|H|TpJ@Egl1Mb~$BD_2#LJ^XVvXNrE?3cV%G*e6Sae`&L2{PFtI5Fo} zu*gvH)_oMicN0n*&q*17sHgQtWujGAyC{~eA?Yx&QL9vnt_YqbL2BE&`0A~@J`h{2VQPX+rj=# N=gRnt8_tJo{{VI7jn4o8