diff --git a/bun.lockb b/bun.lockb index ae96e93d5..61166063c 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/core/src/filetype/definitions/misc.toml b/core/src/filetype/definitions/misc.toml index e5f8e8dab..ae506d250 100644 --- a/core/src/filetype/definitions/misc.toml +++ b/core/src/filetype/definitions/misc.toml @@ -163,6 +163,22 @@ priority = 100 [file_types.metadata] text_based = true +[[file_types]] +id = "model/ply" +name = "PLY 3D Model" +extensions = ["ply"] +mime_types = ["model/ply", "application/ply"] +category = "mesh" +priority = 100 + +[[file_types.magic_bytes]] +pattern = "70 6C 79 0A" +offset = 0 +priority = 100 + +[file_types.metadata] +supports_gaussian_splats = true + # Database [[file_types]] id = "application/x-sqlite3" diff --git a/packages/interface/package.json b/packages/interface/package.json index f1423c458..38b634fd2 100644 --- a/packages/interface/package.json +++ b/packages/interface/package.json @@ -20,10 +20,13 @@ "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", + "@mkkellogg/gaussian-splats-3d": "^0.4.7", "@phosphor-icons/react": "^2.1.0", "@radix-ui/react-dialog": "^1.0.5", "@radix-ui/react-dropdown-menu": "^2.0.6", "@radix-ui/react-tooltip": "^1.0.7", + "@react-three/drei": "^9.122.0", + "@react-three/fiber": "^9.4.2", "@sd/assets": "workspace:*", "@sd/ts-client": "workspace:*", "@sd/ui": "workspace:*", @@ -45,12 +48,14 @@ "rooks": "^9.3.0", "sonner": "^1.0.3", "tailwind-merge": "^1.14.0", + "three": "^0.160.0", "zod": "^3.23", "zustand": "^5.0.8" }, "devDependencies": { "@types/react": "npm:types-react@rc", "@types/react-dom": "npm:types-react-dom@rc", + "@types/three": "^0.182.0", "typescript": "^5.6.2" } } diff --git a/packages/interface/src/components/QuickPreview/ContentRenderer.tsx b/packages/interface/src/components/QuickPreview/ContentRenderer.tsx index 12983ac40..2fa40bd53 100644 --- a/packages/interface/src/components/QuickPreview/ContentRenderer.tsx +++ b/packages/interface/src/components/QuickPreview/ContentRenderer.tsx @@ -3,7 +3,7 @@ import { File as FileComponent } from "../Explorer/File"; import { formatBytes, getContentKind } from "../Explorer/utils"; import { usePlatform } from "../../platform"; import { useServer } from "../../ServerContext"; -import { useState, useEffect, useRef } from "react"; +import { useState, useEffect, useRef, lazy, Suspense } from "react"; import { MagnifyingGlassPlus, MagnifyingGlassMinus, @@ -14,6 +14,8 @@ import { AudioPlayer } from "./AudioPlayer"; import { useZoomPan } from "./useZoomPan"; import { Folder } from "@sd/assets/icons"; +const MeshViewer = lazy(() => import('./MeshViewer').then(m => ({ default: m.MeshViewer }))); + interface ContentRendererProps { file: File; onZoomChange?: (isZoomed: boolean) => void; @@ -390,6 +392,18 @@ export function ContentRenderer({ file, onZoomChange }: ContentRendererProps) { return ; case "audio": return ; + case "mesh": + return ( + + + + } + > + + + ); case "document": case "book": case "spreadsheet": diff --git a/packages/interface/src/components/QuickPreview/MeshViewer.tsx b/packages/interface/src/components/QuickPreview/MeshViewer.tsx new file mode 100644 index 000000000..20565144d --- /dev/null +++ b/packages/interface/src/components/QuickPreview/MeshViewer.tsx @@ -0,0 +1,328 @@ +/// + +import { Canvas } from "@react-three/fiber"; +import { OrbitControls, PerspectiveCamera } from "@react-three/drei"; +import { useState, useEffect, useRef, Suspense } from "react"; +import type { File } from "@sd/ts-client"; +import { usePlatform } from "../../platform"; +import { File as FileComponent } from "../Explorer/File"; +import { PLYLoader } from "three/examples/jsm/loaders/PLYLoader.js"; +import * as GaussianSplats3D from "@mkkellogg/gaussian-splats-3d"; +import * as THREE from "three"; + +interface MeshViewerProps { + file: File; + onZoomChange?: (isZoomed: boolean) => void; +} + +interface MeshSceneProps { + url: string; +} + +function MeshScene({ url }: MeshSceneProps) { + const meshRef = useRef(null); + const [geometry, setGeometry] = useState(null); + + useEffect(() => { + const loader = new PLYLoader(); + loader.load( + url, + (loadedGeometry) => { + loadedGeometry.computeVertexNormals(); + loadedGeometry.center(); + setGeometry(loadedGeometry); + }, + undefined, + (error) => { + console.error("[MeshScene] PLY load error:", error); + }, + ); + + return () => { + if (geometry) { + geometry.dispose(); + } + }; + }, [url]); + + if (!geometry) { + return null; + } + + return ( + // @ts-expect-error - React Three Fiber JSX types + + {/* @ts-expect-error - React Three Fiber JSX types */} + + {/* @ts-expect-error - React Three Fiber JSX types */} + + ); +} + +function GaussianSplatViewer({ + url, + onFallback, +}: { + url: string; + onFallback: () => void; +}) { + const containerRef = useRef(null); + const viewerRef = useRef(null); + + useEffect(() => { + if (!containerRef.current) return; + + let cancelled = false; + + const initViewer = async () => { + try { + const container = containerRef.current; + if (!container) return; + + const viewer = new GaussianSplats3D.Viewer({ + rootElement: container, + cameraUp: [0, -1, 0], + initialCameraPosition: [0, 0, -0.5], + initialCameraLookAt: [0, 0, 0], + selfDrivenMode: true, + sphericalHarmonicsDegree: 2, + sharedMemoryForWorkers: false, + }); + + viewerRef.current = viewer; + + await viewer.addSplatScene(url, { + format: GaussianSplats3D.SceneFormat.Ply, + showLoadingUI: false, + progressiveLoad: true, + splatAlphaRemovalThreshold: 5, + }); + + if (!cancelled) { + // Try to get scene info and adjust camera + const splatMesh = viewer.splatMesh; + if (splatMesh) { + console.log("[GaussianSplatViewer] SplatMesh info:", { + splatCount: splatMesh.getSplatCount?.(), + position: splatMesh.position, + scale: splatMesh.scale, + }); + } + + viewer.start(); + console.log("[GaussianSplatViewer] Viewer started"); + + // Verify canvas was created + const canvas = container.querySelector("canvas"); + if (canvas) { + const styles = window.getComputedStyle(canvas); + console.log("[GaussianSplatViewer] Canvas info:", { + width: canvas.width, + height: canvas.height, + offsetWidth: canvas.offsetWidth, + offsetHeight: canvas.offsetHeight, + display: styles.display, + visibility: styles.visibility, + opacity: styles.opacity, + zIndex: styles.zIndex, + position: styles.position, + }); + } else { + console.error( + "[GaussianSplatViewer] No canvas created!", + ); + } + } + } catch (err) { + if ( + !cancelled && + err instanceof Error && + err.name !== "AbortError" + ) { + console.error("[GaussianSplatViewer] Error:", err); + onFallback(); + } + } + }; + + initViewer(); + + return () => { + cancelled = true; + if (viewerRef.current) { + viewerRef.current.dispose(); + viewerRef.current = null; + } + }; + }, [url, onFallback]); + + return ( +
+ ); +} + +export function MeshViewer({ file, onZoomChange }: MeshViewerProps) { + const platform = usePlatform(); + const [meshUrl, setMeshUrl] = useState(null); + const [isGaussianSplat, setIsGaussianSplat] = useState(false); + const [splatFailed, setSplatFailed] = useState(false); + const [shouldLoad, setShouldLoad] = useState(false); + const [loading, setLoading] = useState(true); + + const fileId = file.content_identity?.uuid || file.id; + + useEffect(() => { + setShouldLoad(false); + setMeshUrl(null); + setLoading(true); + + const timer = setTimeout(() => { + setShouldLoad(true); + }, 50); + + return () => clearTimeout(timer); + }, [fileId]); + + useEffect(() => { + if (!shouldLoad || !platform.convertFileSrc) { + return; + } + + const sdPath = file.sd_path as any; + const physicalPath = sdPath?.Physical?.path; + + if (!physicalPath) { + console.log("[MeshViewer] No physical path available"); + setLoading(false); + return; + } + + const url = platform.convertFileSrc(physicalPath); + setMeshUrl(url); + + // Create an AbortController to cancel the detection fetch if component unmounts + const abortController = new AbortController(); + + fetch(url, { signal: abortController.signal }) + .then((res) => res.arrayBuffer()) + .then((buffer) => { + const header = new TextDecoder().decode(buffer.slice(0, 3000)); + + // Gaussian Splat detection + const hasSH = + header.includes("f_dc_0") || + header.includes("sh0") || + header.includes("sh_0"); + const hasScale = + header.includes("scale_0") || + header.includes("scale_1") || + header.includes("scale_2"); + const hasOpacity = header.includes("opacity"); + const hasRotation = + header.includes("rot_0") || + header.includes("rot_1") || + header.includes("rot_2") || + header.includes("rot_3"); + + const isGS = hasSH && (hasScale || hasOpacity || hasRotation); + + setIsGaussianSplat(isGS); + setLoading(false); + }) + .catch((error) => { + // Ignore abort errors (expected when component unmounts) + if (error.name !== "AbortError") { + console.error( + "[MeshViewer] Error detecting format:", + error, + ); + } + setLoading(false); + }); + + return () => { + abortController.abort(); + }; + }, [shouldLoad, fileId, file.sd_path, platform]); + + if (!meshUrl || loading) { + return ( +
+
+ + {loading && ( +
+ Loading 3D model... +
+ )} +
+
+ ); + } + + return ( +
+ {isGaussianSplat && !splatFailed ? ( + <> + setSplatFailed(true)} + /> +
+ Gaussian Splat +
+ + ) : ( + <> +
+ 3D Mesh +
+ + + {/* @ts-expect-error - React Three Fiber JSX types */} + + {/* @ts-expect-error - React Three Fiber JSX types */} + + {/* @ts-expect-error - React Three Fiber JSX types */} + + + + + + +
+
Left drag: Rotate
+
Right drag: Pan
+
Scroll: Zoom
+
+ + )} +
+ ); +} diff --git a/packages/interface/tsconfig.json b/packages/interface/tsconfig.json index e5f5334ec..d54f2b70e 100644 --- a/packages/interface/tsconfig.json +++ b/packages/interface/tsconfig.json @@ -11,7 +11,8 @@ "baseUrl": ".", "paths": { "~/*": ["./src/*"] - } + }, + "types": ["@react-three/fiber"] }, "include": ["src"] }