Add support for PLY 3D models and integrate Gaussian splats visualization

- Introduced a new file type definition for PLY 3D models in `misc.toml`.
- Updated `package.json` to include dependencies for Gaussian splats and React Three Fiber.
- Implemented a `MeshViewer` component for rendering 3D models, supporting both standard mesh and Gaussian splat formats.
- Enhanced `ContentRenderer` to handle mesh file types with lazy loading for improved performance.
- Updated TypeScript configuration to include types for React Three Fiber.

core/src/filetype/definitions/misc.toml

@@ -163,6 +163,22 @@ priority = 100
 [file_types.metadata]
 text_based = true
 
+[[file_types]]
+id = "model/ply"
+name = "PLY 3D Model"
+extensions = ["ply"]
+mime_types = ["model/ply", "application/ply"]
+category = "mesh"
+priority = 100
+
+[[file_types.magic_bytes]]
+pattern = "70 6C 79 0A"
+offset = 0
+priority = 100
+
+[file_types.metadata]
+supports_gaussian_splats = true
+
 # Database
 [[file_types]]
 id = "application/x-sqlite3"

packages/interface/package.json

@@ -20,10 +20,13 @@
 		"@dnd-kit/core": "^6.3.1",
 		"@dnd-kit/sortable": "^10.0.0",
 		"@dnd-kit/utilities": "^3.2.2",
+		"@mkkellogg/gaussian-splats-3d": "^0.4.7",
 		"@phosphor-icons/react": "^2.1.0",
 		"@radix-ui/react-dialog": "^1.0.5",
 		"@radix-ui/react-dropdown-menu": "^2.0.6",
 		"@radix-ui/react-tooltip": "^1.0.7",
+		"@react-three/drei": "^9.122.0",
+		"@react-three/fiber": "^9.4.2",
 		"@sd/assets": "workspace:*",
 		"@sd/ts-client": "workspace:*",
 		"@sd/ui": "workspace:*",
@@ -45,12 +48,14 @@
 		"rooks": "^9.3.0",
 		"sonner": "^1.0.3",
 		"tailwind-merge": "^1.14.0",
+		"three": "^0.160.0",
 		"zod": "^3.23",
 		"zustand": "^5.0.8"
 	},
 	"devDependencies": {
 		"@types/react": "npm:types-react@rc",
 		"@types/react-dom": "npm:types-react-dom@rc",
+		"@types/three": "^0.182.0",
 		"typescript": "^5.6.2"
 	}
 }

packages/interface/src/components/QuickPreview/ContentRenderer.tsx

@@ -3,7 +3,7 @@ import { File as FileComponent } from "../Explorer/File";
 import { formatBytes, getContentKind } from "../Explorer/utils";
 import { usePlatform } from "../../platform";
 import { useServer } from "../../ServerContext";
-import { useState, useEffect, useRef } from "react";
+import { useState, useEffect, useRef, lazy, Suspense } from "react";
 import {
 	MagnifyingGlassPlus,
 	MagnifyingGlassMinus,
@@ -14,6 +14,8 @@ import { AudioPlayer } from "./AudioPlayer";
 import { useZoomPan } from "./useZoomPan";
 import { Folder } from "@sd/assets/icons";
 
+const MeshViewer = lazy(() => import('./MeshViewer').then(m => ({ default: m.MeshViewer })));
+
 interface ContentRendererProps {
 	file: File;
 	onZoomChange?: (isZoomed: boolean) => void;
@@ -390,6 +392,18 @@ export function ContentRenderer({ file, onZoomChange }: ContentRendererProps) {
 			return <VideoRenderer file={file} onZoomChange={onZoomChange} />;
 		case "audio":
 			return <AudioRenderer file={file} />;
+		case "mesh":
+			return (
+				<Suspense
+					fallback={
+						<div className="w-full h-full flex items-center justify-center">
+							<FileComponent.Thumb file={file} size={200} />
+						</div>
+					}
+				>
+					<MeshViewer file={file} />
+				</Suspense>
+			);
 		case "document":
 		case "book":
 		case "spreadsheet":

packages/interface/src/components/QuickPreview/MeshViewer.tsx

@@ -0,0 +1,328 @@
+/// <reference types="@react-three/fiber" />
+
+import { Canvas } from "@react-three/fiber";
+import { OrbitControls, PerspectiveCamera } from "@react-three/drei";
+import { useState, useEffect, useRef, Suspense } from "react";
+import type { File } from "@sd/ts-client";
+import { usePlatform } from "../../platform";
+import { File as FileComponent } from "../Explorer/File";
+import { PLYLoader } from "three/examples/jsm/loaders/PLYLoader.js";
+import * as GaussianSplats3D from "@mkkellogg/gaussian-splats-3d";
+import * as THREE from "three";
+
+interface MeshViewerProps {
+	file: File;
+	onZoomChange?: (isZoomed: boolean) => void;
+}
+
+interface MeshSceneProps {
+	url: string;
+}
+
+function MeshScene({ url }: MeshSceneProps) {
+	const meshRef = useRef<THREE.Mesh>(null);
+	const [geometry, setGeometry] = useState<THREE.BufferGeometry | null>(null);
+
+	useEffect(() => {
+		const loader = new PLYLoader();
+		loader.load(
+			url,
+			(loadedGeometry) => {
+				loadedGeometry.computeVertexNormals();
+				loadedGeometry.center();
+				setGeometry(loadedGeometry);
+			},
+			undefined,
+			(error) => {
+				console.error("[MeshScene] PLY load error:", error);
+			},
+		);
+
+		return () => {
+			if (geometry) {
+				geometry.dispose();
+			}
+		};
+	}, [url]);
+
+	if (!geometry) {
+		return null;
+	}
+
+	return (
+		// @ts-expect-error - React Three Fiber JSX types
+		<mesh ref={meshRef} geometry={geometry}>
+			{/* @ts-expect-error - React Three Fiber JSX types */}
+			<meshStandardMaterial
+				color="#888888"
+				metalness={0.3}
+				roughness={0.7}
+			/>
+			{/* @ts-expect-error - React Three Fiber JSX types */}
+		</mesh>
+	);
+}
+
+function GaussianSplatViewer({
+	url,
+	onFallback,
+}: {
+	url: string;
+	onFallback: () => void;
+}) {
+	const containerRef = useRef<HTMLDivElement>(null);
+	const viewerRef = useRef<any>(null);
+
+	useEffect(() => {
+		if (!containerRef.current) return;
+
+		let cancelled = false;
+
+		const initViewer = async () => {
+			try {
+				const container = containerRef.current;
+				if (!container) return;
+
+				const viewer = new GaussianSplats3D.Viewer({
+					rootElement: container,
+					cameraUp: [0, -1, 0],
+					initialCameraPosition: [0, 0, -0.5],
+					initialCameraLookAt: [0, 0, 0],
+					selfDrivenMode: true,
+					sphericalHarmonicsDegree: 2,
+					sharedMemoryForWorkers: false,
+				});
+
+				viewerRef.current = viewer;
+
+				await viewer.addSplatScene(url, {
+					format: GaussianSplats3D.SceneFormat.Ply,
+					showLoadingUI: false,
+					progressiveLoad: true,
+					splatAlphaRemovalThreshold: 5,
+				});
+
+				if (!cancelled) {
+					// Try to get scene info and adjust camera
+					const splatMesh = viewer.splatMesh;
+					if (splatMesh) {
+						console.log("[GaussianSplatViewer] SplatMesh info:", {
+							splatCount: splatMesh.getSplatCount?.(),
+							position: splatMesh.position,
+							scale: splatMesh.scale,
+						});
+					}
+
+					viewer.start();
+					console.log("[GaussianSplatViewer] Viewer started");
+
+					// Verify canvas was created
+					const canvas = container.querySelector("canvas");
+					if (canvas) {
+						const styles = window.getComputedStyle(canvas);
+						console.log("[GaussianSplatViewer] Canvas info:", {
+							width: canvas.width,
+							height: canvas.height,
+							offsetWidth: canvas.offsetWidth,
+							offsetHeight: canvas.offsetHeight,
+							display: styles.display,
+							visibility: styles.visibility,
+							opacity: styles.opacity,
+							zIndex: styles.zIndex,
+							position: styles.position,
+						});
+					} else {
+						console.error(
+							"[GaussianSplatViewer] No canvas created!",
+						);
+					}
+				}
+			} catch (err) {
+				if (
+					!cancelled &&
+					err instanceof Error &&
+					err.name !== "AbortError"
+				) {
+					console.error("[GaussianSplatViewer] Error:", err);
+					onFallback();
+				}
+			}
+		};
+
+		initViewer();
+
+		return () => {
+			cancelled = true;
+			if (viewerRef.current) {
+				viewerRef.current.dispose();
+				viewerRef.current = null;
+			}
+		};
+	}, [url, onFallback]);
+
+	return (
+		<div
+			ref={containerRef}
+			style={{
+				width: "100%",
+				height: "100%",
+				position: "relative",
+			}}
+		/>
+	);
+}
+
+export function MeshViewer({ file, onZoomChange }: MeshViewerProps) {
+	const platform = usePlatform();
+	const [meshUrl, setMeshUrl] = useState<string | null>(null);
+	const [isGaussianSplat, setIsGaussianSplat] = useState(false);
+	const [splatFailed, setSplatFailed] = useState(false);
+	const [shouldLoad, setShouldLoad] = useState(false);
+	const [loading, setLoading] = useState(true);
+
+	const fileId = file.content_identity?.uuid || file.id;
+
+	useEffect(() => {
+		setShouldLoad(false);
+		setMeshUrl(null);
+		setLoading(true);
+
+		const timer = setTimeout(() => {
+			setShouldLoad(true);
+		}, 50);
+
+		return () => clearTimeout(timer);
+	}, [fileId]);
+
+	useEffect(() => {
+		if (!shouldLoad || !platform.convertFileSrc) {
+			return;
+		}
+
+		const sdPath = file.sd_path as any;
+		const physicalPath = sdPath?.Physical?.path;
+
+		if (!physicalPath) {
+			console.log("[MeshViewer] No physical path available");
+			setLoading(false);
+			return;
+		}
+
+		const url = platform.convertFileSrc(physicalPath);
+		setMeshUrl(url);
+
+		// Create an AbortController to cancel the detection fetch if component unmounts
+		const abortController = new AbortController();
+
+		fetch(url, { signal: abortController.signal })
+			.then((res) => res.arrayBuffer())
+			.then((buffer) => {
+				const header = new TextDecoder().decode(buffer.slice(0, 3000));
+
+				// Gaussian Splat detection
+				const hasSH =
+					header.includes("f_dc_0") ||
+					header.includes("sh0") ||
+					header.includes("sh_0");
+				const hasScale =
+					header.includes("scale_0") ||
+					header.includes("scale_1") ||
+					header.includes("scale_2");
+				const hasOpacity = header.includes("opacity");
+				const hasRotation =
+					header.includes("rot_0") ||
+					header.includes("rot_1") ||
+					header.includes("rot_2") ||
+					header.includes("rot_3");
+
+				const isGS = hasSH && (hasScale || hasOpacity || hasRotation);
+
+				setIsGaussianSplat(isGS);
+				setLoading(false);
+			})
+			.catch((error) => {
+				// Ignore abort errors (expected when component unmounts)
+				if (error.name !== "AbortError") {
+					console.error(
+						"[MeshViewer] Error detecting format:",
+						error,
+					);
+				}
+				setLoading(false);
+			});
+
+		return () => {
+			abortController.abort();
+		};
+	}, [shouldLoad, fileId, file.sd_path, platform]);
+
+	if (!meshUrl || loading) {
+		return (
+			<div className="flex h-full w-full items-center justify-center">
+				<div className="text-center">
+					<FileComponent.Thumb file={file} size={200} />
+					{loading && (
+						<div className="mt-4 text-sm text-ink-dull">
+							Loading 3D model...
+						</div>
+					)}
+				</div>
+			</div>
+		);
+	}
+
+	return (
+		<div className="absolute inset-0 flex items-center justify-center bg-black">
+			{isGaussianSplat && !splatFailed ? (
+				<>
+					<GaussianSplatViewer
+						url={meshUrl}
+						onFallback={() => setSplatFailed(true)}
+					/>
+					<div className="pointer-events-none absolute left-4 top-4 z-50 rounded-lg bg-black/80 px-3 py-1.5 text-sm font-medium text-white backdrop-blur-xl">
+						Gaussian Splat
+					</div>
+				</>
+			) : (
+				<>
+					<div className="pointer-events-none absolute left-4 top-4 z-10 rounded-lg bg-black/80 px-3 py-1.5 text-sm font-medium text-white backdrop-blur-xl">
+						3D Mesh
+					</div>
+					<Canvas style={{ width: "100%", height: "100%" }}>
+						<PerspectiveCamera makeDefault position={[0, 0, 5]} />
+						{/* @ts-expect-error - React Three Fiber JSX types */}
+						<ambientLight intensity={0.5} />
+						{/* @ts-expect-error - React Three Fiber JSX types */}
+						<spotLight
+							position={[10, 10, 10]}
+							angle={0.3}
+							penumbra={0.5}
+							intensity={1}
+						/>
+						{/* @ts-expect-error - React Three Fiber JSX types */}
+						<spotLight
+							position={[-10, -10, -10]}
+							angle={0.3}
+							penumbra={0.5}
+							intensity={0.5}
+						/>
+						<Suspense fallback={null}>
+							<MeshScene url={meshUrl} />
+						</Suspense>
+						<OrbitControls
+							enableDamping
+							dampingFactor={0.05}
+							minDistance={0.5}
+							maxDistance={100}
+						/>
+					</Canvas>
+					<div className="pointer-events-none absolute bottom-4 left-4 z-10 rounded-lg bg-black/80 px-3 py-2 text-xs text-white/70 backdrop-blur-xl">
+						<div>Left drag: Rotate</div>
+						<div>Right drag: Pan</div>
+						<div>Scroll: Zoom</div>
+					</div>
+				</>
+			)}
+		</div>
+	);
+}

packages/interface/tsconfig.json

@@ -11,7 +11,8 @@
 		"baseUrl": ".",
 		"paths": {
 			"~/*": ["./src/*"]
-		}
+		},
+		"types": ["@react-three/fiber"]
 	},
 	"include": ["src"]
 }