From 8abc966b844ba53dbdfd6c6dfe2c5cb210a0fd35 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Thu, 18 Dec 2025 12:20:54 -0800 Subject: [PATCH] Add Gaussian Splat generation functionality - Introduced a new `GaussianSplat` variant in the `JobOutput` enum to handle output from Gaussian splat jobs. - Implemented `GenerateSplatAction` for initiating Gaussian splat generation, including input and output structures. - Created `GaussianSplatJob` and its associated processing logic for batch image processing and splat generation. - Added `GaussianSplatProcessor` to manage the processing of images into 3D Gaussian splats. - Updated the media module to include new splat-related functionalities and integrated them into the existing workflow. - Enhanced the interface components to support visualization and interaction with Gaussian splats, including a toggle for displaying splat overlays. - Updated TypeScript types to accommodate new splat generation inputs and outputs, ensuring type safety across the application. --- core/src/infra/job/output.rs | 18 + core/src/ops/media/mod.rs | 3 + core/src/ops/media/splat/action.rs | 95 +++++ core/src/ops/media/splat/job.rs | 346 ++++++++++++++++++ core/src/ops/media/splat/mod.rs | 99 +++++ core/src/ops/media/splat/processor.rs | 167 +++++++++ core/src/ops/sidecar/types.rs | 9 + .../QuickPreview/ContentRenderer.tsx | 62 ++++ .../components/QuickPreview/MeshViewer.tsx | 20 +- .../src/inspectors/FileInspector.tsx | 48 +++ packages/ts-client/src/generated/types.ts | 285 ++++++++------- 11 files changed, 1014 insertions(+), 138 deletions(-) create mode 100644 core/src/ops/media/splat/action.rs create mode 100644 core/src/ops/media/splat/job.rs create mode 100644 core/src/ops/media/splat/mod.rs create mode 100644 core/src/ops/media/splat/processor.rs diff --git a/core/src/infra/job/output.rs b/core/src/infra/job/output.rs index dde1663ab..2d7c3c337 100644 --- a/core/src/infra/job/output.rs +++ b/core/src/infra/job/output.rs @@ -82,6 +82,13 @@ pub enum JobOutput { error_count: usize, }, + /// Gaussian splat generation output + GaussianSplat { + total_processed: usize, + success_count: usize, + error_count: usize, + }, + /// Generic output with custom data #[specta(skip)] Custom(serde_json::Value), @@ -269,6 +276,17 @@ impl fmt::Display for JobOutput { total_processed, success_count, error_count ) } + Self::GaussianSplat { + total_processed, + success_count, + error_count, + } => { + write!( + f, + "Gaussian splat: {} processed ({} success, {} errors)", + total_processed, success_count, error_count + ) + } Self::Custom(_) => write!(f, "Custom output"), } } diff --git a/core/src/ops/media/mod.rs b/core/src/ops/media/mod.rs index 546c14e06..f6e766d90 100644 --- a/core/src/ops/media/mod.rs +++ b/core/src/ops/media/mod.rs @@ -4,6 +4,7 @@ //! - Thumbnail generation //! - OCR (text extraction from images/PDFs) //! - Speech-to-text (audio/video transcription) +//! - Gaussian splat generation (3D view synthesis from images) //! - Video transcoding //! - Audio metadata extraction //! - Image optimization @@ -13,6 +14,7 @@ pub mod blurhash; pub mod metadata_extractor; pub mod ocr; pub mod proxy; +pub mod splat; #[cfg(feature = "ffmpeg")] pub mod speech; @@ -29,6 +31,7 @@ pub use metadata_extractor::{ }; pub use ocr::{OcrJob, OcrProcessor}; pub use proxy::{ProxyJob, ProxyProcessor}; +pub use splat::{GaussianSplatJob, GaussianSplatProcessor}; #[cfg(feature = "ffmpeg")] pub use speech::{SpeechToTextJob, SpeechToTextProcessor}; diff --git a/core/src/ops/media/splat/action.rs b/core/src/ops/media/splat/action.rs new file mode 100644 index 000000000..d338731fd --- /dev/null +++ b/core/src/ops/media/splat/action.rs @@ -0,0 +1,95 @@ +//! Gaussian splat action handlers + +use super::{ + job::{GaussianSplatJob, GaussianSplatJobConfig}, + processor::GaussianSplatProcessor, +}; +use crate::{ + context::CoreContext, + infra::action::{error::ActionError, LibraryAction}, + ops::indexing::{path_resolver::PathResolver, processor::ProcessorEntry}, +}; +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::sync::Arc; +use uuid::Uuid; + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct GenerateSplatInput { + pub entry_uuid: Uuid, + pub model_path: Option, // Path to SHARP model checkpoint or None for auto-download +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct GenerateSplatOutput { + /// Job ID for tracking splat generation progress + pub job_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateSplatAction { + input: GenerateSplatInput, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct GaussianSplatJobOutput { + pub total_processed: usize, + pub success_count: usize, + pub error_count: usize, +} + +impl GenerateSplatAction { + pub fn new(input: GenerateSplatInput) -> Self { + Self { input } + } +} + +impl LibraryAction for GenerateSplatAction { + type Input = GenerateSplatInput; + type Output = GenerateSplatOutput; + + fn from_input(input: GenerateSplatInput) -> Result { + Ok(Self::new(input)) + } + + async fn execute( + self, + library: Arc, + _context: Arc, + ) -> Result { + tracing::info!( + "Dispatching Gaussian splat job for entry: {}", + self.input.entry_uuid + ); + + // Create job config for single file + let job_config = super::job::GaussianSplatJobConfig { + location_id: None, + entry_uuid: Some(self.input.entry_uuid), // Single file mode + model_path: self.input.model_path, + reprocess: false, + }; + + // Create job + let job = super::job::GaussianSplatJob::new(job_config); + + // Dispatch job + let job_handle = library + .jobs() + .dispatch(job) + .await + .map_err(|e| ActionError::Internal(format!("Failed to dispatch job: {}", e)))?; + + tracing::info!("Gaussian splat job dispatched: {}", job_handle.id()); + + Ok(GenerateSplatOutput { + job_id: job_handle.id().to_string(), + }) + } + + fn action_kind(&self) -> &'static str { + "media.splat.generate" + } +} + +crate::register_library_action!(GenerateSplatAction, "media.splat.generate"); diff --git a/core/src/ops/media/splat/job.rs b/core/src/ops/media/splat/job.rs new file mode 100644 index 000000000..897a729d3 --- /dev/null +++ b/core/src/ops/media/splat/job.rs @@ -0,0 +1,346 @@ +//! Gaussian splat generation job for batch image processing + +use super::processor::GaussianSplatProcessor; +use crate::{ + infra::{ + db::entities::entry, + job::{prelude::*, traits::DynJob}, + }, + ops::indexing::processor::ProcessorEntry, +}; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::sync::Arc; +use tracing::{info, warn}; +use uuid::Uuid; + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct GaussianSplatJobConfig { + /// Location ID to process (None = all entries in library) + pub location_id: Option, + /// Single entry UUID to process (for UI-triggered single file) + pub entry_uuid: Option, + /// Path to SHARP model checkpoint (None = auto-download) + pub model_path: Option, + /// Reprocess files that already have splats + pub reprocess: bool, +} + +impl Default for GaussianSplatJobConfig { + fn default() -> Self { + Self { + location_id: None, + entry_uuid: None, + model_path: None, + reprocess: false, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SplatJobState { + phase: SplatPhase, + entries: Vec<(i32, std::path::PathBuf, Option)>, + processed: usize, + success_count: usize, + error_count: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum SplatPhase { + Discovery, + Processing, + Complete, +} + +#[derive(Serialize, Deserialize)] +pub struct GaussianSplatJob { + config: GaussianSplatJobConfig, + state: SplatJobState, +} + +impl GaussianSplatJob { + pub fn new(config: GaussianSplatJobConfig) -> Self { + Self { + config, + state: SplatJobState { + phase: SplatPhase::Discovery, + entries: Vec::new(), + processed: 0, + success_count: 0, + error_count: 0, + }, + } + } + + pub fn from_location(location_id: Uuid) -> Self { + Self::new(GaussianSplatJobConfig { + location_id: Some(location_id), + ..Default::default() + }) + } +} + +impl Job for GaussianSplatJob { + const NAME: &'static str = "gaussian_splat"; + const RESUMABLE: bool = true; + const DESCRIPTION: Option<&'static str> = + Some("Generate 3D Gaussian splats from images for view synthesis"); +} + +#[async_trait::async_trait] +impl JobHandler for GaussianSplatJob { + type Output = GaussianSplatJobOutput; + + async fn run(&mut self, ctx: JobContext<'_>) -> JobResult { + match self.state.phase { + SplatPhase::Discovery => { + ctx.log("Starting Gaussian splat discovery phase"); + self.run_discovery(&ctx).await?; + self.state.phase = SplatPhase::Processing; + } + SplatPhase::Processing => {} + SplatPhase::Complete => { + return Ok(GaussianSplatJobOutput { + total_processed: self.state.processed, + success_count: self.state.success_count, + error_count: self.state.error_count, + }); + } + } + + ctx.log(format!( + "Gaussian splat processing {} images", + self.state.entries.len() + )); + + let processor = GaussianSplatProcessor::new(ctx.library_arc()); + let processor = if let Some(ref model_path) = self.config.model_path { + processor.with_model_path(model_path.clone()) + } else { + processor + }; + + let total = self.state.entries.len(); + + while self.state.processed < total { + ctx.check_interrupt().await?; + + let (entry_id, path, mime_type) = &self.state.entries[self.state.processed]; + + // Load entry to get content_id + let entry_model = entry::Entity::find_by_id(*entry_id) + .one(ctx.library_db()) + .await? + .ok_or_else(|| JobError::execution("Entry not found"))?; + + let proc_entry = ProcessorEntry { + id: *entry_id, + uuid: entry_model.uuid, + path: path.clone(), + kind: crate::ops::indexing::state::EntryKind::File, + size: entry_model.size as u64, + content_id: entry_model.content_id, + mime_type: mime_type.clone(), + }; + + if !processor.should_process(&proc_entry) { + self.state.processed += 1; + continue; + } + + // Report progress + ctx.progress(Progress::Indeterminate(format!( + "Generating splat for {}...", + path.file_name().and_then(|n| n.to_str()).unwrap_or("file") + ))); + + let result = processor.process(ctx.library_db(), &proc_entry).await; + + match result { + Ok(result) if result.success => { + ctx.log(format!( + "Generated splat for {}: {} bytes", + path.display(), + result.bytes_processed + )); + self.state.success_count += 1; + } + Ok(_) => { + warn!("Splat generation failed for {}", path.display()); + self.state.error_count += 1; + } + Err(e) => { + ctx.log(format!( + "ERROR: Splat generation error for {}: {}", + path.display(), + e + )); + self.state.error_count += 1; + } + } + + self.state.processed += 1; + + // Report progress with count + ctx.progress(Progress::Count { + current: self.state.processed, + total, + }); + + if self.state.processed % 5 == 0 { + ctx.checkpoint().await?; + } + } + + self.state.phase = SplatPhase::Complete; + ctx.log(format!( + "Gaussian splat generation complete: {} success, {} errors", + self.state.success_count, self.state.error_count + )); + + Ok(GaussianSplatJobOutput { + total_processed: self.state.processed, + success_count: self.state.success_count, + error_count: self.state.error_count, + }) + } +} + +impl GaussianSplatJob { + async fn run_discovery(&mut self, ctx: &JobContext<'_>) -> JobResult<()> { + use crate::infra::db::entities::{content_identity, entry, mime_type}; + + ctx.log("Starting Gaussian splat discovery"); + + // Check if SHARP CLI is available + if !super::check_sharp_available().await.unwrap_or(false) { + return Err(JobError::execution( + "SHARP CLI not found. Please install ml-sharp (pip install -e /path/to/ml-sharp)", + )); + } + + ctx.log("SHARP CLI available"); + + let db = ctx.library_db(); + + // Check if this is single-file mode (from UI action) + if let Some(entry_uuid) = self.config.entry_uuid { + ctx.log(format!("Single file mode: processing entry {}", entry_uuid)); + + // Load the specific entry + let entry_model = entry::Entity::find() + .filter(entry::Column::Uuid.eq(entry_uuid)) + .one(db) + .await? + .ok_or_else(|| JobError::execution("Entry not found"))?; + + if let Some(content_id) = entry_model.content_id { + if let Ok(Some(ci)) = content_identity::Entity::find_by_id(content_id) + .one(db) + .await + { + if let Some(mime_id) = ci.mime_type_id { + if let Ok(Some(mime)) = mime_type::Entity::find_by_id(mime_id).one(db).await + { + if super::is_splat_supported(&mime.mime_type) { + if let Ok(path) = crate::ops::indexing::PathResolver::get_full_path( + db, + entry_model.id, + ) + .await + { + self.state.entries.push(( + entry_model.id, + path, + Some(mime.mime_type), + )); + } + } + } + } + } + } + + ctx.log(format!( + "Single file discovered: {} entries", + self.state.entries.len() + )); + return Ok(()); + } + + // Batch mode - discover all eligible entries + let entries = entry::Entity::find() + .filter(entry::Column::ContentId.is_not_null()) + .all(db) + .await?; + + ctx.log(format!("Found {} entries with content", entries.len())); + + for entry_model in entries { + if let Some(content_id) = entry_model.content_id { + if let Ok(Some(ci)) = content_identity::Entity::find_by_id(content_id) + .one(db) + .await + { + if let Some(mime_id) = ci.mime_type_id { + if let Ok(Some(mime)) = mime_type::Entity::find_by_id(mime_id).one(db).await + { + if super::is_splat_supported(&mime.mime_type) { + if let Ok(path) = crate::ops::indexing::PathResolver::get_full_path( + db, + entry_model.id, + ) + .await + { + self.state.entries.push(( + entry_model.id, + path, + Some(mime.mime_type), + )); + } + } + } + } + } + } + } + + ctx.log(format!( + "Discovery complete: {} image files", + self.state.entries.len() + )); + + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Type)] +pub struct GaussianSplatJobOutput { + pub total_processed: usize, + pub success_count: usize, + pub error_count: usize, +} + +impl From for JobOutput { + fn from(output: GaussianSplatJobOutput) -> Self { + JobOutput::GaussianSplat { + total_processed: output.total_processed, + success_count: output.success_count, + error_count: output.error_count, + } + } +} + +impl DynJob for GaussianSplatJob { + fn job_name(&self) -> &'static str { + "Gaussian Splat Generation" + } +} + +impl From for Box { + fn from(job: GaussianSplatJob) -> Self { + Box::new(job) + } +} diff --git a/core/src/ops/media/splat/mod.rs b/core/src/ops/media/splat/mod.rs new file mode 100644 index 000000000..9ffa8b1db --- /dev/null +++ b/core/src/ops/media/splat/mod.rs @@ -0,0 +1,99 @@ +//! Gaussian Splat generation system +//! +//! Generates 3D Gaussian splats from images using Apple's SHARP model. +//! Generates .ply sidecar files for photorealistic view synthesis. + +pub mod action; +pub mod job; +pub mod processor; + +pub use action::{GenerateSplatAction, GenerateSplatInput, GenerateSplatOutput}; +pub use job::{GaussianSplatJob, GaussianSplatJobConfig}; +pub use processor::GaussianSplatProcessor; + +use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; + +/// Generate a 3D Gaussian splat from an image using SHARP +/// +/// This calls the `sharp` CLI tool as a subprocess. +/// The sharp tool must be installed (e.g., via `pip install -r requirements.txt` in ml-sharp repo) +/// +/// # Arguments +/// * `source_path` - Path to the input image +/// * `output_dir` - Directory where the .ply file will be generated +/// * `model_path` - Optional path to the SHARP model checkpoint +/// +/// # Returns +/// Path to the generated .ply file +pub async fn generate_splat_from_image( + source_path: &Path, + output_dir: &Path, + model_path: Option<&Path>, +) -> Result { + use tokio::process::Command; + + // Ensure output directory exists + tokio::fs::create_dir_all(output_dir).await?; + + // Build command + let mut cmd = Command::new("sharp"); + cmd.arg("predict") + .arg("-i") + .arg(source_path) + .arg("-o") + .arg(output_dir); + + // Add model path if provided + if let Some(model) = model_path { + cmd.arg("-c").arg(model); + } + + // Execute + let output = cmd + .output() + .await + .context("Failed to execute 'sharp' command. Is it installed?")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("SHARP failed: {}", stderr); + } + + // The output file will be named based on input filename with .ply extension + let ply_filename = source_path + .file_stem() + .context("Invalid source filename")? + .to_str() + .context("Non-UTF8 filename")?; + + let ply_path = output_dir.join(format!("{}.ply", ply_filename)); + + if !ply_path.exists() { + anyhow::bail!( + "SHARP did not generate expected output file: {:?}", + ply_path + ); + } + + Ok(ply_path) +} + +/// Check if SHARP CLI is available in PATH +pub async fn check_sharp_available() -> Result { + let output = tokio::process::Command::new("sharp") + .arg("--help") + .output() + .await; + + Ok(output.is_ok()) +} + +/// Check if an image type is supported for splat generation +pub fn is_splat_supported(mime_type: &str) -> bool { + // SHARP supports common image formats + matches!( + mime_type, + "image/jpeg" | "image/png" | "image/webp" | "image/bmp" | "image/tiff" + ) +} diff --git a/core/src/ops/media/splat/processor.rs b/core/src/ops/media/splat/processor.rs new file mode 100644 index 000000000..0ea15ebe7 --- /dev/null +++ b/core/src/ops/media/splat/processor.rs @@ -0,0 +1,167 @@ +//! Gaussian splat processor - generates 3D splats from images + +use crate::library::Library; +use crate::ops::indexing::processor::{ProcessorEntry, ProcessorResult}; +use crate::ops::indexing::state::EntryKind; +use crate::ops::sidecar::types::{SidecarFormat, SidecarKind, SidecarVariant}; +use anyhow::Result; +use serde_json::Value; +use std::sync::Arc; +use tracing::{debug, warn}; + +pub struct GaussianSplatProcessor { + library: Arc, + model_path: Option, +} + +impl GaussianSplatProcessor { + pub fn new(library: Arc) -> Self { + Self { + library, + model_path: None, + } + } + + pub fn with_model_path(mut self, path: String) -> Self { + self.model_path = Some(path); + self + } + + pub fn with_settings(mut self, settings: &Value) -> Result { + if let Some(path) = settings.get("model_path").and_then(|v| v.as_str()) { + self.model_path = Some(path.to_string()); + } + + Ok(self) + } + + pub fn should_process(&self, entry: &ProcessorEntry) -> bool { + if !matches!(entry.kind, EntryKind::File) { + return false; + } + + if entry.content_id.is_none() { + return false; + } + + entry + .mime_type + .as_ref() + .map_or(false, |m| super::is_splat_supported(m)) + } + + pub async fn process( + &self, + db: &sea_orm::DatabaseConnection, + entry: &ProcessorEntry, + ) -> Result { + let content_uuid = if let Some(content_id) = entry.content_id { + use crate::infra::db::entities::content_identity; + use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; + + let ci = content_identity::Entity::find() + .filter(content_identity::Column::Id.eq(content_id)) + .one(db) + .await? + .ok_or_else(|| anyhow::anyhow!("ContentIdentity not found"))?; + + ci.uuid + .ok_or_else(|| anyhow::anyhow!("ContentIdentity missing UUID"))? + } else { + return Ok(ProcessorResult::failure( + "Entry has no content_id".to_string(), + )); + }; + + debug!("→ Generating Gaussian splat for: {}", entry.path.display()); + + // Get sidecar manager + let sidecar_manager = self + .library + .core_context() + .get_sidecar_manager() + .await + .ok_or_else(|| anyhow::anyhow!("SidecarManager not available"))?; + + // Check if splat already exists + if sidecar_manager + .exists( + &self.library.id(), + &content_uuid, + &SidecarKind::GaussianSplat, + &SidecarVariant::new("ply"), + &SidecarFormat::Ply, + ) + .await + .unwrap_or(false) + { + debug!("Gaussian splat already exists for {}", content_uuid); + return Ok(ProcessorResult::success(0, 0)); + } + + // Compute sidecar path + let sidecar_path = sidecar_manager + .compute_path( + &self.library.id(), + &content_uuid, + &SidecarKind::GaussianSplat, + &SidecarVariant::new("ply"), + &SidecarFormat::Ply, + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to compute path: {}", e))?; + + // Create temporary output directory + let temp_dir = std::env::temp_dir().join(format!("sd_splat_{}", content_uuid)); + tokio::fs::create_dir_all(&temp_dir).await?; + + // Generate splat using SHARP + let model_path_ref = self.model_path.as_ref().map(|s| std::path::Path::new(s)); + let ply_path = super::generate_splat_from_image(&entry.path, &temp_dir, model_path_ref) + .await + .map_err(|e| anyhow::anyhow!("Failed to generate Gaussian splat: {}", e))?; + + // Read generated PLY file + let ply_data = tokio::fs::read(&ply_path).await?; + let ply_size = ply_data.len(); + + debug!("Generated splat: {} bytes", ply_size); + + // Ensure sidecar directory exists + if let Some(parent) = sidecar_path.absolute_path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + + // Copy PLY to sidecar location + tokio::fs::copy(&ply_path, &sidecar_path.absolute_path).await?; + + // Clean up temp directory + let _ = tokio::fs::remove_dir_all(&temp_dir).await; + + debug!( + "✓ Generated Gaussian splat: {} ({} bytes)", + sidecar_path.relative_path.display(), + ply_size + ); + + // Register sidecar in database + sidecar_manager + .record_sidecar( + &self.library, + &content_uuid, + &SidecarKind::GaussianSplat, + &SidecarVariant::new("ply"), + &SidecarFormat::Ply, + ply_size as u64, + None, + ) + .await + .map_err(|e| anyhow::anyhow!("Failed to record sidecar: {}", e))?; + + Ok(ProcessorResult::success(1, ply_size as u64)) + } + + pub fn name(&self) -> &'static str { + "gaussian_splat" + } +} diff --git a/core/src/ops/sidecar/types.rs b/core/src/ops/sidecar/types.rs index f3d7102c8..7e74ce3cd 100644 --- a/core/src/ops/sidecar/types.rs +++ b/core/src/ops/sidecar/types.rs @@ -11,6 +11,7 @@ pub enum SidecarKind { Embeddings, Ocr, Transcript, + GaussianSplat, } impl SidecarKind { @@ -22,6 +23,7 @@ impl SidecarKind { Self::Embeddings => "embeddings", Self::Ocr => "ocr", Self::Transcript => "transcript", + Self::GaussianSplat => "gaussian_splat", } } @@ -33,6 +35,7 @@ impl SidecarKind { Self::Embeddings => "embeddings", Self::Ocr => "ocr", Self::Transcript => "transcript", + Self::GaussianSplat => "gaussian_splats", } } } @@ -54,6 +57,7 @@ impl TryFrom<&str> for SidecarKind { "embeddings" => Ok(Self::Embeddings), "ocr" => Ok(Self::Ocr), "transcript" => Ok(Self::Transcript), + "gaussian_splat" => Ok(Self::GaussianSplat), _ => Err(format!("Invalid sidecar kind: {}", value)), } } @@ -98,6 +102,7 @@ impl From for SidecarVariant { /// - Json: Text-based structured data (OCR, transcripts) /// - MessagePack: Binary structured data (embeddings, vectors) /// - Text: Plain text extractions +/// - Ply: 3D model format for Gaussian splats /// /// MessagePack is preferred for embeddings because: /// - 6x smaller than JSON (1.7KB vs 10KB per 384-dim vector) @@ -112,6 +117,7 @@ pub enum SidecarFormat { Json, MessagePack, Text, + Ply, } impl SidecarFormat { @@ -122,6 +128,7 @@ impl SidecarFormat { Self::Json => "json", Self::MessagePack => "msgpack", Self::Text => "txt", + Self::Ply => "ply", } } @@ -132,6 +139,7 @@ impl SidecarFormat { Self::Json => "json", Self::MessagePack => "messagepack", Self::Text => "text", + Self::Ply => "ply", } } } @@ -152,6 +160,7 @@ impl TryFrom<&str> for SidecarFormat { "json" => Ok(Self::Json), "msgpack" | "messagepack" => Ok(Self::MessagePack), "text" | "txt" => Ok(Self::Text), + "ply" => Ok(Self::Ply), _ => Err(format!("Invalid sidecar format: {}", value)), } } diff --git a/packages/interface/src/components/QuickPreview/ContentRenderer.tsx b/packages/interface/src/components/QuickPreview/ContentRenderer.tsx index 2fa40bd53..35c07e06a 100644 --- a/packages/interface/src/components/QuickPreview/ContentRenderer.tsx +++ b/packages/interface/src/components/QuickPreview/ContentRenderer.tsx @@ -8,6 +8,7 @@ import { MagnifyingGlassPlus, MagnifyingGlassMinus, ArrowCounterClockwise, + Cube, } from "@phosphor-icons/react"; import { VideoPlayer } from "./VideoPlayer"; import { AudioPlayer } from "./AudioPlayer"; @@ -28,12 +29,29 @@ function ImageRenderer({ file, onZoomChange }: ContentRendererProps) { const [originalLoaded, setOriginalLoaded] = useState(false); const [originalUrl, setOriginalUrl] = useState(null); const [shouldLoadOriginal, setShouldLoadOriginal] = useState(false); + const [showSplat, setShowSplat] = useState(false); const { zoom, zoomIn, zoomOut, reset, isZoomed, transform } = useZoomPan(containerRef); // Get a stable identifier for the image file itself const imageFileId = file.content_identity?.uuid || file.id; + // Check if Gaussian splat sidecar exists and get URL + const splatSidecar = file.sidecars?.find( + (s) => s.kind === "gaussian_splat" && s.format === "ply" + ); + const hasSplat = !!splatSidecar; + + // Build sidecar URL for the splat + const splatUrl = hasSplat && file.content_identity?.uuid + ? buildSidecarUrl( + file.content_identity.uuid, + splatSidecar!.kind, + splatSidecar!.variant, + splatSidecar!.format, + ) + : null; + // Notify parent of zoom state changes useEffect(() => { onZoomChange?.(isZoomed); @@ -44,6 +62,7 @@ function ImageRenderer({ file, onZoomChange }: ContentRendererProps) { setShouldLoadOriginal(false); setOriginalLoaded(false); setOriginalUrl(null); + setShowSplat(false); // Reset splat view when file changes const timer = setTimeout(() => { setShouldLoadOriginal(true); @@ -107,11 +126,54 @@ function ImageRenderer({ file, onZoomChange }: ContentRendererProps) { const thumbnailUrl = getHighestResThumbnail(); + // Render splat view separately (not overlayed) + if (showSplat && hasSplat && splatUrl) { + return ( + <> + {/* Splat Toggle */} +
+ +
+ + {/* Splat Viewer - matches direct .ply rendering structure */} + + + + } + > + + + + ); + } + + // Render image view with zoom/pan return (
+ {/* Splat Toggle (top-left) */} + {hasSplat && ( +
+ +
+ )} + {/* Zoom Controls */}
)} + {/* Gaussian Splat for images */} + {isImage && ( + + )} + {/* Speech-to-text for audio/video */} {(isVideo || isAudio) && (