mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-31 10:28:43 -04:00
9058a2bb46
* feat(3d): add Generate3D RPC, FLAG_3D capability, and /v1/3d/generations endpoint Adds the plumbing for image-conditioned 3D asset generation (binary glTF / GLB output), modeled on the video generation path: - backend.proto: Generate3D RPC + Generate3DRequest (staged image src, glb dst, seed/step/cfg_scale/texture_steps, quality and background enums, params map for backend-specific extras) - pkg/grpc: thread Generate3D through client, server, embed, base and the backend interfaces; connection-evicting and distributed-node wrappers (in-flight tracking + file staging) included - core/config: FLAG_3D usecase (guessed only for the trellis2cpp backend), '3d' canonical usecase string mapped to the Generate3D method, and a '3d' output modality - REST: POST /v1/3d/generations (+ unversioned alias) returning OpenAIResponse with a /generated-3d URL or b64_json; conditioning image accepted as URL, base64, or data URI; quality/background validated at the edge; .glb served as model/gltf-binary - auth: '3d' route feature (default ON); /api/instructions entry Assisted-by: Claude:claude-fable-5 [Claude Code] Signed-off-by: Richard Palethorpe <io@richiejp.com> * feat(trellis2cpp): add the trellis2.cpp image-to-3D backend Wraps localai-org/trellis2cpp (C++/GGML port of Microsoft TRELLIS.2, pbr-textures branch) as a Go+purego backend, following the stablediffusion-ggml pattern: - backend/go/trellis2cpp: purego bindings to the flat C ABI (v9, asserted at startup), eager pipeline load with model-set validation (refuses non-trellis GGUFs; degrades coarse/geometry-only/textured exactly like the upstream demo), Generate3D via t2_generate + t2_bake_glb writing a binary glTF to dst. Weight-free unit tests cover resolution/validation/param mapping — CI never downloads the multi-GB GGUF set or runs inference. - CPU SIMD variants build into per-variant directories (the shared libggml sonames collide across variants, unlike sd-ggml's flat renamed-.so scheme); run.sh picks one via /proc/cpuinfo. - CI wiring: backend-matrix entries (cpu, cuda12/13, vulkan amd64+arm64, l4t, l4t-cuda13, darwin metal), index.yaml meta + latest/master image entries, bump_deps tracking of the pbr-textures branch, changed-backends.js mapping, top-level Makefile targets. - Importer: auto-detects trellis GGUF repos/URIs (registered before llama-cpp so the .gguf match isn't stolen) and expands any trellis URI to the full 10-file component set spanning the three LocalAI-io HF repos. - Gallery: trellis2-4b (full PBR + 1024 cascade) and trellis2-4b-geometry (512 untextured) with verified sha256s. Assisted-by: Claude:claude-fable-5 [Claude Code] Signed-off-by: Richard Palethorpe <io@richiejp.com> * feat(ui): 3D generation page with native GLB viewer and IndexedDB history Adds a Studio tab + /app/3d page for the new image-to-3D endpoint: - GlbViewer ports the trellis2cpp demo's dependency-free WebGL2 renderer (quaternion trackball, metallic-roughness PBR, ACES, hidden-line wireframe with a bounded index budget) and pairs it with a minimal GLB parser for the two forms t2_bake_glb emits — dense vertex-PBR (linear COLOR_0 + _METALLIC_ROUGHNESS, uploaded as normalized integers) and the opt-in UV-atlas textured form. Parsing happens before any GL so stats and errors render without WebGL2. - use3DHistory stores past generations (params, input thumbnail, and the GLB blob itself) in IndexedDB with keep-newest-20 eviction — GLBs are multi-MB binaries localStorage can't hold — and the page offers a download button for the active GLB. - Wiring: CAP_3D capability constant (FLAG_3D — the exact string /api/models/capabilities serves), threeDApi, router entries, Studio tab, vite dev proxy, en locale keys. - e2e: render-smoke entry plus a focused spec that feeds a real one-triangle vertex-PBR GLB through the parser/viewer and exercises IndexedDB persistence, selection, deletion, and API errors. Assisted-by: Claude:claude-fable-5 [Claude Code] Signed-off-by: Richard Palethorpe <io@richiejp.com> * fix(3d): address API correctness and UX issues Keep 3D generation on the LocalAI-specific /3d/generations route and ensure authentication and permissions cover it. Propagate distributed transfer failures, publish a portable ARM64 backend image, honor importer overrides, and align discovery, upload validation, and touch controls. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe <io@richiejp.com> * feat(3d): add previewable print remeshing Add a single-detail CGAL Alpha Wrap workflow for existing Trellis GLBs, including PBR reprojection, API documentation, tracing, and an in-browser preview before download. Allow the remesh route to enforce its 512 MiB upload cap independently of the smaller global default so generated high-resolution meshes can be processed. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe <io@richiejp.com> * build(trellis2cpp): centralize remesh dependency pins Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Richard Palethorpe <io@richiejp.com> * fix(kokoros): implement Generate3D stub for new proto RPC The Generate3D RPC added to backend.proto for the trellis2cpp backend made tonic's generated Backend trait require generate3_d, breaking the kokoros-grpc build. Return unimplemented like the other unsupported modalities. Assisted-by: Claude Code:claude-fable-5 Signed-off-by: Richard Palethorpe <io@richiejp.com> --------- Signed-off-by: Richard Palethorpe <io@richiejp.com> Co-authored-by: localai-org-maint-bot <bot-opensource@localaisrl.com>
75 lines
3.0 KiB
Go
75 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
"math"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
func tinyVertexGLB() []byte {
|
|
bin := make([]byte, 80)
|
|
positions := []float32{1, 2, 3, 4, 5, 6, 7, 8, 9}
|
|
for i, value := range positions {
|
|
binary.LittleEndian.PutUint32(bin[i*4:], math.Float32bits(value))
|
|
}
|
|
colors := []uint16{
|
|
65535, 0, 0, 65535,
|
|
0, 65535, 0, 32768,
|
|
0, 0, 65535, 65535,
|
|
}
|
|
for i, value := range colors {
|
|
binary.LittleEndian.PutUint16(bin[36+i*2:], value)
|
|
}
|
|
copy(bin[60:], []byte{0, 153, 64, 128, 255, 32})
|
|
for i, value := range []uint32{0, 1, 2} {
|
|
binary.LittleEndian.PutUint32(bin[68+i*4:], value)
|
|
}
|
|
|
|
jsonChunk := []byte(fmt.Sprintf(`{"asset":{"version":"2.0"},"meshes":[{"primitives":[{"attributes":{"POSITION":0,"COLOR_0":1,"_METALLIC_ROUGHNESS":2},"indices":3}]}],"accessors":[{"bufferView":0,"componentType":5126,"count":3,"type":"VEC3"},{"bufferView":1,"componentType":5123,"normalized":true,"count":3,"type":"VEC4"},{"bufferView":2,"componentType":5121,"normalized":true,"count":3,"type":"VEC2"},{"bufferView":3,"componentType":5125,"count":3,"type":"SCALAR"}],"bufferViews":[{"buffer":0,"byteOffset":0,"byteLength":36},{"buffer":0,"byteOffset":36,"byteLength":24},{"buffer":0,"byteOffset":60,"byteLength":6},{"buffer":0,"byteOffset":68,"byteLength":12}],"buffers":[{"byteLength":%d}]}`, len(bin)))
|
|
for len(jsonChunk)%4 != 0 {
|
|
jsonChunk = append(jsonChunk, ' ')
|
|
}
|
|
total := 12 + 8 + len(jsonChunk) + 8 + len(bin)
|
|
glb := make([]byte, total)
|
|
binary.LittleEndian.PutUint32(glb[0:], glbMagic)
|
|
binary.LittleEndian.PutUint32(glb[4:], 2)
|
|
binary.LittleEndian.PutUint32(glb[8:], uint32(total))
|
|
binary.LittleEndian.PutUint32(glb[12:], uint32(len(jsonChunk)))
|
|
binary.LittleEndian.PutUint32(glb[16:], glbJSONChunk)
|
|
copy(glb[20:], jsonChunk)
|
|
binHeader := 20 + len(jsonChunk)
|
|
binary.LittleEndian.PutUint32(glb[binHeader:], uint32(len(bin)))
|
|
binary.LittleEndian.PutUint32(glb[binHeader+4:], glbBINChunk)
|
|
copy(glb[binHeader+8:], bin)
|
|
return glb
|
|
}
|
|
|
|
var _ = Describe("vertex GLB parsing for print remeshing", func() {
|
|
It("restores trellis coordinates, topology, and PBR values", func() {
|
|
mesh, err := parseVertexGLB(tinyVertexGLB())
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(mesh.verts).To(Equal([]float32{1, -3, 2, 4, -6, 5, 7, -9, 8}))
|
|
Expect(mesh.tris).To(Equal([]int32{0, 1, 2}))
|
|
Expect(mesh.pbr).To(HaveLen(18))
|
|
Expect(mesh.pbr[0]).To(BeNumerically("~", 1, 1e-5))
|
|
Expect(mesh.pbr[3]).To(BeNumerically("~", 0, 1e-5))
|
|
Expect(mesh.pbr[4]).To(BeNumerically("~", 0.6, 0.01))
|
|
Expect(mesh.pbr[11]).To(BeNumerically("~", 32768.0/65535.0, 1e-5))
|
|
})
|
|
|
|
It("rejects indices outside the source vertex buffer", func() {
|
|
glb := tinyVertexGLB()
|
|
binary.LittleEndian.PutUint32(glb[len(glb)-12:], 3)
|
|
_, err := parseVertexGLB(glb)
|
|
Expect(err).To(MatchError(ContainSubstring("outside the vertex buffer")))
|
|
})
|
|
|
|
It("rejects non-GLB input", func() {
|
|
_, err := parseVertexGLB([]byte("not a mesh"))
|
|
Expect(err).To(MatchError("input is not a GLB file"))
|
|
})
|
|
})
|