From 89ee62b2af72ac5ac0d651c3930ec37274253ede Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Wed, 29 Jul 2026 15:35:51 +0200 Subject: [PATCH 1/8] gallery: add KAT-Coder V2.5 Dev GGUF variants (#11186) * gallery: add KAT-Coder V2.5 Dev GGUF variants Add Q4_K_M and Q8_0 builds of the newly released KAT-Coder-V2.5-Dev agentic coding model. Assisted-by: Codex:gpt-5 [Hugging Face API] * gallery: add KAT-Coder APEX variants Assisted-by: Codex:gpt-5 [web] * gallery: add KAT-Coder APEX checksums Assisted-by: Codex:gpt-5 --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 226 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 1144c7e4b..67e219c51 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,230 @@ --- +- &kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev" + variants: + - model: kat-coder-v2.5-dev-q8 + - model: kat-coder-v2.5-dev-apex-i-quality + - model: kat-coder-v2.5-dev-apex-i-balanced + - model: kat-coder-v2.5-dev-apex-i-compact + - model: kat-coder-v2.5-dev-apex-i-mini + - model: kat-coder-v2.5-dev-apex-quality + - model: kat-coder-v2.5-dev-apex-balanced + - model: kat-coder-v2.5-dev-apex-compact + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev + - https://huggingface.co/bartowski/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF + - https://huggingface.co/mudler/KAT-Coder-V2.5-Dev-APEX-GGUF + description: | + KAT-Coder-V2.5-Dev is an Apache-2.0 agentic coding model from Kwaipilot, + post-trained from Qwen3.6-35B-A3B. It has 35 billion total parameters with + 3 billion activated per token, a 262K-token context window, and text-only + weights tuned for repository-level coding and tool use. + + This entry offers standard Q4_K_M and Q8_0 GGUF quantizations alongside + APEX mixed-precision variants with quality, balanced, compact, and mini + profiles. The APEX I-profiles use importance-matrix calibration. + license: "apache-2.0" + icon: https://huggingface.co/Kwaipilot/KAT-Coder-V2.5-Dev/resolve/main/kat_logo_hd.png + tags: + - llm + - gguf + - qwen + - coding + last_checked: "2026-07-29" + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF/Kwaipilot_KAT-Coder-V2.5-Dev-Q4_K_M.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF/Kwaipilot_KAT-Coder-V2.5-Dev-Q4_K_M.gguf + sha256: 4221c26e5663502d1c96fc901c9967d0e70ce2dcfaa5a9fb9280a46bd19e3c07 + uri: huggingface://bartowski/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF/Kwaipilot_KAT-Coder-V2.5-Dev-Q4_K_M.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-q8" + variants: [] + description: | + KAT-Coder-V2.5-Dev is an Apache-2.0 agentic coding model from Kwaipilot, + post-trained from Qwen3.6-35B-A3B. It has 35 billion total parameters with + 3 billion activated per token, a 262K-token context window, and text-only + weights tuned for repository-level coding and tool use. + + This entry uses the higher-quality Q8_0 GGUF quantization. + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF/Kwaipilot_KAT-Coder-V2.5-Dev-Q8_0.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF/Kwaipilot_KAT-Coder-V2.5-Dev-Q8_0.gguf + sha256: 5fa510f44779b0e3d38a6678985f417a1c65e3000405ca5d6dcf7fd065e47a15 + uri: huggingface://bartowski/Kwaipilot_KAT-Coder-V2.5-Dev-GGUF/Kwaipilot_KAT-Coder-V2.5-Dev-Q8_0.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-apex-i-quality" + variants: [] + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Quality.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Quality.gguf + sha256: e1cf7f33e13ee787a8557effee41eef5261b24696e283ef9d320830ba39f6784 + uri: huggingface://mudler/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Quality.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-apex-i-balanced" + variants: [] + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Balanced.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Balanced.gguf + sha256: ee6e0ec15964c42ba91831d13e9709239f1b74da3dc49dd3edec4ad6aed8029f + uri: huggingface://mudler/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Balanced.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-apex-i-compact" + variants: [] + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Compact.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Compact.gguf + sha256: 5235ac39e7989d9fcf08078acfa755611d42a1f80060d226e4aba04a3595d0d8 + uri: huggingface://mudler/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Compact.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-apex-i-mini" + variants: [] + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Mini.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Mini.gguf + sha256: 9d901bf1c44946840e15e6f73a66782f56ae2f1a41d6508995ddcd976e9af878 + uri: huggingface://mudler/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-I-Mini.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-apex-quality" + variants: [] + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Quality.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Quality.gguf + sha256: 849737baf59183ed518d34f2460f0dcd86190953ccc3213072bdcb1d7f8d2882 + uri: huggingface://mudler/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Quality.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-apex-balanced" + variants: [] + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Balanced.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Balanced.gguf + sha256: 9a5d31b110a95bc085d9851c420dd7a1ffeaec7c9263073c8850d8fb81c4f8ba + uri: huggingface://mudler/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Balanced.gguf +- !!merge <<: *kat-coder-v2-5-dev + name: "kat-coder-v2.5-dev-apex-compact" + variants: [] + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Compact.gguf + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Compact.gguf + sha256: 98dfee53102bbf01e67a768066543a707b40f365ff71e35132ac432c58ad99fd + uri: huggingface://mudler/KAT-Coder-V2.5-Dev-APEX-GGUF/KAT-Coder-V2.5-Dev-APEX-Compact.gguf - name: "inkling" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" urls: From 8089b2bf0998428975da076d06faab592fd415e6 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:58:27 +0200 Subject: [PATCH 2/8] fix(ci): only rebuild the full backend matrix on breaking backend.proto edits (#11192) backend/backend.proto is consumed by every language, so its SHARED_BUILD_INPUTS rule could only ever be always/always: 417 Linux plus 56 Darwin builds. It fires on ~1.3% of commits (10 of 767 over six months), which made it the single largest CI cost driver in the repo. On 2026-07-29 the queue reached 2178 jobs against 8 concurrent runners. Four runs totalling 935 of those jobs were triggered by nothing but a proto edit. The largest, 378 jobs on master, came from PR #11158, whose entire proto diff was six lines adding `bool cache_prompt = 8;` to one message. No backend that does not read that field behaves any differently for it. Make the rule content-aware. changed-backends.js resolves backend.proto at the base revision (the contents-API pattern already used for backend-matrix.yml) and hands both texts to protoChangeIsAdditive(), which compares them structurally so a comment reflow, reindent or field reorder does not read as a change. An additive-only edit (new field with an unused number, new message, new enum value, new RPC) suppresses the rule and rebuilds nothing; a removed, renumbered, retyped or renamed field, a dropped RPC or a changed option still rebuilds everything, as does an unresolvable base revision. Every other matched rule is untouched, so a PR that edits the proto and scripts/build/ is still a full rebuild, and the weekly full-matrix cron remains the backstop for stale wheels. Verified against all ten proto commits of the preceding six months: the nine with a resolvable parent all classify as additive, and controls covering a retyped-and-renumbered field, a deleted RPC, identical revisions and a reindent-plus-comment-reflow all classify correctly. Assisted-by: Claude:opus-5 [claude-code] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .agents/ci-caching.md | 13 ++- scripts/changed-backends.js | 46 ++++++++- scripts/lib/backend-filter.mjs | 142 ++++++++++++++++++++++++++- scripts/lib/backend-filter_test.mjs | 143 ++++++++++++++++++++++++++++ 4 files changed, 339 insertions(+), 5 deletions(-) diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 17cc6001d..26aba82e6 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -122,7 +122,7 @@ The per-backend prefix match only sees files under a backend's own directory, so | Changed path | Rebuilds | |---|---| -| `backend/backend.proto` | everything (all languages compile or copy it) | +| `backend/backend.proto` | nothing if the edit is additive-only, otherwise everything (see below) | | `backend/Dockerfile.` | the Linux entries whose `dockerfile:` names it | | `backend/python/common/` | Python, Linux + Darwin | | `scripts/build/package-gpu-libs.sh` | Python, Linux only | @@ -132,6 +132,17 @@ The per-backend prefix match only sees files under a backend's own directory, so Deliberately excluded: `backend/index.yaml` (gallery metadata, never enters an image), `.github/backend-matrix.yml` (adding a backend would rebuild all of them), `backend/Dockerfile.base-grpc-builder` (owned by `base-images.yml`), and the root `Makefile` (touched in ~11% of commits, and its backend-relevant edits arrive alongside the backend directory anyway). `make test-ci-scripts` pins all of this. +#### `backend/backend.proto` is content-filtered, not path-filtered + +Every language consumes the proto, so a path rule for it can only ever say "rebuild all 473 images". It changes in ~1.3% of commits, and that was enough to make it the single largest CI cost driver in the repo: on 2026-07-29 four runs totalling 935 queued jobs traced to nothing but a proto edit, one of which (#11158) was a six-line diff adding `bool cache_prompt = 8;`. + +An additive proto edit cannot change how a backend that never references the new symbol behaves, so `filterMatrix()` suppresses the rule for one. `changed-backends.js` fetches `backend/backend.proto` at the base revision (same contents-API pattern as `.github/backend-matrix.yml`) and hands both texts to `protoChangeIsAdditive()`, which compares them structurally rather than textually: + +- **Additive, rebuilds nothing**: a new field with an unused number, a new message, a new enum value, a new RPC. Comment, whitespace and ordering changes also land here. +- **Breaking, rebuilds everything**: a removed, renumbered, retyped or renamed field, a dropped RPC, a changed `option` or `package`. So does an unresolvable base revision, matching the run-all posture used for a truncated diff. + +Checked against every proto commit in the preceding six months, all nine resolvable ones classify as additive. Note the tradeoff this accepts: generated stubs do change for an additive edit, so image bytes would differ on a rebuild even though behavior does not. That is the same standard already applied when the filter declines to rebuild on unrelated `pkg/` changes, and the weekly cron remains the backstop. + The Sunday 06:00 UTC cron on `backend.yml` exists specifically because path filtering can leave Python backends frozen on stale wheels. `DEPS_REFRESH` (below) only fires when the build actually runs, so an untouched Python backend would never re-resolve its unpinned deps. The weekly cron is the safety net. ## The `DEPS_REFRESH` cache-buster (Python backends) diff --git a/scripts/changed-backends.js b/scripts/changed-backends.js index 1c87d0247..08a024fc6 100644 --- a/scripts/changed-backends.js +++ b/scripts/changed-backends.js @@ -6,6 +6,7 @@ import { getAllBackendPaths, filterMatrix, BACKEND_MATRIX_FILE, + BACKEND_PROTO_FILE, } from "./lib/backend-filter.mjs"; // Matrix data lives in a small data-only YAML so both backend.yml (master push) @@ -116,6 +117,42 @@ async function getPreviousMatrix(event) { } } +// backend.proto at the base revision plus the checked-out copy, so filterMatrix +// can tell an additive edit (a new field, message or RPC, which invalidates no +// existing image) from a breaking one. Returning null means "rebuild +// everything", the same posture as an unresolvable matrix diff. +// +// Only called when the changed-file list names the proto, so the common path +// costs no extra API request. +async function getProtoRevisions(event) { + const ref = event.pull_request ? event.pull_request.base.sha : event.before; + if (!ref || /^0+$/.test(ref)) return null; + const owner = event.repository.owner.login; + const repo = event.repository.name; + try { + const res = await octokit.request('GET /repos/{owner}/{repo}/contents/{path}', { + owner, + repo, + path: BACKEND_PROTO_FILE, + ref, + mediaType: { format: 'raw' }, + }); + const previous = typeof res.data === 'string' + ? res.data + : Buffer.from(res.data.content, 'base64').toString('utf8'); + return { + previous, + current: fs.readFileSync(BACKEND_PROTO_FILE, "utf8"), + }; + } catch (err) { + console.log( + `could not read ${BACKEND_PROTO_FILE} at ${ref}, falling back to run-all:`, + err.message + ); + return null; + } +} + // Group matrix entries by tag-suffix and emit a merge-matrix entry per group. // Both multi-leg groups (per-arch fan-out) and singletons get one entry each: // the build job pushes by digest only with no tags applied, so every backend @@ -240,7 +277,7 @@ function emitFullMatrix() { } } -function emitFilteredMatrix(changedFiles, previousMatrix) { +function emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions) { console.log("Changed files:", changedFiles); const { filtered, filteredDarwin, changedBackends } = filterMatrix({ @@ -248,6 +285,7 @@ function emitFilteredMatrix(changedFiles, previousMatrix) { includesDarwin, changedFiles, previousMatrix, + protoRevisions, }); console.log("Filtered files:", filtered); @@ -306,5 +344,9 @@ function emitFilteredMatrix(changedFiles, previousMatrix) { ? await getPreviousMatrix(event) : null; - emitFilteredMatrix(changedFiles, previousMatrix); + const protoRevisions = changedFiles.includes(BACKEND_PROTO_FILE) + ? await getProtoRevisions(event) + : null; + + emitFilteredMatrix(changedFiles, previousMatrix, protoRevisions); })(); diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs index 50edd1726..ccded8d53 100644 --- a/scripts/lib/backend-filter.mjs +++ b/scripts/lib/backend-filter.mjs @@ -178,6 +178,126 @@ const GO_BACKEND_PKG_PREFIXES = [ "pkg/utils/", ]; +export const BACKEND_PROTO_FILE = "backend/backend.proto"; +const PROTO_RULE_ID = "backend-proto"; + +// Split a .proto into a map of symbol -> fingerprint so two revisions can be +// compared structurally instead of textually. A comment reflow, a reindent or a +// reordered field must not read as a change, and a renumbered or retyped field +// must. +// +// The scanner is deliberately syntax-light: it tracks brace depth to build a +// container path and records one entry per declaration (`message`, `enum`, +// `service`, `oneof`, `rpc`) and one per statement (fields, enum values, +// `option`, `reserved`). It never needs to understand types, only to notice +// when the text describing one stops being identical. +function protoSymbols(text) { + const symbols = new Map(); + const stack = []; + const norm = s => s.trim().replace(/\s+/g, " "); + + // A declaration is identified by its kind and name, so that changing its body + // shows up as changed members rather than as a wholesale replacement. + const declKey = header => { + const rpc = header.match(/^rpc\s+([A-Za-z_]\w*)/); + if (rpc) return `rpc ${rpc[1]}`; + const decl = header.match(/^(message|enum|service|oneof|extend)\s+([A-Za-z_]\w*)/); + if (decl) return `${decl[1]} ${decl[2]}`; + return header; + }; + + // `bool cache_prompt = 8` is identified by `cache_prompt`, so renumbering or + // retyping it changes the fingerprint under a stable key, while renaming it + // reads as a removal plus an addition. Statements with no `=` (`reserved 4;`) + // are their own identity. + const stmtKey = stmt => { + const eq = stmt.indexOf("="); + if (eq === -1) return stmt; + const lhs = norm(stmt.slice(0, eq)).split(" "); + return lhs[lhs.length - 1] || stmt; + }; + + let buf = ""; + let i = 0; + while (i < text.length) { + const c = text[i]; + + // String literals first: `option go_package = "github.com/..."` contains a + // `//` that is not a comment. + if (c === '"' || c === "'") { + buf += c; + i++; + while (i < text.length) { + if (text[i] === "\\") { + buf += text.slice(i, i + 2); + i += 2; + continue; + } + buf += text[i]; + i++; + if (text[i - 1] === c) break; + } + continue; + } + if (c === "/" && text[i + 1] === "/") { + while (i < text.length && text[i] !== "\n") i++; + continue; + } + if (c === "/" && text[i + 1] === "*") { + i += 2; + while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++; + i += 2; + continue; + } + if (c === "{") { + const header = norm(buf); + buf = ""; + i++; + const key = header ? declKey(header) : ""; + if (header) symbols.set(`${stack.join("/")}|${key}`, header); + stack.push(key); + continue; + } + if (c === "}") { + stack.pop(); + buf = ""; + i++; + continue; + } + if (c === ";") { + const stmt = norm(buf); + buf = ""; + i++; + if (stmt) symbols.set(`${stack.join("/")}|${stmtKey(stmt)}`, stmt); + continue; + } + buf += c; + i++; + } + return symbols; +} + +// True when every symbol the previous revision declared survives unchanged into +// the current one. New symbols are free: a backend that never references a new +// field, message or RPC produces the same behavior with or without it, so its +// image does not need rebuilding. +// +// Anything else (a removed, renumbered, retyped or renamed field, a dropped +// RPC, a changed option) is treated as breaking and rebuilds the full matrix. +// Either text being unresolvable is breaking too, matching the run-all posture +// changed-backends.js takes for a diff it cannot compute. +export function protoChangeIsAdditive(previousText, currentText) { + if (typeof previousText !== "string" || typeof currentText !== "string") { + return false; + } + const before = protoSymbols(previousText); + const after = protoSymbols(currentText); + for (const [key, fingerprint] of before) { + if (after.get(key) !== fingerprint) return false; + } + return true; +} + // Shared build inputs: files that end up in, or decide the contents of, images // belonging to backends whose own directory they do not live under. The // per-backend prefix match in filterMatrix() structurally cannot see these, so @@ -200,7 +320,14 @@ export const SHARED_BUILD_INPUTS = [ // backends regenerate their stubs from it via `make protogen-go`, the C++ // CMakeLists compile backend.pb.cc from it, and the Rust crate Makefile // copies it in before build.rs runs. - matches: file => file === "backend/backend.proto", + // + // Which is why this rule is always/always, and why it is the single most + // expensive entry in the table: 417 Linux plus 56 Darwin builds. It fires + // on 1.3% of commits (10 of 767 over six months), and every one of those + // ten was purely additive. filterMatrix() suppresses this rule for an + // additive-only diff, so `id` exists to identify it there. + id: PROTO_RULE_ID, + matches: file => file === BACKEND_PROTO_FILE, linux: always, darwin: always, }, @@ -362,8 +489,19 @@ export function filterMatrix({ includesDarwin, changedFiles, previousMatrix, + protoRevisions, }) { - const sharedRules = matchedSharedRules(changedFiles); + // An additive-only backend.proto edit invalidates no existing image, so drop + // its always/always rule. Every other matched rule still applies: a PR that + // touches the proto and scripts/build/ is still a full rebuild. + const protoAdditiveOnly = + changedFiles.includes(BACKEND_PROTO_FILE) && + !!protoRevisions && + protoChangeIsAdditive(protoRevisions.previous, protoRevisions.current); + + const sharedRules = matchedSharedRules(changedFiles).filter( + rule => !(rule.id === PROTO_RULE_ID && protoAdditiveOnly) + ); const matrixFileChanged = changedFiles.includes(BACKEND_MATRIX_FILE); // The matrix file changed but we could not resolve what it used to say (API diff --git a/scripts/lib/backend-filter_test.mjs b/scripts/lib/backend-filter_test.mjs index 0d99e1e10..5b62807a7 100644 --- a/scripts/lib/backend-filter_test.mjs +++ b/scripts/lib/backend-filter_test.mjs @@ -385,3 +385,146 @@ test("an unavailable previous matrix conservatively rebuilds everything", () => assert.equal(filtered.length, includes.length); assert.equal(filteredDarwin.length, includesDarwin.length); }); + +// --- backend.proto: additive changes must not rebuild the world ------------- +// +// backend/backend.proto is consumed by every language, so the SHARED_BUILD_INPUTS +// rule for it is always/always: a full 417-entry Linux matrix plus all 56 Darwin +// entries. It fires on 1.3% of commits, and in practice every one of those has +// been purely additive (a new field with an unused number, a new message, a new +// RPC). Adding `bool cache_prompt = 8;` (PR #11158) cannot change the behavior of +// a backend that never reads it, yet it rebuilt all 473 images. +// +// So the rule becomes content-aware rather than path-aware, using the same shape +// as previousMatrix above: changed-backends.js resolves the base revision and +// hands the two texts in, and everything here stays pure. + +const protoWith = body => ` +syntax = "proto3"; + +package backend; + +service Backend { + rpc Health(HealthMessage) returns (Reply) {} + rpc Predict(PredictOptions) returns (Reply) {} +} + +message HealthMessage {} + +message PredictOptions { +${body} +} +`; + +const BASE_FIELDS = ` string Prompt = 1; + int32 Tokens = 2; + bool UseTokenizerTemplate = 3;`; + +const runProto = (previous, current) => + filterMatrix({ + includes, + includesDarwin, + changedFiles: ["backend/backend.proto"], + protoRevisions: { previous, current }, + }); + +test("an added proto field rebuilds nothing", () => { + // The real PR #11158 diff: six added lines, one new field, unused number. + const { filtered, filteredDarwin, changedBackends } = runProto( + protoWith(BASE_FIELDS), + protoWith(`${BASE_FIELDS}\n bool cache_prompt = 8;`) + ); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); + assert.equal(changedBackends.size, 0); +}); + +test("an added proto message and RPC rebuild nothing", () => { + const current = protoWith(BASE_FIELDS).replace( + "message HealthMessage {}", + "message HealthMessage {}\n\nmessage ScoreRequest {\n string Text = 1;\n}" + ).replace( + " rpc Predict(PredictOptions) returns (Reply) {}", + " rpc Predict(PredictOptions) returns (Reply) {}\n rpc Score(ScoreRequest) returns (Reply) {}" + ); + + const { filtered, filteredDarwin } = runProto(protoWith(BASE_FIELDS), current); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); +}); + +test("a removed proto field rebuilds every backend on every OS", () => { + const { filtered, filteredDarwin } = runProto( + protoWith(BASE_FIELDS), + protoWith(` string Prompt = 1;\n bool UseTokenizerTemplate = 3;`) + ); + + assert.equal(filtered.length, includes.length); + assert.equal(filteredDarwin.length, includesDarwin.length); +}); + +test("a renumbered proto field rebuilds every backend on every OS", () => { + // Wire-incompatible: an old backend reading field 2 gets nothing. + const { filtered, filteredDarwin } = runProto( + protoWith(BASE_FIELDS), + protoWith(` string Prompt = 1;\n int32 Tokens = 9;\n bool UseTokenizerTemplate = 3;`) + ); + + assert.equal(filtered.length, includes.length); + assert.equal(filteredDarwin.length, includesDarwin.length); +}); + +test("a retyped proto field rebuilds every backend on every OS", () => { + const { filtered, filteredDarwin } = runProto( + protoWith(BASE_FIELDS), + protoWith(` string Prompt = 1;\n int64 Tokens = 2;\n bool UseTokenizerTemplate = 3;`) + ); + + assert.equal(filtered.length, includes.length); + assert.equal(filteredDarwin.length, includesDarwin.length); +}); + +test("a renamed proto field rebuilds every backend on every OS", () => { + // Same number and type, but every generated accessor changes name. + const { filtered, filteredDarwin } = runProto( + protoWith(BASE_FIELDS), + protoWith(` string Prompt = 1;\n int32 MaxTokens = 2;\n bool UseTokenizerTemplate = 3;`) + ); + + assert.equal(filtered.length, includes.length); + assert.equal(filteredDarwin.length, includesDarwin.length); +}); + +test("a removed proto RPC rebuilds every backend on every OS", () => { + const { filtered, filteredDarwin } = runProto( + protoWith(BASE_FIELDS), + protoWith(BASE_FIELDS).replace( + " rpc Predict(PredictOptions) returns (Reply) {}\n", + "" + ) + ); + + assert.equal(filtered.length, includes.length); + assert.equal(filteredDarwin.length, includesDarwin.length); +}); + +test("a comment-only proto change rebuilds nothing", () => { + const { filtered, filteredDarwin } = runProto( + protoWith(BASE_FIELDS), + protoWith(` string Prompt = 1;\n // how many tokens to emit\n int32 Tokens = 2;\n bool UseTokenizerTemplate = 3;`) + ); + + assert.deepEqual(filtered, []); + assert.deepEqual(filteredDarwin, []); +}); + +test("unresolvable proto revisions conservatively rebuild everything", () => { + // Same posture as the previousMatrix fallback: if we cannot resolve what the + // proto used to say, we must not claim the change was additive. + const { filtered, filteredDarwin } = runProto(null, protoWith(BASE_FIELDS)); + + assert.equal(filtered.length, includes.length); + assert.equal(filteredDarwin.length, includesDarwin.length); +}); From 9058a2bb46a4a764319abde979bc5b1713a5100d Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Wed, 29 Jul 2026 15:15:04 +0100 Subject: [PATCH 3/8] feat: Add 3d generation UI/API and trellis2cpp backend (#10979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(3d): add Generate3D RPC, FLAG_3D capability, and /v1/3d/generations endpoint Adds the plumbing for image-conditioned 3D asset generation (binary glTF / GLB output), modeled on the video generation path: - backend.proto: Generate3D RPC + Generate3DRequest (staged image src, glb dst, seed/step/cfg_scale/texture_steps, quality and background enums, params map for backend-specific extras) - pkg/grpc: thread Generate3D through client, server, embed, base and the backend interfaces; connection-evicting and distributed-node wrappers (in-flight tracking + file staging) included - core/config: FLAG_3D usecase (guessed only for the trellis2cpp backend), '3d' canonical usecase string mapped to the Generate3D method, and a '3d' output modality - REST: POST /v1/3d/generations (+ unversioned alias) returning OpenAIResponse with a /generated-3d URL or b64_json; conditioning image accepted as URL, base64, or data URI; quality/background validated at the edge; .glb served as model/gltf-binary - auth: '3d' route feature (default ON); /api/instructions entry Assisted-by: Claude:claude-fable-5 [Claude Code] Signed-off-by: Richard Palethorpe * feat(trellis2cpp): add the trellis2.cpp image-to-3D backend Wraps localai-org/trellis2cpp (C++/GGML port of Microsoft TRELLIS.2, pbr-textures branch) as a Go+purego backend, following the stablediffusion-ggml pattern: - backend/go/trellis2cpp: purego bindings to the flat C ABI (v9, asserted at startup), eager pipeline load with model-set validation (refuses non-trellis GGUFs; degrades coarse/geometry-only/textured exactly like the upstream demo), Generate3D via t2_generate + t2_bake_glb writing a binary glTF to dst. Weight-free unit tests cover resolution/validation/param mapping — CI never downloads the multi-GB GGUF set or runs inference. - CPU SIMD variants build into per-variant directories (the shared libggml sonames collide across variants, unlike sd-ggml's flat renamed-.so scheme); run.sh picks one via /proc/cpuinfo. - CI wiring: backend-matrix entries (cpu, cuda12/13, vulkan amd64+arm64, l4t, l4t-cuda13, darwin metal), index.yaml meta + latest/master image entries, bump_deps tracking of the pbr-textures branch, changed-backends.js mapping, top-level Makefile targets. - Importer: auto-detects trellis GGUF repos/URIs (registered before llama-cpp so the .gguf match isn't stolen) and expands any trellis URI to the full 10-file component set spanning the three LocalAI-io HF repos. - Gallery: trellis2-4b (full PBR + 1024 cascade) and trellis2-4b-geometry (512 untextured) with verified sha256s. Assisted-by: Claude:claude-fable-5 [Claude Code] Signed-off-by: Richard Palethorpe * feat(ui): 3D generation page with native GLB viewer and IndexedDB history Adds a Studio tab + /app/3d page for the new image-to-3D endpoint: - GlbViewer ports the trellis2cpp demo's dependency-free WebGL2 renderer (quaternion trackball, metallic-roughness PBR, ACES, hidden-line wireframe with a bounded index budget) and pairs it with a minimal GLB parser for the two forms t2_bake_glb emits — dense vertex-PBR (linear COLOR_0 + _METALLIC_ROUGHNESS, uploaded as normalized integers) and the opt-in UV-atlas textured form. Parsing happens before any GL so stats and errors render without WebGL2. - use3DHistory stores past generations (params, input thumbnail, and the GLB blob itself) in IndexedDB with keep-newest-20 eviction — GLBs are multi-MB binaries localStorage can't hold — and the page offers a download button for the active GLB. - Wiring: CAP_3D capability constant (FLAG_3D — the exact string /api/models/capabilities serves), threeDApi, router entries, Studio tab, vite dev proxy, en locale keys. - e2e: render-smoke entry plus a focused spec that feeds a real one-triangle vertex-PBR GLB through the parser/viewer and exercises IndexedDB persistence, selection, deletion, and API errors. Assisted-by: Claude:claude-fable-5 [Claude Code] Signed-off-by: Richard Palethorpe * fix(3d): address API correctness and UX issues Keep 3D generation on the LocalAI-specific /3d/generations route and ensure authentication and permissions cover it. Propagate distributed transfer failures, publish a portable ARM64 backend image, honor importer overrides, and align discovery, upload validation, and touch controls. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * feat(3d): add previewable print remeshing Add a single-detail CGAL Alpha Wrap workflow for existing Trellis GLBs, including PBR reprojection, API documentation, tracing, and an in-browser preview before download. Allow the remesh route to enforce its 512 MiB upload cap independently of the smaller global default so generated high-resolution meshes can be processed. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe * build(trellis2cpp): centralize remesh dependency pins Assisted-by: Codex:GPT-5 [apply_patch] [exec_command] Signed-off-by: Richard Palethorpe * fix(kokoros): implement Generate3D stub for new proto RPC The Generate3D RPC added to backend.proto for the trellis2cpp backend made tonic's generated Backend trait require generate3_d, breaking the kokoros-grpc build. Return unimplemented like the other unsupported modalities. Assisted-by: Claude Code:claude-fable-5 Signed-off-by: Richard Palethorpe --------- Signed-off-by: Richard Palethorpe Co-authored-by: localai-org-maint-bot --- .github/backend-matrix.yml | 113 ++++ .github/workflows/bump_deps.yaml | 4 + .github/workflows/test-extra.yml | 36 + Makefile | 12 +- README.md | 1 + backend/backend.proto | 15 + backend/go/trellis2cpp/.gitignore | 6 + backend/go/trellis2cpp/Makefile | 132 ++++ backend/go/trellis2cpp/glb_mesh.go | 252 +++++++ backend/go/trellis2cpp/glb_mesh_test.go | 74 ++ backend/go/trellis2cpp/main.go | 50 ++ backend/go/trellis2cpp/package.sh | 63 ++ backend/go/trellis2cpp/run.sh | 61 ++ backend/go/trellis2cpp/trellis2.go | 511 ++++++++++++++ backend/go/trellis2cpp/trellis2_test.go | 247 +++++++ backend/index.yaml | 109 +++ backend/rust/kokoros/src/service.rs | 7 + core/backend/model3d.go | 104 +++ core/config/backend_capabilities.go | 15 + core/config/model_capabilities.go | 11 +- core/config/model_capabilities_test.go | 19 + core/config/model_config.go | 15 +- core/gallery/importers/importers.go | 5 + core/gallery/importers/trellis2cpp.go | 170 +++++ core/gallery/importers/trellis2cpp_test.go | 105 +++ core/http/app.go | 19 +- core/http/auth/features.go | 5 + core/http/auth/middleware.go | 1 + core/http/auth/middleware_test.go | 10 + core/http/auth/permissions.go | 3 +- core/http/body_limit_test.go | 66 ++ .../endpoints/localai/api_instructions.go | 6 + .../localai/api_instructions_test.go | 13 +- core/http/endpoints/localai/model3d.go | 186 +++++ .../localai/model3d_internal_test.go | 72 ++ core/http/endpoints/localai/model3d_remesh.go | 163 +++++ .../localai/model3d_remesh_internal_test.go | 68 ++ .../react-ui/e2e/page-render-smoke.spec.js | 1 + core/http/react-ui/e2e/threed-gen.spec.js | 319 +++++++++ .../react-ui/public/locales/en/media.json | 56 +- .../react-ui/public/locales/en/models.json | 1 + core/http/react-ui/src/App.css | 39 ++ .../react-ui/src/components/GlbViewer.jsx | 636 ++++++++++++++++++ .../react-ui/src/components/ThreeDHistory.jsx | 76 +++ core/http/react-ui/src/hooks/use3DHistory.js | 132 ++++ core/http/react-ui/src/pages/Models.jsx | 1 + core/http/react-ui/src/pages/Studio.jsx | 21 +- core/http/react-ui/src/pages/ThreeDGen.jsx | 285 ++++++++ core/http/react-ui/src/pages/Traces.jsx | 2 + core/http/react-ui/src/router.jsx | 3 + core/http/react-ui/src/utils/api.js | 16 + core/http/react-ui/src/utils/capabilities.js | 5 + core/http/react-ui/src/utils/config.js | 2 + core/http/react-ui/src/utils/glb.js | 144 ++++ core/http/react-ui/vite.config.js | 2 + core/http/routes/localai.go | 13 + core/http/routes/ui_api.go | 1 + core/http/routes/ui_api_internal_test.go | 13 + core/schema/localai.go | 23 + core/schema/request.go | 2 +- core/services/nodes/file_staging_3d_test.go | 46 ++ core/services/nodes/file_staging_client.go | 36 + core/services/nodes/health_mock_test.go | 3 + core/services/nodes/inflight.go | 6 + core/services/nodes/inflight_test.go | 4 + core/trace/backend_trace.go | 2 + docs/content/features/3d-generation.md | 122 ++++ docs/content/features/backends.md | 1 + gallery/index.yaml | 95 +++ gallery/trellis2cpp.yaml | 9 + pkg/grpc/backend.go | 1 + pkg/grpc/base/base.go | 4 + pkg/grpc/client.go | 17 + pkg/grpc/embed.go | 4 + pkg/grpc/interface.go | 1 + pkg/grpc/server.go | 12 + pkg/model/connection_evicting_client.go | 6 + pkg/model/initializers.go | 1 + scripts/lib/backend-filter.mjs | 6 + scripts/lib/backend-filter_test.mjs | 12 +- swagger/docs.go | 119 ++++ swagger/swagger.json | 119 ++++ swagger/swagger.yaml | 82 +++ 83 files changed, 5231 insertions(+), 19 deletions(-) create mode 100644 backend/go/trellis2cpp/.gitignore create mode 100644 backend/go/trellis2cpp/Makefile create mode 100644 backend/go/trellis2cpp/glb_mesh.go create mode 100644 backend/go/trellis2cpp/glb_mesh_test.go create mode 100644 backend/go/trellis2cpp/main.go create mode 100755 backend/go/trellis2cpp/package.sh create mode 100755 backend/go/trellis2cpp/run.sh create mode 100644 backend/go/trellis2cpp/trellis2.go create mode 100644 backend/go/trellis2cpp/trellis2_test.go create mode 100644 core/backend/model3d.go create mode 100644 core/gallery/importers/trellis2cpp.go create mode 100644 core/gallery/importers/trellis2cpp_test.go create mode 100644 core/http/body_limit_test.go create mode 100644 core/http/endpoints/localai/model3d.go create mode 100644 core/http/endpoints/localai/model3d_internal_test.go create mode 100644 core/http/endpoints/localai/model3d_remesh.go create mode 100644 core/http/endpoints/localai/model3d_remesh_internal_test.go create mode 100644 core/http/react-ui/e2e/threed-gen.spec.js create mode 100644 core/http/react-ui/src/components/GlbViewer.jsx create mode 100644 core/http/react-ui/src/components/ThreeDHistory.jsx create mode 100644 core/http/react-ui/src/hooks/use3DHistory.js create mode 100644 core/http/react-ui/src/pages/ThreeDGen.jsx create mode 100644 core/http/react-ui/src/utils/glb.js create mode 100644 core/http/routes/ui_api_internal_test.go create mode 100644 core/services/nodes/file_staging_3d_test.go create mode 100644 docs/content/features/3d-generation.md create mode 100644 gallery/trellis2cpp.yaml diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 2ae8a26af..dd033fa85 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -756,6 +756,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1716,6 +1729,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -1729,6 +1755,19 @@ include: backend: "stablediffusion-ggml" dockerfile: "./backend/Dockerfile.golang" context: "./" + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-cuda-13-arm64-trellis2cpp' + base-image: "ubuntu:24.04" + ubuntu-version: '2404' + runs-on: 'ubuntu-24.04-arm' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -3267,6 +3306,35 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + # trellis2cpp + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-trellis2cpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' # sam3-cpp - build-type: '' cuda-major-version: "" @@ -3592,6 +3660,34 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-trellis2cpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -3605,6 +3701,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2204' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-arm64-trellis2cpp' + base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + runs-on: 'ubuntu-24.04-arm' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2204' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -5977,6 +6086,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-stablediffusion-ggml" build-type: "metal" lang: "go" + - backend: "trellis2cpp" + tag-suffix: "-metal-darwin-arm64-trellis2cpp" + build-type: "metal" + lang: "go" - backend: "whisper" tag-suffix: "-metal-darwin-arm64-whisper" build-type: "metal" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index 7cd472f4e..b6dbbb449 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -78,6 +78,10 @@ jobs: variable: "STABLEDIFFUSION_GGML_VERSION" branch: "master" file: "backend/go/stablediffusion-ggml/Makefile" + - repository: "localai-org/trellis2cpp" + variable: "TRELLIS2CPP_VERSION" + branch: "pbr-textures" + file: "backend/go/trellis2cpp/Makefile" - repository: "mudler/go-piper" variable: "PIPER_VERSION" branch: "master" diff --git a/.github/workflows/test-extra.yml b/.github/workflows/test-extra.yml index 899b98e60..96eb57155 100644 --- a/.github/workflows/test-extra.yml +++ b/.github/workflows/test-extra.yml @@ -38,6 +38,7 @@ jobs: acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }} qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }} magpie-tts-cpp: ${{ steps.detect.outputs.magpie-tts-cpp }} + trellis2cpp: ${{ steps.detect.outputs.trellis2cpp }} rfdetr-cpp: ${{ steps.detect.outputs.rfdetr-cpp }} locate-anything-cpp: ${{ steps.detect.outputs.locate-anything-cpp }} vibevoice-cpp: ${{ steps.detect.outputs.vibevoice-cpp }} @@ -935,6 +936,41 @@ jobs: - name: Test rfdetr-cpp run: | make --jobs=5 --output-sync=target -C backend/go/rfdetr-cpp test + # Weight-free packaged-backend smoke for trellis2cpp. Starting run.sh loads + # libtrellis2 + ggml, resolves the complete C ABI (including remeshing), and + # answers gRPC Health without downloading or loading the multi-GB model set. + tests-trellis2cpp: + needs: detect-changes + if: needs.detect-changes.outputs.trellis2cpp == 'true' || needs.detect-changes.outputs.run-all == 'true' + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake curl unzip + - name: Setup Go + uses: actions/setup-go@v5 + - name: Display Go version + run: go version + - name: Proto Dependencies + run: | + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \ + unzip -j -d /usr/local/bin protoc.zip bin/protoc && \ + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go + - name: Build trellis2cpp + run: | + make --jobs=5 --output-sync=target -C backend/go/trellis2cpp + - name: Test trellis2cpp + run: | + make --jobs=5 --output-sync=target -C backend/go/trellis2cpp test # Per-backend e2e for locate-anything-cpp: builds the .so + Go binary and # runs `make -C backend/go/locate-anything-cpp test`. test.sh fetches the # locate-anything-q8_0 GGUF (~6.3 GB, NVIDIA LocateAnything-3B) from the diff --git a/Makefile b/Makefile index 81b00d7c8..423ee8605 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -594,6 +594,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/rust/kokoros kokoros-grpc $(MAKE) -C backend/go/rfdetr-cpp $(MAKE) -C backend/go/locate-anything-cpp + $(MAKE) -C backend/go/trellis2cpp test-extra: prepare-test-extra $(MAKE) -C backend/python/transformers test @@ -626,6 +627,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/go/depth-anything-cpp test $(MAKE) -C backend/go/supertonic test $(MAKE) -C backend/go/vllm-cpp test + $(MAKE) -C backend/go/trellis2cpp test ## ## End-to-end gRPC tests that exercise a built backend container image. @@ -1218,6 +1220,10 @@ backends/stablediffusion-ggml-darwin: BACKEND=stablediffusion-ggml BUILD_TYPE=metal $(MAKE) build-darwin-go-backend ./local-ai backends install "ocifile://$(abspath ./backend-images/stablediffusion-ggml.tar)" +backends/trellis2cpp-darwin: + BACKEND=trellis2cpp BUILD_TYPE=metal $(MAKE) build-darwin-go-backend + ./local-ai backends install "ocifile://$(abspath ./backend-images/trellis2cpp.tar)" + backend-images: mkdir -p backend-images @@ -1249,6 +1255,7 @@ BACKEND_CLOUD_PROXY = cloud-proxy|golang|.|false|true BACKEND_HUGGINGFACE = huggingface|golang|.|false|true BACKEND_SILERO_VAD = silero-vad|golang|.|false|true BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true +BACKEND_TRELLIS2CPP = trellis2cpp|golang|.|--progress=plain|true BACKEND_WHISPER = whisper|golang|.|false|true BACKEND_CRISPASR = crispasr|golang|.|false|true BACKEND_PARAKEET_CPP = parakeet-cpp|golang|.|false|true @@ -1348,6 +1355,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_CLOUD_PROXY))) $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE))) $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD))) $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML))) +$(eval $(call generate-docker-build-target,$(BACKEND_TRELLIS2CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_WHISPER))) $(eval $(call generate-docker-build-target,$(BACKEND_CRISPASR))) $(eval $(call generate-docker-build-target,$(BACKEND_PARAKEET_CPP))) @@ -1408,7 +1416,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp ######################################################## ### Mock Backend for E2E Tests diff --git a/README.md b/README.md index d2d0c1479..cbdc44b68 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,7 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native | [depth-anything.cpp](https://github.com/mudler/depth-anything.cpp) | Depth Anything 3 monocular metric depth + camera pose estimation | | [face-detect.cpp](https://github.com/mudler/face-detect.cpp) | Face detection, recognition, demographics and anti-spoofing (SCRFD/ArcFace, YuNet/SFace), replacing the Python insightface backend | | [free-splatter.cpp](https://github.com/localai-org/free-splatter.cpp) | Pose-free 3D reconstruction (FreeSplatter): turns a handful of plain photos into 3D Gaussians, no camera poses or GPU required | +| [trellis2.cpp](https://github.com/localai-org/trellis2cpp) | C++/GGML port of Microsoft TRELLIS.2: single-image to textured 3D mesh (GLB with PBR materials) | | [privacy-filter.cpp](https://github.com/localai-org/privacy-filter.cpp) | Standalone GGML PII/NER token-classification engine powering LocalAI's PII redaction tier | | [LocalVQE](https://github.com/localai-org/LocalVQE) | Joint acoustic echo cancellation, noise suppression, and dereverberation | | [local-store](https://github.com/mudler/LocalAI) | Local-first vector database for embeddings (shipped in-tree) | diff --git a/backend/backend.proto b/backend/backend.proto index 711e9465c..2935d1a62 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -16,6 +16,7 @@ service Backend { rpc Embedding(PredictOptions) returns (EmbeddingResult) {} rpc GenerateImage(GenerateImageRequest) returns (Result) {} rpc GenerateVideo(GenerateVideoRequest) returns (Result) {} + rpc Generate3D(Generate3DRequest) returns (Result) {} rpc AudioTranscription(TranscriptRequest) returns (TranscriptResult) {} rpc AudioTranscriptionStream(TranscriptRequest) returns (stream TranscriptStreamResponse) {} // AudioTranscriptionLive is the bidirectional live-microphone ASR RPC. The @@ -658,6 +659,20 @@ message GenerateVideoRequest { string ModelIdentity = 15; } +message Generate3DRequest { + string src = 1; // Path to the staged conditioning image (3D generation is image-conditioned) + string dst = 2; // Output path for the generated binary glTF (.glb) asset + int32 seed = 3; // <=0 lets the backend pick a random seed + int32 step = 4; // Flow sampling steps; <=0 uses the backend default + float cfg_scale = 5; // Classifier-free guidance scale; <=0 uses the backend default + int32 texture_steps = 6; // Texture flow sampling steps; <=0 uses the backend default + string quality = 7; // Mesh pipeline: ""|"auto"|"coarse"|"512"|"1024" + string background = 8; // Conditioning-image background handling: ""|"auto"|"keep"|"black"|"white" + // Backend-specific per-request generation parameters. Values are strings + // and are validated/coerced by the selected backend. + map params = 9; +} + message TTSRequest { string text = 1; string model = 2; diff --git a/backend/go/trellis2cpp/.gitignore b/backend/go/trellis2cpp/.gitignore new file mode 100644 index 000000000..b5cfa6aba --- /dev/null +++ b/backend/go/trellis2cpp/.gitignore @@ -0,0 +1,6 @@ +package/ +sources/ +.cache/ +build-*/ +variants/ +trellis2cpp diff --git a/backend/go/trellis2cpp/Makefile b/backend/go/trellis2cpp/Makefile new file mode 100644 index 000000000..c20a991a5 --- /dev/null +++ b/backend/go/trellis2cpp/Makefile @@ -0,0 +1,132 @@ +CMAKE_ARGS?= +BUILD_TYPE?= +NATIVE?=false + +CURRENT_DIR=$(abspath ./) +GOCMD?=go +GO_TAGS?= +JOBS?=$(shell nproc --ignore=1) + +# trellis2.cpp — C++/ggml port of Microsoft TRELLIS.2 (image -> 3D GLB). +# The ggml submodule is pinned by trellis2cpp's .gitmodules and fetched via +# --recursive. The commit pin lives here so bump_deps.yaml can update it. +TRELLIS2CPP_REPO?=https://github.com/localai-org/trellis2cpp +TRELLIS2CPP_VERSION?=73dfbe5dfc2cbefd0950853718086556c6d9b043 + +# libtrellis2 + ggml as shared libraries; no example/test binaries. +CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release +CMAKE_ARGS+=-DBUILD_SHARED_LIBS=ON +CMAKE_ARGS+=-DTRELLIS2_BUILD_EXAMPLES=OFF +CMAKE_ARGS+=-DTRELLIS2_BUILD_TESTS=OFF +# Print remeshing is part of trellis2cpp's ABI, so upstream owns the tested +# CGAL/Boost versions, checksums, fetch logic, and update automation. LocalAI +# only opts into that dependency set and pins the trellis2cpp commit above. +CMAKE_ARGS+=-DTRELLIS2_FETCH_PRINT_REMESH_DEPS=ON +CMAKE_ARGS+=-DTRELLIS2_PRINT_REMESH_DEPS_DIR=$(CURRENT_DIR)/sources/print-remesh-deps + +ifeq ($(NATIVE),false) + CMAKE_ARGS+=-DGGML_NATIVE=OFF +endif + +ifeq ($(BUILD_TYPE),cublas) + CMAKE_ARGS+=-DGGML_CUDA=ON +else ifeq ($(BUILD_TYPE),vulkan) + CMAKE_ARGS+=-DGGML_VULKAN=ON +else ifeq ($(BUILD_TYPE),hipblas) + ROCM_HOME ?= /opt/rocm + ROCM_PATH ?= /opt/rocm + export CXX=$(ROCM_HOME)/llvm/bin/clang++ + export CC=$(ROCM_HOME)/llvm/bin/clang + AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201 + CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) +else ifeq ($(OS),Darwin) + ifneq ($(BUILD_TYPE),metal) + CMAKE_ARGS+=-DTRELLIS2_METAL=OFF -DGGML_METAL=OFF + else + # trellis2cpp turns on GGML_METAL(+EMBED_LIBRARY) itself when + # TRELLIS2_METAL is enabled on Apple platforms. + CMAKE_ARGS+=-DTRELLIS2_METAL=ON + endif + # Dependent libggml*.dylib resolve next to libtrellis2.dylib even + # without DYLD_LIBRARY_PATH being exported. + CMAKE_ARGS+=-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH=@loader_path +endif + +ifeq ($(BUILD_TYPE),sycl_f16) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DGGML_SYCL_F16=ON +endif + +ifeq ($(BUILD_TYPE),sycl_f32) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx +endif + +sources/trellis2cpp: + git clone --recursive $(TRELLIS2CPP_REPO) sources/trellis2cpp && \ + cd sources/trellis2cpp && \ + git checkout $(TRELLIS2CPP_VERSION) && \ + git submodule update --init --recursive --depth 1 --single-branch + +# Detect OS +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +# The AVX variants are x86-only. ARM64 images use the portable fallback while +# still enabling the selected GPU backend (Vulkan/CUDA) through CMAKE_ARGS. +ifeq ($(UNAME_S),Linux) + ifneq (,$(filter x86_64 amd64,$(UNAME_M))) + VARIANTS = avx avx2 avx512 fallback + else + VARIANTS = fallback + endif +else + # On non-Linux (e.g., Darwin), build only the fallback variant + VARIANTS = fallback +endif +VARIANT_TARGETS = $(foreach v,$(VARIANTS),variants/$(v)/.built) + +VARIANT_FLAGS_avx = -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off +VARIANT_FLAGS_avx2 = -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on +VARIANT_FLAGS_avx512 = -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on +VARIANT_FLAGS_fallback = -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# libtrellis2 links libggml/libggml-base/libggml-cpu (+ the GPU backend) by +# soname, and those sonames collide across SIMD variants — so each variant +# lives in its own directory and run.sh selects one via LD_LIBRARY_PATH, +# unlike stablediffusion-ggml's flat renamed-.so scheme. +variants/%/.built: sources/trellis2cpp + rm -rf build-$* variants/$* + mkdir -p build-$* variants/$* + cd build-$* && cmake ../sources/trellis2cpp $(CMAKE_ARGS) $(VARIANT_FLAGS_$*) && \ + cmake --build . --config Release -j$(JOBS) + @for f in build-$*/libtrellis2.so build-$*/libtrellis2.dylib; do \ + if [ -e $$f ]; then cp -a $$f variants/$*/; fi; done + find build-$*/ggml \( -name 'libggml*.so*' -o -name 'libggml*.dylib' \) -exec cp -a {} variants/$*/ \; + rm -rf build-$* + touch $@ + +trellis2cpp: main.go trellis2.go $(VARIANT_TARGETS) + CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o trellis2cpp ./ + +package: trellis2cpp + bash package.sh + +build: package + +clean: purge + rm -rf variants trellis2cpp package sources + +purge: + rm -rf build-* + +# Weight-free by construction: pure-Go unit tests over model-path resolution, +# validation, and request-parameter mapping. The multi-GB GGUF weights are +# never downloaded in CI; end-to-end generation is exercised manually. +test: + $(GOCMD) test -v ./... + +all: trellis2cpp package diff --git a/backend/go/trellis2cpp/glb_mesh.go b/backend/go/trellis2cpp/glb_mesh.go new file mode 100644 index 000000000..144c9243a --- /dev/null +++ b/backend/go/trellis2cpp/glb_mesh.go @@ -0,0 +1,252 @@ +package main + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "math" +) + +const ( + glbMagic = 0x46546c67 + glbJSONChunk = 0x4e4f534a + glbBINChunk = 0x004e4942 +) + +type glbAccessor struct { + BufferView int `json:"bufferView"` + ByteOffset int `json:"byteOffset"` + ComponentType int `json:"componentType"` + Count int `json:"count"` + Type string `json:"type"` + Normalized bool `json:"normalized"` +} + +type glbBufferView struct { + Buffer int `json:"buffer"` + ByteOffset int `json:"byteOffset"` + ByteLength int `json:"byteLength"` + ByteStride int `json:"byteStride"` +} + +type glbPrimitive struct { + Attributes map[string]int `json:"attributes"` + Indices *int `json:"indices"` +} + +type glbDocument struct { + Accessors []glbAccessor `json:"accessors"` + BufferViews []glbBufferView `json:"bufferViews"` + Meshes []struct { + Primitives []glbPrimitive `json:"primitives"` + } `json:"meshes"` +} + +type glbVertexMesh struct { + verts []float32 + tris []int32 + pbr []float32 +} + +func glbLayout(componentType int, accessorType string) (componentBytes, components int, err error) { + switch componentType { + case 5121: + componentBytes = 1 + case 5123: + componentBytes = 2 + case 5125, 5126: + componentBytes = 4 + default: + return 0, 0, fmt.Errorf("unsupported GLB component type %d", componentType) + } + switch accessorType { + case "SCALAR": + components = 1 + case "VEC2": + components = 2 + case "VEC3": + components = 3 + case "VEC4": + components = 4 + default: + return 0, 0, fmt.Errorf("unsupported GLB accessor type %q", accessorType) + } + return componentBytes, components, nil +} + +func glbAccessorData(doc *glbDocument, binChunk []byte, index int) (glbAccessor, []byte, error) { + if index < 0 || index >= len(doc.Accessors) { + return glbAccessor{}, nil, fmt.Errorf("missing GLB accessor %d", index) + } + a := doc.Accessors[index] + if a.BufferView < 0 || a.BufferView >= len(doc.BufferViews) { + return glbAccessor{}, nil, fmt.Errorf("missing GLB buffer view %d", a.BufferView) + } + v := doc.BufferViews[a.BufferView] + if v.Buffer != 0 || v.ByteStride != 0 { + return glbAccessor{}, nil, fmt.Errorf("interleaved or external GLB buffers are unsupported") + } + componentBytes, components, err := glbLayout(a.ComponentType, a.Type) + if err != nil { + return glbAccessor{}, nil, err + } + if a.Count <= 0 || a.Count > math.MaxInt/(componentBytes*components) { + return glbAccessor{}, nil, fmt.Errorf("invalid GLB accessor count %d", a.Count) + } + length := a.Count * componentBytes * components + if v.ByteOffset < 0 || v.ByteLength < 0 || a.ByteOffset < 0 || + a.ByteOffset > v.ByteLength || length > v.ByteLength-a.ByteOffset || + length > len(binChunk) || v.ByteOffset > len(binChunk)-length-a.ByteOffset { + return glbAccessor{}, nil, fmt.Errorf("GLB accessor %d is outside the BIN chunk", index) + } + start := v.ByteOffset + a.ByteOffset + return a, binChunk[start : start+length], nil +} + +// parseVertexGLB reads the dense vertex-PBR form emitted by trellis2.cpp. GLB +// coordinates and linear COLOR_0 values are converted back to the native +// trellis coordinate/material convention before CGAL remeshing and rebaking. +func parseVertexGLB(data []byte) (*glbVertexMesh, error) { + if len(data) < 20 || binary.LittleEndian.Uint32(data[0:4]) != glbMagic { + return nil, fmt.Errorf("input is not a GLB file") + } + if binary.LittleEndian.Uint32(data[4:8]) != 2 { + return nil, fmt.Errorf("unsupported GLB version") + } + total := int(binary.LittleEndian.Uint32(data[8:12])) + if total != len(data) { + return nil, fmt.Errorf("invalid GLB length") + } + + var jsonChunk, binChunk []byte + for offset := 12; offset <= len(data)-8; { + length := int(binary.LittleEndian.Uint32(data[offset : offset+4])) + chunkType := binary.LittleEndian.Uint32(data[offset+4 : offset+8]) + start := offset + 8 + if length < 0 || start > len(data)-length { + return nil, fmt.Errorf("invalid GLB chunk length") + } + switch chunkType { + case glbJSONChunk: + if jsonChunk == nil { + jsonChunk = data[start : start+length] + } + case glbBINChunk: + if binChunk == nil { + binChunk = data[start : start+length] + } + } + offset = start + length + } + if jsonChunk == nil || binChunk == nil { + return nil, fmt.Errorf("GLB must contain JSON and BIN chunks") + } + + var doc glbDocument + if err := json.Unmarshal(jsonChunk, &doc); err != nil { + return nil, fmt.Errorf("parsing GLB JSON: %w", err) + } + if len(doc.Meshes) != 1 || len(doc.Meshes[0].Primitives) != 1 { + return nil, fmt.Errorf("GLB must contain one mesh primitive") + } + primitive := doc.Meshes[0].Primitives[0] + positionIndex, ok := primitive.Attributes["POSITION"] + if !ok { + return nil, fmt.Errorf("GLB mesh has no POSITION attribute") + } + position, positionData, err := glbAccessorData(&doc, binChunk, positionIndex) + if err != nil { + return nil, err + } + if position.ComponentType != 5126 || position.Type != "VEC3" { + return nil, fmt.Errorf("GLB POSITION must be float32 VEC3") + } + + mesh := &glbVertexMesh{verts: make([]float32, position.Count*3)} + for i := 0; i < position.Count; i++ { + x := math.Float32frombits(binary.LittleEndian.Uint32(positionData[(i*3)*4:])) + y := math.Float32frombits(binary.LittleEndian.Uint32(positionData[(i*3+1)*4:])) + z := math.Float32frombits(binary.LittleEndian.Uint32(positionData[(i*3+2)*4:])) + if math.IsNaN(float64(x)) || math.IsNaN(float64(y)) || math.IsNaN(float64(z)) || + math.IsInf(float64(x), 0) || math.IsInf(float64(y), 0) || math.IsInf(float64(z), 0) { + return nil, fmt.Errorf("GLB POSITION contains a non-finite value") + } + mesh.verts[i*3] = x + mesh.verts[i*3+1] = -z + mesh.verts[i*3+2] = y + } + + if primitive.Indices == nil { + if position.Count%3 != 0 { + return nil, fmt.Errorf("unindexed GLB vertex count is not divisible by three") + } + mesh.tris = make([]int32, position.Count) + for i := range mesh.tris { + mesh.tris[i] = int32(i) + } + } else { + indices, indexData, err := glbAccessorData(&doc, binChunk, *primitive.Indices) + if err != nil { + return nil, err + } + if indices.Type != "SCALAR" || indices.Count%3 != 0 || (indices.ComponentType != 5123 && indices.ComponentType != 5125) { + return nil, fmt.Errorf("GLB indices must be uint16/uint32 triangles") + } + mesh.tris = make([]int32, indices.Count) + for i := range mesh.tris { + var value uint32 + if indices.ComponentType == 5123 { + value = uint32(binary.LittleEndian.Uint16(indexData[i*2:])) + } else { + value = binary.LittleEndian.Uint32(indexData[i*4:]) + } + if value >= uint32(position.Count) || value > math.MaxInt32 { + return nil, fmt.Errorf("GLB index %d is outside the vertex buffer", value) + } + mesh.tris[i] = int32(value) + } + } + + colorIndex, hasColor := primitive.Attributes["COLOR_0"] + if !hasColor { + return mesh, nil + } + color, colorData, err := glbAccessorData(&doc, binChunk, colorIndex) + if err != nil { + return nil, err + } + if color.ComponentType != 5123 || color.Type != "VEC4" || !color.Normalized || color.Count != position.Count { + return nil, fmt.Errorf("GLB COLOR_0 must be normalized uint16 VEC4 aligned with POSITION") + } + metalRoughIndex, hasMetalRough := primitive.Attributes["_METALLIC_ROUGHNESS"] + var metalRoughData []byte + if hasMetalRough { + metalRough, data, err := glbAccessorData(&doc, binChunk, metalRoughIndex) + if err != nil { + return nil, err + } + if metalRough.ComponentType != 5121 || metalRough.Type != "VEC2" || !metalRough.Normalized || metalRough.Count != position.Count { + return nil, fmt.Errorf("GLB _METALLIC_ROUGHNESS must be normalized uint8 VEC2 aligned with POSITION") + } + metalRoughData = data + } + + mesh.pbr = make([]float32, position.Count*6) + for i := 0; i < position.Count; i++ { + for channel := 0; channel < 3; channel++ { + linear := float32(binary.LittleEndian.Uint16(colorData[(i*4+channel)*2:])) / 65535 + if linear <= 0.0031308 { + mesh.pbr[i*6+channel] = linear * 12.92 + } else { + mesh.pbr[i*6+channel] = 1.055*float32(math.Pow(float64(linear), 1.0/2.4)) - 0.055 + } + } + mesh.pbr[i*6+5] = float32(binary.LittleEndian.Uint16(colorData[(i*4+3)*2:])) / 65535 + mesh.pbr[i*6+4] = 0.6 + if hasMetalRough { + mesh.pbr[i*6+3] = float32(metalRoughData[i*2]) / 255 + mesh.pbr[i*6+4] = float32(metalRoughData[i*2+1]) / 255 + } + } + return mesh, nil +} diff --git a/backend/go/trellis2cpp/glb_mesh_test.go b/backend/go/trellis2cpp/glb_mesh_test.go new file mode 100644 index 000000000..8ba4f9264 --- /dev/null +++ b/backend/go/trellis2cpp/glb_mesh_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "encoding/binary" + "fmt" + "math" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func tinyVertexGLB() []byte { + bin := make([]byte, 80) + positions := []float32{1, 2, 3, 4, 5, 6, 7, 8, 9} + for i, value := range positions { + binary.LittleEndian.PutUint32(bin[i*4:], math.Float32bits(value)) + } + colors := []uint16{ + 65535, 0, 0, 65535, + 0, 65535, 0, 32768, + 0, 0, 65535, 65535, + } + for i, value := range colors { + binary.LittleEndian.PutUint16(bin[36+i*2:], value) + } + copy(bin[60:], []byte{0, 153, 64, 128, 255, 32}) + for i, value := range []uint32{0, 1, 2} { + binary.LittleEndian.PutUint32(bin[68+i*4:], value) + } + + jsonChunk := []byte(fmt.Sprintf(`{"asset":{"version":"2.0"},"meshes":[{"primitives":[{"attributes":{"POSITION":0,"COLOR_0":1,"_METALLIC_ROUGHNESS":2},"indices":3}]}],"accessors":[{"bufferView":0,"componentType":5126,"count":3,"type":"VEC3"},{"bufferView":1,"componentType":5123,"normalized":true,"count":3,"type":"VEC4"},{"bufferView":2,"componentType":5121,"normalized":true,"count":3,"type":"VEC2"},{"bufferView":3,"componentType":5125,"count":3,"type":"SCALAR"}],"bufferViews":[{"buffer":0,"byteOffset":0,"byteLength":36},{"buffer":0,"byteOffset":36,"byteLength":24},{"buffer":0,"byteOffset":60,"byteLength":6},{"buffer":0,"byteOffset":68,"byteLength":12}],"buffers":[{"byteLength":%d}]}`, len(bin))) + for len(jsonChunk)%4 != 0 { + jsonChunk = append(jsonChunk, ' ') + } + total := 12 + 8 + len(jsonChunk) + 8 + len(bin) + glb := make([]byte, total) + binary.LittleEndian.PutUint32(glb[0:], glbMagic) + binary.LittleEndian.PutUint32(glb[4:], 2) + binary.LittleEndian.PutUint32(glb[8:], uint32(total)) + binary.LittleEndian.PutUint32(glb[12:], uint32(len(jsonChunk))) + binary.LittleEndian.PutUint32(glb[16:], glbJSONChunk) + copy(glb[20:], jsonChunk) + binHeader := 20 + len(jsonChunk) + binary.LittleEndian.PutUint32(glb[binHeader:], uint32(len(bin))) + binary.LittleEndian.PutUint32(glb[binHeader+4:], glbBINChunk) + copy(glb[binHeader+8:], bin) + return glb +} + +var _ = Describe("vertex GLB parsing for print remeshing", func() { + It("restores trellis coordinates, topology, and PBR values", func() { + mesh, err := parseVertexGLB(tinyVertexGLB()) + Expect(err).NotTo(HaveOccurred()) + Expect(mesh.verts).To(Equal([]float32{1, -3, 2, 4, -6, 5, 7, -9, 8})) + Expect(mesh.tris).To(Equal([]int32{0, 1, 2})) + Expect(mesh.pbr).To(HaveLen(18)) + Expect(mesh.pbr[0]).To(BeNumerically("~", 1, 1e-5)) + Expect(mesh.pbr[3]).To(BeNumerically("~", 0, 1e-5)) + Expect(mesh.pbr[4]).To(BeNumerically("~", 0.6, 0.01)) + Expect(mesh.pbr[11]).To(BeNumerically("~", 32768.0/65535.0, 1e-5)) + }) + + It("rejects indices outside the source vertex buffer", func() { + glb := tinyVertexGLB() + binary.LittleEndian.PutUint32(glb[len(glb)-12:], 3) + _, err := parseVertexGLB(glb) + Expect(err).To(MatchError(ContainSubstring("outside the vertex buffer"))) + }) + + It("rejects non-GLB input", func() { + _, err := parseVertexGLB([]byte("not a mesh")) + Expect(err).To(MatchError("input is not a GLB file")) + }) +}) diff --git a/backend/go/trellis2cpp/main.go b/backend/go/trellis2cpp/main.go new file mode 100644 index 000000000..14938bdd6 --- /dev/null +++ b/backend/go/trellis2cpp/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + + "github.com/ebitengine/purego" + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func registerLibFuncs(lib uintptr) { + registerLibFuncsWith(func(fptr any, name string) { + purego.RegisterLibFunc(fptr, lib, name) + }) +} + +func main() { + // run.sh selects the CPU-variant directory and points TRELLIS2_LIBRARY at it. + libName := os.Getenv("TRELLIS2_LIBRARY") + if libName == "" { + if runtime.GOOS == "darwin" { + libName = "./variants/fallback/libtrellis2.dylib" + } else { + libName = "./variants/fallback/libtrellis2.so" + } + } + + lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + panic(err) + } + + registerLibFuncs(lib) + + if got := t2AbiVersion(); got != abiVersion { + panic(fmt.Sprintf("trellis2 ABI mismatch: library reports %d, backend built for %d", got, abiVersion)) + } + + flag.Parse() + + if err := grpc.StartServer(*addr, &Trellis2{}); err != nil { + panic(err) + } +} diff --git a/backend/go/trellis2cpp/package.sh b/backend/go/trellis2cpp/package.sh new file mode 100755 index 000000000..20d7dedc2 --- /dev/null +++ b/backend/go/trellis2cpp/package.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Script to copy the appropriate libraries based on architecture +# This script is used in the final stage of the Dockerfile + +set -e + +CURDIR=$(dirname "$(realpath $0)") +REPO_ROOT="${CURDIR}/../../.." + +# Create lib directory +mkdir -p $CURDIR/package/lib + +# Each CPU variant keeps its libtrellis2 + libggml* set in its own directory +# (their sonames collide across variants); run.sh selects one at startup. +cp -a $CURDIR/variants $CURDIR/package/ +cp -avf $CURDIR/trellis2cpp $CURDIR/package/ +cp -fv $CURDIR/run.sh $CURDIR/package/ + +# Detect architecture and copy appropriate libraries +if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then + # x86_64 architecture + echo "Detected x86_64 architecture, copying x86_64 libraries..." + cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so + cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then + # ARM64 architecture + echo "Detected ARM64 architecture, copying ARM64 libraries..." + cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so + cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ $(uname -s) = "Darwin" ]; then + echo "Detected Darwin" +else + echo "Error: Could not detect architecture" + exit 1 +fi + +# Package GPU libraries based on BUILD_TYPE +# The GPU library packaging script will detect BUILD_TYPE and copy appropriate GPU libraries +GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh" +if [ -f "$GPU_LIB_SCRIPT" ]; then + echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..." + source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib" + package_gpu_libs +fi + +echo "Packaging completed successfully" +ls -liah $CURDIR/package/ +ls -liah $CURDIR/package/lib/ diff --git a/backend/go/trellis2cpp/run.sh b/backend/go/trellis2cpp/run.sh new file mode 100755 index 000000000..78f5ec362 --- /dev/null +++ b/backend/go/trellis2cpp/run.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -ex + +# Get the absolute current dir where the script is located +CURDIR=$(dirname "$(realpath "$0")") + +cd / + +echo "CPU info:" +if [ "$(uname)" != "Darwin" ]; then + grep -e "model\sname" /proc/cpuinfo | head -1 + grep -e "flags" /proc/cpuinfo | head -1 +fi + +# Each variant directory bundles libtrellis2 plus its libggml* set (the ggml +# sonames collide across SIMD variants, so they can't share one directory). +VARIANT=fallback + +if [ "$(uname)" = "Darwin" ]; then + LIBRARY="$CURDIR/variants/$VARIANT/libtrellis2.dylib" + if [ ! -e "$LIBRARY" ]; then + LIBRARY="$CURDIR/variants/$VARIANT/libtrellis2.so" + fi + export DYLD_LIBRARY_PATH="$CURDIR/variants/$VARIANT:$CURDIR/lib:$DYLD_LIBRARY_PATH" +else + if grep -q -e "\savx\s" /proc/cpuinfo ; then + echo "CPU: AVX found OK" + if [ -d "$CURDIR/variants/avx" ]; then + VARIANT=avx + fi + fi + + if grep -q -e "\savx2\s" /proc/cpuinfo ; then + echo "CPU: AVX2 found OK" + if [ -d "$CURDIR/variants/avx2" ]; then + VARIANT=avx2 + fi + fi + + if grep -q -e "\savx512f\s" /proc/cpuinfo ; then + echo "CPU: AVX512F found OK" + if [ -d "$CURDIR/variants/avx512" ]; then + VARIANT=avx512 + fi + fi + + LIBRARY="$CURDIR/variants/$VARIANT/libtrellis2.so" + export LD_LIBRARY_PATH="$CURDIR/variants/$VARIANT:$CURDIR/lib:$LD_LIBRARY_PATH" +fi + +export TRELLIS2_LIBRARY=$LIBRARY + +# If there is a lib/ld.so, use it +if [ -f "$CURDIR"/lib/ld.so ]; then + echo "Using lib/ld.so" + echo "Using library: $LIBRARY" + exec "$CURDIR"/lib/ld.so "$CURDIR"/trellis2cpp "$@" +fi + +echo "Using library: $LIBRARY" +exec "$CURDIR"/trellis2cpp "$@" diff --git a/backend/go/trellis2cpp/trellis2.go b/backend/go/trellis2cpp/trellis2.go new file mode 100644 index 000000000..addbf1bb0 --- /dev/null +++ b/backend/go/trellis2cpp/trellis2.go @@ -0,0 +1,511 @@ +package main + +// trellis2.go — purego bindings to libtrellis2's flat C ABI (trellis2_capi.h) +// plus the LocalAI backend implementation. Adapted from the upstream demo +// server's engine.go; the t2_abi_version binding guards against header/library +// drift. + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "unsafe" + + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/utils" +) + +const abiVersion = 11 + +// Pipeline types (enum t2_pipeline_type) and background modes +// (enum t2_background_mode). +const ( + pipeAuto = 0 + pipeCoarse = 1 + pipe512 = 2 + pipe1024 = 3 + + backgroundAuto = 0 + backgroundKeep = 1 + backgroundBlack = 2 + backgroundWhite = 3 +) + +// The bindings use typed pointers (*float32/*int32/*byte) rather than uintptr +// for C-owned buffers so no uintptr->unsafe.Pointer conversions are needed; +// only the opaque t2_pipeline / t2_mesh_result handles stay uintptr. +var ( + t2AbiVersion func() int32 + t2PipelineLoad func(dino, ssFlow, ssDec, slatFlow, slatFlowHR, shapeDec, + shapeEnc, texDec, texFlow, texFlowHR string, + flags int32, err *byte, errLen int32) uintptr + t2PipelineFree func(p uintptr) + t2PipelineBackend func(p uintptr) string + t2PipelineCaps func(p uintptr) int32 + t2Generate func(p uintptr, img *byte, imgLen int32, + pipelineType, backgroundMode int32, seed uint64, steps int32, + guidance float32, textureSteps int32, + progress, user, preview, previewUser uintptr, + err *byte, errLen int32) uintptr + t2MeshNVerts func(r uintptr) int32 + t2MeshNTris func(r uintptr) int32 + t2MeshVerts func(r uintptr) *float32 + t2MeshTris func(r uintptr) *int32 + t2MeshHasPBR func(r uintptr) int32 + t2MeshPBR func(r uintptr) *float32 + t2MeshFree func(r uintptr) + t2BakeGLB func(verts *float32, nv int32, tris *int32, nt int32, + pbr *float32, texSize, componentFilter int32, + outLen *int32, err *byte, errLen int32) *byte + // CGAL Alpha Wrap print remeshing — availability is fixed at library build + // time, so gate every use on t2_print_remesh_available. + t2PrintRemeshAvailable func() int32 + t2PreparePrintMesh func(verts *float32, nv int32, tris *int32, nt int32, + pbr *float32, componentFilter int32, alphaRatio, offsetRatio float32, + err *byte, errLen int32) uintptr + t2BakeProjectedGLB func(targetVerts *float32, targetNV int32, + targetTris *int32, targetNT int32, + sourceVerts *float32, sourceNV int32, + sourceTris *int32, sourceNT int32, + sourcePBR *float32, texSize, sourceComponentFilter int32, + outLen *int32, err *byte, errLen int32) *byte + t2FreeBuffer func(buf *byte) +) + +type libFunc struct { + funcPtr any + name string +} + +func registerLibFuncsWith(register func(fptr any, name string)) { + for _, lf := range []libFunc{ + {&t2AbiVersion, "t2_abi_version"}, + {&t2PipelineLoad, "t2_pipeline_load"}, + {&t2PipelineFree, "t2_pipeline_free"}, + {&t2PipelineBackend, "t2_pipeline_backend"}, + {&t2PipelineCaps, "t2_pipeline_caps"}, + {&t2Generate, "t2_generate"}, + {&t2MeshNVerts, "t2_mesh_n_verts"}, + {&t2MeshNTris, "t2_mesh_n_tris"}, + {&t2MeshVerts, "t2_mesh_verts"}, + {&t2MeshTris, "t2_mesh_tris"}, + {&t2MeshHasPBR, "t2_mesh_has_pbr"}, + {&t2MeshPBR, "t2_mesh_pbr"}, + {&t2MeshFree, "t2_mesh_free"}, + {&t2BakeGLB, "t2_bake_glb"}, + {&t2PrintRemeshAvailable, "t2_print_remesh_available"}, + {&t2PreparePrintMesh, "t2_prepare_print_mesh"}, + {&t2BakeProjectedGLB, "t2_bake_projected_glb"}, + {&t2FreeBuffer, "t2_free_buffer"}, + } { + register(lf.funcPtr, lf.name) + } +} + +// modelSet holds the resolved path for every pipeline role; optional roles are +// "" when disabled (the C side treats NULL/"" as "omit"). +type modelSet struct { + dino, ssFlow, ssDec string + slatFlow, slatFlow1024 string + shapeDec string + shapeEnc, texDec string + texSlatFlow512, texSlatFlow1024 string +} + +// role → (option key, default filename) in t2_pipeline_load argument order. +// The option keys follow the sd-ggml `*_path` convention; the default +// filenames are the ones the upstream converters emit and the demo server +// looks up, so a gallery install needs no options at all. +type modelRole struct { + key string + filename string + required bool + assign func(*modelSet, string) +} + +var modelRoles = []modelRole{ + {"dino_path", "dino_f16.gguf", true, func(s *modelSet, p string) { s.dino = p }}, + {"ss_flow_path", "ss_flow_f16.gguf", true, func(s *modelSet, p string) { s.ssFlow = p }}, + {"ss_dec_path", "ss_dec_f16.gguf", true, func(s *modelSet, p string) { s.ssDec = p }}, + {"slat_flow_path", "slat_flow_f16.gguf", false, func(s *modelSet, p string) { s.slatFlow = p }}, + {"slat_flow_1024_path", "slat_flow_1024_f16.gguf", false, func(s *modelSet, p string) { s.slatFlow1024 = p }}, + {"shape_dec_path", "shape_dec_f16.gguf", false, func(s *modelSet, p string) { s.shapeDec = p }}, + {"shape_enc_path", "shape_enc_f16.gguf", false, func(s *modelSet, p string) { s.shapeEnc = p }}, + {"tex_dec_path", "tex_dec_f16.gguf", false, func(s *modelSet, p string) { s.texDec = p }}, + {"tex_slat_flow_512_path", "tex_slat_flow_512_f16.gguf", false, func(s *modelSet, p string) { s.texSlatFlow512 = p }}, + {"tex_slat_flow_1024_path", "tex_slat_flow_1024_f16.gguf", false, func(s *modelSet, p string) { s.texSlatFlow1024 = p }}, +} + +// resolveModels maps LocalAI's model file + options onto the ten pipeline +// roles. The model file only anchors the GGUF directory; each role resolves +// to an explicit `_path` option when given, else to its default +// filename in that directory. Missing required files refuse the load (a +// backend must not capture arbitrary GGUFs — see issue #9287); missing +// optional files degrade capabilities the same way the upstream demo does. +func resolveModels(modelFile, modelPath string, options []string) (modelSet, error) { + base := modelFile + if !filepath.IsAbs(base) { + base = filepath.Join(modelPath, base) + } + ggufDir := filepath.Dir(base) + + overrides := map[string]string{} + for _, op := range options { + key, value, found := strings.Cut(op, ":") + if !found || !strings.HasSuffix(key, "_path") { + continue + } + if !filepath.IsAbs(value) { + value = filepath.Join(modelPath, value) + if err := utils.VerifyPath(value, modelPath); err != nil { + return modelSet{}, fmt.Errorf("option %s: %w", key, err) + } + } + overrides[key] = value + } + + var set modelSet + var missingRequired []string + for _, role := range modelRoles { + path, explicit := overrides[role.key] + if !explicit { + path = filepath.Join(ggufDir, role.filename) + } + if _, err := os.Stat(path); err != nil { + if explicit { + return modelSet{}, fmt.Errorf("option %s points at a missing file: %s", role.key, path) + } + if role.required { + missingRequired = append(missingRequired, role.filename) + } + path = "" + } + role.assign(&set, path) + } + if len(missingRequired) > 0 { + return modelSet{}, fmt.Errorf("not a trellis2 model set: missing required %s in %s", strings.Join(missingRequired, ", "), ggufDir) + } + + // Degradation mirrors the upstream demo: the 512 pair enables everything + // finer than coarse; texturing needs its three-model set; a textured 1024 + // cascade additionally needs the HR texture flow. + if set.slatFlow == "" || set.shapeDec == "" { + set.slatFlow, set.shapeDec = "", "" + set.slatFlow1024 = "" + set.shapeEnc, set.texDec, set.texSlatFlow512, set.texSlatFlow1024 = "", "", "", "" + return set, nil + } + if set.shapeEnc == "" || set.texDec == "" || set.texSlatFlow512 == "" { + set.shapeEnc, set.texDec, set.texSlatFlow512, set.texSlatFlow1024 = "", "", "", "" + } else if set.texSlatFlow1024 == "" { + set.slatFlow1024 = "" + } + return set, nil +} + +func pipelineForQuality(quality string) int32 { + switch quality { + case "coarse": + return pipeCoarse + case "512": + return pipe512 + case "1024": + return pipe1024 + default: + return pipeAuto + } +} + +func backgroundForMode(background string) int32 { + switch background { + case "keep": + return backgroundKeep + case "black": + return backgroundBlack + case "white": + return backgroundWhite + default: + return backgroundAuto + } +} + +func componentFilterFor(components string) int32 { + switch components { + case "tiny": + return 0 // remove only tiny islands + case "largest": + return 1 // keep the largest connected component + default: + return 2 // preserve every connected component (demo default) + } +} + +func atoiOr(s string, fallback int32) int32 { + if s == "" { + return fallback + } + n, err := strconv.Atoi(s) + if err != nil { + return fallback + } + return int32(n) +} + +func boolParam(s string) bool { + return s == "1" || strings.EqualFold(s, "true") +} + +// ratioOr parses a fraction-of-bounding-box-diagonal parameter. Out-of-range +// or unparseable values fall back rather than error, mirroring atoiOr; the +// accepted range matches what the upstream demo clamps to. +func ratioOr(s string, fallback float32) float32 { + if s == "" { + return fallback + } + f, err := strconv.ParseFloat(s, 32) + if err != nil || f < 0.00001 || f > 0.5 { + return fallback + } + return float32(f) +} + +type Trellis2 struct { + base.SingleThread + // t2_generate is not thread-safe per pipeline. The gRPC server already + // serializes calls via Locking(), but keep a local mutex too so the + // invariant doesn't depend on the transport. + mu sync.Mutex + pipeline uintptr +} + +func (t *Trellis2) Load(opts *pb.ModelOptions) error { + set, err := resolveModels(opts.ModelFile, opts.ModelPath, opts.Options) + if err != nil { + return err + } + + errBuf := make([]byte, 512) + p := t2PipelineLoad(set.dino, set.ssFlow, set.ssDec, + set.slatFlow, set.slatFlow1024, set.shapeDec, + set.shapeEnc, set.texDec, set.texSlatFlow512, set.texSlatFlow1024, + 0 /*flags*/, &errBuf[0], int32(len(errBuf))) + if p == 0 { + return fmt.Errorf("trellis2 pipeline load: %s", cstr(errBuf)) + } + + t.mu.Lock() + if t.pipeline != 0 { + t2PipelineFree(t.pipeline) + } + t.pipeline = p + t.mu.Unlock() + + fmt.Fprintf(os.Stderr, "trellis2 pipeline loaded: backend=%s caps=%#x\n", + t2PipelineBackend(p), t2PipelineCaps(p)) + return nil +} + +func (t *Trellis2) Free() error { + t.mu.Lock() + defer t.mu.Unlock() + if t.pipeline != 0 { + t2PipelineFree(t.pipeline) + t.pipeline = 0 + } + return nil +} + +func (t *Trellis2) Generate3D(opts *pb.Generate3DRequest) error { + if opts.Dst == "" { + return fmt.Errorf("dst is empty") + } + if opts.GetParams()["operation"] == "print_remesh" { + t.mu.Lock() + defer t.mu.Unlock() + return remeshGLB(opts) + } + img, err := os.ReadFile(opts.Src) + if err != nil { + return fmt.Errorf("reading conditioning image: %w", err) + } + if len(img) == 0 { + return fmt.Errorf("conditioning image is empty") + } + + seed := uint64(opts.Seed) + if opts.Seed <= 0 { + seed = rand.Uint64() + } + guidance := opts.CfgScale + if guidance <= 0 { + guidance = -1 // <0 selects the pipeline default (7.5) + } + texSize := atoiOr(opts.GetParams()["texture_size"], 0) // <=0 selects the bake default + componentFilter := componentFilterFor(opts.GetParams()["components"]) + + // Optional CGAL Alpha Wrap: wrap the generated mesh into a watertight, + // intersection-free 2-manifold for 3D printing. Ratios are fractions of + // the bounding-box diagonal; offset defaults to alpha/30 per the CGAL + // guideline the upstream demo uses. Offset is deliberately not an + // independent parameter: looser values produce puffy or degenerate wraps. + printRemesh := boolParam(opts.GetParams()["print_remesh"]) + alphaRatio := ratioOr(opts.GetParams()["alpha_ratio"], 0.005) + offsetRatio := alphaRatio / 30 + if printRemesh && t2PrintRemeshAvailable() == 0 { + return fmt.Errorf("print_remesh requested but libtrellis2 was built without CGAL Alpha Wrap") + } + + t.mu.Lock() + defer t.mu.Unlock() + if t.pipeline == 0 { + return fmt.Errorf("model not loaded") + } + + errBuf := make([]byte, 512) + r := t2Generate(t.pipeline, &img[0], int32(len(img)), + pipelineForQuality(opts.Quality), backgroundForMode(opts.Background), + seed, opts.Step, guidance, opts.TextureSteps, + 0, 0, 0, 0, // no progress/preview callbacks + &errBuf[0], int32(len(errBuf))) + if r == 0 { + return fmt.Errorf("trellis2 generate: %s", cstr(errBuf)) + } + defer t2MeshFree(r) + + nv := t2MeshNVerts(r) + nt := t2MeshNTris(r) + if nv == 0 || nt == 0 { + return fmt.Errorf("empty mesh") + } + var pbr *float32 + if t2MeshHasPBR(r) != 0 { + pbr = t2MeshPBR(r) + } + + // Bake straight from the mesh accessor buffers — they stay valid until + // t2_mesh_free, so no copies are needed. + var outLen int32 + var glb *byte + if printRemesh { + wrap := t2PreparePrintMesh(t2MeshVerts(r), nv, t2MeshTris(r), nt, pbr, + componentFilter, alphaRatio, offsetRatio, + &errBuf[0], int32(len(errBuf))) + if wrap == 0 { + return fmt.Errorf("trellis2 print remesh: %s", cstr(errBuf)) + } + defer t2MeshFree(wrap) + wnv, wnt := t2MeshNVerts(wrap), t2MeshNTris(wrap) + if wnv == 0 || wnt == 0 { + return fmt.Errorf("empty print mesh") + } + if pbr != nil { + // Wrapping creates new vertices, so the source material is + // reprojected per texel onto the wrap's UV atlas (demo handleGLB). + glb = t2BakeProjectedGLB(t2MeshVerts(wrap), wnv, t2MeshTris(wrap), wnt, + t2MeshVerts(r), nv, t2MeshTris(r), nt, pbr, + texSize, componentFilter, + &outLen, &errBuf[0], int32(len(errBuf))) + } else { + glb = t2BakeGLB(t2MeshVerts(wrap), wnv, t2MeshTris(wrap), wnt, nil, + texSize, 2, // the wrap output is already component-filtered + &outLen, &errBuf[0], int32(len(errBuf))) + } + } else { + glb = t2BakeGLB(t2MeshVerts(r), nv, t2MeshTris(r), nt, pbr, + texSize, componentFilter, + &outLen, &errBuf[0], int32(len(errBuf))) + } + if glb == nil { + return fmt.Errorf("trellis2 GLB bake: %s", cstr(errBuf)) + } + defer t2FreeBuffer(glb) + + out := make([]byte, int(outLen)) + copy(out, unsafe.Slice(glb, int(outLen))) + return os.WriteFile(opts.Dst, out, 0600) +} + +// remeshGLB applies the demo's post-generation print workflow to an existing +// dense vertex-PBR GLB. It does not touch the inference pipeline: CGAL wrapping, +// UV unwrapping, and PBR projection are CPU-only post-processing operations. +func remeshGLB(opts *pb.Generate3DRequest) error { + if opts.Src == "" { + return fmt.Errorf("src is empty") + } + if t2PrintRemeshAvailable() == 0 { + return fmt.Errorf("print remeshing is unavailable (libtrellis2 was built without CGAL Alpha Wrap)") + } + data, err := os.ReadFile(opts.Src) + if err != nil { + return fmt.Errorf("reading source GLB: %w", err) + } + mesh, err := parseVertexGLB(data) + if err != nil { + return fmt.Errorf("reading source GLB: %w", err) + } + + params := opts.GetParams() + alphaRatio := ratioOr(params["alpha_ratio"], 0.005) + offsetRatio := alphaRatio / 30 + componentFilter := componentFilterFor(params["components"]) + textureSize := atoiOr(params["texture_size"], 2048) + var sourcePBR *float32 + if len(mesh.pbr) != 0 { + sourcePBR = &mesh.pbr[0] + } + errBuf := make([]byte, 512) + wrap := t2PreparePrintMesh( + &mesh.verts[0], int32(len(mesh.verts)/3), + &mesh.tris[0], int32(len(mesh.tris)/3), + sourcePBR, componentFilter, alphaRatio, offsetRatio, + &errBuf[0], int32(len(errBuf)), + ) + if wrap == 0 { + return fmt.Errorf("trellis2 print remesh: %s", cstr(errBuf)) + } + defer t2MeshFree(wrap) + + wrappedVerts, wrappedTris := t2MeshNVerts(wrap), t2MeshNTris(wrap) + if wrappedVerts == 0 || wrappedTris == 0 { + return fmt.Errorf("empty print mesh") + } + var outLen int32 + var glb *byte + if sourcePBR != nil { + glb = t2BakeProjectedGLB( + t2MeshVerts(wrap), wrappedVerts, t2MeshTris(wrap), wrappedTris, + &mesh.verts[0], int32(len(mesh.verts)/3), + &mesh.tris[0], int32(len(mesh.tris)/3), sourcePBR, + int32(textureSize), componentFilter, + &outLen, &errBuf[0], int32(len(errBuf)), + ) + } else { + glb = t2BakeGLB( + t2MeshVerts(wrap), wrappedVerts, t2MeshTris(wrap), wrappedTris, + nil, int32(textureSize), 2, + &outLen, &errBuf[0], int32(len(errBuf)), + ) + } + if glb == nil || outLen <= 0 { + return fmt.Errorf("trellis2 GLB bake: %s", cstr(errBuf)) + } + defer t2FreeBuffer(glb) + + out := make([]byte, int(outLen)) + copy(out, unsafe.Slice(glb, int(outLen))) + return os.WriteFile(opts.Dst, out, 0o600) +} + +func cstr(b []byte) string { + for i, c := range b { + if c == 0 { + return string(b[:i]) + } + } + return string(b) +} diff --git a/backend/go/trellis2cpp/trellis2_test.go b/backend/go/trellis2cpp/trellis2_test.go new file mode 100644 index 000000000..d492756e4 --- /dev/null +++ b/backend/go/trellis2cpp/trellis2_test.go @@ -0,0 +1,247 @@ +package main + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func TestTrellis2Cpp(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "trellis2cpp backend suite") +} + +// touch creates empty files — resolveModels only checks existence, so the +// tests never need real GGUF weights. +func touch(dir string, names ...string) { + for _, name := range names { + Expect(os.WriteFile(filepath.Join(dir, name), nil, 0o600)).To(Succeed()) + } +} + +var requiredFiles = []string{"dino_f16.gguf", "ss_flow_f16.gguf", "ss_dec_f16.gguf"} + +var fullSet = append(append([]string{}, requiredFiles...), + "slat_flow_f16.gguf", "slat_flow_1024_f16.gguf", "shape_dec_f16.gguf", + "shape_enc_f16.gguf", "tex_dec_f16.gguf", + "tex_slat_flow_512_f16.gguf", "tex_slat_flow_1024_f16.gguf") + +var _ = Describe("resolveModels", func() { + var dir string + + BeforeEach(func() { + dir = GinkgoT().TempDir() + }) + + It("refuses a directory without the trellis2 component files", func() { + touch(dir, "some-llm.gguf") + + _, err := resolveModels("some-llm.gguf", dir, nil) + Expect(err).To(MatchError(ContainSubstring("not a trellis2 model set"))) + }) + + It("resolves every role from the full default-named set", func() { + touch(dir, fullSet...) + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + for name, path := range map[string]string{ + "dino": set.dino, + "ss_flow": set.ssFlow, + "ss_dec": set.ssDec, + "slat_flow": set.slatFlow, + "slat_flow_1024": set.slatFlow1024, + "shape_dec": set.shapeDec, + "shape_enc": set.shapeEnc, + "tex_dec": set.texDec, + "tex_slat_flow_512": set.texSlatFlow512, + "tex_slat_flow_1024": set.texSlatFlow1024, + } { + Expect(path).NotTo(BeEmpty(), "role %s", name) + } + }) + + It("degrades to coarse-only without the 512 pair, even when texture files exist", func() { + touch(dir, requiredFiles...) + touch(dir, "shape_enc_f16.gguf", "tex_dec_f16.gguf", "tex_slat_flow_512_f16.gguf", "slat_flow_1024_f16.gguf") + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(set.slatFlow).To(BeEmpty()) + Expect(set.shapeDec).To(BeEmpty()) + Expect(set.slatFlow1024).To(BeEmpty()) + Expect(set.shapeEnc).To(BeEmpty()) + Expect(set.texDec).To(BeEmpty()) + Expect(set.texSlatFlow512).To(BeEmpty()) + Expect(set.texSlatFlow1024).To(BeEmpty()) + }) + + It("disables texturing but keeps fine geometry when the texture set is incomplete", func() { + touch(dir, requiredFiles...) + touch(dir, "slat_flow_f16.gguf", "slat_flow_1024_f16.gguf", "shape_dec_f16.gguf", "tex_dec_f16.gguf") + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(set.slatFlow).NotTo(BeEmpty()) + Expect(set.shapeDec).NotTo(BeEmpty()) + Expect(set.slatFlow1024).NotTo(BeEmpty()) + Expect(set.shapeEnc).To(BeEmpty()) + Expect(set.texDec).To(BeEmpty()) + Expect(set.texSlatFlow512).To(BeEmpty()) + Expect(set.texSlatFlow1024).To(BeEmpty()) + }) + + It("drops the 1024 cascade when texturing lacks the HR texture flow", func() { + touch(dir, fullSet...) + Expect(os.Remove(filepath.Join(dir, "tex_slat_flow_1024_f16.gguf"))).To(Succeed()) + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(set.slatFlow1024).To(BeEmpty()) + Expect(set.shapeEnc).NotTo(BeEmpty()) + Expect(set.texDec).NotTo(BeEmpty()) + Expect(set.texSlatFlow512).NotTo(BeEmpty()) + }) + + It("honors explicit *_path option overrides", func() { + touch(dir, fullSet...) + custom := filepath.Join(dir, "custom") + Expect(os.Mkdir(custom, 0o750)).To(Succeed()) + touch(custom, "my-dino.gguf") + + set, err := resolveModels("ss_flow_f16.gguf", dir, []string{"dino_path:custom/my-dino.gguf"}) + Expect(err).NotTo(HaveOccurred()) + Expect(set.dino).To(Equal(filepath.Join(custom, "my-dino.gguf"))) + }) + + It("fails when an explicitly configured file is missing", func() { + touch(dir, fullSet...) + + _, err := resolveModels("ss_flow_f16.gguf", dir, []string{"tex_dec_path:nope.gguf"}) + Expect(err).To(MatchError(ContainSubstring("missing file"))) + }) + + It("rejects option paths escaping the model directory", func() { + touch(dir, fullSet...) + + _, err := resolveModels("ss_flow_f16.gguf", dir, []string{"dino_path:../outside.gguf"}) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = DescribeTable("request parameter mapping", + func(got, want int32) { + Expect(got).To(Equal(want)) + }, + Entry("quality empty", pipelineForQuality(""), int32(pipeAuto)), + Entry("quality auto", pipelineForQuality("auto"), int32(pipeAuto)), + Entry("quality coarse", pipelineForQuality("coarse"), int32(pipeCoarse)), + Entry("quality 512", pipelineForQuality("512"), int32(pipe512)), + Entry("quality 1024", pipelineForQuality("1024"), int32(pipe1024)), + Entry("background empty", backgroundForMode(""), int32(backgroundAuto)), + Entry("background auto", backgroundForMode("auto"), int32(backgroundAuto)), + Entry("background keep", backgroundForMode("keep"), int32(backgroundKeep)), + Entry("background black", backgroundForMode("black"), int32(backgroundBlack)), + Entry("background white", backgroundForMode("white"), int32(backgroundWhite)), + Entry("components default", componentFilterFor(""), int32(2)), + Entry("components all", componentFilterFor("all"), int32(2)), + Entry("components largest", componentFilterFor("largest"), int32(1)), + Entry("components tiny", componentFilterFor("tiny"), int32(0)), + Entry("atoi empty", atoiOr("", 0), int32(0)), + Entry("atoi value", atoiOr("2048", 0), int32(2048)), + Entry("atoi junk", atoiOr("junk", 7), int32(7)), +) + +var _ = Describe("print remesh parameters", func() { + It("parses the print_remesh toggle", func() { + Expect(boolParam("1")).To(BeTrue()) + Expect(boolParam("true")).To(BeTrue()) + Expect(boolParam("TRUE")).To(BeTrue()) + Expect(boolParam("")).To(BeFalse()) + Expect(boolParam("0")).To(BeFalse()) + Expect(boolParam("no")).To(BeFalse()) + }) + + It("parses ratios and clamps junk to the fallback", func() { + Expect(ratioOr("", 0.005)).To(BeNumerically("~", 0.005, 1e-6)) + Expect(ratioOr("0.01", 0.005)).To(BeNumerically("~", 0.01, 1e-6)) + Expect(ratioOr("junk", 0.005)).To(BeNumerically("~", 0.005, 1e-6)) + Expect(ratioOr("-1", 0.005)).To(BeNumerically("~", 0.005, 1e-6)) + Expect(ratioOr("0.9", 0.005)).To(BeNumerically("~", 0.005, 1e-6), "above the demo's 50% cap") + }) +}) + +var _ = Describe("packaged backend", func() { + It("starts and answers Health without loading model weights", func() { + runScript := os.Getenv("TRELLIS2CPP_SMOKE_RUN") + if runScript == "" { + runScript = filepath.Join("package", "run.sh") + } + if _, err := os.Stat(runScript); os.IsNotExist(err) { + Skip("packaged backend is not present; run make before the smoke test") + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + addr := listener.Addr().String() + Expect(listener.Close()).To(Succeed()) + + cmd := exec.Command("bash", runScript, "--addr="+addr) + cmd.Stdout = GinkgoWriter + cmd.Stderr = GinkgoWriter + Expect(cmd.Start()).To(Succeed()) + processDone := make(chan error, 1) + go func() { processDone <- cmd.Wait() }() + processExited := false + DeferCleanup(func() { + if cmd.Process != nil && !processExited { + _ = cmd.Process.Kill() + <-processDone + } + }) + + Eventually(func() error { + select { + case err := <-processDone: + processExited = true + if err != nil { + return StopTrying("backend exited before Health succeeded").Wrap(err) + } + return StopTrying("backend exited before Health succeeded") + default: + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + ) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + + reply, err := pb.NewBackendClient(conn).Health(ctx, &pb.HealthMessage{}) + if err != nil { + return err + } + if string(reply.GetMessage()) != "OK" { + return fmt.Errorf("unexpected Health reply %q", reply.GetMessage()) + } + return nil + }, 30*time.Second, 200*time.Millisecond).Should(Succeed()) + }) +}) diff --git a/backend/index.yaml b/backend/index.yaml index 2d002d690..c1c1e471e 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -425,6 +425,32 @@ nvidia-cuda-12: "cuda12-stablediffusion-ggml" nvidia-l4t-cuda-12: "nvidia-l4t-arm64-stablediffusion-ggml" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-stablediffusion-ggml" +- &trellis2cpp + name: "trellis2cpp" + alias: "trellis2cpp" + license: mit + description: | + TRELLIS.2 image-to-3D generation (GLB meshes with PBR textures) in C++/ggml + urls: + - https://github.com/localai-org/trellis2cpp + - https://github.com/microsoft/TRELLIS.2 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Metal + capabilities: + default: "cpu-trellis2cpp" + nvidia: "cuda12-trellis2cpp" + vulkan: "vulkan-trellis2cpp" + nvidia-l4t: "nvidia-l4t-arm64-trellis2cpp" + metal: "metal-trellis2cpp" + nvidia-cuda-13: "cuda13-trellis2cpp" + nvidia-cuda-12: "cuda12-trellis2cpp" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-trellis2cpp" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-trellis2cpp" - &rfdetr name: "rfdetr" alias: "rfdetr" @@ -1972,6 +1998,18 @@ nvidia-cuda-12: "cuda12-stablediffusion-ggml-development" nvidia-l4t-cuda-12: "nvidia-l4t-arm64-stablediffusion-ggml-development" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-stablediffusion-ggml-development" +- !!merge <<: *trellis2cpp + name: "trellis2cpp-development" + capabilities: + default: "cpu-trellis2cpp-development" + nvidia: "cuda12-trellis2cpp-development" + vulkan: "vulkan-trellis2cpp-development" + nvidia-l4t: "nvidia-l4t-arm64-trellis2cpp-development" + metal: "metal-trellis2cpp-development" + nvidia-cuda-13: "cuda13-trellis2cpp-development" + nvidia-cuda-12: "cuda12-trellis2cpp-development" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-trellis2cpp-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-trellis2cpp-development" - !!merge <<: *neutts name: "cpu-neutts" uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-neutts" @@ -3688,6 +3726,77 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-stablediffusion-ggml" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-13-stablediffusion-ggml +## trellis2cpp +- !!merge <<: *trellis2cpp + name: "cpu-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-trellis2cpp" + mirrors: + - localai/localai-backends:latest-cpu-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cpu-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-trellis2cpp" + mirrors: + - localai/localai-backends:master-cpu-trellis2cpp +- !!merge <<: *trellis2cpp + name: "metal-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "metal-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "vulkan-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-trellis2cpp" + mirrors: + - localai/localai-backends:latest-gpu-vulkan-trellis2cpp +- !!merge <<: *trellis2cpp + name: "vulkan-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-trellis2cpp" + mirrors: + - localai/localai-backends:master-gpu-vulkan-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda12-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-trellis2cpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda12-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-trellis2cpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-trellis2cpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-trellis2cpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-trellis2cpp +- !!merge <<: *trellis2cpp + name: "nvidia-l4t-arm64-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "nvidia-l4t-arm64-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:master-nvidia-l4t-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-nvidia-l4t-arm64-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-nvidia-l4t-arm64-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-trellis2cpp ## privacy-filter - !!merge <<: *privacyfilter name: "cpu-privacy-filter" diff --git a/backend/rust/kokoros/src/service.rs b/backend/rust/kokoros/src/service.rs index 495b0a423..80fd3ceb8 100644 --- a/backend/rust/kokoros/src/service.rs +++ b/backend/rust/kokoros/src/service.rs @@ -334,6 +334,13 @@ impl Backend for KokorosService { Err(Status::unimplemented("Not supported")) } + async fn generate3_d( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not supported")) + } + async fn audio_transcription( &self, _: Request, diff --git a/core/backend/model3d.go b/core/backend/model3d.go new file mode 100644 index 000000000..1952501d9 --- /dev/null +++ b/core/backend/model3d.go @@ -0,0 +1,104 @@ +package backend + +import ( + "maps" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/trace" + "github.com/mudler/LocalAI/pkg/grpc/proto" + model "github.com/mudler/LocalAI/pkg/model" +) + +// Model3DGenerationOptions is the backend-neutral request passed to 3D +// generators. Image contains a staged local path by the time it reaches +// this layer. +type Model3DGenerationOptions struct { + Image string + Destination string + Seed int32 + Step int32 + CFGScale float32 + TextureSteps int32 + Quality string + Background string + Params map[string]string +} + +func Model3DGeneration(options Model3DGenerationOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) { + opts := ModelOptions(modelConfig, appConfig) + inferenceModel, err := loader.Load(opts...) + if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) + return nil, err + } + + fn := func() error { + _, err := inferenceModel.Generate3D( + appConfig.Context, + &proto.Generate3DRequest{ + Src: options.Image, + Dst: options.Destination, + Seed: options.Seed, + Step: options.Step, + CfgScale: options.CFGScale, + TextureSteps: options.TextureSteps, + Quality: options.Quality, + Background: options.Background, + Params: maps.Clone(options.Params), + }, + ) + return err + } + + if appConfig.EnableTracing { + trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) + + traceType := trace.BackendTrace3DGeneration + traceSummary := "3d: " + options.Quality + traceData := map[string]any{} + if options.Params["operation"] == "print_remesh" { + traceType = trace.BackendTrace3DRemesh + traceSummary = "3d: remesh" + traceData["detail_percent"] = options.Params["detail_percent"] + traceData["has_mesh"] = options.Image != "" + } else { + traceData = map[string]any{ + "seed": options.Seed, + "step": options.Step, + "cfg_scale": options.CFGScale, + "texture_steps": options.TextureSteps, + "quality": options.Quality, + "background": options.Background, + "has_image": options.Image != "", + } + } + + startTime := time.Now() + originalFn := fn + fn = func() error { + err := originalFn() + duration := time.Since(startTime) + + errStr := "" + if err != nil { + errStr = err.Error() + } + + trace.RecordBackendTrace(trace.BackendTrace{ + Timestamp: startTime, + Duration: duration, + Type: traceType, + ModelName: modelConfig.Name, + Backend: modelConfig.Backend, + Summary: trace.TruncateString(traceSummary, 200), + Error: errStr, + Data: traceData, + }) + + return err + } + } + + return fn, nil +} diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index ad8b4d622..3de119bbb 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -16,6 +16,7 @@ const ( UsecaseTokenize = "tokenize" UsecaseImage = "image" UsecaseVideo = "video" + Usecase3D = "3d" UsecaseTranscript = "transcript" UsecaseTTS = "tts" UsecaseSoundGeneration = "sound_generation" @@ -42,6 +43,7 @@ const ( MethodEmbedding GRPCMethod = "Embedding" MethodGenerateImage GRPCMethod = "GenerateImage" MethodGenerateVideo GRPCMethod = "GenerateVideo" + MethodGenerate3D GRPCMethod = "Generate3D" MethodAudioTranscription GRPCMethod = "AudioTranscription" MethodTTS GRPCMethod = "TTS" MethodTTSStream GRPCMethod = "TTSStream" @@ -124,6 +126,11 @@ var UsecaseInfoMap = map[string]UsecaseInfo{ GRPCMethod: MethodGenerateVideo, Description: "Video generation via the GenerateVideo RPC, with optional image or audio conditioning when supported by the backend.", }, + Usecase3D: { + Flag: FLAG_3D, + GRPCMethod: MethodGenerate3D, + Description: "Image-conditioned 3D asset generation via the Generate3D RPC — a binary glTF (GLB) mesh with optional PBR material (TRELLIS.2).", + }, UsecaseTranscript: { Flag: FLAG_TRANSCRIPT, GRPCMethod: MethodAudioTranscription, @@ -351,6 +358,14 @@ var BackendCapabilities = map[string]BackendCapability{ Description: "Stable Diffusion via GGML quantized models", }, + // --- 3D generation backends --- + "trellis2cpp": { + GRPCMethods: []GRPCMethod{MethodGenerate3D}, + PossibleUsecases: []string{Usecase3D}, + DefaultUsecases: []string{Usecase3D}, + Description: "trellis2.cpp — C++/GGML port of Microsoft TRELLIS.2: single-image to textured 3D mesh (GLB)", + }, + // --- Speech-to-text backends --- "whisper": { GRPCMethods: []GRPCMethod{MethodAudioTranscription, MethodVAD}, diff --git a/core/config/model_capabilities.go b/core/config/model_capabilities.go index 79a117b4a..6b432bddc 100644 --- a/core/config/model_capabilities.go +++ b/core/config/model_capabilities.go @@ -15,9 +15,10 @@ const ( ModalityImage = "image" ModalityAudio = "audio" ModalityVideo = "video" + Modality3D = "3d" ) -var modalityOrder = []string{ModalityText, ModalityImage, ModalityAudio, ModalityVideo} +var modalityOrder = []string{ModalityText, ModalityImage, ModalityAudio, ModalityVideo, Modality3D} func declaredModalities(modalities []string) map[string]bool { declared := make(map[string]bool, len(modalities)) @@ -154,6 +155,7 @@ func (c *ModelConfig) Capabilities() []string { add(c.HasUsecases(FLAG_SOUND_GENERATION), UsecaseSoundGeneration) add(c.HasUsecases(FLAG_IMAGE), UsecaseImage) add(c.HasUsecases(FLAG_VIDEO), UsecaseVideo) + add(c.HasUsecases(FLAG_3D), Usecase3D) add(c.HasUsecases(FLAG_VAD), UsecaseVAD) add(c.HasUsecases(FLAG_DETECTION), UsecaseDetection) add(c.HasUsecases(FLAG_DEPTH), UsecaseDepth) @@ -181,9 +183,10 @@ func (c *ModelConfig) InputModalities() []string { c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) || imageGen || videoGen // Image input via a chat model requires vision (gated on chat, like the - // Ollama surface); detection/depth/face models consume images directly. + // Ollama surface); detection/depth/face/3D models consume images directly. imageIn := (chatish && c.VisionSupported()) || c.LimitMMPerPrompt.LimitImagePerPrompt > 0 || - c.HasUsecases(FLAG_DETECTION) || c.HasUsecases(FLAG_DEPTH) || c.HasUsecases(FLAG_FACE_RECOGNITION) + c.HasUsecases(FLAG_DETECTION) || c.HasUsecases(FLAG_DEPTH) || c.HasUsecases(FLAG_FACE_RECOGNITION) || + c.HasUsecases(FLAG_3D) audioIn := c.AudioInputSupported() || c.HasUsecases(FLAG_TRANSCRIPT) || c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO) || c.HasUsecases(FLAG_VAD) || c.HasUsecases(FLAG_DIARIZATION) || @@ -208,10 +211,12 @@ func (c *ModelConfig) OutputModalities() []string { audioOut := c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) || c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO) videoOut := c.HasUsecases(FLAG_VIDEO) + threeDOut := c.HasUsecases(FLAG_3D) modalities[ModalityText] = modalities[ModalityText] || textOut modalities[ModalityImage] = modalities[ModalityImage] || imageOut modalities[ModalityAudio] = modalities[ModalityAudio] || audioOut modalities[ModalityVideo] = modalities[ModalityVideo] || videoOut + modalities[Modality3D] = modalities[Modality3D] || threeDOut return orderedModalities(modalities) } diff --git a/core/config/model_capabilities_test.go b/core/config/model_capabilities_test.go index 545bad50b..84f01af7b 100644 --- a/core/config/model_capabilities_test.go +++ b/core/config/model_capabilities_test.go @@ -109,6 +109,25 @@ var _ = Describe("Model capabilities derivation", func() { Expect(cfg.OutputModalities()).To(Equal([]string{"image"})) }) + It("guesses the 3d usecase from the trellis2cpp backend and only that backend", func() { + cfg := &ModelConfig{Backend: "trellis2cpp"} + Expect(cfg.HasUsecases(FLAG_3D)).To(BeTrue()) + Expect(cfg.Capabilities()).To(ContainElement(Usecase3D)) + + other := &ModelConfig{Backend: "llama-cpp"} + Expect(other.HasUsecases(FLAG_3D)).To(BeFalse()) + }) + + It("a 3D-generation model reads an image and writes a 3D asset", func() { + // Pins the wire strings the UI depends on: capability "3d", + // input modality "image" (no text prompt — TRELLIS.2 is + // image-conditioned only), output modality "3d". + cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_3D), Backend: "trellis2cpp"} + Expect(cfg.Capabilities()).To(Equal([]string{Usecase3D})) + Expect(cfg.InputModalities()).To(Equal([]string{ModalityImage})) + Expect(cfg.OutputModalities()).To(Equal([]string{Modality3D})) + }) + It("conditioned video uses declared modalities without backend-specific inference", func() { cfg := &ModelConfig{ KnownUsecases: usecaseBits(FLAG_VIDEO), diff --git a/core/config/model_config.go b/core/config/model_config.go index 3a32ac2c8..21bc42fac 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -1673,6 +1673,11 @@ const ( // labels via the SoundDetection RPC, e.g. ced). FLAG_SOUND_CLASSIFICATION ModelConfigUsecase = 0b10000000000000000000000 + // Marks a model as wired for the Generate3D gRPC primitive + // (image-conditioned 3D asset generation — a binary glTF mesh with + // optional PBR material, e.g. trellis2cpp). + FLAG_3D ModelConfigUsecase = 0b100000000000000000000000 + // Common Subsets FLAG_LLM ModelConfigUsecase = FLAG_CHAT | FLAG_COMPLETION | FLAG_EDIT ) @@ -1686,7 +1691,7 @@ var ModalityGroups = []ModelConfigUsecase{ FLAG_TRANSCRIPT | FLAG_REALTIME_AUDIO | FLAG_SOUND_CLASSIFICATION, // audio input — realtime_audio is any-to-any, so it counts here too FLAG_TTS | FLAG_SOUND_GENERATION | FLAG_REALTIME_AUDIO, // audio output — and here, so a lone realtime_audio flag still reads as multimodal FLAG_AUDIO_TRANSFORM, // audio in/out transforms - FLAG_IMAGE | FLAG_VIDEO, // visual generation + FLAG_IMAGE | FLAG_VIDEO | FLAG_3D, // visual generation } // IsMultimodal returns true if the given usecases span two or more orthogonal @@ -1733,6 +1738,7 @@ func GetAllModelConfigUsecases() map[string]ModelConfigUsecase { "FLAG_SCORE": FLAG_SCORE, "FLAG_DEPTH": FLAG_DEPTH, "FLAG_TOKEN_CLASSIFY": FLAG_TOKEN_CLASSIFY, + "FLAG_3D": FLAG_3D, } } @@ -1884,6 +1890,13 @@ func (c *ModelConfig) GuessUsecases(u ModelConfigUsecase) bool { } } + if (u & FLAG_3D) == FLAG_3D { + threeDBackends := []string{"trellis2cpp"} + if !slices.Contains(threeDBackends, c.Backend) { + return false + } + } + if (u & FLAG_FACE_RECOGNITION) == FLAG_FACE_RECOGNITION { faceBackends := []string{"insightface"} if !slices.Contains(faceBackends, c.Backend) { diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 8c156144d..a86e86530 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -143,6 +143,11 @@ var defaultImporters = []Importer{ &CoquiImporter{}, // Image/Video (Batch 3) &StableDiffusionGGMLImporter{}, + // Trellis2CppImporter (TRELLIS.2 image-to-3D, native C++/ggml port) must + // run before LlamaCPPImporter so its GGUF sets aren't claimed by the + // generic .gguf importer; matches only trellis-named URIs/repos or the + // distinctive component filenames, so arbitrary GGUFs are never claimed. + &Trellis2CppImporter{}, &ACEStepImporter{}, // LongCat repositories carry generic Diffusers metadata, so this exact // owner/repo matcher must run before DiffuserImporter. diff --git a/core/gallery/importers/trellis2cpp.go b/core/gallery/importers/trellis2cpp.go new file mode 100644 index 000000000..b94bb904d --- /dev/null +++ b/core/gallery/importers/trellis2cpp.go @@ -0,0 +1,170 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "go.yaml.in/yaml/v2" +) + +var _ Importer = &Trellis2CppImporter{} + +// trellis2File describes one component of the TRELLIS.2 GGUF set hosted on +// the LocalAI-io HuggingFace org. The pipeline spans three source repos +// (TRELLIS.2-4B, TRELLIS-image-large for the SS decoder, and a DINOv3 +// mirror), so a single import URI always expands to this full set — no one +// repo can describe it alone. Filenames follow the trellis2cpp converter +// defaults, which the backend resolves without any options. +type trellis2File struct { + filename string + uri string + sha256 string +} + +var trellis2Files = []trellis2File{ + {"dino_f16.gguf", "https://huggingface.co/LocalAI-io/dinov3-vitl16-pretrain-lvd1689m-GGUF/resolve/main/dino_f16.gguf", "385d8186a38a2328ec740fb2ac1f33f9194d8774efc7ccafd4aa2e51cf5f6450"}, + {"ss_flow_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/ss_flow_f16.gguf", "1dded5b74237d24e6876a642a26f90b43742e3554418573860f810e3bbe61e8c"}, + {"ss_dec_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS-image-large-GGUF/resolve/main/ss_dec_f16.gguf", "9c2210b7ed830fdc8286961a8189878ff5bcfd3bfc83ab4eacee005d293d2185"}, + {"slat_flow_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/slat_flow_f16.gguf", "2f94bad7b1c524ad8c01943bc38fcc0c314e7d482ce896f3c6e96eb6e7cec15c"}, + {"slat_flow_1024_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/slat_flow_1024_f16.gguf", "b6a2270131e2e9235e9b6cb525193eb85ae132fa5af3274322aacd39e40a6bc5"}, + {"shape_dec_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/shape_dec_f16.gguf", "6fe53f1d7763dabf7c8d72bc38f4053d87fde6f65bf17a9d378d27edb39d3530"}, + {"shape_enc_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/shape_enc_f16.gguf", "3ec80ff580987fcdb9bc594fc8b6fda890d63101ca442eb2b26f5dc315e8696c"}, + {"tex_dec_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/tex_dec_f16.gguf", "afd304f4dfcb8c94df851b85519b415b99f04070f7d29de1320c50631b1be4e0"}, + {"tex_slat_flow_512_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/tex_slat_flow_512_f16.gguf", "89a081b7f5487a5b31f03d240e4d959a56db0cc2c46c327230097a2554da52ae"}, + {"tex_slat_flow_1024_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/tex_slat_flow_1024_f16.gguf", "bbb55b0910c7929aac5e0612a9bb15113837a2c674cafb9f0f170eda8b5558a8"}, +} + +// trellis2ComponentNames are the distinctive default component filenames. A +// raw .gguf URL with one of these basenames is a strong trellis2 signal. +// dino_f16.gguf is deliberately absent — DINO checkpoints are common enough +// that the bare name would over-claim. +var trellis2ComponentNames = map[string]struct{}{ + "ss_flow_f16.gguf": {}, + "ss_dec_f16.gguf": {}, + "slat_flow_f16.gguf": {}, + "slat_flow_1024_f16.gguf": {}, + "shape_dec_f16.gguf": {}, + "shape_enc_f16.gguf": {}, + "tex_dec_f16.gguf": {}, + "tex_slat_flow_512_f16.gguf": {}, + "tex_slat_flow_1024_f16.gguf": {}, +} + +// Trellis2CppImporter recognises Microsoft TRELLIS.2 image-to-3D GGUF sets +// (the trellis2.cpp converter outputs hosted under LocalAI-io). It must be +// registered BEFORE LlamaCPPImporter so llama-cpp does not steal the .gguf +// match. preferences.backend="trellis2cpp" overrides detection. +type Trellis2CppImporter struct{} + +func (i *Trellis2CppImporter) Name() string { return "trellis2cpp" } +func (i *Trellis2CppImporter) Modality() string { return "3d" } +func (i *Trellis2CppImporter) AutoDetects() bool { return true } + +// containsTrellisToken reports whether s (compared case-insensitively) +// carries a TRELLIS marker ("trellis" covers TRELLIS.2 / trellis2 too). +func containsTrellisToken(s string) bool { + return strings.Contains(strings.ToLower(s), "trellis") +} + +func (i *Trellis2CppImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return false + } + } + + if b, ok := preferencesMap["backend"].(string); ok && b != "" { + return b == "trellis2cpp" + } + + // Raw .gguf URL named after a distinctive pipeline component. + if strings.HasSuffix(strings.ToLower(details.URI), ".gguf") { + base := strings.ToLower(filepath.Base(details.URI)) + if _, ok := trellis2ComponentNames[base]; ok { + return true + } + } + + // A trellis-named URI or HF repo carrying GGUFs. + if containsTrellisToken(details.URI) { + if strings.HasSuffix(strings.ToLower(details.URI), ".gguf") { + return true + } + if details.HuggingFace != nil && hasGGUF(details.HuggingFace.Files) { + return true + } + // HF details may be nil (tree-listing quirk) — decide from the + // owner/repo alone. + if _, repo, ok := HFOwnerRepoFromURI(details.URI); ok && containsTrellisToken(repo) { + return true + } + } + + return false +} + +func (i *Trellis2CppImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return gallery.ModelConfig{}, err + } + } + + name, ok := preferencesMap["name"].(string) + if !ok { + name = "trellis2-4b" + } + + description, ok := preferencesMap["description"].(string) + if !ok { + description = "TRELLIS.2 image-to-3D (GLB with PBR textures) — imported from " + details.URI + } + + cfg := gallery.ModelConfig{ + Name: name, + Description: description, + } + // The full pipeline spans three HF repos, so any trellis URI imports the + // complete known-good set rather than whatever single repo was pasted. + for _, f := range trellis2Files { + cfg.Files = append(cfg.Files, gallery.File{ + URI: f.uri, + Filename: f.filename, + SHA256: f.sha256, + }) + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + Backend: "trellis2cpp", + KnownUsecaseStrings: []string{"FLAG_3D"}, + PredictionOptions: schema.PredictionOptions{ + // ss_flow anchors the GGUF directory; the backend resolves the + // other components from their default filenames next to it. + BasicModelRequest: schema.BasicModelRequest{Model: "ss_flow_f16.gguf"}, + }, + } + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + + cfg.ConfigFile = string(data) + return cfg, nil +} diff --git a/core/gallery/importers/trellis2cpp_test.go b/core/gallery/importers/trellis2cpp_test.go new file mode 100644 index 000000000..00251526f --- /dev/null +++ b/core/gallery/importers/trellis2cpp_test.go @@ -0,0 +1,105 @@ +package importers_test + +import ( + "encoding/json" + "fmt" + + "github.com/mudler/LocalAI/core/gallery/importers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Trellis2CppImporter", func() { + Context("detection from HuggingFace", func() { + // LocalAI-io/TRELLIS.2-4B-GGUF is the canonical GGUF conversion of + // microsoft/TRELLIS.2-4B produced by the trellis2cpp converters. + // Detection must route it to trellis2cpp (and NOT to llama-cpp, + // which otherwise steals every .gguf repo). + It("matches the TRELLIS.2 GGUF repo and imports the full component set", func() { + uri := "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF" + preferences := json.RawMessage(`{}`) + + modelConfig, err := importers.DiscoverModelConfig(uri, preferences) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: trellis2cpp"), fmt.Sprintf("Model config: %+v", modelConfig)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("FLAG_3D")) + // The pipeline spans three repos; the import must carry the whole + // set, anchored on ss_flow. + Expect(modelConfig.Files).To(HaveLen(10)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: ss_flow_f16.gguf")) + }) + + It("matches a raw .gguf URL named after a distinctive pipeline component", func() { + uri := "https://example.com/models/tex_slat_flow_512_f16.gguf" + preferences := json.RawMessage(`{}`) + + modelConfig, err := importers.DiscoverModelConfig(uri, preferences) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: trellis2cpp"), fmt.Sprintf("Model config: %+v", modelConfig)) + }) + }) + + Context("preference override", func() { + It("honours preferences.backend=trellis2cpp for arbitrary URIs", func() { + uri := "https://example.com/some-unrelated-model" + preferences := json.RawMessage(`{"backend": "trellis2cpp"}`) + + modelConfig, err := importers.DiscoverModelConfig(uri, preferences) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: trellis2cpp"), fmt.Sprintf("Model config: %+v", modelConfig)) + }) + + It("does not override a different explicit backend", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/tex_slat_flow_512_f16.gguf", + Preferences: json.RawMessage(`{"backend": "llama-cpp"}`), + }) + + Expect(match).To(BeFalse()) + }) + + It("still auto-detects when the backend preference is empty", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/tex_slat_flow_512_f16.gguf", + Preferences: json.RawMessage(`{"backend": ""}`), + }) + + Expect(match).To(BeTrue()) + }) + }) + + Context("negative detection", func() { + It("does not claim an unrelated raw .gguf URL", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/llama-3-8b-Q4_K.gguf", + Preferences: json.RawMessage(`{}`), + }) + Expect(match).To(BeFalse()) + }) + + It("does not claim a bare dino_f16.gguf (too generic a name)", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/dino_f16.gguf", + Preferences: json.RawMessage(`{}`), + }) + Expect(match).To(BeFalse()) + }) + }) + + Context("Importer interface metadata", func() { + It("exposes name/modality/autodetect", func() { + imp := &importers.Trellis2CppImporter{} + Expect(imp.Name()).To(Equal("trellis2cpp")) + Expect(imp.Modality()).To(Equal("3d")) + Expect(imp.AutoDetects()).To(BeTrue()) + }) + }) +}) diff --git a/core/http/app.go b/core/http/app.go index bee47013b..8b08af4ad 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -55,6 +55,12 @@ var quietPaths = []string{"/api/operations", "/api/resources", "/healthz", "/rea // conditional revalidation round-trip. const immutableAssetCacheControl = "public, max-age=31536000, immutable" +func defaultBodyLimitSkipper(c echo.Context) bool { + // Remeshing accepts generated GLBs that routinely exceed the default + // upload limit. The route has its own tighter, format-specific limit. + return c.Request().Method == http.MethodPost && c.Path() == "/3d/remesh" +} + // applyModelLoadCooldown maps a ModelLoadCooldownError anywhere in err's chain // to HTTP 503 with a Retry-After header (whole seconds, floor 1), so a client // polling a model whose load recently failed backs off instead of triggering a @@ -123,7 +129,10 @@ func API(application *application.Application) (*echo.Echo, error) { // Set body limit if application.ApplicationConfig().UploadLimitMB > 0 { - e.Use(middleware.BodyLimit(fmt.Sprintf("%dM", application.ApplicationConfig().UploadLimitMB))) + e.Use(middleware.BodyLimitWithConfig(middleware.BodyLimitConfig{ + Limit: fmt.Sprintf("%dM", application.ApplicationConfig().UploadLimitMB), + Skipper: defaultBodyLimitSkipper, + })) } // SPA fallback handler, set later when React UI is available @@ -305,14 +314,22 @@ func API(application *application.Application) (*echo.Echo, error) { audioPath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "audio") imagePath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "images") videoPath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "videos") + threeDPath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "3d") os.MkdirAll(audioPath, 0750) os.MkdirAll(imagePath, 0750) os.MkdirAll(videoPath, 0750) + _ = os.MkdirAll(threeDPath, 0750) + + // Go's built-in MIME table has no .glb entry and minimal containers + // ship no /etc/mime.types, so generated GLBs would otherwise be + // served as application/octet-stream. + _ = mime.AddExtensionType(".glb", "model/gltf-binary") e.Static("/generated-audio", audioPath) e.Static("/generated-images", imagePath) e.Static("/generated-videos", videoPath) + e.Static("/generated-3d", threeDPath) } // Usage recording is initialised in application/startup.go and diff --git a/core/http/auth/features.go b/core/http/auth/features.go index a43eef992..063b4d76e 100644 --- a/core/http/auth/features.go +++ b/core/http/auth/features.go @@ -91,6 +91,10 @@ var RouteFeatureRegistry = []RouteFeature{ // Video {"POST", "/video", FeatureVideo}, + // 3D generation + {"POST", "/3d/generations", Feature3D}, + {"POST", "/3d/remesh", Feature3D}, + // Sound generation {"POST", "/v1/sound-generation", FeatureSound}, @@ -182,6 +186,7 @@ func APIFeatureMetas() []FeatureMeta { {FeatureVAD, "Voice Activity Detection", true}, {FeatureDetection, "Detection", true}, {FeatureVideo, "Video Generation", true}, + {Feature3D, "3D Generation", true}, {FeatureEmbeddings, "Embeddings", true}, {FeatureSound, "Sound Generation", true}, {FeatureRealtime, "Realtime", true}, diff --git a/core/http/auth/middleware.go b/core/http/auth/middleware.go index c67954640..dd76bdf09 100644 --- a/core/http/auth/middleware.go +++ b/core/http/auth/middleware.go @@ -584,6 +584,7 @@ func isAPIPath(path string) bool { strings.HasPrefix(path, "/tts") || strings.HasPrefix(path, "/vad") || strings.HasPrefix(path, "/video") || + strings.HasPrefix(path, "/3d/") || strings.HasPrefix(path, "/stores/") || strings.HasPrefix(path, "/system") || strings.HasPrefix(path, "/ws/") || diff --git a/core/http/auth/middleware_test.go b/core/http/auth/middleware_test.go index 5137851e1..94be9336a 100644 --- a/core/http/auth/middleware_test.go +++ b/core/http/auth/middleware_test.go @@ -156,6 +156,16 @@ var _ = Describe("Auth Middleware", func() { Expect(rec.Code).To(Equal(http.StatusUnauthorized)) }) + It("returns 401 for unauthenticated 3D generation requests", func() { + rec := doRequest(app, http.MethodPost, "/3d/generations") + Expect(rec.Code).To(Equal(http.StatusUnauthorized)) + }) + + It("returns 401 for unauthenticated 3D remesh requests", func() { + rec := doRequest(app, http.MethodPost, "/3d/remesh") + Expect(rec.Code).To(Equal(http.StatusUnauthorized)) + }) + It("allows unauthenticated access to non-API paths when no legacy keys", func() { rec := doRequest(app, http.MethodGet, "/app") Expect(rec.Code).To(Equal(http.StatusOK)) diff --git a/core/http/auth/permissions.go b/core/http/auth/permissions.go index 1795792f9..6c01e03a3 100644 --- a/core/http/auth/permissions.go +++ b/core/http/auth/permissions.go @@ -47,6 +47,7 @@ const ( FeatureVAD = "vad" FeatureDetection = "detection" FeatureVideo = "video" + Feature3D = "3d" FeatureEmbeddings = "embeddings" FeatureSound = "sound" FeatureRealtime = "realtime" @@ -73,7 +74,7 @@ var GeneralFeatures = []string{FeatureFineTuning, FeatureQuantization} var APIFeatures = []string{ FeatureChat, FeatureImages, FeatureAudioSpeech, FeatureAudioTranscription, FeatureAudioDiarization, FeatureAudioClassification, - FeatureVAD, FeatureDetection, FeatureVideo, FeatureEmbeddings, FeatureSound, + FeatureVAD, FeatureDetection, FeatureVideo, Feature3D, FeatureEmbeddings, FeatureSound, FeatureRealtime, FeatureRerank, FeatureTokenize, FeatureMCP, FeatureStores, FeatureFaceRecognition, FeatureVoiceRecognition, FeatureAudioTransform, FeaturePIIFilter, diff --git a/core/http/body_limit_test.go b/core/http/body_limit_test.go new file mode 100644 index 000000000..7d398766c --- /dev/null +++ b/core/http/body_limit_test.go @@ -0,0 +1,66 @@ +package http_test + +import ( + "bytes" + "context" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/http" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Request body limits", func() { + It("lets the remesh route apply its larger limit without weakening other routes", func() { + dir := GinkgoT().TempDir() + models := filepath.Join(dir, "models") + backends := filepath.Join(dir, "backends") + Expect(os.Mkdir(models, 0o750)).To(Succeed()) + Expect(os.Mkdir(backends, 0o750)).To(Succeed()) + + state, err := system.GetSystemState( + system.WithModelPath(models), + system.WithBackendPath(backends), + ) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + localApp, err := application.New( + config.WithContext(ctx), + config.WithSystemState(state), + config.WithUploadLimitMB(1), + ) + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = localApp.Shutdown() }() + app, err := API(localApp) + Expect(err).NotTo(HaveOccurred()) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + Expect(writer.WriteField("model", "missing-model")).To(Succeed()) + part, err := writer.CreateFormFile("mesh", "large.glb") + Expect(err).NotTo(HaveOccurred()) + _, err = part.Write(bytes.Repeat([]byte{'x'}, 2<<20)) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + request := httptest.NewRequest(http.MethodPost, "/3d/remesh", body) + request.Header.Set("Content-Type", writer.FormDataContentType()) + response := httptest.NewRecorder() + app.ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusNotFound), response.Body.String()) + + request = httptest.NewRequest(http.MethodPost, "/3d/generations", bytes.NewReader(make([]byte, 2<<20))) + request.Header.Set("Content-Type", "application/json") + response = httptest.NewRecorder() + app.ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusRequestEntityTooLarge), response.Body.String()) + }) +}) diff --git a/core/http/endpoints/localai/api_instructions.go b/core/http/endpoints/localai/api_instructions.go index 02ff85d1d..62f264cf0 100644 --- a/core/http/endpoints/localai/api_instructions.go +++ b/core/http/endpoints/localai/api_instructions.go @@ -81,6 +81,12 @@ var instructionDefs = []instructionDef{ Tags: []string{"video"}, Intro: "POST /video accepts start_image, end_image, and audio as public URL, base64, or data URI. Backend-specific tuning is passed as string values in params.", }, + { + Name: "3d", + Description: "Image-to-3D asset generation (binary glTF / GLB) via TRELLIS.2", + Tags: []string{"3d"}, + Intro: "POST /3d/generations accepts a conditioning image as public URL, base64, or data URI (no text prompt) and returns one .glb asset as a URL under /generated-3d or as b64_json. quality selects the mesh pipeline (auto|coarse|512|1024); background controls solid-background removal (auto|keep|black|white); step, texture_steps, and cfg_scale tune the flow sampling. POST /3d/remesh accepts multipart model, mesh (GLB), and a single detail percentage to return a watertight print-ready GLB; the enclosing offset is derived automatically.", + }, { Name: "face-recognition", Description: "Face verification (1:1), identification (1:N), embedding, and demographic analysis", diff --git a/core/http/endpoints/localai/api_instructions_test.go b/core/http/endpoints/localai/api_instructions_test.go index 43caa9ee1..9648e674a 100644 --- a/core/http/endpoints/localai/api_instructions_test.go +++ b/core/http/endpoints/localai/api_instructions_test.go @@ -39,7 +39,7 @@ var _ = Describe("API Instructions Endpoints", func() { instructions, ok := resp["instructions"].([]any) Expect(ok).To(BeTrue()) - Expect(instructions).To(HaveLen(17)) + Expect(instructions).To(HaveLen(18)) // Verify each instruction has required fields and correct URL format for _, s := range instructions { @@ -79,6 +79,7 @@ var _ = Describe("API Instructions Endpoints", func() { "middleware-admin", "intelligent-routing", "voice-library", + "3d", )) }) }) @@ -123,6 +124,16 @@ var _ = Describe("API Instructions Endpoints", func() { Expect(string(body)).To(ContainSubstring("stream")) }) + It("should advertise the LocalAI 3D generation path", func() { + req := httptest.NewRequest(http.MethodGet, "/api/instructions/3d", nil) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + + body, _ := io.ReadAll(rec.Body) + Expect(string(body)).To(ContainSubstring("POST /3d/generations")) + Expect(string(body)).NotTo(ContainSubstring("/v1/3d/generations")) + }) + It("should return JSON fragment when format=json", func() { req := httptest.NewRequest(http.MethodGet, "/api/instructions/chat-inference?format=json", nil) rec := httptest.NewRecorder() diff --git a/core/http/endpoints/localai/model3d.go b/core/http/endpoints/localai/model3d.go new file mode 100644 index 000000000..fd514c7db --- /dev/null +++ b/core/http/endpoints/localai/model3d.go @@ -0,0 +1,186 @@ +package localai + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "time" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + + "github.com/mudler/xlog" + + model "github.com/mudler/LocalAI/pkg/model" +) + +// Conditioning images are single frames, so a much tighter cap than the +// video-input limit is enough. +const max3DInputBytes = 32 << 20 + +var ( + valid3DQualities = []string{"", "auto", "coarse", "512", "1024"} + valid3DBackgrounds = []string{"", "auto", "keep", "black", "white"} +) + +// Model3DEndpoint +// @Summary Creates a 3D asset (binary glTF / GLB) from a conditioning image. +// @Tags 3d +// @Param request body schema.Model3DRequest true "query params" +// @Success 200 {object} schema.OpenAIResponse "Response" +// @Router /3d/generations [post] +func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { + return func(c echo.Context) error { + input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.Model3DRequest) + if !ok || input.Model == "" { + xlog.Error("3D Endpoint - Invalid Input") + return echo.ErrBadRequest + } + + config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + if !ok || config == nil { + xlog.Error("3D Endpoint - Invalid Config") + return echo.ErrBadRequest + } + + if input.Image == "" { + return echo.NewHTTPError(http.StatusBadRequest, "image is required: 3D generation is image-conditioned") + } + // Reject unknown enum values here rather than surfacing an opaque + // backend error after a model load. + if !slices.Contains(valid3DQualities, input.Quality) { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid quality %q: must be one of auto, coarse, 512, 1024", input.Quality)) + } + if !slices.Contains(valid3DBackgrounds, input.Background) { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid background %q: must be one of auto, keep, black, white", input.Background)) + } + + src, err := stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, input.Image, max3DInputBytes) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid image: %v", err)) + } + defer func() { _ = os.Remove(src) }() + + xlog.Debug("Parameter Config", "config", config) + + if config.Backend == "" { + config.Backend = model.Trellis2CppBackend + } + + step := input.Step + if step == 0 && config.Step != 0 { + step = int32(config.Step) + } + cfgScale := input.CFGScale + if cfgScale == 0 && config.CFGScale != 0 { + cfgScale = config.CFGScale + } + + b64JSON := input.ResponseFormat == "b64_json" + + tempDir := "" + if !b64JSON { + tempDir = filepath.Join(appConfig.GeneratedContentDir, "3d") + if err := os.MkdirAll(tempDir, 0o750); err != nil { + return err + } + } + // Create a temporary file + outputFile, err := os.CreateTemp(tempDir, "b64") + if err != nil { + return err + } + if err := outputFile.Close(); err != nil { + _ = os.Remove(outputFile.Name()) + return err + } + + output := outputFile.Name() + ".glb" + + // Rename the temporary file + err = os.Rename(outputFile.Name(), output) + if err != nil { + _ = os.Remove(outputFile.Name()) + return err + } + preserveOutput := false + defer func() { + if !preserveOutput { + _ = os.Remove(output) + } + }() + + baseURL := middleware.BaseURL(c) + + xlog.Debug("Model3DEndpoint: Calling Model3DGeneration", + "quality", input.Quality, + "background", input.Background, + "cfg_scale", cfgScale, + "step", step, + "texture_steps", input.TextureSteps, + "seed", input.Seed) + + fn, err := backend.Model3DGeneration( + backend.Model3DGenerationOptions{ + Image: src, + Destination: output, + Seed: input.Seed, + Step: step, + CFGScale: cfgScale, + TextureSteps: input.TextureSteps, + Quality: input.Quality, + Background: input.Background, + Params: input.Params, + }, + ml, + *config, + appConfig, + ) + if err != nil { + return mapBackendError(err) + } + if err := fn(); err != nil { + return mapBackendError(err) + } + + item := &schema.Item{} + + if b64JSON { + data, err := os.ReadFile(output) + if err != nil { + return err + } + item.B64JSON = base64.StdEncoding.EncodeToString(data) + } else { + base := filepath.Base(output) + item.URL, err = url.JoinPath(baseURL, "generated-3d", base) + if err != nil { + return err + } + preserveOutput = true + } + + id := uuid.New().String() + created := int(time.Now().Unix()) + resp := &schema.OpenAIResponse{ + ID: id, + Created: created, + Data: []schema.Item{*item}, + } + + jsonResult, _ := json.Marshal(resp) + xlog.Debug("Response", "response", string(jsonResult)) + + return c.JSON(200, resp) + } +} diff --git a/core/http/endpoints/localai/model3d_internal_test.go b/core/http/endpoints/localai/model3d_internal_test.go new file mode 100644 index 000000000..4437a2a3a --- /dev/null +++ b/core/http/endpoints/localai/model3d_internal_test.go @@ -0,0 +1,72 @@ +package localai + +import ( + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" +) + +// The validation branches all return before any model load, so the handler can +// be driven with nil loaders. +var _ = Describe("3D endpoint request validation", func() { + call := func(input *schema.Model3DRequest) error { + appConfig := &config.ApplicationConfig{GeneratedContentDir: GinkgoT().TempDir()} + handler := Model3DEndpoint(nil, nil, appConfig) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/3d/generations", strings.NewReader("{}")) + c := e.NewContext(req, httptest.NewRecorder()) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, input) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Name: "test-3d"}) + return handler(c) + } + + expectBadRequest := func(err error, substr string) { + var httpErr *echo.HTTPError + Expect(err).To(BeAssignableToTypeOf(httpErr)) + httpErr = err.(*echo.HTTPError) + Expect(httpErr.Code).To(Equal(http.StatusBadRequest)) + Expect(httpErr.Message).To(ContainSubstring(substr)) + } + + It("requires a conditioning image", func() { + err := call(&schema.Model3DRequest{BasicModelRequest: schema.BasicModelRequest{Model: "m"}}) + expectBadRequest(err, "image is required") + }) + + It("rejects unknown quality values", func() { + err := call(&schema.Model3DRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "m"}, + Image: "aGk=", + Quality: "2048", + }) + expectBadRequest(err, "invalid quality") + }) + + It("rejects unknown background values", func() { + err := call(&schema.Model3DRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "m"}, + Image: "aGk=", + Background: "transparent", + }) + expectBadRequest(err, "invalid background") + }) + + It("rejects undecodable image payloads", func() { + err := call(&schema.Model3DRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "m"}, + Image: "not%%%base64", + Quality: "512", + Background: "auto", + }) + expectBadRequest(err, "invalid image") + }) +}) diff --git a/core/http/endpoints/localai/model3d_remesh.go b/core/http/endpoints/localai/model3d_remesh.go new file mode 100644 index 000000000..a0a2b004e --- /dev/null +++ b/core/http/endpoints/localai/model3d_remesh.go @@ -0,0 +1,163 @@ +package localai + +import ( + "fmt" + "io" + "math" + "net/http" + "os" + "path/filepath" + "strconv" + + "github.com/labstack/echo/v4" + + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + model "github.com/mudler/LocalAI/pkg/model" +) + +const ( + max3DRemeshBytes = 512 << 20 + defaultRemeshDetailPct = float32(0.5) + minRemeshDetailPct = float32(0.35) + maxRemeshDetailPct = float32(2.5) +) + +func normalizedRemeshDetail(detail float32) (float32, error) { + if detail == 0 { + return defaultRemeshDetailPct, nil + } + if math.IsNaN(float64(detail)) || math.IsInf(float64(detail), 0) || detail < minRemeshDetailPct || detail > maxRemeshDetailPct { + return 0, fmt.Errorf("detail must be between %.2f and %.2f percent", minRemeshDetailPct, maxRemeshDetailPct) + } + return detail, nil +} + +func saveRemeshUpload(c echo.Context, dir string) (string, error) { + header, err := c.FormFile("mesh") + if err != nil { + return "", fmt.Errorf("mesh is required") + } + if header.Size > max3DRemeshBytes { + return "", fmt.Errorf("mesh exceeds the 512 MiB limit") + } + source, err := header.Open() + if err != nil { + return "", fmt.Errorf("opening mesh: %w", err) + } + defer func() { _ = source.Close() }() + + if err := os.MkdirAll(dir, 0o750); err != nil { + return "", err + } + temp, err := os.CreateTemp(dir, "remesh-input-*.glb") + if err != nil { + return "", err + } + path := temp.Name() + defer func() { + if err != nil { + _ = os.Remove(path) + } + }() + + written, copyErr := io.Copy(temp, io.LimitReader(source, max3DRemeshBytes+1)) + closeErr := temp.Close() + if copyErr != nil { + err = fmt.Errorf("saving mesh: %w", copyErr) + return "", err + } + if closeErr != nil { + err = closeErr + return "", err + } + if written == 0 { + err = fmt.Errorf("mesh is empty") + return "", err + } + if written > max3DRemeshBytes { + err = fmt.Errorf("mesh exceeds the 512 MiB limit") + return "", err + } + return path, nil +} + +// Model3DRemeshEndpoint rebuilds an existing generated GLB as a watertight mesh. +// @Summary Applies watertight print remeshing to an existing 3D asset. +// @Tags 3d +// @Accept multipart/form-data +// @Produce model/gltf-binary +// @Param model formData string true "3D model name" +// @Param mesh formData file true "Source GLB" +// @Param detail formData number false "Detail size as percent of the source bounding-box diagonal (0.35–2.5; default 0.5)" +// @Success 200 {file} binary "Remeshed GLB" +// @Router /3d/remesh [post] +func Model3DRemeshEndpoint(ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { + return func(c echo.Context) error { + input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.Model3DRemeshRequest) + if !ok || input.Model == "" { + return echo.NewHTTPError(http.StatusBadRequest, "model is required") + } + modelConfig, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + if !ok || modelConfig == nil { + return echo.ErrBadRequest + } + detail, err := normalizedRemeshDetail(input.Detail) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + + source, err := saveRemeshUpload(c, appConfig.GeneratedContentDir) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + defer func() { _ = os.Remove(source) }() + + outputFile, err := os.CreateTemp(appConfig.GeneratedContentDir, "remeshed-*.glb") + if err != nil { + return err + } + output := outputFile.Name() + if err := outputFile.Close(); err != nil { + _ = os.Remove(output) + return err + } + defer func() { _ = os.Remove(output) }() + + if modelConfig.Backend == "" { + modelConfig.Backend = model.Trellis2CppBackend + } + fn, err := backend.Model3DGeneration( + backend.Model3DGenerationOptions{ + Image: source, + Destination: output, + Params: map[string]string{ + "operation": "print_remesh", + "alpha_ratio": strconv.FormatFloat(float64(detail/100), 'g', -1, 32), + "detail_percent": strconv.FormatFloat(float64(detail), 'g', -1, 32), + "texture_size": "2048", + }, + }, + ml, + *modelConfig, + appConfig, + ) + if err != nil { + return mapBackendError(err) + } + if err := fn(); err != nil { + return mapBackendError(err) + } + + file, err := os.Open(filepath.Clean(output)) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + c.Response().Header().Set(echo.HeaderContentDisposition, `attachment; filename="remeshed.glb"`) + c.Response().Header().Set(echo.HeaderCacheControl, "no-store") + return c.Stream(http.StatusOK, "model/gltf-binary", file) + } +} diff --git a/core/http/endpoints/localai/model3d_remesh_internal_test.go b/core/http/endpoints/localai/model3d_remesh_internal_test.go new file mode 100644 index 000000000..d3e4108e2 --- /dev/null +++ b/core/http/endpoints/localai/model3d_remesh_internal_test.go @@ -0,0 +1,68 @@ +package localai + +import ( + "bytes" + "math" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("3D print remeshing request handling", func() { + DescribeTable("normalizes the demo detail range", + func(input, expected float32, valid bool) { + value, err := normalizedRemeshDetail(input) + if valid { + Expect(err).NotTo(HaveOccurred()) + Expect(value).To(BeNumerically("~", expected, 1e-6)) + } else { + Expect(err).To(HaveOccurred()) + } + }, + Entry("default", float32(0), float32(0.5), true), + Entry("fine endpoint", float32(0.35), float32(0.35), true), + Entry("coarse endpoint", float32(2.5), float32(2.5), true), + Entry("too fine", float32(0.1), float32(0), false), + Entry("too coarse", float32(3), float32(0), false), + Entry("not a number", float32(math.NaN()), float32(0), false), + ) + + It("streams the multipart GLB to a bounded temporary file", func() { + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + Expect(writer.WriteField("model", "trellis-test-model")).To(Succeed()) + Expect(writer.WriteField("detail", "0.35")).To(Succeed()) + part, err := writer.CreateFormFile("mesh", "source.glb") + Expect(err).NotTo(HaveOccurred()) + _, err = part.Write([]byte("glTF-test")) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/3d/remesh", body) + req.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + ctx := e.NewContext(req, httptest.NewRecorder()) + input := new(schema.Model3DRemeshRequest) + Expect(ctx.Bind(input)).To(Succeed()) + Expect(input.Model).To(Equal("trellis-test-model")) + Expect(input.Detail).To(BeNumerically("~", 0.35, 1e-6)) + path, err := saveRemeshUpload(ctx, GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.Remove(path) }() + Expect(os.ReadFile(path)).To(Equal([]byte("glTF-test"))) + }) + + It("requires a mesh part", func() { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/3d/remesh", bytes.NewReader(nil)) + ctx := e.NewContext(req, httptest.NewRecorder()) + _, err := saveRemeshUpload(ctx, GinkgoT().TempDir()) + Expect(err).To(MatchError("mesh is required")) + }) +}) diff --git a/core/http/react-ui/e2e/page-render-smoke.spec.js b/core/http/react-ui/e2e/page-render-smoke.spec.js index 27f006b98..3f24752bf 100644 --- a/core/http/react-ui/e2e/page-render-smoke.spec.js +++ b/core/http/react-ui/e2e/page-render-smoke.spec.js @@ -14,6 +14,7 @@ import { test, expect } from './coverage-fixtures.js' // not bounce to a gate redirect (/login or back to /app home). const PAGES = [ ['/app/talk', 'Talk'], + ['/app/3d', '3D Generation'], ['/app/usage', 'Usage'], ['/app/account', 'Account'], ['/app/studio', 'Studio'], diff --git a/core/http/react-ui/e2e/threed-gen.spec.js b/core/http/react-ui/e2e/threed-gen.spec.js new file mode 100644 index 000000000..b10270e19 --- /dev/null +++ b/core/http/react-ui/e2e/threed-gen.spec.js @@ -0,0 +1,319 @@ +import { test, expect } from './coverage-fixtures.js' + +// 3D generation page: mock the capabilities + generation endpoints, feed a +// real (tiny) Form-B GLB through the parser/viewer, and exercise the +// IndexedDB-backed history. All assertions are DOM/text — never pixels — so +// the suite passes with or without working WebGL2 in headless Chromium. + +function mockCapabilities(page) { + return page.route('**/api/models/capabilities', (route) => { + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ data: [{ id: 'trellis-test-model', capabilities: ['FLAG_3D'] }] }), + }) + }) +} + +// A valid one-triangle GLB in trellis2cpp's vertex-PBR form: POSITION/NORMAL +// f32, COLOR_0 u16-normalized VEC4, _METALLIC_ROUGHNESS u8-normalized VEC2, +// u32 indices — the layout mesh_export.cpp's write_vertex_glb emits. +function buildTinyGlb() { + const positions = new Float32Array([0, 0, 0, 1, 0, 0, 0, 1, 0]) + const normals = new Float32Array([0, 0, 1, 0, 0, 1, 0, 0, 1]) + const colors = new Uint16Array([ + 65535, 0, 0, 65535, + 0, 65535, 0, 65535, + 0, 0, 65535, 65535, + ]) + const metalRough = new Uint8Array([0, 153, 0, 153, 0, 153]) + const indices = new Uint32Array([0, 1, 2]) + + const views = [] + let binLength = 0 + const addView = (typed) => { + const byteOffset = binLength + views.push({ buffer: 0, byteOffset, byteLength: typed.byteLength, target: 34962 }) + binLength += typed.byteLength + binLength += (4 - (binLength % 4)) % 4 + return views.length - 1 + } + addView(positions); addView(normals); addView(colors); addView(metalRough) + const idxView = addView(indices) + views[idxView].target = 34963 + + const json = { + asset: { version: '2.0', generator: 'threed-gen.spec' }, + scene: 0, + scenes: [{ nodes: [0] }], + nodes: [{ mesh: 0 }], + meshes: [{ primitives: [{ attributes: { POSITION: 0, NORMAL: 1, COLOR_0: 2, _METALLIC_ROUGHNESS: 3 }, indices: 4, material: 0 }] }], + materials: [{ pbrMetallicRoughness: { baseColorFactor: [1, 1, 1, 1], metallicFactor: 0, roughnessFactor: 0.6 }, doubleSided: true }], + accessors: [ + { bufferView: 0, componentType: 5126, count: 3, type: 'VEC3', min: [0, 0, 0], max: [1, 1, 0] }, + { bufferView: 1, componentType: 5126, count: 3, type: 'VEC3' }, + { bufferView: 2, componentType: 5123, normalized: true, count: 3, type: 'VEC4' }, + { bufferView: 3, componentType: 5121, normalized: true, count: 3, type: 'VEC2' }, + { bufferView: 4, componentType: 5125, count: 3, type: 'SCALAR' }, + ], + bufferViews: views, + buffers: [{ byteLength: binLength }], + } + + const bin = Buffer.alloc(binLength) + const parts = [positions, normals, colors, metalRough, indices] + for (let i = 0; i < parts.length; i++) { + Buffer.from(parts[i].buffer, parts[i].byteOffset, parts[i].byteLength).copy(bin, views[i].byteOffset) + } + + let jsonText = JSON.stringify(json) + while (jsonText.length % 4 !== 0) jsonText += ' ' + const jsonBuf = Buffer.from(jsonText) + + const total = 12 + 8 + jsonBuf.length + 8 + bin.length + const glb = Buffer.alloc(total) + glb.writeUInt32LE(0x46546c67, 0) // magic 'glTF' + glb.writeUInt32LE(2, 4) + glb.writeUInt32LE(total, 8) + glb.writeUInt32LE(jsonBuf.length, 12) + glb.writeUInt32LE(0x4e4f534a, 16) // 'JSON' + jsonBuf.copy(glb, 20) + const binHeader = 20 + jsonBuf.length + glb.writeUInt32LE(bin.length, binHeader) + glb.writeUInt32LE(0x004e4942, binHeader + 4) // 'BIN\0' + bin.copy(glb, binHeader + 8) + return glb +} + +// 1x1 transparent PNG for the conditioning-image upload. +const TINY_PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', + 'base64', +) + +function mockGeneration(page, onRequest) { + return page.route('**/3d/generations', (route) => { + if (route.request().method() !== 'POST') return route.continue() + onRequest?.(route.request().postDataJSON()) + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + created: Math.floor(Date.now() / 1000), + data: [{ url: '/generated-3d/test.glb' }], + }), + }) + }) +} + +function mockGlbDownload(page) { + return page.route('**/generated-3d/test.glb', (route) => { + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'model/gltf-binary' }, + body: buildTinyGlb(), + }) + }) +} + +async function generateOnce(page) { + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'trellis-test-model' })).toBeVisible({ timeout: 10_000 }) + await page.locator('#threed-image-file').setInputFiles({ + name: 'input.png', + mimeType: 'image/png', + buffer: TINY_PNG, + }) + await page.locator('button[type="submit"]').click() +} + +test.describe('3D generation', () => { + test.beforeEach(async ({ page }) => { + await mockCapabilities(page) + await mockGlbDownload(page) + }) + + test('generates, shows mesh stats, and offers a GLB download', async ({ page }) => { + let requestBody = null + await mockGeneration(page, (body) => { requestBody = body }) + + await generateOnce(page) + + // Stats render from the parsed GLB even without working GL. + await expect(page.getByTestId('glb-stats')).toContainText('3', { timeout: 15_000 }) + await expect(page.getByTestId('glb-stats')).toContainText('1') + await expect(page.getByTestId('glb-stats')).toContainText('PBR') + + const download = page.getByTestId('glb-download') + await expect(download).toBeVisible() + await expect(download).toHaveAttribute('href', /^blob:/) + await expect(download).toHaveAttribute('download', /\.glb$/) + + expect(requestBody.model).toBe('trellis-test-model') + expect(requestBody.image).toBeTruthy() + expect(requestBody.quality).toBe('auto') + expect(requestBody.background).toBe('auto') + expect(requestBody.response_format).toBe('url') + }) + + test('advanced settings map to step/texture_steps/cfg_scale/seed', async ({ page }) => { + let requestBody = null + await mockGeneration(page, (body) => { requestBody = body }) + + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'trellis-test-model' })).toBeVisible({ timeout: 10_000 }) + await page.locator('#threed-image-file').setInputFiles({ name: 'input.png', mimeType: 'image/png', buffer: TINY_PNG }) + + await page.locator('select').first().selectOption('512') + await page.getByRole('button', { name: /Advanced Settings/ }).click() + const advanced = page.locator('#threed-advanced-options') + await advanced.locator('input').nth(0).fill('20') // steps + await advanced.locator('input').nth(1).fill('8') // texture steps + await advanced.locator('input').nth(2).fill('5.5') // guidance + await advanced.locator('input').nth(3).fill('42') // seed + await page.locator('button[type="submit"]').click() + + await expect(page.getByTestId('glb-download')).toBeVisible({ timeout: 15_000 }) + expect(requestBody.quality).toBe('512') + expect(requestBody.step).toBe(20) + expect(requestBody.texture_steps).toBe(8) + expect(requestBody.cfg_scale).toBe(5.5) + expect(requestBody.seed).toBe(42) + }) + + test('applies a single-detail watertight remesh and previews it before download', async ({ page }) => { + await mockGeneration(page) + let remeshRequest = null + await page.route('**/3d/remesh', (route) => { + remeshRequest = route.request() + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'model/gltf-binary' }, + body: buildTinyGlb(), + }) + }) + + await generateOnce(page) + await expect(page.getByTestId('glb-remesh')).toBeVisible({ timeout: 15_000 }) + await page.getByLabel('Remesh detail').fill('100') + await page.getByTestId('glb-remesh').click() + + await expect(page.getByTestId('glb-remesh')).toContainText('Show original') + await expect(page.getByTestId('glb-download')).toHaveAttribute('download', /-remeshed\.glb$/) + expect(remeshRequest).not.toBeNull() + expect(remeshRequest.method()).toBe('POST') + expect(remeshRequest.headers()['content-type']).toContain('multipart/form-data') + const multipart = remeshRequest.postDataBuffer().toString() + expect(multipart).toContain('trellis-test-model') + expect(multipart).toContain('0.35') + + await page.getByTestId('glb-remesh').click() + await expect(page.getByTestId('glb-remesh')).toContainText('Apply remeshing') + await expect(page.getByTestId('glb-download')).not.toHaveAttribute('download', /-remeshed\.glb$/) + }) + + test('history entry persists across navigation and reloads into the viewer', async ({ page }) => { + await mockGeneration(page) + await generateOnce(page) + await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 15_000 }) + + // IndexedDB persists within the browser context — navigate away and back. + await page.goto('/app') + await page.goto('/app/3d') + await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 15_000 }) + + // Selecting the entry loads the stored Blob back into the viewer. + await page.getByTestId('media-history-item').click() + await expect(page.getByTestId('glb-stats')).toBeVisible({ timeout: 15_000 }) + await expect(page.getByTestId('glb-download')).toHaveAttribute('href', /^blob:/) + }) + + test('deleting a history entry removes it', async ({ page }) => { + await mockGeneration(page) + await generateOnce(page) + await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 15_000 }) + + await page.getByTestId('media-history-delete').click() + await expect(page.getByTestId('media-history-item')).toHaveCount(0) + }) + + test('API errors surface through the trace link error box', async ({ page }) => { + await page.route('**/3d/generations', (route) => { + route.fulfill({ + status: 500, + contentType: 'application/json', + body: JSON.stringify({ error: { message: 'trellis2 pipeline load: missing files' } }), + }) + }) + + await generateOnce(page) + await expect(page.locator('.media-result')).toContainText(/missing files|error/i, { timeout: 15_000 }) + }) + + test('rejects oversized conditioning images before making an API request', async ({ page }) => { + let requested = false + await mockGeneration(page, () => { requested = true }) + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'trellis-test-model' })).toBeVisible({ timeout: 10_000 }) + + await page.locator('#threed-image-file').setInputFiles({ + name: 'oversized.png', + mimeType: 'image/png', + buffer: Buffer.alloc(32 * 1024 * 1024 + 1), + }) + + await expect(page.locator('.toast')).toContainText('32 MiB limit') + expect(requested).toBe(false) + }) + + test('hides and guards 3D generation when the feature is disabled', async ({ page }) => { + await page.route('**/api/auth/status', (route) => { + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + authEnabled: true, + staticApiKeyRequired: false, + user: { id: 'restricted-user', role: 'user', permissions: { '3d': false } }, + }), + }) + }) + + await page.goto('/app/studio?tab=threed') + await expect(page.getByRole('button', { name: '3D', exact: true })).toHaveCount(0) + await expect(page.locator('.studio-tab', { hasText: 'Images' })).toHaveClass(/studio-tab-active/) + + await page.goto('/app/3d') + await expect(page).toHaveURL(/\/app\/?$/) + }) + + test.describe('touch input', () => { + test.use({ hasTouch: true }) + + test('accepts two-finger touch gestures in the GLB viewer', async ({ page }) => { + await mockGeneration(page) + await generateOnce(page) + const canvas = page.getByTestId('glb-canvas') + await expect(canvas).toBeVisible({ timeout: 15_000 }) + await canvas.scrollIntoViewIfNeeded() + const box = await canvas.boundingBox() + expect(box).not.toBeNull() + + const client = await page.context().newCDPSession(page) + await client.send('Input.dispatchTouchEvent', { + type: 'touchStart', + touchPoints: [ + { id: 1, x: box.x + box.width * 0.35, y: box.y + box.height * 0.5 }, + { id: 2, x: box.x + box.width * 0.65, y: box.y + box.height * 0.5 }, + ], + }) + await client.send('Input.dispatchTouchEvent', { + type: 'touchMove', + touchPoints: [ + { id: 1, x: box.x + box.width * 0.3, y: box.y + box.height * 0.45 }, + { id: 2, x: box.x + box.width * 0.7, y: box.y + box.height * 0.55 }, + ], + }) + await client.send('Input.dispatchTouchEvent', { type: 'touchEnd', touchPoints: [] }) + + await expect(page.getByRole('button', { name: 'Auto-rotate' })).toHaveClass(/btn-secondary/) + }) + }) +}) diff --git a/core/http/react-ui/public/locales/en/media.json b/core/http/react-ui/public/locales/en/media.json index 7beefe10b..d3e9484f2 100644 --- a/core/http/react-ui/public/locales/en/media.json +++ b/core/http/react-ui/public/locales/en/media.json @@ -5,7 +5,8 @@ "video": "Video", "tts": "TTS", "sound": "Sound", - "transform": "Transform" + "transform": "Transform", + "threed": "3D" } }, "image": { @@ -70,6 +71,59 @@ "noResults": "No video generated" } }, + "threed": { + "title": "3D Generation", + "labels": { + "model": "Model", + "image": "Input image", + "quality": "Quality", + "quality_auto": "Auto (best available)", + "quality_coarse": "Coarse preview (fast)", + "quality_512": "512³ fine", + "quality_1024": "1024³ cascade (slow)", + "background": "Background", + "background_auto": "Auto-remove solid background", + "background_keep": "Keep original", + "background_black": "Remove black", + "background_white": "Remove white", + "advanced": "Advanced Settings", + "steps": "Shape steps", + "textureSteps": "Material steps", + "guidance": "Guidance", + "seed": "Seed", + "seedPlaceholder": "Random" + }, + "actions": { + "generate": "Generate", + "generating": "Generating...", + "remesh": "Apply remeshing", + "remeshing": "Applying remeshing...", + "showOriginal": "Show original", + "download": "Download GLB" + }, + "remesh": { + "title": "Watertight print remesh", + "detail": "Remesh detail", + "coarser": "Coarser · faster", + "finer": "Finer · slower", + "hint": "Detail controls the smallest preserved features. The enclosing offset follows it automatically.", + "ready": "Previewing the watertight remeshed model. This is the version that will be downloaded." + }, + "viewer": { + "wireframe": "Wireframe", + "autoRotate": "Auto-rotate", + "noWebgl": "WebGL2 is not available in this browser — download the GLB to view it elsewhere.", + "contextLost": "The 3D view lost the GPU context — reload the page to restore it.", + "stats": "{{verts}} vertices · {{tris}} triangles", + "hint": "drag: rotate · pinch/wheel: zoom · two-finger/right-drag: pan · double-click: reset" + }, + "empty": "Generated 3D model will appear here", + "toasts": { + "noImage": "Please provide an input image", + "noModel": "Please select a model", + "noResults": "No model generated" + } + }, "tts": { "title": "Text to Speech", "labels": { diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index c9a7ab4e9..2e2f21e57 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -29,6 +29,7 @@ "llm": "Chat", "image": "Image", "video": "Video", + "threed": "3D", "multimodal": "Multimodal", "vision": "Vision", "tts": "TTS", diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index 68a7e4d5a..b27ec01a7 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -5166,6 +5166,45 @@ button.collapsible-header:focus-visible { border-radius: var(--radius-md); } +.threed-remesh-controls { + display: flex; + flex-direction: column; + gap: var(--spacing-xs); + padding: var(--spacing-md); + background: var(--color-surface-raised); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-md); +} + +.threed-remesh-heading, +.threed-remesh-scale { + display: flex; + justify-content: space-between; + gap: var(--spacing-md); +} + +.threed-remesh-heading { + color: var(--color-text-primary); + font-size: var(--text-sm); + font-weight: var(--font-weight-medium); +} + +.threed-remesh-heading output, +.threed-remesh-scale, +.threed-remesh-ready { + color: var(--color-text-muted); + font-size: var(--text-xs); +} + +.threed-remesh-controls input[type="range"] { + width: 100%; +} + +.threed-remesh-ready { + margin: var(--spacing-xs) 0 0; + color: var(--color-success); +} + .media-result-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); diff --git a/core/http/react-ui/src/components/GlbViewer.jsx b/core/http/react-ui/src/components/GlbViewer.jsx new file mode 100644 index 000000000..ead0c8a32 --- /dev/null +++ b/core/http/react-ui/src/components/GlbViewer.jsx @@ -0,0 +1,636 @@ +import { useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { parseGlb } from '../utils/glb' + +/* ── WebGL2 GLB viewer ────────────────────────────────────────────────────── + * Ported from the trellis2cpp demo server's hand-rolled viewer + * (localai-org/trellis2cpp server/web/index.html): quaternion trackball, + * metallic-roughness PBR with a procedural environment, ACES tonemapping, + * hidden-line wireframe. Kept dependency-free — the renderer is tuned for + * the dense unoriented dual-grid meshes TRELLIS.2 produces, which generic + * GLTF viewers shade poorly. + * + * Deltas from the demo: input is a parsed GLB (see utils/glb.js) instead of + * the demo's T2MESH stream; COLOR_0 arrives LINEAR (no gamma decode — the + * demo's pow(2.2) would double-darken it); vertex attributes upload as + * normalized integers straight from the GLB buffers; an optional UV-texture + * path covers the atlas-baked form; the base orientation drops the demo's + * Z-up -> Y-up turn because baked GLBs are already Y-up. + */ + +const VS = `#version 300 es +layout(location=0) in vec3 pos; +layout(location=1) in vec3 nrm; +layout(location=2) in vec4 baseColor; +layout(location=3) in vec2 metalRough; +layout(location=4) in vec2 uv; +uniform mat4 mvp, viewModel; +out vec3 vN; out vec3 vV; out vec4 vCol; out vec2 vMR; out vec2 vUV; +void main() { + gl_Position = mvp * vec4(pos, 1.0); + vec4 vp = viewModel * vec4(pos, 1.0); + vV = -vp.xyz; // view-space: to-camera (camera at origin) + vN = mat3(viewModel) * nrm; // view-space normal (orbits with camera) + vCol = baseColor; + vMR = metalRough; + vUV = uv; +}` + +// Lightweight metallic-roughness PBR in view space. The dual-grid mesh has +// unoriented winding (faithful to TRELLIS.2), so every normal is faced toward +// the camera. No IBL — a procedural environment + one studio key light; +// ACES tonemapping at the end. +const FS = `#version 300 es +precision highp float; +in vec3 vN; in vec3 vV; in vec4 vCol; in vec2 vMR; in vec2 vUV; +uniform int wire; +uniform int colorMode; // 0 = untextured grey, 1 = vertex PBR (linear COLOR_0), 2 = UV textures +uniform sampler2D baseColorTex; +uniform sampler2D mrTex; +uniform float uMetallicFactor, uRoughnessFactor; +out vec4 frag; +// procedural environment radiance in a direction (view space; +y = up) +vec3 envColor(vec3 d) { + vec3 sky = mix(vec3(0.32, 0.40, 0.55), vec3(0.72, 0.80, 0.98), clamp(d.y, 0.0, 1.0)); + vec3 gnd = vec3(0.14, 0.13, 0.12); + return mix(gnd, sky, smoothstep(-0.30, 0.12, d.y)); +} +void main() { + if (wire == 1) { frag = vec4(0.30, 0.64, 1.00, 1.0); return; } + vec3 N = normalize(vN), V = normalize(vV); + if (dot(N, V) < 0.0) N = -N; // face the camera (mesh is unoriented) + vec3 base = vec3(0.62, 0.66, 0.72); + float metal = 0.0, rough = 0.5, opacity = 1.0; + if (colorMode == 1) { + // COLOR_0 is stored linear in the GLB — use it directly. + base = clamp(vCol.rgb, 0.0, 1.0); + metal = clamp(vMR.x, 0.0, 1.0); + rough = clamp(vMR.y, 0.06, 1.0); + opacity = clamp(vCol.a, 0.0, 1.0); + } else if (colorMode == 2) { + vec4 bc = texture(baseColorTex, vUV); + base = pow(clamp(bc.rgb, 0.0, 1.0), vec3(2.2)); // baseColorTexture is sRGB + opacity = bc.a; + vec3 mr = texture(mrTex, vUV).rgb; // G=roughness, B=metallic + metal = clamp(mr.b * uMetallicFactor, 0.0, 1.0); + rough = clamp(mr.g * uRoughnessFactor, 0.06, 1.0); + } + if (opacity < 0.01) discard; + float nv = max(dot(N, V), 1e-3); + + // Schlick Fresnel (grazing reflectance rises to 1 - roughness for metals). + vec3 F0 = mix(vec3(0.04), base, metal); + vec3 F = F0 + (max(vec3(1.0 - rough), F0) - F0) * pow(1.0 - nv, 5.0); + + // diffuse: hemispheric environment irradiance (metals have no diffuse) + vec3 irr = envColor(N) * 0.55 + vec3(0.12); + vec3 diffuse = base * (1.0 - metal) * irr; + + // specular: environment reflection, blurred toward a flat tint by roughness + vec3 refl = mix(envColor(reflect(-V, N)), vec3(0.34, 0.37, 0.44), rough * rough); + vec3 specular = refl * F; + + // one crisp studio key light for a lively highlight + vec3 L = normalize(vec3(0.45, 0.70, 0.55)), H = normalize(L + V); + float shin = mix(8.0, 260.0, pow(1.0 - rough, 2.0)); + float sp = pow(max(dot(N, H), 0.0), shin) * (shin + 2.0) / 6.2831853; + vec3 kc = vec3(1.00, 0.96, 0.88); + float ndl = max(dot(N, L), 0.0); + specular += kc * sp * F * ndl; + diffuse += base * (1.0 - metal) * kc * ndl * 0.28; + + vec3 color = diffuse + specular; + color = (color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14); // ACES + frag = vec4(pow(clamp(color, 0.0, 1.0), vec3(1.0/2.2)), opacity); +}` + +/* trackball orientation (quaternion): each drag composes a small rotation + * about the screen axes onto the current orientation — no gimbal lock. */ +const Q = { + axisAngle(x, y, z, a) { const h = a * 0.5, s = Math.sin(h); return [x * s, y * s, z * s, Math.cos(h)] }, + mul(a, b) { // Hamilton product a·b (apply b, then a) + return [ + a[3] * b[0] + a[0] * b[3] + a[1] * b[2] - a[2] * b[1], + a[3] * b[1] - a[0] * b[2] + a[1] * b[3] + a[2] * b[0], + a[3] * b[2] + a[0] * b[1] - a[1] * b[0] + a[2] * b[3], + a[3] * b[3] - a[0] * b[0] - a[1] * b[1] - a[2] * b[2], + ] + }, + norm(q) { const n = Math.hypot(q[0], q[1], q[2], q[3]) || 1; return [q[0] / n, q[1] / n, q[2] / n, q[3] / n] }, + toMat4(q) { // column-major rotation matrix + const [x, y, z, w] = q + const xx = x * x, yy = y * y, zz = z * z, xy = x * y, xz = x * z, yz = y * z, wx = w * x, wy = w * y, wz = w * z + return new Float32Array([ + 1 - 2 * (yy + zz), 2 * (xy + wz), 2 * (xz - wy), 0, + 2 * (xy - wz), 1 - 2 * (xx + zz), 2 * (yz + wx), 0, + 2 * (xz + wy), 2 * (yz - wx), 1 - 2 * (xx + yy), 0, + 0, 0, 0, 1, + ]) + }, +} + +// GLBs are already Y-up (the baker swaps axes on export), so unlike the demo +// there is no Z-up correction here — just a gentle 3/4 default view. +const QBASE = Q.norm(Q.mul(Q.axisAngle(1, 0, 0, -0.30), Q.axisAngle(0, 1, 0, 0.55))) + +/* minimal mat4 helpers (column-major) */ +const M = { + mul(a, b) { + const o = new Float32Array(16) + for (let c = 0; c < 4; ++c) for (let r = 0; r < 4; ++r) + o[c * 4 + r] = a[r] * b[c * 4] + a[4 + r] * b[c * 4 + 1] + a[8 + r] * b[c * 4 + 2] + a[12 + r] * b[c * 4 + 3] + return o + }, + persp(fov, asp, near, far) { + const f = 1 / Math.tan(fov / 2), o = new Float32Array(16) + o[0] = f / asp; o[5] = f + o[10] = (far + near) / (near - far); o[11] = -1 + o[14] = 2 * far * near / (near - far) + return o + }, + trans(x, y, z) { + const o = new Float32Array(16) + o[0] = o[5] = o[10] = o[15] = 1 + o[12] = x; o[13] = y; o[14] = z + return o + }, + scale(s) { + const o = new Float32Array(16) + o[0] = o[5] = o[10] = s; o[15] = 1 + return o + }, +} + +function glType(gl, array) { + if (array instanceof Uint8Array) return gl.UNSIGNED_BYTE + if (array instanceof Uint16Array) return gl.UNSIGNED_SHORT + return gl.FLOAT +} + +export function createGlbViewer(canvas, { onContextLost } = {}) { + const gl = canvas.getContext('webgl2', { antialias: true }) + if (!gl) return null + + // If the GPU context is lost (driver reset / out of memory), preventDefault + // keeps it recoverable and the page shows a notice instead of a dead canvas. + const contextLost = (e) => { + e.preventDefault() + if (onContextLost) onContextLost() + } + canvas.addEventListener('webglcontextlost', contextLost, false) + + function shader(type, src) { + const s = gl.createShader(type) + gl.shaderSource(s, src); gl.compileShader(s) + if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s)) + return s + } + const prog = gl.createProgram() + gl.attachShader(prog, shader(gl.VERTEX_SHADER, VS)) + gl.attachShader(prog, shader(gl.FRAGMENT_SHADER, FS)) + gl.linkProgram(prog) + if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) throw new Error(gl.getProgramInfoLog(prog)) + const uMVP = gl.getUniformLocation(prog, 'mvp') + const uViewModel = gl.getUniformLocation(prog, 'viewModel') + const uWire = gl.getUniformLocation(prog, 'wire') + const uColorMode = gl.getUniformLocation(prog, 'colorMode') + const uBaseColorTex = gl.getUniformLocation(prog, 'baseColorTex') + const uMrTex = gl.getUniformLocation(prog, 'mrTex') + const uMetallicFactor = gl.getUniformLocation(prog, 'uMetallicFactor') + const uRoughnessFactor = gl.getUniformLocation(prog, 'uRoughnessFactor') + + const vao = gl.createVertexArray() + const vbo = gl.createBuffer(), nbo = gl.createBuffer(), cbo = gl.createBuffer() + const mbo = gl.createBuffer(), ubo = gl.createBuffer(), ibo = gl.createBuffer() + const wireIbo = gl.createBuffer() + let nIndices = 0, nWire = 0 + let colorMode = 0 + let baseColorTexture = null, mrTexture = null + let metallicFactor = 1, roughnessFactor = 1 + // fit-to-view: model = rotation * scale * translate(-center) keeps the demo's + // camera constants valid for any GLB extent (trellis meshes are ~unit cube). + let center = [0, 0, 0], fitScale = 1 + + let rot = QBASE.slice(), dist = 1.8, panX = 0, panY = 0 + let wire = false, spin = true + let disposed = false + + function makeTexture(bitmap) { + const tex = gl.createTexture() + gl.bindTexture(gl.TEXTURE_2D, tex) + // Matches the baker's sampler: linear filtering, clamp, no mipmaps. + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, bitmap) + gl.bindTexture(gl.TEXTURE_2D, null) + return tex + } + + function dropTextures() { + if (baseColorTexture) { gl.deleteTexture(baseColorTexture); baseColorTexture = null } + if (mrTexture) { gl.deleteTexture(mrTexture); mrTexture = null } + } + + async function setMesh(mesh) { + gl.bindVertexArray(vao) + gl.bindBuffer(gl.ARRAY_BUFFER, vbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.positions, gl.STATIC_DRAW) + gl.enableVertexAttribArray(0) + gl.vertexAttribPointer(0, 3, gl.FLOAT, false, 0, 0) + if (mesh.normals) { + gl.bindBuffer(gl.ARRAY_BUFFER, nbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.normals, gl.STATIC_DRAW) + gl.enableVertexAttribArray(1) + gl.vertexAttribPointer(1, 3, gl.FLOAT, false, 0, 0) + } else { + gl.disableVertexAttribArray(1) + gl.vertexAttrib3f(1, 0, 1, 0) + } + + dropTextures() + colorMode = 0 + if (mesh.color0) { + // Upload COLOR_0 / _METALLIC_ROUGHNESS as normalized integers straight + // from the GLB buffers — no CPU conversion of multi-million-vertex data. + gl.bindBuffer(gl.ARRAY_BUFFER, cbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.color0.array, gl.STATIC_DRAW) + gl.enableVertexAttribArray(2) + gl.vertexAttribPointer(2, mesh.color0.size, glType(gl, mesh.color0.array), mesh.color0.normalized, 0, 0) + if (mesh.metalRough) { + gl.bindBuffer(gl.ARRAY_BUFFER, mbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.metalRough.array, gl.STATIC_DRAW) + gl.enableVertexAttribArray(3) + gl.vertexAttribPointer(3, mesh.metalRough.size, glType(gl, mesh.metalRough.array), mesh.metalRough.normalized, 0, 0) + } else { + gl.disableVertexAttribArray(3) + gl.vertexAttrib2f(3, 0, 0.6) + } + gl.disableVertexAttribArray(4) + colorMode = 1 + } else if (mesh.uv && mesh.baseColorPng) { + gl.bindBuffer(gl.ARRAY_BUFFER, ubo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.uv, gl.STATIC_DRAW) + gl.enableVertexAttribArray(4) + gl.vertexAttribPointer(4, 2, gl.FLOAT, false, 0, 0) + gl.disableVertexAttribArray(2) + gl.disableVertexAttribArray(3) + const bitmaps = await Promise.all([ + createImageBitmap(new Blob([mesh.baseColorPng], { type: 'image/png' })), + mesh.metalRoughPng ? createImageBitmap(new Blob([mesh.metalRoughPng], { type: 'image/png' })) : null, + ]) + if (disposed) return + gl.bindVertexArray(vao) + baseColorTexture = makeTexture(bitmaps[0]) + mrTexture = bitmaps[1] ? makeTexture(bitmaps[1]) : makeTexture(bitmaps[0]) + metallicFactor = mesh.material.metallicFactor + roughnessFactor = mesh.material.roughnessFactor + colorMode = 2 + } else { + gl.disableVertexAttribArray(2) + gl.disableVertexAttribArray(3) + gl.disableVertexAttribArray(4) + } + + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo) + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, mesh.indices, gl.STATIC_DRAW) + nIndices = mesh.indices.length + // wireframe index buffer: 3 edges per triangle, bounded to WIRE_BUDGET — + // a full wireframe of a multi-million-triangle mesh would OOM the GPU and + // lose the context (sub-pixel lines also fill in as a solid mass). + const WIRE_BUDGET = 12_000_000 // ~6M segments, ~48 MB + const nTri = nIndices / 3 + const wireStride = Math.max(1, Math.ceil(nTri * 6 / WIRE_BUDGET)) + const wireIdx = new Uint32Array(Math.ceil(nTri / wireStride) * 6) + let o = 0 + const idx = mesh.indices + for (let t = 0; t < nIndices; t += 3 * wireStride) { + wireIdx[o++] = idx[t]; wireIdx[o++] = idx[t + 1] + wireIdx[o++] = idx[t + 1]; wireIdx[o++] = idx[t + 2] + wireIdx[o++] = idx[t + 2]; wireIdx[o++] = idx[t] + } + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, wireIbo) + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, wireIdx.subarray(0, o), gl.STATIC_DRAW) + nWire = o + gl.bindVertexArray(null) + + center = [ + (mesh.bboxMin[0] + mesh.bboxMax[0]) / 2, + (mesh.bboxMin[1] + mesh.bboxMax[1]) / 2, + (mesh.bboxMin[2] + mesh.bboxMax[2]) / 2, + ] + const radius = Math.hypot( + mesh.bboxMax[0] - mesh.bboxMin[0], + mesh.bboxMax[1] - mesh.bboxMin[1], + mesh.bboxMax[2] - mesh.bboxMin[2], + ) / 2 || 1 + fitScale = 0.866 / radius + resetView() + } + + function clear() { + nIndices = 0 + nWire = 0 + dropTextures() + } + + function resetView() { + rot = QBASE.slice(); dist = 1.8; panX = panY = 0 + } + + /* input */ + let dragging = false, panning = false, lx = 0, ly = 0 + let pinchDistance = 0, pinchX = 0, pinchY = 0 + const pointers = new Map() + const pointerPair = () => Array.from(pointers.values()).slice(0, 2) + const beginPinch = () => { + const [a, b] = pointerPair() + if (!a || !b) return + pinchDistance = Math.hypot(b.x - a.x, b.y - a.y) + pinchX = (a.x + b.x) / 2 + pinchY = (a.y + b.y) / 2 + } + const stopSpin = () => { + spin = false + if (onSpinChange) onSpinChange(false) + } + const onPointerDown = (e) => { + e.preventDefault() + canvas.setPointerCapture(e.pointerId) + pointers.set(e.pointerId, { x: e.clientX, y: e.clientY }) + if (pointers.size === 1) { + dragging = true + panning = e.button === 2 || e.shiftKey + lx = e.clientX; ly = e.clientY + } else { + dragging = false + beginPinch() + stopSpin() + } + } + const onPointerUp = (e) => { + pointers.delete(e.pointerId) + if (canvas.hasPointerCapture(e.pointerId)) canvas.releasePointerCapture(e.pointerId) + pinchDistance = 0 + if (pointers.size === 1) { + const remaining = pointers.values().next().value + dragging = true + panning = false + lx = remaining.x; ly = remaining.y + } else { + dragging = false + } + } + const onPointerMove = (e) => { + if (!pointers.has(e.pointerId)) return + pointers.set(e.pointerId, { x: e.clientX, y: e.clientY }) + + if (pointers.size >= 2) { + const [a, b] = pointerPair() + const nextDistance = Math.hypot(b.x - a.x, b.y - a.y) + const nextX = (a.x + b.x) / 2 + const nextY = (a.y + b.y) / 2 + if (pinchDistance > 0 && nextDistance > 0) { + dist *= pinchDistance / nextDistance + dist = Math.max(0.3, Math.min(8, dist)) + panX += (nextX - pinchX) * 0.0015 * dist + panY -= (nextY - pinchY) * 0.0015 * dist + } + pinchDistance = nextDistance + pinchX = nextX + pinchY = nextY + return + } + + if (!dragging) return + const dx = e.clientX - lx, dy = e.clientY - ly + lx = e.clientX; ly = e.clientY + if (panning) { + panX += dx * 0.0015 * dist; panY -= dy * 0.0015 * dist + } else { + // Compose screen-axis turns onto the current orientation (fixed camera + // frame), so rotation stays screen-relative and never locks up. + const k = 0.008 + rot = Q.norm(Q.mul(Q.axisAngle(1, 0, 0, dy * k), Q.mul(Q.axisAngle(0, 1, 0, dx * k), rot))) + stopSpin() + } + } + const onContextMenu = (e) => e.preventDefault() + const onWheel = (e) => { + e.preventDefault() + dist *= Math.exp(e.deltaY * 0.001) + dist = Math.max(0.3, Math.min(8, dist)) + } + const onDblClick = () => resetView() + let onSpinChange = null + + canvas.addEventListener('pointerdown', onPointerDown) + canvas.addEventListener('pointerup', onPointerUp) + canvas.addEventListener('pointercancel', onPointerUp) + canvas.addEventListener('pointermove', onPointerMove) + canvas.addEventListener('contextmenu', onContextMenu) + canvas.addEventListener('wheel', onWheel, { passive: false }) + canvas.addEventListener('dblclick', onDblClick) + + gl.enable(gl.DEPTH_TEST) + gl.enable(gl.BLEND) + gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA) + gl.clearColor(0.063, 0.078, 0.094, 1) + + let rafId = 0 + let last = performance.now() + function frame(now) { + if (disposed) return + const dt = (now - last) / 1000; last = now + // auto-rotate: a slow turn about the screen-vertical axis (turntable feel) + if (spin) rot = Q.norm(Q.mul(Q.axisAngle(0, 1, 0, dt * 0.4), rot)) + + const w = canvas.clientWidth, h = canvas.clientHeight + if (w > 0 && h > 0 && (canvas.width !== w * devicePixelRatio || canvas.height !== h * devicePixelRatio)) { + canvas.width = w * devicePixelRatio; canvas.height = h * devicePixelRatio + } + gl.viewport(0, 0, canvas.width, canvas.height) + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT) + + if (nIndices) { + const rotation = Q.toMat4(rot) + const model = M.mul(rotation, M.mul(M.scale(fitScale), M.trans(-center[0], -center[1], -center[2]))) + const view = M.trans(panX, panY, -dist) + const viewModel = M.mul(view, model) + const proj = M.persp(0.9, (w || 1) / (h || 1), 0.05, 100) + const mvp = M.mul(proj, viewModel) + + gl.useProgram(prog) + gl.uniformMatrix4fv(uMVP, false, mvp) + gl.uniformMatrix4fv(uViewModel, false, viewModel) + gl.uniform1f(uMetallicFactor, metallicFactor) + gl.uniform1f(uRoughnessFactor, roughnessFactor) + if (colorMode === 2) { + gl.activeTexture(gl.TEXTURE0) + gl.bindTexture(gl.TEXTURE_2D, baseColorTexture) + gl.uniform1i(uBaseColorTex, 0) + gl.activeTexture(gl.TEXTURE1) + gl.bindTexture(gl.TEXTURE_2D, mrTexture) + gl.uniform1i(uMrTex, 1) + } + gl.bindVertexArray(vao) + if (wire) { + // hidden-line wireframe: a depth-only prepass (pushed back a hair via + // polygon offset) occludes back-facing edges, so the front surface's + // edges show instead of a see-through blob. + gl.enable(gl.POLYGON_OFFSET_FILL) + gl.polygonOffset(1.0, 1.0) + gl.colorMask(false, false, false, false) + gl.uniform1i(uWire, 0) + gl.uniform1i(uColorMode, 0) + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo) + gl.drawElements(gl.TRIANGLES, nIndices, gl.UNSIGNED_INT, 0) + gl.colorMask(true, true, true, true) + gl.disable(gl.POLYGON_OFFSET_FILL) + gl.uniform1i(uWire, 1) + // Front-surface edges sit at ~equal depth to the offset-back fill, so + // LEQUAL lets them pass while occluded edges (greater depth) fail. + gl.depthFunc(gl.LEQUAL) + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, wireIbo) + gl.drawElements(gl.LINES, nWire, gl.UNSIGNED_INT, 0) + gl.depthFunc(gl.LESS) + } else { + gl.uniform1i(uWire, 0) + gl.uniform1i(uColorMode, colorMode) + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo) + gl.drawElements(gl.TRIANGLES, nIndices, gl.UNSIGNED_INT, 0) + } + gl.bindVertexArray(null) + } + rafId = requestAnimationFrame(frame) + } + rafId = requestAnimationFrame(frame) + + function dispose() { + disposed = true + cancelAnimationFrame(rafId) + canvas.removeEventListener('pointerdown', onPointerDown) + canvas.removeEventListener('pointerup', onPointerUp) + canvas.removeEventListener('pointercancel', onPointerUp) + canvas.removeEventListener('pointermove', onPointerMove) + canvas.removeEventListener('contextmenu', onContextMenu) + canvas.removeEventListener('wheel', onWheel) + canvas.removeEventListener('dblclick', onDblClick) + canvas.removeEventListener('webglcontextlost', contextLost) + dropTextures() + gl.deleteBuffer(vbo); gl.deleteBuffer(nbo); gl.deleteBuffer(cbo) + gl.deleteBuffer(mbo); gl.deleteBuffer(ubo); gl.deleteBuffer(ibo) + gl.deleteBuffer(wireIbo) + gl.deleteVertexArray(vao) + gl.deleteProgram(prog) + gl.getExtension('WEBGL_lose_context')?.loseContext() + } + + return { + setMesh, + clear, + dispose, + resetView, + setWire(v) { wire = v }, + setSpin(v) { spin = v }, + onSpinChanged(fn) { onSpinChange = fn }, + } +} + +export default function GlbViewer({ blob }) { + const { t } = useTranslation('media') + const canvasRef = useRef(null) + const viewerRef = useRef(null) + const [glError, setGlError] = useState(null) // 'no-webgl2' | 'context-lost' | parse error text + const [stats, setStats] = useState(null) + const [wire, setWire] = useState(false) + const [spin, setSpin] = useState(true) + + useEffect(() => { // GL lifecycle — once per mount + let viewer = null + try { + viewer = createGlbViewer(canvasRef.current, { onContextLost: () => setGlError('context-lost') }) + } catch { + viewer = null + } + if (!viewer) { + setGlError('no-webgl2') + return undefined + } + viewer.onSpinChanged(setSpin) + viewerRef.current = viewer + return () => { + viewerRef.current = null + viewer.dispose() + } + }, []) + + useEffect(() => { // (re)load when the blob changes + if (!blob) { + viewerRef.current?.clear() + setStats(null) + return undefined + } + let cancelled = false + ;(async () => { + try { + // Parse before touching GL: stats and parse errors surface even when + // WebGL2 is unavailable (e.g. headless CI), and the download button + // keeps working either way. + const mesh = parseGlb(await blob.arrayBuffer()) + if (cancelled) return + setStats({ + nVerts: mesh.nVerts, + nTris: mesh.nTris, + pbr: !!(mesh.color0 || mesh.baseColorPng), + }) + if (viewerRef.current) await viewerRef.current.setMesh(mesh) + } catch (err) { + if (!cancelled) setGlError(err.message) + } + })() + return () => { cancelled = true } + }, [blob]) + + const toggleWire = () => { + const next = !wire + setWire(next) + viewerRef.current?.setWire(next) + } + const toggleSpin = () => { + const next = !spin + setSpin(next) + viewerRef.current?.setSpin(next) + } + + return ( +
+ +
+ + + {stats && ( + + {t('threed.viewer.stats', { verts: stats.nVerts.toLocaleString(), tris: stats.nTris.toLocaleString() })} + {stats.pbr ? ' · PBR' : ''} + + )} +
+ {glError === 'no-webgl2' &&

{t('threed.viewer.noWebgl')}

} + {glError === 'context-lost' &&

{t('threed.viewer.contextLost')}

} + {glError && glError !== 'no-webgl2' && glError !== 'context-lost' && ( +

{glError}

+ )} +

{t('threed.viewer.hint')}

+
+ ) +} diff --git a/core/http/react-ui/src/components/ThreeDHistory.jsx b/core/http/react-ui/src/components/ThreeDHistory.jsx new file mode 100644 index 000000000..7686669d5 --- /dev/null +++ b/core/http/react-ui/src/components/ThreeDHistory.jsx @@ -0,0 +1,76 @@ +import { memo, useState } from 'react' +import { relativeTime } from '../utils/format' +import { useTranslation } from 'react-i18next' + +// ThreeDHistory — sibling of MediaHistory for IndexedDB-backed 3D entries +// (see use3DHistory). Reuses MediaHistory's markup, CSS classes, and testids +// so styling and e2e helpers carry over; the differences are the entry shape +// (input-image thumbnail, quality subtitle) and the Blob-backed source. +// Deliberately a plain vertical list — no showcase/gallery mode. +export default memo(function ThreeDHistory({ entries, selectedId, onSelect, onDelete, onClearAll }) { + const { t } = useTranslation('media') + const [expanded, setExpanded] = useState(true) + + return ( +
+
setExpanded(!expanded)} + style={{ display: 'flex', alignItems: 'center' }} + > + + {t('history.title')} ({entries.length}) + {entries.length > 0 && ( + + )} +
+ {expanded && ( +
+ {entries.length === 0 ? ( +
{t('history.empty')}
+ ) : ( + entries.map(entry => ( +
onSelect(entry.id)} + data-testid="media-history-item" + > +
+ {entry.inputThumb ? ( + + ) : ( + + )} +
+
+
+ {entry.name || entry.model} + {relativeTime(entry.createdAt)} +
+
+ {entry.params?.quality ? `${entry.params.quality} · ` : ''}{entry.model} +
+
+ +
+ )) + )} +
+ )} +
+ ) +}) diff --git a/core/http/react-ui/src/hooks/use3DHistory.js b/core/http/react-ui/src/hooks/use3DHistory.js new file mode 100644 index 000000000..b15e331ae --- /dev/null +++ b/core/http/react-ui/src/hooks/use3DHistory.js @@ -0,0 +1,132 @@ +import { useState, useEffect, useCallback, useMemo } from 'react' +import { generateId } from '../utils/format' + +// use3DHistory — IndexedDB-backed history of 3D generations. Unlike +// useMediaHistory (localStorage, URL-only), entries here carry the generated +// GLB as a Blob: GLBs are multi-megabyte binaries the server may eventually +// clean up, and IndexedDB is the only browser store that handles blobs of +// that size. Blobs read back from IndexedDB are lazy handles (the bytes are +// not materialized into JS memory until used), so a single store is fine. +// +// Entry: { id, createdAt, name, model, +// params: { seed, steps, textureSteps, guidance, quality, background }, +// inputThumb, // small dataURL of the conditioning image +// glb } // Blob + +const DB_NAME = 'localai-3d-history' +const DB_VERSION = 1 +const STORE = 'generations' +const MAX_ENTRIES = 20 + +function openDb() { + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME, DB_VERSION) + req.onupgradeneeded = () => { + const db = req.result + if (!db.objectStoreNames.contains(STORE)) { + db.createObjectStore(STORE, { keyPath: 'id' }).createIndex('createdAt', 'createdAt') + } + } + req.onsuccess = () => resolve(req.result) + req.onerror = () => reject(req.error) + }) +} + +const txDone = (tx) => new Promise((resolve, reject) => { + tx.oncomplete = resolve + tx.onerror = () => reject(tx.error) + tx.onabort = () => reject(tx.error) +}) + +async function withStore(mode, fn) { + const db = await openDb() + try { + const tx = db.transaction(STORE, mode) + const result = fn(tx.objectStore(STORE)) + await txDone(tx) + return result + } finally { + db.close() + } +} + +async function idbGetAll() { + const req = await withStore('readonly', (store) => store.getAll()) + return (req.result || []).sort((a, b) => b.createdAt - a.createdAt) +} + +// Insert + keep-newest-N eviction in one transaction so a crash between the +// two can't leave the store unbounded. +async function idbPutAndEvict(entry) { + await withStore('readwrite', (store) => { + store.put(entry) + // getAllKeys on the createdAt index yields primary keys oldest-first. + const keysReq = store.index('createdAt').getAllKeys() + keysReq.onsuccess = () => { + const excess = keysReq.result.length - MAX_ENTRIES + for (let i = 0; i < excess; i++) store.delete(keysReq.result[i]) + } + }) +} + +const idbDelete = (id) => withStore('readwrite', (store) => store.delete(id)) +const idbClear = () => withStore('readwrite', (store) => store.clear()) + +export function use3DHistory() { + const [entries, setEntries] = useState([]) + const [selectedId, setSelectedId] = useState(null) + + const refresh = useCallback(async () => { + try { + setEntries(await idbGetAll()) + } catch { + // IndexedDB unavailable (private mode etc.) — degrade to session-only. + setEntries((prev) => prev) + } + }, []) + + useEffect(() => { refresh() }, [refresh]) + + const addEntry = useCallback(async ({ model, params, inputThumb, glb, name }) => { + const entry = { id: generateId(), createdAt: Date.now(), model, params, inputThumb, glb, name } + try { + await idbPutAndEvict(entry) + await refresh() + } catch { + setEntries((prev) => [entry, ...prev].slice(0, MAX_ENTRIES)) + } + return entry + }, [refresh]) + + const deleteEntry = useCallback(async (id) => { + setSelectedId((prev) => (prev === id ? null : prev)) + try { + await idbDelete(id) + await refresh() + } catch { + setEntries((prev) => prev.filter((e) => e.id !== id)) + } + }, [refresh]) + + const clearAll = useCallback(async () => { + setSelectedId(null) + try { + await idbClear() + } catch { + // fall through to the local reset below + } + setEntries([]) + }, []) + + // Toggles: clicking the selected entry deselects it (back to latest result). + const selectEntry = useCallback((id) => { + setSelectedId((prev) => (prev === id ? null : id)) + }, []) + + const selectedEntry = useMemo( + () => entries.find((e) => e.id === selectedId) || null, + [entries, selectedId], + ) + + return { entries, addEntry, deleteEntry, clearAll, selectEntry, selectedId, selectedEntry } +} diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index 53dfccf2f..f7491805a 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -56,6 +56,7 @@ const FILTERS = [ { key: 'chat', labelKey: 'filters.llm', icon: 'fa-brain' }, { key: 'image', labelKey: 'filters.image', icon: 'fa-image' }, { key: 'video', labelKey: 'filters.video', icon: 'fa-video' }, + { key: '3d', labelKey: 'filters.threed', icon: 'fa-cube' }, { key: 'multimodal', labelKey: 'filters.multimodal', icon: 'fa-shapes' }, { key: 'vision', labelKey: 'filters.vision', icon: 'fa-eye' }, { key: 'tts', labelKey: 'filters.tts', icon: 'fa-microphone' }, diff --git a/core/http/react-ui/src/pages/Studio.jsx b/core/http/react-ui/src/pages/Studio.jsx index 5926078c2..43692cea7 100644 --- a/core/http/react-ui/src/pages/Studio.jsx +++ b/core/http/react-ui/src/pages/Studio.jsx @@ -2,6 +2,7 @@ import { useSearchParams } from 'react-router-dom' import { useTranslation } from 'react-i18next' import ImageGen from './ImageGen' import VideoGen from './VideoGen' +import ThreeDGen from './ThreeDGen' import TTS from './TTS' import Sound from './Sound' import AudioTransform from './AudioTransform' @@ -10,6 +11,7 @@ import { useAuth } from '../context/AuthContext' const BASE_TABS = [ { key: 'images', labelKey: 'studio.tabs.images', icon: 'fas fa-image' }, { key: 'video', labelKey: 'studio.tabs.video', icon: 'fas fa-video' }, + { key: 'threed', labelKey: 'studio.tabs.threed', icon: 'fas fa-cube' }, { key: 'tts', labelKey: 'studio.tabs.tts', icon: 'fas fa-headphones' }, { key: 'sound', labelKey: 'studio.tabs.sound', icon: 'fas fa-music' }, ] @@ -19,6 +21,7 @@ const TRANSFORM_TAB = { key: 'transform', labelKey: 'studio.tabs.transform', ico const TAB_COMPONENTS = { images: ImageGen, video: VideoGen, + threed: ThreeDGen, tts: TTS, sound: Sound, transform: AudioTransform, @@ -28,19 +31,23 @@ export default function Studio() { const { t } = useTranslation('media') const { hasFeature } = useAuth() const [searchParams, setSearchParams] = useSearchParams() - const activeTab = searchParams.get('tab') || 'images' + const requestedTab = searchParams.get('tab') || 'images' + const threeDEnabled = hasFeature('3d') + const transformEnabled = hasFeature('audio_transform') + const activeTab = + ((requestedTab === 'threed' && !threeDEnabled) || + (requestedTab === 'transform' && !transformEnabled)) + ? 'images' + : requestedTab - // Transform is a distinct capability; only show its tab when enabled. - const tabs = hasFeature('audio_transform') ? [...BASE_TABS, TRANSFORM_TAB] : BASE_TABS + const enabledTabs = BASE_TABS.filter(tab => tab.key !== 'threed' || threeDEnabled) + const tabs = transformEnabled ? [...enabledTabs, TRANSFORM_TAB] : enabledTabs const setTab = (key) => { setSearchParams({ tab: key }, { replace: true }) } - const ActiveComponent = - (activeTab === 'transform' && !hasFeature('audio_transform')) - ? ImageGen - : (TAB_COMPONENTS[activeTab] || ImageGen) + const ActiveComponent = TAB_COMPONENTS[activeTab] || ImageGen return (
diff --git a/core/http/react-ui/src/pages/ThreeDGen.jsx b/core/http/react-ui/src/pages/ThreeDGen.jsx new file mode 100644 index 000000000..f4a0e3775 --- /dev/null +++ b/core/http/react-ui/src/pages/ThreeDGen.jsx @@ -0,0 +1,285 @@ +import { useState } from 'react' +import { useParams, useOutletContext } from 'react-router-dom' +import { useTranslation } from 'react-i18next' +import ModelSelector from '../components/ModelSelector' +import PageHeader from '../components/PageHeader' +import { CAP_3D } from '../utils/capabilities' +import LoadingSpinner from '../components/LoadingSpinner' +import GenerationProgress from '../components/GenerationProgress' +import ErrorWithTraceLink from '../components/ErrorWithTraceLink' +import ThreeDHistory from '../components/ThreeDHistory' +import GlbViewer from '../components/GlbViewer' +import MediaInput from '../components/biometrics/MediaInput' +import { threeDApi } from '../utils/api' +import { apiUrl } from '../utils/basePath' +import { use3DHistory } from '../hooks/use3DHistory' +import useObjectUrl from '../hooks/useObjectUrl' + +const QUALITIES = ['auto', 'coarse', '512', '1024'] +const BACKGROUNDS = ['auto', 'keep', 'black', 'white'] +const MAX_3D_INPUT_BYTES = 32 * 1024 * 1024 +const REMESH_DETAIL_COARSE = 2.5 +const REMESH_DETAIL_FINE = 0.35 + +function remeshDetail(sliderValue) { + const position = Number(sliderValue) / 100 + return REMESH_DETAIL_COARSE * Math.pow(REMESH_DETAIL_FINE / REMESH_DETAIL_COARSE, position) +} + +function remeshedName(name = '3d-model.glb') { + return `${name.replace(/\.glb$/i, '')}-remeshed.glb` +} + +// Small thumbnail of the conditioning image for the history list — full-size +// data URLs would bloat every IndexedDB entry for no visual gain. +async function makeThumb(dataUrl, size = 96) { + try { + const img = new Image() + await new Promise((resolve, reject) => { + img.onload = resolve + img.onerror = reject + img.src = dataUrl + }) + const scale = size / Math.max(img.width, img.height, 1) + const canvas = document.createElement('canvas') + canvas.width = Math.max(1, Math.round(img.width * scale)) + canvas.height = Math.max(1, Math.round(img.height * scale)) + canvas.getContext('2d').drawImage(img, 0, 0, canvas.width, canvas.height) + return canvas.toDataURL('image/jpeg', 0.7) + } catch { + return null + } +} + +export default function ThreeDGen() { + const { model: urlModel } = useParams() + const { addToast } = useOutletContext() + const { t } = useTranslation('media') + const [model, setModel] = useState(urlModel || '') + const [image, setImage] = useState(null) + const [quality, setQuality] = useState('auto') + const [background, setBackground] = useState('auto') + const [steps, setSteps] = useState('') + const [textureSteps, setTextureSteps] = useState('') + const [guidance, setGuidance] = useState('') + const [seed, setSeed] = useState('') + const [showAdvanced, setShowAdvanced] = useState(false) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + const [result, setResult] = useState(null) // { blob, name, model } + const [remeshSlider, setRemeshSlider] = useState(82) + const [remeshState, setRemeshState] = useState(null) // { sourceBlob, blob?, name?, error? } + const [remeshLoading, setRemeshLoading] = useState(false) + const { entries, addEntry, deleteEntry, clearAll, selectEntry, selectedId, selectedEntry } = use3DHistory() + + const source = selectedEntry + ? { blob: selectedEntry.glb, name: selectedEntry.name, model: selectedEntry.model } + : result + const showingRemesh = !!source && remeshState?.sourceBlob === source.blob && !!remeshState.blob + const active = showingRemesh ? { blob: remeshState.blob, name: remeshState.name } : source + const remeshError = source && remeshState?.sourceBlob === source.blob ? remeshState.error : null + const detail = remeshDetail(remeshSlider) + const downloadUrl = useObjectUrl(active?.blob) + + const handleGenerate = async (e) => { + e.preventDefault() + if (!image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return } + if (!model) { addToast(t('threed.toasts.noModel'), 'warning'); return } + + setLoading(true) + setResult(null) + setRemeshState(null) + setError(null) + + const body = { model, image: image.base64, quality, background, response_format: 'url' } + if (steps) body.step = parseInt(steps) + if (textureSteps) body.texture_steps = parseInt(textureSteps) + if (guidance) body.cfg_scale = parseFloat(guidance) + if (seed) body.seed = parseInt(seed) + + try { + const data = await threeDApi.generate(body) + const url = data?.data?.[0]?.url + if (!url) { + addToast(t('threed.toasts.noResults'), 'warning') + return + } + const glbResp = await fetch(apiUrl(url)) + if (!glbResp.ok) throw new Error(`fetching the generated GLB failed: HTTP ${glbResp.status}`) + const glb = await glbResp.blob() + const name = url.split('/').pop() + setResult({ blob: glb, name, model }) + selectEntry(null) + const inputThumb = image.dataUrl ? await makeThumb(image.dataUrl) : null + await addEntry({ + model, + params: { quality, background, steps, textureSteps, guidance, seed }, + inputThumb, + glb, + name, + }) + } catch (err) { + setError(err.message) + } finally { + setLoading(false) + } + } + + const handleRemesh = async () => { + if (!source?.blob || !source.model) return + if (showingRemesh) { + setRemeshState(null) + return + } + + const sourceBlob = source.blob + setRemeshLoading(true) + setRemeshState({ sourceBlob, error: null }) + try { + const blob = await threeDApi.remesh(sourceBlob, source.model, detail) + setRemeshState({ sourceBlob, blob, name: remeshedName(source.name), error: null }) + } catch (err) { + setRemeshState({ sourceBlob, error: err.message }) + } finally { + setRemeshLoading(false) + } + } + + const handleRemeshDetail = (value) => { + setRemeshSlider(value) + if (source && remeshState?.sourceBlob === source.blob) setRemeshState(null) + } + + return ( +
+
+ {t('threed.title')}} /> + +
+
+ + +
+ + addToast(err.message, 'error')} + maxBytes={MAX_3D_INPUT_BYTES} + idPrefix="threed" + /> + +
+
+ + +
+
+ + +
+
+ +