diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 17cc6001d..26aba82e6 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -122,7 +122,7 @@ The per-backend prefix match only sees files under a backend's own directory, so | Changed path | Rebuilds | |---|---| -| `backend/backend.proto` | everything (all languages compile or copy it) | +| `backend/backend.proto` | nothing if the edit is additive-only, otherwise everything (see below) | | `backend/Dockerfile.` | the Linux entries whose `dockerfile:` names it | | `backend/python/common/` | Python, Linux + Darwin | | `scripts/build/package-gpu-libs.sh` | Python, Linux only | @@ -132,6 +132,17 @@ The per-backend prefix match only sees files under a backend's own directory, so Deliberately excluded: `backend/index.yaml` (gallery metadata, never enters an image), `.github/backend-matrix.yml` (adding a backend would rebuild all of them), `backend/Dockerfile.base-grpc-builder` (owned by `base-images.yml`), and the root `Makefile` (touched in ~11% of commits, and its backend-relevant edits arrive alongside the backend directory anyway). `make test-ci-scripts` pins all of this. +#### `backend/backend.proto` is content-filtered, not path-filtered + +Every language consumes the proto, so a path rule for it can only ever say "rebuild all 473 images". It changes in ~1.3% of commits, and that was enough to make it the single largest CI cost driver in the repo: on 2026-07-29 four runs totalling 935 queued jobs traced to nothing but a proto edit, one of which (#11158) was a six-line diff adding `bool cache_prompt = 8;`. + +An additive proto edit cannot change how a backend that never references the new symbol behaves, so `filterMatrix()` suppresses the rule for one. `changed-backends.js` fetches `backend/backend.proto` at the base revision (same contents-API pattern as `.github/backend-matrix.yml`) and hands both texts to `protoChangeIsAdditive()`, which compares them structurally rather than textually: + +- **Additive, rebuilds nothing**: a new field with an unused number, a new message, a new enum value, a new RPC. Comment, whitespace and ordering changes also land here. +- **Breaking, rebuilds everything**: a removed, renumbered, retyped or renamed field, a dropped RPC, a changed `option` or `package`. So does an unresolvable base revision, matching the run-all posture used for a truncated diff. + +Checked against every proto commit in the preceding six months, all nine resolvable ones classify as additive. Note the tradeoff this accepts: generated stubs do change for an additive edit, so image bytes would differ on a rebuild even though behavior does not. That is the same standard already applied when the filter declines to rebuild on unrelated `pkg/` changes, and the weekly cron remains the backstop. + The Sunday 06:00 UTC cron on `backend.yml` exists specifically because path filtering can leave Python backends frozen on stale wheels. `DEPS_REFRESH` (below) only fires when the build actually runs, so an untouched Python backend would never re-resolve its unpinned deps. The weekly cron is the safety net. ## The `DEPS_REFRESH` cache-buster (Python backends) diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 2ae8a26af..b549e4648 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -756,6 +756,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1716,6 +1729,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -1729,6 +1755,19 @@ include: backend: "stablediffusion-ggml" dockerfile: "./backend/Dockerfile.golang" context: "./" + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-cuda-13-arm64-trellis2cpp' + base-image: "ubuntu:24.04" + ubuntu-version: '2404' + runs-on: 'ubuntu-24.04-arm' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -3267,6 +3306,35 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + # trellis2cpp + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-trellis2cpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' # sam3-cpp - build-type: '' cuda-major-version: "" @@ -3592,6 +3660,34 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-trellis2cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-trellis2cpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -3605,6 +3701,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2204' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-arm64-trellis2cpp' + base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + runs-on: 'ubuntu-24.04-arm' + backend: "trellis2cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2204' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -5436,6 +5545,35 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + # valkey-store + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-valkey-store' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "valkey-store" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-valkey-store' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "valkey-store" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' # rfdetr - build-type: '' cuda-major-version: "" @@ -5977,6 +6115,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-stablediffusion-ggml" build-type: "metal" lang: "go" + - backend: "trellis2cpp" + tag-suffix: "-metal-darwin-arm64-trellis2cpp" + build-type: "metal" + lang: "go" - backend: "whisper" tag-suffix: "-metal-darwin-arm64-whisper" build-type: "metal" @@ -6154,6 +6296,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-cloud-proxy" build-type: "metal" lang: "go" + - backend: "valkey-store" + tag-suffix: "-metal-darwin-arm64-valkey-store" + build-type: "metal" + lang: "go" - backend: "llama-cpp-quantization" tag-suffix: "-metal-darwin-arm64-llama-cpp-quantization" build-type: "mps" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index 7cd472f4e..b6dbbb449 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -78,6 +78,10 @@ jobs: variable: "STABLEDIFFUSION_GGML_VERSION" branch: "master" file: "backend/go/stablediffusion-ggml/Makefile" + - repository: "localai-org/trellis2cpp" + variable: "TRELLIS2CPP_VERSION" + branch: "pbr-textures" + file: "backend/go/trellis2cpp/Makefile" - repository: "mudler/go-piper" variable: "PIPER_VERSION" branch: "master" diff --git a/.github/workflows/test-extra.yml b/.github/workflows/test-extra.yml index 899b98e60..96eb57155 100644 --- a/.github/workflows/test-extra.yml +++ b/.github/workflows/test-extra.yml @@ -38,6 +38,7 @@ jobs: acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }} qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }} magpie-tts-cpp: ${{ steps.detect.outputs.magpie-tts-cpp }} + trellis2cpp: ${{ steps.detect.outputs.trellis2cpp }} rfdetr-cpp: ${{ steps.detect.outputs.rfdetr-cpp }} locate-anything-cpp: ${{ steps.detect.outputs.locate-anything-cpp }} vibevoice-cpp: ${{ steps.detect.outputs.vibevoice-cpp }} @@ -935,6 +936,41 @@ jobs: - name: Test rfdetr-cpp run: | make --jobs=5 --output-sync=target -C backend/go/rfdetr-cpp test + # Weight-free packaged-backend smoke for trellis2cpp. Starting run.sh loads + # libtrellis2 + ggml, resolves the complete C ABI (including remeshing), and + # answers gRPC Health without downloading or loading the multi-GB model set. + tests-trellis2cpp: + needs: detect-changes + if: needs.detect-changes.outputs.trellis2cpp == 'true' || needs.detect-changes.outputs.run-all == 'true' + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake curl unzip + - name: Setup Go + uses: actions/setup-go@v5 + - name: Display Go version + run: go version + - name: Proto Dependencies + run: | + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \ + unzip -j -d /usr/local/bin protoc.zip bin/protoc && \ + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go + - name: Build trellis2cpp + run: | + make --jobs=5 --output-sync=target -C backend/go/trellis2cpp + - name: Test trellis2cpp + run: | + make --jobs=5 --output-sync=target -C backend/go/trellis2cpp test # Per-backend e2e for locate-anything-cpp: builds the .so + Go binary and # runs `make -C backend/go/locate-anything-cpp test`. test.sh fetches the # locate-anything-q8_0 GGUF (~6.3 GB, NVIDIA LocateAnything-3B) from the diff --git a/.gitignore b/.gitignore index 5286461b3..4fb77f49e 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ LocalAI # Go backend packages whose main lives under backend/go/. /cloud-proxy /local-store +/valkey-store # prevent above rules from omitting the helm chart !charts/* # prevent above rules from omitting the api/localai folder diff --git a/Makefile b/Makefile index 81b00d7c8..bd3b21d9d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -69,7 +69,7 @@ else GORELEASER=$(shell which goreleaser) endif -TEST_PATHS?=./api/... ./pkg/... ./core/... ./backend/go/cloud-proxy/... ./backend/go/local-store/... +TEST_PATHS?=./api/... ./pkg/... ./core/... ./backend/go/cloud-proxy/... ./backend/go/local-store/... ./backend/go/valkey-store/... ## Coverage output and the committed baseline that CI compares against. ## The gate is strict: total coverage must never decrease (no tolerance). @@ -386,6 +386,15 @@ test-stores: backends/local-store BACKENDS_PATH=$(abspath ./)/backends \ $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r tests/integration +## Valkey-backed vector-store integration. Requires a running Valkey Search +## server (valkey/valkey-bundle:9.1.0) reachable at $$VALKEY_ADDR — the suite +## skips itself when VALKEY_ADDR is unset. Builds the backend on demand and +## points the model loader at it via BACKENDS_PATH. Label-filtered to the +## valkey specs so it does not also run the in-memory local-store suite. +test-valkey-store: backends/valkey-store + BACKENDS_PATH=$(abspath ./)/backends \ + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --label-filter='valkey' -v -r tests/integration + test-opus: @echo 'Running opus backend tests' $(MAKE) -C backend/go/opus libopusshim.so @@ -594,6 +603,8 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/rust/kokoros kokoros-grpc $(MAKE) -C backend/go/rfdetr-cpp $(MAKE) -C backend/go/locate-anything-cpp + $(MAKE) -C backend/go/trellis2cpp + $(MAKE) -C backend/go/valkey-store test-extra: prepare-test-extra $(MAKE) -C backend/python/transformers test @@ -626,6 +637,8 @@ test-extra: prepare-test-extra $(MAKE) -C backend/go/depth-anything-cpp test $(MAKE) -C backend/go/supertonic test $(MAKE) -C backend/go/vllm-cpp test + $(MAKE) -C backend/go/trellis2cpp test + $(MAKE) -C backend/go/valkey-store test ## ## End-to-end gRPC tests that exercise a built backend container image. @@ -1218,6 +1231,10 @@ backends/stablediffusion-ggml-darwin: BACKEND=stablediffusion-ggml BUILD_TYPE=metal $(MAKE) build-darwin-go-backend ./local-ai backends install "ocifile://$(abspath ./backend-images/stablediffusion-ggml.tar)" +backends/trellis2cpp-darwin: + BACKEND=trellis2cpp BUILD_TYPE=metal $(MAKE) build-darwin-go-backend + ./local-ai backends install "ocifile://$(abspath ./backend-images/trellis2cpp.tar)" + backend-images: mkdir -p backend-images @@ -1245,10 +1262,12 @@ BACKEND_PRIVACY_FILTER = privacy-filter|privacy-filter|.|false|false # Golang backends BACKEND_PIPER = piper|golang|.|false|true BACKEND_LOCAL_STORE = local-store|golang|.|false|true +BACKEND_VALKEY_STORE = valkey-store|golang|.|false|true BACKEND_CLOUD_PROXY = cloud-proxy|golang|.|false|true BACKEND_HUGGINGFACE = huggingface|golang|.|false|true BACKEND_SILERO_VAD = silero-vad|golang|.|false|true BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|true +BACKEND_TRELLIS2CPP = trellis2cpp|golang|.|--progress=plain|true BACKEND_WHISPER = whisper|golang|.|false|true BACKEND_CRISPASR = crispasr|golang|.|false|true BACKEND_PARAKEET_CPP = parakeet-cpp|golang|.|false|true @@ -1344,10 +1363,12 @@ $(eval $(call generate-docker-build-target,$(BACKEND_DS4))) $(eval $(call generate-docker-build-target,$(BACKEND_PRIVACY_FILTER))) $(eval $(call generate-docker-build-target,$(BACKEND_PIPER))) $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE))) +$(eval $(call generate-docker-build-target,$(BACKEND_VALKEY_STORE))) $(eval $(call generate-docker-build-target,$(BACKEND_CLOUD_PROXY))) $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE))) $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD))) $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML))) +$(eval $(call generate-docker-build-target,$(BACKEND_TRELLIS2CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_WHISPER))) $(eval $(call generate-docker-build-target,$(BACKEND_CRISPASR))) $(eval $(call generate-docker-build-target,$(BACKEND_PARAKEET_CPP))) @@ -1408,7 +1429,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store ######################################################## ### Mock Backend for E2E Tests diff --git a/README.md b/README.md index d2d0c1479..cbdc44b68 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,7 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native | [depth-anything.cpp](https://github.com/mudler/depth-anything.cpp) | Depth Anything 3 monocular metric depth + camera pose estimation | | [face-detect.cpp](https://github.com/mudler/face-detect.cpp) | Face detection, recognition, demographics and anti-spoofing (SCRFD/ArcFace, YuNet/SFace), replacing the Python insightface backend | | [free-splatter.cpp](https://github.com/localai-org/free-splatter.cpp) | Pose-free 3D reconstruction (FreeSplatter): turns a handful of plain photos into 3D Gaussians, no camera poses or GPU required | +| [trellis2.cpp](https://github.com/localai-org/trellis2cpp) | C++/GGML port of Microsoft TRELLIS.2: single-image to textured 3D mesh (GLB with PBR materials) | | [privacy-filter.cpp](https://github.com/localai-org/privacy-filter.cpp) | Standalone GGML PII/NER token-classification engine powering LocalAI's PII redaction tier | | [LocalVQE](https://github.com/localai-org/LocalVQE) | Joint acoustic echo cancellation, noise suppression, and dereverberation | | [local-store](https://github.com/mudler/LocalAI) | Local-first vector database for embeddings (shipped in-tree) | diff --git a/backend/README.md b/backend/README.md index 0e92a0f03..a458da35f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -56,6 +56,7 @@ The backend system provides language-specific Dockerfiles that handle the build - **stablediffusion-ggml**: Stable Diffusion in Go with GGML Cpp backend - **piper**: Text-to-speech synthesis Golang with C bindings using rhaspy/piper - **local-store**: Vector storage backend +- **valkey-store**: Durable vector storage backend backed by Valkey Search (FT.*) #### C++ Backends (`cpp/`) - **llama-cpp**: Llama.cpp integration diff --git a/backend/backend.proto b/backend/backend.proto index 711e9465c..2935d1a62 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -16,6 +16,7 @@ service Backend { rpc Embedding(PredictOptions) returns (EmbeddingResult) {} rpc GenerateImage(GenerateImageRequest) returns (Result) {} rpc GenerateVideo(GenerateVideoRequest) returns (Result) {} + rpc Generate3D(Generate3DRequest) returns (Result) {} rpc AudioTranscription(TranscriptRequest) returns (TranscriptResult) {} rpc AudioTranscriptionStream(TranscriptRequest) returns (stream TranscriptStreamResponse) {} // AudioTranscriptionLive is the bidirectional live-microphone ASR RPC. The @@ -658,6 +659,20 @@ message GenerateVideoRequest { string ModelIdentity = 15; } +message Generate3DRequest { + string src = 1; // Path to the staged conditioning image (3D generation is image-conditioned) + string dst = 2; // Output path for the generated binary glTF (.glb) asset + int32 seed = 3; // <=0 lets the backend pick a random seed + int32 step = 4; // Flow sampling steps; <=0 uses the backend default + float cfg_scale = 5; // Classifier-free guidance scale; <=0 uses the backend default + int32 texture_steps = 6; // Texture flow sampling steps; <=0 uses the backend default + string quality = 7; // Mesh pipeline: ""|"auto"|"coarse"|"512"|"1024" + string background = 8; // Conditioning-image background handling: ""|"auto"|"keep"|"black"|"white" + // Backend-specific per-request generation parameters. Values are strings + // and are validated/coerced by the selected backend. + map params = 9; +} + message TTSRequest { string text = 1; string model = 2; diff --git a/backend/go/trellis2cpp/.gitignore b/backend/go/trellis2cpp/.gitignore new file mode 100644 index 000000000..b5cfa6aba --- /dev/null +++ b/backend/go/trellis2cpp/.gitignore @@ -0,0 +1,6 @@ +package/ +sources/ +.cache/ +build-*/ +variants/ +trellis2cpp diff --git a/backend/go/trellis2cpp/Makefile b/backend/go/trellis2cpp/Makefile new file mode 100644 index 000000000..c20a991a5 --- /dev/null +++ b/backend/go/trellis2cpp/Makefile @@ -0,0 +1,132 @@ +CMAKE_ARGS?= +BUILD_TYPE?= +NATIVE?=false + +CURRENT_DIR=$(abspath ./) +GOCMD?=go +GO_TAGS?= +JOBS?=$(shell nproc --ignore=1) + +# trellis2.cpp — C++/ggml port of Microsoft TRELLIS.2 (image -> 3D GLB). +# The ggml submodule is pinned by trellis2cpp's .gitmodules and fetched via +# --recursive. The commit pin lives here so bump_deps.yaml can update it. +TRELLIS2CPP_REPO?=https://github.com/localai-org/trellis2cpp +TRELLIS2CPP_VERSION?=73dfbe5dfc2cbefd0950853718086556c6d9b043 + +# libtrellis2 + ggml as shared libraries; no example/test binaries. +CMAKE_ARGS+=-DCMAKE_BUILD_TYPE=Release +CMAKE_ARGS+=-DBUILD_SHARED_LIBS=ON +CMAKE_ARGS+=-DTRELLIS2_BUILD_EXAMPLES=OFF +CMAKE_ARGS+=-DTRELLIS2_BUILD_TESTS=OFF +# Print remeshing is part of trellis2cpp's ABI, so upstream owns the tested +# CGAL/Boost versions, checksums, fetch logic, and update automation. LocalAI +# only opts into that dependency set and pins the trellis2cpp commit above. +CMAKE_ARGS+=-DTRELLIS2_FETCH_PRINT_REMESH_DEPS=ON +CMAKE_ARGS+=-DTRELLIS2_PRINT_REMESH_DEPS_DIR=$(CURRENT_DIR)/sources/print-remesh-deps + +ifeq ($(NATIVE),false) + CMAKE_ARGS+=-DGGML_NATIVE=OFF +endif + +ifeq ($(BUILD_TYPE),cublas) + CMAKE_ARGS+=-DGGML_CUDA=ON +else ifeq ($(BUILD_TYPE),vulkan) + CMAKE_ARGS+=-DGGML_VULKAN=ON +else ifeq ($(BUILD_TYPE),hipblas) + ROCM_HOME ?= /opt/rocm + ROCM_PATH ?= /opt/rocm + export CXX=$(ROCM_HOME)/llvm/bin/clang++ + export CC=$(ROCM_HOME)/llvm/bin/clang + AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201 + CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) +else ifeq ($(OS),Darwin) + ifneq ($(BUILD_TYPE),metal) + CMAKE_ARGS+=-DTRELLIS2_METAL=OFF -DGGML_METAL=OFF + else + # trellis2cpp turns on GGML_METAL(+EMBED_LIBRARY) itself when + # TRELLIS2_METAL is enabled on Apple platforms. + CMAKE_ARGS+=-DTRELLIS2_METAL=ON + endif + # Dependent libggml*.dylib resolve next to libtrellis2.dylib even + # without DYLD_LIBRARY_PATH being exported. + CMAKE_ARGS+=-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH=@loader_path +endif + +ifeq ($(BUILD_TYPE),sycl_f16) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DGGML_SYCL_F16=ON +endif + +ifeq ($(BUILD_TYPE),sycl_f32) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx +endif + +sources/trellis2cpp: + git clone --recursive $(TRELLIS2CPP_REPO) sources/trellis2cpp && \ + cd sources/trellis2cpp && \ + git checkout $(TRELLIS2CPP_VERSION) && \ + git submodule update --init --recursive --depth 1 --single-branch + +# Detect OS +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +# The AVX variants are x86-only. ARM64 images use the portable fallback while +# still enabling the selected GPU backend (Vulkan/CUDA) through CMAKE_ARGS. +ifeq ($(UNAME_S),Linux) + ifneq (,$(filter x86_64 amd64,$(UNAME_M))) + VARIANTS = avx avx2 avx512 fallback + else + VARIANTS = fallback + endif +else + # On non-Linux (e.g., Darwin), build only the fallback variant + VARIANTS = fallback +endif +VARIANT_TARGETS = $(foreach v,$(VARIANTS),variants/$(v)/.built) + +VARIANT_FLAGS_avx = -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off +VARIANT_FLAGS_avx2 = -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on +VARIANT_FLAGS_avx512 = -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on +VARIANT_FLAGS_fallback = -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off + +# libtrellis2 links libggml/libggml-base/libggml-cpu (+ the GPU backend) by +# soname, and those sonames collide across SIMD variants — so each variant +# lives in its own directory and run.sh selects one via LD_LIBRARY_PATH, +# unlike stablediffusion-ggml's flat renamed-.so scheme. +variants/%/.built: sources/trellis2cpp + rm -rf build-$* variants/$* + mkdir -p build-$* variants/$* + cd build-$* && cmake ../sources/trellis2cpp $(CMAKE_ARGS) $(VARIANT_FLAGS_$*) && \ + cmake --build . --config Release -j$(JOBS) + @for f in build-$*/libtrellis2.so build-$*/libtrellis2.dylib; do \ + if [ -e $$f ]; then cp -a $$f variants/$*/; fi; done + find build-$*/ggml \( -name 'libggml*.so*' -o -name 'libggml*.dylib' \) -exec cp -a {} variants/$*/ \; + rm -rf build-$* + touch $@ + +trellis2cpp: main.go trellis2.go $(VARIANT_TARGETS) + CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o trellis2cpp ./ + +package: trellis2cpp + bash package.sh + +build: package + +clean: purge + rm -rf variants trellis2cpp package sources + +purge: + rm -rf build-* + +# Weight-free by construction: pure-Go unit tests over model-path resolution, +# validation, and request-parameter mapping. The multi-GB GGUF weights are +# never downloaded in CI; end-to-end generation is exercised manually. +test: + $(GOCMD) test -v ./... + +all: trellis2cpp package diff --git a/backend/go/trellis2cpp/glb_mesh.go b/backend/go/trellis2cpp/glb_mesh.go new file mode 100644 index 000000000..144c9243a --- /dev/null +++ b/backend/go/trellis2cpp/glb_mesh.go @@ -0,0 +1,252 @@ +package main + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "math" +) + +const ( + glbMagic = 0x46546c67 + glbJSONChunk = 0x4e4f534a + glbBINChunk = 0x004e4942 +) + +type glbAccessor struct { + BufferView int `json:"bufferView"` + ByteOffset int `json:"byteOffset"` + ComponentType int `json:"componentType"` + Count int `json:"count"` + Type string `json:"type"` + Normalized bool `json:"normalized"` +} + +type glbBufferView struct { + Buffer int `json:"buffer"` + ByteOffset int `json:"byteOffset"` + ByteLength int `json:"byteLength"` + ByteStride int `json:"byteStride"` +} + +type glbPrimitive struct { + Attributes map[string]int `json:"attributes"` + Indices *int `json:"indices"` +} + +type glbDocument struct { + Accessors []glbAccessor `json:"accessors"` + BufferViews []glbBufferView `json:"bufferViews"` + Meshes []struct { + Primitives []glbPrimitive `json:"primitives"` + } `json:"meshes"` +} + +type glbVertexMesh struct { + verts []float32 + tris []int32 + pbr []float32 +} + +func glbLayout(componentType int, accessorType string) (componentBytes, components int, err error) { + switch componentType { + case 5121: + componentBytes = 1 + case 5123: + componentBytes = 2 + case 5125, 5126: + componentBytes = 4 + default: + return 0, 0, fmt.Errorf("unsupported GLB component type %d", componentType) + } + switch accessorType { + case "SCALAR": + components = 1 + case "VEC2": + components = 2 + case "VEC3": + components = 3 + case "VEC4": + components = 4 + default: + return 0, 0, fmt.Errorf("unsupported GLB accessor type %q", accessorType) + } + return componentBytes, components, nil +} + +func glbAccessorData(doc *glbDocument, binChunk []byte, index int) (glbAccessor, []byte, error) { + if index < 0 || index >= len(doc.Accessors) { + return glbAccessor{}, nil, fmt.Errorf("missing GLB accessor %d", index) + } + a := doc.Accessors[index] + if a.BufferView < 0 || a.BufferView >= len(doc.BufferViews) { + return glbAccessor{}, nil, fmt.Errorf("missing GLB buffer view %d", a.BufferView) + } + v := doc.BufferViews[a.BufferView] + if v.Buffer != 0 || v.ByteStride != 0 { + return glbAccessor{}, nil, fmt.Errorf("interleaved or external GLB buffers are unsupported") + } + componentBytes, components, err := glbLayout(a.ComponentType, a.Type) + if err != nil { + return glbAccessor{}, nil, err + } + if a.Count <= 0 || a.Count > math.MaxInt/(componentBytes*components) { + return glbAccessor{}, nil, fmt.Errorf("invalid GLB accessor count %d", a.Count) + } + length := a.Count * componentBytes * components + if v.ByteOffset < 0 || v.ByteLength < 0 || a.ByteOffset < 0 || + a.ByteOffset > v.ByteLength || length > v.ByteLength-a.ByteOffset || + length > len(binChunk) || v.ByteOffset > len(binChunk)-length-a.ByteOffset { + return glbAccessor{}, nil, fmt.Errorf("GLB accessor %d is outside the BIN chunk", index) + } + start := v.ByteOffset + a.ByteOffset + return a, binChunk[start : start+length], nil +} + +// parseVertexGLB reads the dense vertex-PBR form emitted by trellis2.cpp. GLB +// coordinates and linear COLOR_0 values are converted back to the native +// trellis coordinate/material convention before CGAL remeshing and rebaking. +func parseVertexGLB(data []byte) (*glbVertexMesh, error) { + if len(data) < 20 || binary.LittleEndian.Uint32(data[0:4]) != glbMagic { + return nil, fmt.Errorf("input is not a GLB file") + } + if binary.LittleEndian.Uint32(data[4:8]) != 2 { + return nil, fmt.Errorf("unsupported GLB version") + } + total := int(binary.LittleEndian.Uint32(data[8:12])) + if total != len(data) { + return nil, fmt.Errorf("invalid GLB length") + } + + var jsonChunk, binChunk []byte + for offset := 12; offset <= len(data)-8; { + length := int(binary.LittleEndian.Uint32(data[offset : offset+4])) + chunkType := binary.LittleEndian.Uint32(data[offset+4 : offset+8]) + start := offset + 8 + if length < 0 || start > len(data)-length { + return nil, fmt.Errorf("invalid GLB chunk length") + } + switch chunkType { + case glbJSONChunk: + if jsonChunk == nil { + jsonChunk = data[start : start+length] + } + case glbBINChunk: + if binChunk == nil { + binChunk = data[start : start+length] + } + } + offset = start + length + } + if jsonChunk == nil || binChunk == nil { + return nil, fmt.Errorf("GLB must contain JSON and BIN chunks") + } + + var doc glbDocument + if err := json.Unmarshal(jsonChunk, &doc); err != nil { + return nil, fmt.Errorf("parsing GLB JSON: %w", err) + } + if len(doc.Meshes) != 1 || len(doc.Meshes[0].Primitives) != 1 { + return nil, fmt.Errorf("GLB must contain one mesh primitive") + } + primitive := doc.Meshes[0].Primitives[0] + positionIndex, ok := primitive.Attributes["POSITION"] + if !ok { + return nil, fmt.Errorf("GLB mesh has no POSITION attribute") + } + position, positionData, err := glbAccessorData(&doc, binChunk, positionIndex) + if err != nil { + return nil, err + } + if position.ComponentType != 5126 || position.Type != "VEC3" { + return nil, fmt.Errorf("GLB POSITION must be float32 VEC3") + } + + mesh := &glbVertexMesh{verts: make([]float32, position.Count*3)} + for i := 0; i < position.Count; i++ { + x := math.Float32frombits(binary.LittleEndian.Uint32(positionData[(i*3)*4:])) + y := math.Float32frombits(binary.LittleEndian.Uint32(positionData[(i*3+1)*4:])) + z := math.Float32frombits(binary.LittleEndian.Uint32(positionData[(i*3+2)*4:])) + if math.IsNaN(float64(x)) || math.IsNaN(float64(y)) || math.IsNaN(float64(z)) || + math.IsInf(float64(x), 0) || math.IsInf(float64(y), 0) || math.IsInf(float64(z), 0) { + return nil, fmt.Errorf("GLB POSITION contains a non-finite value") + } + mesh.verts[i*3] = x + mesh.verts[i*3+1] = -z + mesh.verts[i*3+2] = y + } + + if primitive.Indices == nil { + if position.Count%3 != 0 { + return nil, fmt.Errorf("unindexed GLB vertex count is not divisible by three") + } + mesh.tris = make([]int32, position.Count) + for i := range mesh.tris { + mesh.tris[i] = int32(i) + } + } else { + indices, indexData, err := glbAccessorData(&doc, binChunk, *primitive.Indices) + if err != nil { + return nil, err + } + if indices.Type != "SCALAR" || indices.Count%3 != 0 || (indices.ComponentType != 5123 && indices.ComponentType != 5125) { + return nil, fmt.Errorf("GLB indices must be uint16/uint32 triangles") + } + mesh.tris = make([]int32, indices.Count) + for i := range mesh.tris { + var value uint32 + if indices.ComponentType == 5123 { + value = uint32(binary.LittleEndian.Uint16(indexData[i*2:])) + } else { + value = binary.LittleEndian.Uint32(indexData[i*4:]) + } + if value >= uint32(position.Count) || value > math.MaxInt32 { + return nil, fmt.Errorf("GLB index %d is outside the vertex buffer", value) + } + mesh.tris[i] = int32(value) + } + } + + colorIndex, hasColor := primitive.Attributes["COLOR_0"] + if !hasColor { + return mesh, nil + } + color, colorData, err := glbAccessorData(&doc, binChunk, colorIndex) + if err != nil { + return nil, err + } + if color.ComponentType != 5123 || color.Type != "VEC4" || !color.Normalized || color.Count != position.Count { + return nil, fmt.Errorf("GLB COLOR_0 must be normalized uint16 VEC4 aligned with POSITION") + } + metalRoughIndex, hasMetalRough := primitive.Attributes["_METALLIC_ROUGHNESS"] + var metalRoughData []byte + if hasMetalRough { + metalRough, data, err := glbAccessorData(&doc, binChunk, metalRoughIndex) + if err != nil { + return nil, err + } + if metalRough.ComponentType != 5121 || metalRough.Type != "VEC2" || !metalRough.Normalized || metalRough.Count != position.Count { + return nil, fmt.Errorf("GLB _METALLIC_ROUGHNESS must be normalized uint8 VEC2 aligned with POSITION") + } + metalRoughData = data + } + + mesh.pbr = make([]float32, position.Count*6) + for i := 0; i < position.Count; i++ { + for channel := 0; channel < 3; channel++ { + linear := float32(binary.LittleEndian.Uint16(colorData[(i*4+channel)*2:])) / 65535 + if linear <= 0.0031308 { + mesh.pbr[i*6+channel] = linear * 12.92 + } else { + mesh.pbr[i*6+channel] = 1.055*float32(math.Pow(float64(linear), 1.0/2.4)) - 0.055 + } + } + mesh.pbr[i*6+5] = float32(binary.LittleEndian.Uint16(colorData[(i*4+3)*2:])) / 65535 + mesh.pbr[i*6+4] = 0.6 + if hasMetalRough { + mesh.pbr[i*6+3] = float32(metalRoughData[i*2]) / 255 + mesh.pbr[i*6+4] = float32(metalRoughData[i*2+1]) / 255 + } + } + return mesh, nil +} diff --git a/backend/go/trellis2cpp/glb_mesh_test.go b/backend/go/trellis2cpp/glb_mesh_test.go new file mode 100644 index 000000000..8ba4f9264 --- /dev/null +++ b/backend/go/trellis2cpp/glb_mesh_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "encoding/binary" + "fmt" + "math" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func tinyVertexGLB() []byte { + bin := make([]byte, 80) + positions := []float32{1, 2, 3, 4, 5, 6, 7, 8, 9} + for i, value := range positions { + binary.LittleEndian.PutUint32(bin[i*4:], math.Float32bits(value)) + } + colors := []uint16{ + 65535, 0, 0, 65535, + 0, 65535, 0, 32768, + 0, 0, 65535, 65535, + } + for i, value := range colors { + binary.LittleEndian.PutUint16(bin[36+i*2:], value) + } + copy(bin[60:], []byte{0, 153, 64, 128, 255, 32}) + for i, value := range []uint32{0, 1, 2} { + binary.LittleEndian.PutUint32(bin[68+i*4:], value) + } + + jsonChunk := []byte(fmt.Sprintf(`{"asset":{"version":"2.0"},"meshes":[{"primitives":[{"attributes":{"POSITION":0,"COLOR_0":1,"_METALLIC_ROUGHNESS":2},"indices":3}]}],"accessors":[{"bufferView":0,"componentType":5126,"count":3,"type":"VEC3"},{"bufferView":1,"componentType":5123,"normalized":true,"count":3,"type":"VEC4"},{"bufferView":2,"componentType":5121,"normalized":true,"count":3,"type":"VEC2"},{"bufferView":3,"componentType":5125,"count":3,"type":"SCALAR"}],"bufferViews":[{"buffer":0,"byteOffset":0,"byteLength":36},{"buffer":0,"byteOffset":36,"byteLength":24},{"buffer":0,"byteOffset":60,"byteLength":6},{"buffer":0,"byteOffset":68,"byteLength":12}],"buffers":[{"byteLength":%d}]}`, len(bin))) + for len(jsonChunk)%4 != 0 { + jsonChunk = append(jsonChunk, ' ') + } + total := 12 + 8 + len(jsonChunk) + 8 + len(bin) + glb := make([]byte, total) + binary.LittleEndian.PutUint32(glb[0:], glbMagic) + binary.LittleEndian.PutUint32(glb[4:], 2) + binary.LittleEndian.PutUint32(glb[8:], uint32(total)) + binary.LittleEndian.PutUint32(glb[12:], uint32(len(jsonChunk))) + binary.LittleEndian.PutUint32(glb[16:], glbJSONChunk) + copy(glb[20:], jsonChunk) + binHeader := 20 + len(jsonChunk) + binary.LittleEndian.PutUint32(glb[binHeader:], uint32(len(bin))) + binary.LittleEndian.PutUint32(glb[binHeader+4:], glbBINChunk) + copy(glb[binHeader+8:], bin) + return glb +} + +var _ = Describe("vertex GLB parsing for print remeshing", func() { + It("restores trellis coordinates, topology, and PBR values", func() { + mesh, err := parseVertexGLB(tinyVertexGLB()) + Expect(err).NotTo(HaveOccurred()) + Expect(mesh.verts).To(Equal([]float32{1, -3, 2, 4, -6, 5, 7, -9, 8})) + Expect(mesh.tris).To(Equal([]int32{0, 1, 2})) + Expect(mesh.pbr).To(HaveLen(18)) + Expect(mesh.pbr[0]).To(BeNumerically("~", 1, 1e-5)) + Expect(mesh.pbr[3]).To(BeNumerically("~", 0, 1e-5)) + Expect(mesh.pbr[4]).To(BeNumerically("~", 0.6, 0.01)) + Expect(mesh.pbr[11]).To(BeNumerically("~", 32768.0/65535.0, 1e-5)) + }) + + It("rejects indices outside the source vertex buffer", func() { + glb := tinyVertexGLB() + binary.LittleEndian.PutUint32(glb[len(glb)-12:], 3) + _, err := parseVertexGLB(glb) + Expect(err).To(MatchError(ContainSubstring("outside the vertex buffer"))) + }) + + It("rejects non-GLB input", func() { + _, err := parseVertexGLB([]byte("not a mesh")) + Expect(err).To(MatchError("input is not a GLB file")) + }) +}) diff --git a/backend/go/trellis2cpp/main.go b/backend/go/trellis2cpp/main.go new file mode 100644 index 000000000..14938bdd6 --- /dev/null +++ b/backend/go/trellis2cpp/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + + "github.com/ebitengine/purego" + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func registerLibFuncs(lib uintptr) { + registerLibFuncsWith(func(fptr any, name string) { + purego.RegisterLibFunc(fptr, lib, name) + }) +} + +func main() { + // run.sh selects the CPU-variant directory and points TRELLIS2_LIBRARY at it. + libName := os.Getenv("TRELLIS2_LIBRARY") + if libName == "" { + if runtime.GOOS == "darwin" { + libName = "./variants/fallback/libtrellis2.dylib" + } else { + libName = "./variants/fallback/libtrellis2.so" + } + } + + lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + panic(err) + } + + registerLibFuncs(lib) + + if got := t2AbiVersion(); got != abiVersion { + panic(fmt.Sprintf("trellis2 ABI mismatch: library reports %d, backend built for %d", got, abiVersion)) + } + + flag.Parse() + + if err := grpc.StartServer(*addr, &Trellis2{}); err != nil { + panic(err) + } +} diff --git a/backend/go/trellis2cpp/package.sh b/backend/go/trellis2cpp/package.sh new file mode 100755 index 000000000..20d7dedc2 --- /dev/null +++ b/backend/go/trellis2cpp/package.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Script to copy the appropriate libraries based on architecture +# This script is used in the final stage of the Dockerfile + +set -e + +CURDIR=$(dirname "$(realpath $0)") +REPO_ROOT="${CURDIR}/../../.." + +# Create lib directory +mkdir -p $CURDIR/package/lib + +# Each CPU variant keeps its libtrellis2 + libggml* set in its own directory +# (their sonames collide across variants); run.sh selects one at startup. +cp -a $CURDIR/variants $CURDIR/package/ +cp -avf $CURDIR/trellis2cpp $CURDIR/package/ +cp -fv $CURDIR/run.sh $CURDIR/package/ + +# Detect architecture and copy appropriate libraries +if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then + # x86_64 architecture + echo "Detected x86_64 architecture, copying x86_64 libraries..." + cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so + cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then + # ARM64 architecture + echo "Detected ARM64 architecture, copying ARM64 libraries..." + cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so + cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ $(uname -s) = "Darwin" ]; then + echo "Detected Darwin" +else + echo "Error: Could not detect architecture" + exit 1 +fi + +# Package GPU libraries based on BUILD_TYPE +# The GPU library packaging script will detect BUILD_TYPE and copy appropriate GPU libraries +GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh" +if [ -f "$GPU_LIB_SCRIPT" ]; then + echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..." + source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib" + package_gpu_libs +fi + +echo "Packaging completed successfully" +ls -liah $CURDIR/package/ +ls -liah $CURDIR/package/lib/ diff --git a/backend/go/trellis2cpp/run.sh b/backend/go/trellis2cpp/run.sh new file mode 100755 index 000000000..78f5ec362 --- /dev/null +++ b/backend/go/trellis2cpp/run.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -ex + +# Get the absolute current dir where the script is located +CURDIR=$(dirname "$(realpath "$0")") + +cd / + +echo "CPU info:" +if [ "$(uname)" != "Darwin" ]; then + grep -e "model\sname" /proc/cpuinfo | head -1 + grep -e "flags" /proc/cpuinfo | head -1 +fi + +# Each variant directory bundles libtrellis2 plus its libggml* set (the ggml +# sonames collide across SIMD variants, so they can't share one directory). +VARIANT=fallback + +if [ "$(uname)" = "Darwin" ]; then + LIBRARY="$CURDIR/variants/$VARIANT/libtrellis2.dylib" + if [ ! -e "$LIBRARY" ]; then + LIBRARY="$CURDIR/variants/$VARIANT/libtrellis2.so" + fi + export DYLD_LIBRARY_PATH="$CURDIR/variants/$VARIANT:$CURDIR/lib:$DYLD_LIBRARY_PATH" +else + if grep -q -e "\savx\s" /proc/cpuinfo ; then + echo "CPU: AVX found OK" + if [ -d "$CURDIR/variants/avx" ]; then + VARIANT=avx + fi + fi + + if grep -q -e "\savx2\s" /proc/cpuinfo ; then + echo "CPU: AVX2 found OK" + if [ -d "$CURDIR/variants/avx2" ]; then + VARIANT=avx2 + fi + fi + + if grep -q -e "\savx512f\s" /proc/cpuinfo ; then + echo "CPU: AVX512F found OK" + if [ -d "$CURDIR/variants/avx512" ]; then + VARIANT=avx512 + fi + fi + + LIBRARY="$CURDIR/variants/$VARIANT/libtrellis2.so" + export LD_LIBRARY_PATH="$CURDIR/variants/$VARIANT:$CURDIR/lib:$LD_LIBRARY_PATH" +fi + +export TRELLIS2_LIBRARY=$LIBRARY + +# If there is a lib/ld.so, use it +if [ -f "$CURDIR"/lib/ld.so ]; then + echo "Using lib/ld.so" + echo "Using library: $LIBRARY" + exec "$CURDIR"/lib/ld.so "$CURDIR"/trellis2cpp "$@" +fi + +echo "Using library: $LIBRARY" +exec "$CURDIR"/trellis2cpp "$@" diff --git a/backend/go/trellis2cpp/trellis2.go b/backend/go/trellis2cpp/trellis2.go new file mode 100644 index 000000000..addbf1bb0 --- /dev/null +++ b/backend/go/trellis2cpp/trellis2.go @@ -0,0 +1,511 @@ +package main + +// trellis2.go — purego bindings to libtrellis2's flat C ABI (trellis2_capi.h) +// plus the LocalAI backend implementation. Adapted from the upstream demo +// server's engine.go; the t2_abi_version binding guards against header/library +// drift. + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "unsafe" + + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/utils" +) + +const abiVersion = 11 + +// Pipeline types (enum t2_pipeline_type) and background modes +// (enum t2_background_mode). +const ( + pipeAuto = 0 + pipeCoarse = 1 + pipe512 = 2 + pipe1024 = 3 + + backgroundAuto = 0 + backgroundKeep = 1 + backgroundBlack = 2 + backgroundWhite = 3 +) + +// The bindings use typed pointers (*float32/*int32/*byte) rather than uintptr +// for C-owned buffers so no uintptr->unsafe.Pointer conversions are needed; +// only the opaque t2_pipeline / t2_mesh_result handles stay uintptr. +var ( + t2AbiVersion func() int32 + t2PipelineLoad func(dino, ssFlow, ssDec, slatFlow, slatFlowHR, shapeDec, + shapeEnc, texDec, texFlow, texFlowHR string, + flags int32, err *byte, errLen int32) uintptr + t2PipelineFree func(p uintptr) + t2PipelineBackend func(p uintptr) string + t2PipelineCaps func(p uintptr) int32 + t2Generate func(p uintptr, img *byte, imgLen int32, + pipelineType, backgroundMode int32, seed uint64, steps int32, + guidance float32, textureSteps int32, + progress, user, preview, previewUser uintptr, + err *byte, errLen int32) uintptr + t2MeshNVerts func(r uintptr) int32 + t2MeshNTris func(r uintptr) int32 + t2MeshVerts func(r uintptr) *float32 + t2MeshTris func(r uintptr) *int32 + t2MeshHasPBR func(r uintptr) int32 + t2MeshPBR func(r uintptr) *float32 + t2MeshFree func(r uintptr) + t2BakeGLB func(verts *float32, nv int32, tris *int32, nt int32, + pbr *float32, texSize, componentFilter int32, + outLen *int32, err *byte, errLen int32) *byte + // CGAL Alpha Wrap print remeshing — availability is fixed at library build + // time, so gate every use on t2_print_remesh_available. + t2PrintRemeshAvailable func() int32 + t2PreparePrintMesh func(verts *float32, nv int32, tris *int32, nt int32, + pbr *float32, componentFilter int32, alphaRatio, offsetRatio float32, + err *byte, errLen int32) uintptr + t2BakeProjectedGLB func(targetVerts *float32, targetNV int32, + targetTris *int32, targetNT int32, + sourceVerts *float32, sourceNV int32, + sourceTris *int32, sourceNT int32, + sourcePBR *float32, texSize, sourceComponentFilter int32, + outLen *int32, err *byte, errLen int32) *byte + t2FreeBuffer func(buf *byte) +) + +type libFunc struct { + funcPtr any + name string +} + +func registerLibFuncsWith(register func(fptr any, name string)) { + for _, lf := range []libFunc{ + {&t2AbiVersion, "t2_abi_version"}, + {&t2PipelineLoad, "t2_pipeline_load"}, + {&t2PipelineFree, "t2_pipeline_free"}, + {&t2PipelineBackend, "t2_pipeline_backend"}, + {&t2PipelineCaps, "t2_pipeline_caps"}, + {&t2Generate, "t2_generate"}, + {&t2MeshNVerts, "t2_mesh_n_verts"}, + {&t2MeshNTris, "t2_mesh_n_tris"}, + {&t2MeshVerts, "t2_mesh_verts"}, + {&t2MeshTris, "t2_mesh_tris"}, + {&t2MeshHasPBR, "t2_mesh_has_pbr"}, + {&t2MeshPBR, "t2_mesh_pbr"}, + {&t2MeshFree, "t2_mesh_free"}, + {&t2BakeGLB, "t2_bake_glb"}, + {&t2PrintRemeshAvailable, "t2_print_remesh_available"}, + {&t2PreparePrintMesh, "t2_prepare_print_mesh"}, + {&t2BakeProjectedGLB, "t2_bake_projected_glb"}, + {&t2FreeBuffer, "t2_free_buffer"}, + } { + register(lf.funcPtr, lf.name) + } +} + +// modelSet holds the resolved path for every pipeline role; optional roles are +// "" when disabled (the C side treats NULL/"" as "omit"). +type modelSet struct { + dino, ssFlow, ssDec string + slatFlow, slatFlow1024 string + shapeDec string + shapeEnc, texDec string + texSlatFlow512, texSlatFlow1024 string +} + +// role → (option key, default filename) in t2_pipeline_load argument order. +// The option keys follow the sd-ggml `*_path` convention; the default +// filenames are the ones the upstream converters emit and the demo server +// looks up, so a gallery install needs no options at all. +type modelRole struct { + key string + filename string + required bool + assign func(*modelSet, string) +} + +var modelRoles = []modelRole{ + {"dino_path", "dino_f16.gguf", true, func(s *modelSet, p string) { s.dino = p }}, + {"ss_flow_path", "ss_flow_f16.gguf", true, func(s *modelSet, p string) { s.ssFlow = p }}, + {"ss_dec_path", "ss_dec_f16.gguf", true, func(s *modelSet, p string) { s.ssDec = p }}, + {"slat_flow_path", "slat_flow_f16.gguf", false, func(s *modelSet, p string) { s.slatFlow = p }}, + {"slat_flow_1024_path", "slat_flow_1024_f16.gguf", false, func(s *modelSet, p string) { s.slatFlow1024 = p }}, + {"shape_dec_path", "shape_dec_f16.gguf", false, func(s *modelSet, p string) { s.shapeDec = p }}, + {"shape_enc_path", "shape_enc_f16.gguf", false, func(s *modelSet, p string) { s.shapeEnc = p }}, + {"tex_dec_path", "tex_dec_f16.gguf", false, func(s *modelSet, p string) { s.texDec = p }}, + {"tex_slat_flow_512_path", "tex_slat_flow_512_f16.gguf", false, func(s *modelSet, p string) { s.texSlatFlow512 = p }}, + {"tex_slat_flow_1024_path", "tex_slat_flow_1024_f16.gguf", false, func(s *modelSet, p string) { s.texSlatFlow1024 = p }}, +} + +// resolveModels maps LocalAI's model file + options onto the ten pipeline +// roles. The model file only anchors the GGUF directory; each role resolves +// to an explicit `_path` option when given, else to its default +// filename in that directory. Missing required files refuse the load (a +// backend must not capture arbitrary GGUFs — see issue #9287); missing +// optional files degrade capabilities the same way the upstream demo does. +func resolveModels(modelFile, modelPath string, options []string) (modelSet, error) { + base := modelFile + if !filepath.IsAbs(base) { + base = filepath.Join(modelPath, base) + } + ggufDir := filepath.Dir(base) + + overrides := map[string]string{} + for _, op := range options { + key, value, found := strings.Cut(op, ":") + if !found || !strings.HasSuffix(key, "_path") { + continue + } + if !filepath.IsAbs(value) { + value = filepath.Join(modelPath, value) + if err := utils.VerifyPath(value, modelPath); err != nil { + return modelSet{}, fmt.Errorf("option %s: %w", key, err) + } + } + overrides[key] = value + } + + var set modelSet + var missingRequired []string + for _, role := range modelRoles { + path, explicit := overrides[role.key] + if !explicit { + path = filepath.Join(ggufDir, role.filename) + } + if _, err := os.Stat(path); err != nil { + if explicit { + return modelSet{}, fmt.Errorf("option %s points at a missing file: %s", role.key, path) + } + if role.required { + missingRequired = append(missingRequired, role.filename) + } + path = "" + } + role.assign(&set, path) + } + if len(missingRequired) > 0 { + return modelSet{}, fmt.Errorf("not a trellis2 model set: missing required %s in %s", strings.Join(missingRequired, ", "), ggufDir) + } + + // Degradation mirrors the upstream demo: the 512 pair enables everything + // finer than coarse; texturing needs its three-model set; a textured 1024 + // cascade additionally needs the HR texture flow. + if set.slatFlow == "" || set.shapeDec == "" { + set.slatFlow, set.shapeDec = "", "" + set.slatFlow1024 = "" + set.shapeEnc, set.texDec, set.texSlatFlow512, set.texSlatFlow1024 = "", "", "", "" + return set, nil + } + if set.shapeEnc == "" || set.texDec == "" || set.texSlatFlow512 == "" { + set.shapeEnc, set.texDec, set.texSlatFlow512, set.texSlatFlow1024 = "", "", "", "" + } else if set.texSlatFlow1024 == "" { + set.slatFlow1024 = "" + } + return set, nil +} + +func pipelineForQuality(quality string) int32 { + switch quality { + case "coarse": + return pipeCoarse + case "512": + return pipe512 + case "1024": + return pipe1024 + default: + return pipeAuto + } +} + +func backgroundForMode(background string) int32 { + switch background { + case "keep": + return backgroundKeep + case "black": + return backgroundBlack + case "white": + return backgroundWhite + default: + return backgroundAuto + } +} + +func componentFilterFor(components string) int32 { + switch components { + case "tiny": + return 0 // remove only tiny islands + case "largest": + return 1 // keep the largest connected component + default: + return 2 // preserve every connected component (demo default) + } +} + +func atoiOr(s string, fallback int32) int32 { + if s == "" { + return fallback + } + n, err := strconv.Atoi(s) + if err != nil { + return fallback + } + return int32(n) +} + +func boolParam(s string) bool { + return s == "1" || strings.EqualFold(s, "true") +} + +// ratioOr parses a fraction-of-bounding-box-diagonal parameter. Out-of-range +// or unparseable values fall back rather than error, mirroring atoiOr; the +// accepted range matches what the upstream demo clamps to. +func ratioOr(s string, fallback float32) float32 { + if s == "" { + return fallback + } + f, err := strconv.ParseFloat(s, 32) + if err != nil || f < 0.00001 || f > 0.5 { + return fallback + } + return float32(f) +} + +type Trellis2 struct { + base.SingleThread + // t2_generate is not thread-safe per pipeline. The gRPC server already + // serializes calls via Locking(), but keep a local mutex too so the + // invariant doesn't depend on the transport. + mu sync.Mutex + pipeline uintptr +} + +func (t *Trellis2) Load(opts *pb.ModelOptions) error { + set, err := resolveModels(opts.ModelFile, opts.ModelPath, opts.Options) + if err != nil { + return err + } + + errBuf := make([]byte, 512) + p := t2PipelineLoad(set.dino, set.ssFlow, set.ssDec, + set.slatFlow, set.slatFlow1024, set.shapeDec, + set.shapeEnc, set.texDec, set.texSlatFlow512, set.texSlatFlow1024, + 0 /*flags*/, &errBuf[0], int32(len(errBuf))) + if p == 0 { + return fmt.Errorf("trellis2 pipeline load: %s", cstr(errBuf)) + } + + t.mu.Lock() + if t.pipeline != 0 { + t2PipelineFree(t.pipeline) + } + t.pipeline = p + t.mu.Unlock() + + fmt.Fprintf(os.Stderr, "trellis2 pipeline loaded: backend=%s caps=%#x\n", + t2PipelineBackend(p), t2PipelineCaps(p)) + return nil +} + +func (t *Trellis2) Free() error { + t.mu.Lock() + defer t.mu.Unlock() + if t.pipeline != 0 { + t2PipelineFree(t.pipeline) + t.pipeline = 0 + } + return nil +} + +func (t *Trellis2) Generate3D(opts *pb.Generate3DRequest) error { + if opts.Dst == "" { + return fmt.Errorf("dst is empty") + } + if opts.GetParams()["operation"] == "print_remesh" { + t.mu.Lock() + defer t.mu.Unlock() + return remeshGLB(opts) + } + img, err := os.ReadFile(opts.Src) + if err != nil { + return fmt.Errorf("reading conditioning image: %w", err) + } + if len(img) == 0 { + return fmt.Errorf("conditioning image is empty") + } + + seed := uint64(opts.Seed) + if opts.Seed <= 0 { + seed = rand.Uint64() + } + guidance := opts.CfgScale + if guidance <= 0 { + guidance = -1 // <0 selects the pipeline default (7.5) + } + texSize := atoiOr(opts.GetParams()["texture_size"], 0) // <=0 selects the bake default + componentFilter := componentFilterFor(opts.GetParams()["components"]) + + // Optional CGAL Alpha Wrap: wrap the generated mesh into a watertight, + // intersection-free 2-manifold for 3D printing. Ratios are fractions of + // the bounding-box diagonal; offset defaults to alpha/30 per the CGAL + // guideline the upstream demo uses. Offset is deliberately not an + // independent parameter: looser values produce puffy or degenerate wraps. + printRemesh := boolParam(opts.GetParams()["print_remesh"]) + alphaRatio := ratioOr(opts.GetParams()["alpha_ratio"], 0.005) + offsetRatio := alphaRatio / 30 + if printRemesh && t2PrintRemeshAvailable() == 0 { + return fmt.Errorf("print_remesh requested but libtrellis2 was built without CGAL Alpha Wrap") + } + + t.mu.Lock() + defer t.mu.Unlock() + if t.pipeline == 0 { + return fmt.Errorf("model not loaded") + } + + errBuf := make([]byte, 512) + r := t2Generate(t.pipeline, &img[0], int32(len(img)), + pipelineForQuality(opts.Quality), backgroundForMode(opts.Background), + seed, opts.Step, guidance, opts.TextureSteps, + 0, 0, 0, 0, // no progress/preview callbacks + &errBuf[0], int32(len(errBuf))) + if r == 0 { + return fmt.Errorf("trellis2 generate: %s", cstr(errBuf)) + } + defer t2MeshFree(r) + + nv := t2MeshNVerts(r) + nt := t2MeshNTris(r) + if nv == 0 || nt == 0 { + return fmt.Errorf("empty mesh") + } + var pbr *float32 + if t2MeshHasPBR(r) != 0 { + pbr = t2MeshPBR(r) + } + + // Bake straight from the mesh accessor buffers — they stay valid until + // t2_mesh_free, so no copies are needed. + var outLen int32 + var glb *byte + if printRemesh { + wrap := t2PreparePrintMesh(t2MeshVerts(r), nv, t2MeshTris(r), nt, pbr, + componentFilter, alphaRatio, offsetRatio, + &errBuf[0], int32(len(errBuf))) + if wrap == 0 { + return fmt.Errorf("trellis2 print remesh: %s", cstr(errBuf)) + } + defer t2MeshFree(wrap) + wnv, wnt := t2MeshNVerts(wrap), t2MeshNTris(wrap) + if wnv == 0 || wnt == 0 { + return fmt.Errorf("empty print mesh") + } + if pbr != nil { + // Wrapping creates new vertices, so the source material is + // reprojected per texel onto the wrap's UV atlas (demo handleGLB). + glb = t2BakeProjectedGLB(t2MeshVerts(wrap), wnv, t2MeshTris(wrap), wnt, + t2MeshVerts(r), nv, t2MeshTris(r), nt, pbr, + texSize, componentFilter, + &outLen, &errBuf[0], int32(len(errBuf))) + } else { + glb = t2BakeGLB(t2MeshVerts(wrap), wnv, t2MeshTris(wrap), wnt, nil, + texSize, 2, // the wrap output is already component-filtered + &outLen, &errBuf[0], int32(len(errBuf))) + } + } else { + glb = t2BakeGLB(t2MeshVerts(r), nv, t2MeshTris(r), nt, pbr, + texSize, componentFilter, + &outLen, &errBuf[0], int32(len(errBuf))) + } + if glb == nil { + return fmt.Errorf("trellis2 GLB bake: %s", cstr(errBuf)) + } + defer t2FreeBuffer(glb) + + out := make([]byte, int(outLen)) + copy(out, unsafe.Slice(glb, int(outLen))) + return os.WriteFile(opts.Dst, out, 0600) +} + +// remeshGLB applies the demo's post-generation print workflow to an existing +// dense vertex-PBR GLB. It does not touch the inference pipeline: CGAL wrapping, +// UV unwrapping, and PBR projection are CPU-only post-processing operations. +func remeshGLB(opts *pb.Generate3DRequest) error { + if opts.Src == "" { + return fmt.Errorf("src is empty") + } + if t2PrintRemeshAvailable() == 0 { + return fmt.Errorf("print remeshing is unavailable (libtrellis2 was built without CGAL Alpha Wrap)") + } + data, err := os.ReadFile(opts.Src) + if err != nil { + return fmt.Errorf("reading source GLB: %w", err) + } + mesh, err := parseVertexGLB(data) + if err != nil { + return fmt.Errorf("reading source GLB: %w", err) + } + + params := opts.GetParams() + alphaRatio := ratioOr(params["alpha_ratio"], 0.005) + offsetRatio := alphaRatio / 30 + componentFilter := componentFilterFor(params["components"]) + textureSize := atoiOr(params["texture_size"], 2048) + var sourcePBR *float32 + if len(mesh.pbr) != 0 { + sourcePBR = &mesh.pbr[0] + } + errBuf := make([]byte, 512) + wrap := t2PreparePrintMesh( + &mesh.verts[0], int32(len(mesh.verts)/3), + &mesh.tris[0], int32(len(mesh.tris)/3), + sourcePBR, componentFilter, alphaRatio, offsetRatio, + &errBuf[0], int32(len(errBuf)), + ) + if wrap == 0 { + return fmt.Errorf("trellis2 print remesh: %s", cstr(errBuf)) + } + defer t2MeshFree(wrap) + + wrappedVerts, wrappedTris := t2MeshNVerts(wrap), t2MeshNTris(wrap) + if wrappedVerts == 0 || wrappedTris == 0 { + return fmt.Errorf("empty print mesh") + } + var outLen int32 + var glb *byte + if sourcePBR != nil { + glb = t2BakeProjectedGLB( + t2MeshVerts(wrap), wrappedVerts, t2MeshTris(wrap), wrappedTris, + &mesh.verts[0], int32(len(mesh.verts)/3), + &mesh.tris[0], int32(len(mesh.tris)/3), sourcePBR, + int32(textureSize), componentFilter, + &outLen, &errBuf[0], int32(len(errBuf)), + ) + } else { + glb = t2BakeGLB( + t2MeshVerts(wrap), wrappedVerts, t2MeshTris(wrap), wrappedTris, + nil, int32(textureSize), 2, + &outLen, &errBuf[0], int32(len(errBuf)), + ) + } + if glb == nil || outLen <= 0 { + return fmt.Errorf("trellis2 GLB bake: %s", cstr(errBuf)) + } + defer t2FreeBuffer(glb) + + out := make([]byte, int(outLen)) + copy(out, unsafe.Slice(glb, int(outLen))) + return os.WriteFile(opts.Dst, out, 0o600) +} + +func cstr(b []byte) string { + for i, c := range b { + if c == 0 { + return string(b[:i]) + } + } + return string(b) +} diff --git a/backend/go/trellis2cpp/trellis2_test.go b/backend/go/trellis2cpp/trellis2_test.go new file mode 100644 index 000000000..d492756e4 --- /dev/null +++ b/backend/go/trellis2cpp/trellis2_test.go @@ -0,0 +1,247 @@ +package main + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func TestTrellis2Cpp(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "trellis2cpp backend suite") +} + +// touch creates empty files — resolveModels only checks existence, so the +// tests never need real GGUF weights. +func touch(dir string, names ...string) { + for _, name := range names { + Expect(os.WriteFile(filepath.Join(dir, name), nil, 0o600)).To(Succeed()) + } +} + +var requiredFiles = []string{"dino_f16.gguf", "ss_flow_f16.gguf", "ss_dec_f16.gguf"} + +var fullSet = append(append([]string{}, requiredFiles...), + "slat_flow_f16.gguf", "slat_flow_1024_f16.gguf", "shape_dec_f16.gguf", + "shape_enc_f16.gguf", "tex_dec_f16.gguf", + "tex_slat_flow_512_f16.gguf", "tex_slat_flow_1024_f16.gguf") + +var _ = Describe("resolveModels", func() { + var dir string + + BeforeEach(func() { + dir = GinkgoT().TempDir() + }) + + It("refuses a directory without the trellis2 component files", func() { + touch(dir, "some-llm.gguf") + + _, err := resolveModels("some-llm.gguf", dir, nil) + Expect(err).To(MatchError(ContainSubstring("not a trellis2 model set"))) + }) + + It("resolves every role from the full default-named set", func() { + touch(dir, fullSet...) + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + for name, path := range map[string]string{ + "dino": set.dino, + "ss_flow": set.ssFlow, + "ss_dec": set.ssDec, + "slat_flow": set.slatFlow, + "slat_flow_1024": set.slatFlow1024, + "shape_dec": set.shapeDec, + "shape_enc": set.shapeEnc, + "tex_dec": set.texDec, + "tex_slat_flow_512": set.texSlatFlow512, + "tex_slat_flow_1024": set.texSlatFlow1024, + } { + Expect(path).NotTo(BeEmpty(), "role %s", name) + } + }) + + It("degrades to coarse-only without the 512 pair, even when texture files exist", func() { + touch(dir, requiredFiles...) + touch(dir, "shape_enc_f16.gguf", "tex_dec_f16.gguf", "tex_slat_flow_512_f16.gguf", "slat_flow_1024_f16.gguf") + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(set.slatFlow).To(BeEmpty()) + Expect(set.shapeDec).To(BeEmpty()) + Expect(set.slatFlow1024).To(BeEmpty()) + Expect(set.shapeEnc).To(BeEmpty()) + Expect(set.texDec).To(BeEmpty()) + Expect(set.texSlatFlow512).To(BeEmpty()) + Expect(set.texSlatFlow1024).To(BeEmpty()) + }) + + It("disables texturing but keeps fine geometry when the texture set is incomplete", func() { + touch(dir, requiredFiles...) + touch(dir, "slat_flow_f16.gguf", "slat_flow_1024_f16.gguf", "shape_dec_f16.gguf", "tex_dec_f16.gguf") + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(set.slatFlow).NotTo(BeEmpty()) + Expect(set.shapeDec).NotTo(BeEmpty()) + Expect(set.slatFlow1024).NotTo(BeEmpty()) + Expect(set.shapeEnc).To(BeEmpty()) + Expect(set.texDec).To(BeEmpty()) + Expect(set.texSlatFlow512).To(BeEmpty()) + Expect(set.texSlatFlow1024).To(BeEmpty()) + }) + + It("drops the 1024 cascade when texturing lacks the HR texture flow", func() { + touch(dir, fullSet...) + Expect(os.Remove(filepath.Join(dir, "tex_slat_flow_1024_f16.gguf"))).To(Succeed()) + + set, err := resolveModels("ss_flow_f16.gguf", dir, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(set.slatFlow1024).To(BeEmpty()) + Expect(set.shapeEnc).NotTo(BeEmpty()) + Expect(set.texDec).NotTo(BeEmpty()) + Expect(set.texSlatFlow512).NotTo(BeEmpty()) + }) + + It("honors explicit *_path option overrides", func() { + touch(dir, fullSet...) + custom := filepath.Join(dir, "custom") + Expect(os.Mkdir(custom, 0o750)).To(Succeed()) + touch(custom, "my-dino.gguf") + + set, err := resolveModels("ss_flow_f16.gguf", dir, []string{"dino_path:custom/my-dino.gguf"}) + Expect(err).NotTo(HaveOccurred()) + Expect(set.dino).To(Equal(filepath.Join(custom, "my-dino.gguf"))) + }) + + It("fails when an explicitly configured file is missing", func() { + touch(dir, fullSet...) + + _, err := resolveModels("ss_flow_f16.gguf", dir, []string{"tex_dec_path:nope.gguf"}) + Expect(err).To(MatchError(ContainSubstring("missing file"))) + }) + + It("rejects option paths escaping the model directory", func() { + touch(dir, fullSet...) + + _, err := resolveModels("ss_flow_f16.gguf", dir, []string{"dino_path:../outside.gguf"}) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = DescribeTable("request parameter mapping", + func(got, want int32) { + Expect(got).To(Equal(want)) + }, + Entry("quality empty", pipelineForQuality(""), int32(pipeAuto)), + Entry("quality auto", pipelineForQuality("auto"), int32(pipeAuto)), + Entry("quality coarse", pipelineForQuality("coarse"), int32(pipeCoarse)), + Entry("quality 512", pipelineForQuality("512"), int32(pipe512)), + Entry("quality 1024", pipelineForQuality("1024"), int32(pipe1024)), + Entry("background empty", backgroundForMode(""), int32(backgroundAuto)), + Entry("background auto", backgroundForMode("auto"), int32(backgroundAuto)), + Entry("background keep", backgroundForMode("keep"), int32(backgroundKeep)), + Entry("background black", backgroundForMode("black"), int32(backgroundBlack)), + Entry("background white", backgroundForMode("white"), int32(backgroundWhite)), + Entry("components default", componentFilterFor(""), int32(2)), + Entry("components all", componentFilterFor("all"), int32(2)), + Entry("components largest", componentFilterFor("largest"), int32(1)), + Entry("components tiny", componentFilterFor("tiny"), int32(0)), + Entry("atoi empty", atoiOr("", 0), int32(0)), + Entry("atoi value", atoiOr("2048", 0), int32(2048)), + Entry("atoi junk", atoiOr("junk", 7), int32(7)), +) + +var _ = Describe("print remesh parameters", func() { + It("parses the print_remesh toggle", func() { + Expect(boolParam("1")).To(BeTrue()) + Expect(boolParam("true")).To(BeTrue()) + Expect(boolParam("TRUE")).To(BeTrue()) + Expect(boolParam("")).To(BeFalse()) + Expect(boolParam("0")).To(BeFalse()) + Expect(boolParam("no")).To(BeFalse()) + }) + + It("parses ratios and clamps junk to the fallback", func() { + Expect(ratioOr("", 0.005)).To(BeNumerically("~", 0.005, 1e-6)) + Expect(ratioOr("0.01", 0.005)).To(BeNumerically("~", 0.01, 1e-6)) + Expect(ratioOr("junk", 0.005)).To(BeNumerically("~", 0.005, 1e-6)) + Expect(ratioOr("-1", 0.005)).To(BeNumerically("~", 0.005, 1e-6)) + Expect(ratioOr("0.9", 0.005)).To(BeNumerically("~", 0.005, 1e-6), "above the demo's 50% cap") + }) +}) + +var _ = Describe("packaged backend", func() { + It("starts and answers Health without loading model weights", func() { + runScript := os.Getenv("TRELLIS2CPP_SMOKE_RUN") + if runScript == "" { + runScript = filepath.Join("package", "run.sh") + } + if _, err := os.Stat(runScript); os.IsNotExist(err) { + Skip("packaged backend is not present; run make before the smoke test") + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + addr := listener.Addr().String() + Expect(listener.Close()).To(Succeed()) + + cmd := exec.Command("bash", runScript, "--addr="+addr) + cmd.Stdout = GinkgoWriter + cmd.Stderr = GinkgoWriter + Expect(cmd.Start()).To(Succeed()) + processDone := make(chan error, 1) + go func() { processDone <- cmd.Wait() }() + processExited := false + DeferCleanup(func() { + if cmd.Process != nil && !processExited { + _ = cmd.Process.Kill() + <-processDone + } + }) + + Eventually(func() error { + select { + case err := <-processDone: + processExited = true + if err != nil { + return StopTrying("backend exited before Health succeeded").Wrap(err) + } + return StopTrying("backend exited before Health succeeded") + default: + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + conn, err := grpc.DialContext(ctx, addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + ) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + + reply, err := pb.NewBackendClient(conn).Health(ctx, &pb.HealthMessage{}) + if err != nil { + return err + } + if string(reply.GetMessage()) != "OK" { + return fmt.Errorf("unexpected Health reply %q", reply.GetMessage()) + } + return nil + }, 30*time.Second, 200*time.Millisecond).Should(Succeed()) + }) +}) diff --git a/backend/go/valkey-store/Makefile b/backend/go/valkey-store/Makefile new file mode 100644 index 000000000..0d7ddb327 --- /dev/null +++ b/backend/go/valkey-store/Makefile @@ -0,0 +1,17 @@ +GOCMD=go + +valkey-store: + CGO_ENABLED=0 $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o valkey-store ./ + +package: + bash package.sh + +build: valkey-store package + +## Runs the backend's Ginkgo suite. The unit (mock) specs run without a +## container; the integration specs skip automatically unless VALKEY_ADDR is set. +test: + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./ + +clean: + rm -f valkey-store diff --git a/backend/go/valkey-store/config.go b/backend/go/valkey-store/config.go new file mode 100644 index 000000000..02c5711b4 --- /dev/null +++ b/backend/go/valkey-store/config.go @@ -0,0 +1,261 @@ +package main + +// Connection + index configuration for the Valkey-backed vector store. +// +// Configuration is read from the model config `options:` list (a repeated +// `key:value` string carried over gRPC in ModelOptions.Options) rather than +// from process-wide environment variables. Driving it from the model config is +// the LocalAI convention and, crucially, lets multiple stores each have their +// own Valkey config (a face registry on one server, a router cache on another) +// within a single LocalAI process — something a single VALKEY_* env surface +// could never express. Every default lives as a named constant below — no +// magic literals sprinkled through the store logic — so the defaults can be +// audited in one place and referenced by the unit tests. +// +// Example model YAML: +// +// name: my-vector-store +// backend: valkey-store +// options: +// - addr:valkey.internal:6379 +// - index_algo:HNSW +// - distance_metric:COSINE + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" +) + +const ( + // _defaultAddr is the single-node Valkey address used when the `addr` + // option is unset. Matches the port Phase 0 reserved for integration tests. + _defaultAddr = "localhost:6379" + + // _defaultClientName is mandatory: every connection identifies itself with + // this name so operators can spot LocalAI's traffic via CLIENT LIST. It is + // always set on the client, even if the operator clears the client_name option. + _defaultClientName = "localai-valkey-store" + + // _defaultIndexAlgo is FLAT (exact brute-force KNN) to preserve parity with + // local-store's linear scan and keep the exact-cosine test expectations. + _defaultIndexAlgo = indexAlgoFlat + + // _defaultDistanceMetric is COSINE so similarities match local-store + // (sim = 1 - cosine_distance). L2/IP are opt-in. + _defaultDistanceMetric = distanceCosine + + // HNSW graph defaults (only used when index_algo=HNSW). Values follow + // the Valkey Search documented defaults. + _defaultHNSWM = 16 + _defaultHNSWEFConstruction = 200 + _defaultHNSWEFRuntime = 10 + + // _defaultRequestTimeoutMS bounds every command. We deliberately do NOT rely + // on the client's built-in write timeout: index back-fill or a slow KNN can + // exceed a short default, so we thread this explicit deadline into every + // command context. + _defaultRequestTimeoutMS = 5000 + + // Valkey Search index algorithms. + indexAlgoFlat = "FLAT" + indexAlgoHNSW = "HNSW" + + // Supported distance metrics. + distanceCosine = "COSINE" + distanceL2 = "L2" + distanceIP = "IP" + + // Option keys recognised in the model config `options:` list. They mirror + // the previous VALKEY_* env var names without the prefix and lower-cased, so + // operators migrating a config have an obvious 1:1 mapping. + optAddr = "addr" + optUsername = "username" + optPassword = "password" + optUsernameEnv = "username_env" + optPasswordEnv = "password_env" + optTLS = "tls" + optTLSSkipVerify = "tls_skip_verify" + optTLSCACert = "tls_ca_cert" + optClientName = "client_name" + optDB = "db" + optIndexAlgo = "index_algo" + optDistanceMetric = "distance_metric" + optHNSWM = "hnsw_m" + optHNSWEFConstruction = "hnsw_ef_construction" + optHNSWEFRuntime = "hnsw_ef_runtime" + optRequestTimeoutMS = "request_timeout_ms" +) + +// hnswParams holds the HNSW-only tuning knobs. They are ignored unless +// IndexAlgo == indexAlgoHNSW. +type hnswParams struct { + M int + EFConstruction int + EFRuntime int +} + +// Config is the fully-resolved store configuration produced by loadConfig(). +type Config struct { + Addr string + Username string + Password string + UseTLS bool + TLSSkipVerify bool + TLSCACert string + ClientName string + DB int + IndexAlgo string + DistanceMetric string + HNSW hnswParams + RequestTimeout time.Duration +} + +// parseOptions turns the repeated `key:value` ModelOptions.Options list into a +// lookup map. The split is on the FIRST ':' via strings.Cut, so values that +// themselves contain a colon (e.g. `addr:host:6379`) are preserved intact. A +// malformed entry with no ':' is warned about and skipped rather than silently +// dropped, so an operator typo is visible in the logs. +func parseOptions(opts *pb.ModelOptions) map[string]string { + m := make(map[string]string) + if opts == nil { + return m + } + for _, o := range opts.GetOptions() { + k, v, ok := strings.Cut(o, ":") + if !ok { + xlog.Warn("valkey-store: ignoring malformed option (want key:value)", "option", o) + continue + } + m[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v) + } + return m +} + +// loadConfig resolves the store configuration from the model config options and +// returns a validated Config. It fails fast on an unknown index algorithm or +// distance metric (and on a malformed integer) so a misconfiguration surfaces +// at Load() rather than silently degrading search. +func loadConfig(opts *pb.ModelOptions) (Config, error) { + o := parseOptions(opts) + + // intOr parses an integer option, failing fast on a malformed value the same + // way an invalid index algo or distance metric does. A typo like + // `hnsw_m:1x6` must surface at Load() rather than silently degrading to the + // default and producing subtly wrong (and hard-to-diagnose) index + // behaviour. The first parse error wins and is returned below. + var parseErr error + intOr := func(key string, fallback int) int { + v, ok := o[key] + if !ok || v == "" { + return fallback + } + n, err := strconv.Atoi(v) + if err != nil { + if parseErr == nil { + parseErr = fmt.Errorf("valkey-store: invalid option %s %q: %w", key, v, err) + } + return fallback + } + return n + } + + cfg := Config{ + Addr: strOr(o, optAddr, _defaultAddr), + Username: resolveCredential(o, optUsername, optUsernameEnv), + Password: resolveCredential(o, optPassword, optPasswordEnv), + UseTLS: boolOr(o, optTLS, false), + TLSSkipVerify: boolOr(o, optTLSSkipVerify, false), + TLSCACert: o[optTLSCACert], + ClientName: strOr(o, optClientName, _defaultClientName), + DB: intOr(optDB, 0), + IndexAlgo: strings.ToUpper(strOr(o, optIndexAlgo, _defaultIndexAlgo)), + DistanceMetric: strings.ToUpper(strOr(o, optDistanceMetric, _defaultDistanceMetric)), + HNSW: hnswParams{ + M: intOr(optHNSWM, _defaultHNSWM), + EFConstruction: intOr(optHNSWEFConstruction, _defaultHNSWEFConstruction), + EFRuntime: intOr(optHNSWEFRuntime, _defaultHNSWEFRuntime), + }, + RequestTimeout: time.Duration(intOr(optRequestTimeoutMS, _defaultRequestTimeoutMS)) * time.Millisecond, + } + if parseErr != nil { + return Config{}, parseErr + } + + // ClientName is mandatory. Restore the default if the operator blanked it, + // so the connection is always identifiable. + if cfg.ClientName == "" { + cfg.ClientName = _defaultClientName + } + + if cfg.DB < 0 { + return Config{}, fmt.Errorf("valkey-store: invalid option %s %d (must be >= 0)", optDB, cfg.DB) + } + + switch cfg.IndexAlgo { + case indexAlgoFlat, indexAlgoHNSW: + default: + return Config{}, fmt.Errorf("valkey-store: invalid option %s %q (want FLAT or HNSW)", optIndexAlgo, cfg.IndexAlgo) + } + + switch cfg.DistanceMetric { + case distanceCosine, distanceL2, distanceIP: + default: + return Config{}, fmt.Errorf("valkey-store: invalid option %s %q (want COSINE, L2 or IP)", optDistanceMetric, cfg.DistanceMetric) + } + + if cfg.RequestTimeout <= 0 { + cfg.RequestTimeout = time.Duration(_defaultRequestTimeoutMS) * time.Millisecond + } + + return cfg, nil +} + +// strOr returns the option value for key, or fallback when it is unset/empty. +func strOr(o map[string]string, key, fallback string) string { + if v, ok := o[key]; ok && v != "" { + return v + } + return fallback +} + +// boolOr parses a boolean option, falling back to the default on an unset or +// unparseable value. A typo is surfaced via a warning (like the previous env +// behaviour) rather than failing Load for a coarse on/off switch. +func boolOr(o map[string]string, key string, fallback bool) bool { + v, ok := o[key] + if !ok || v == "" { + return fallback + } + b, err := strconv.ParseBool(v) + if err != nil { + xlog.Warn("valkey-store: ignoring unparseable option, using default", "key", key, "value", v, "default", fallback) + return fallback + } + return b +} + +// resolveCredential resolves a credential value with the following priority: +// 1. Direct value from the model config option (e.g. `username:admin`) +// 2. Env-indirection: if `username_env` names an env var, read the credential +// from that variable (e.g. `username_env:MY_VALKEY_USER` → os.Getenv("MY_VALKEY_USER")) +// +// The env-indirection pattern (same as cloud-proxy's api_key_env) avoids putting +// secrets directly in model YAML: distinct store configs can each reference a +// different credential env var without any plaintext passwords in the config. +func resolveCredential(o map[string]string, directKey, envKey string) string { + // Direct value takes precedence (backward compatible). + if v := o[directKey]; v != "" { + return v + } + // Env indirection: the option names an env var that holds the credential. + if envVar := o[envKey]; envVar != "" { + return os.Getenv(envVar) + } + return "" +} diff --git a/backend/go/valkey-store/encoding.go b/backend/go/valkey-store/encoding.go new file mode 100644 index 000000000..cec086070 --- /dev/null +++ b/backend/go/valkey-store/encoding.go @@ -0,0 +1,80 @@ +package main + +// Vector⇄key encoding: the "vector IS the key" resolution. +// +// local-store keys entries *by* the vector itself (a []float32). Valkey hashes +// are keyed by strings, so we synthesise a deterministic, lossless key: +// +// key = prefix + hex(little-endian float32 bytes of the vector) +// +// The same vector always produces the same bytes, so HSET is an upsert and +// HGET/DEL are exact matches — and the encoding is reversible, so we can hand +// the original []float32 back on Get/Find. +// +// Divergence from local-store (documented and tested): local-store compares +// keys with slices.Compare, which treats -0.0 == +0.0 and orders NaN, so those +// collapse to the same logical key. Byte-encoding makes -0.0 and +0.0 (and any +// distinct NaN bit-pattern) *distinct* keys. We accept this on purpose: a +// lossless, deterministic, exact round-trip is more valuable for a persistent +// store than reproducing local-store's float-equality quirk, and callers never +// rely on -0.0/+0.0 aliasing. + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "math" + "strings" +) + +// _float32Bytes is the wire width of a single FLOAT32 component. +const _float32Bytes = 4 + +// vecToBytes encodes a vector as little-endian float32 bytes. This is byte-for +// -byte identical to valkey.VectorString32, so the value we store in the hash +// `vec` field and the bytes we hash into the key share one encoding. +func vecToBytes(v []float32) []byte { + b := make([]byte, len(v)*_float32Bytes) + for i, e := range v { + off := i * _float32Bytes + binary.LittleEndian.PutUint32(b[off:off+_float32Bytes], math.Float32bits(e)) + } + return b +} + +// bytesToVec reverses vecToBytes. It rejects a payload whose length is not a +// multiple of the float32 width, which would indicate a corrupted/foreign value. +func bytesToVec(b []byte) ([]float32, error) { + if len(b)%_float32Bytes != 0 { + return nil, fmt.Errorf("valkey-store: vector byte length %d is not a multiple of %d", len(b), _float32Bytes) + } + v := make([]float32, len(b)/_float32Bytes) + for i := range v { + off := i * _float32Bytes + v[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[off : off+_float32Bytes])) + } + return v, nil +} + +// encodeKey builds the Valkey hash key for a vector: prefix + hex(bytes). +// Hex keeps the key printable (so it is safe in FT.CREATE PREFIX and in logs) +// while staying lossless. +func encodeKey(prefix string, v []float32) string { + return prefix + hex.EncodeToString(vecToBytes(v)) +} + +// decodeKey reverses encodeKey. It is intentionally retained as the tested, +// symmetric inverse of encodeKey — it is NOT on the hot Find path (StoresFind +// decodes the returned `vec` bytes via bytesToVec directly), but keeping the +// key↔vector mapping provably invertible guards the encoding contract and is +// exercised by the round-trip unit tests. +func decodeKey(prefix, key string) ([]float32, error) { + if !strings.HasPrefix(key, prefix) { + return nil, fmt.Errorf("valkey-store: key %q does not have expected prefix %q", key, prefix) + } + b, err := hex.DecodeString(strings.TrimPrefix(key, prefix)) + if err != nil { + return nil, fmt.Errorf("valkey-store: decode key hex: %w", err) + } + return bytesToVec(b) +} diff --git a/backend/go/valkey-store/encoding_test.go b/backend/go/valkey-store/encoding_test.go new file mode 100644 index 000000000..1c89f1ab1 --- /dev/null +++ b/backend/go/valkey-store/encoding_test.go @@ -0,0 +1,77 @@ +package main + +// Unit tests for the vector⇄key encoding. These need no Valkey server: they +// exercise the pure lossless-encoding contract that the whole store relies on, +// including the documented edge cases (-0.0/+0.0 and NaN) where this encoding +// intentionally diverges from local-store's slices.Compare float equality. + +import ( + "math" + "math/rand/v2" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + valkey "github.com/valkey-io/valkey-go" +) + +var _ = Describe("vector⇄bytes encoding", func() { + It("round-trips vectors of varying dimensions", func() { + r := rand.New(rand.NewPCG(1, 2)) + for _, dim := range []int{1, 3, 4, 16, 128, 768} { + v := make([]float32, dim) + for i := range v { + v[i] = float32(r.NormFloat64()) + } + got, err := bytesToVec(vecToBytes(v)) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(v)) + } + }) + + It("matches valkey.VectorString32 byte-for-byte", func() { + // The stored `vec` field uses valkey.VectorString32; the key uses + // vecToBytes. They must be the same encoding or Get/Find break. + v := []float32{0.1, -0.2, 3.5, 0} + Expect(valkey.BinaryString(vecToBytes(v))).To(Equal(valkey.VectorString32(v))) + }) + + It("rejects a byte payload that is not a multiple of 4", func() { + _, err := bytesToVec([]byte{1, 2, 3}) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("key encoding", func() { + const prefix = "vs:test:" + + It("round-trips key encode/decode", func() { + v := []float32{0.5, 0.5, 0.5} + key := encodeKey(prefix, v) + Expect(key).To(HavePrefix(prefix)) + got, err := decodeKey(prefix, key) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(v)) + }) + + It("produces distinct keys for -0.0 and +0.0 (documented divergence)", func() { + negZero := float32(math.Copysign(0, -1)) + posZero := float32(0) + Expect(math.Signbit(float64(negZero))).To(BeTrue()) + Expect(encodeKey(prefix, []float32{negZero})).NotTo(Equal(encodeKey(prefix, []float32{posZero}))) + }) + + It("produces a stable, distinct key for a NaN component", func() { + nan := float32(math.NaN()) + k1 := encodeKey(prefix, []float32{nan}) + k2 := encodeKey(prefix, []float32{nan}) + // Deterministic: same NaN bit-pattern → same key. + Expect(k1).To(Equal(k2)) + // Distinct from a normal value. + Expect(k1).NotTo(Equal(encodeKey(prefix, []float32{0}))) + }) + + It("rejects a key without the expected prefix", func() { + _, err := decodeKey(prefix, "wrong:deadbeef") + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/backend/go/valkey-store/main.go b/backend/go/valkey-store/main.go new file mode 100644 index 000000000..ba030899a --- /dev/null +++ b/backend/go/valkey-store/main.go @@ -0,0 +1,25 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each store + +import ( + "flag" + "os" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + "github.com/mudler/xlog" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func main() { + xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(os.Getenv("LOCALAI_LOG_LEVEL")), os.Getenv("LOCALAI_LOG_FORMAT"))) + + flag.Parse() + + if err := grpc.StartServer(*addr, NewValkeyStore()); err != nil { + panic(err) + } +} diff --git a/backend/go/valkey-store/package.sh b/backend/go/valkey-store/package.sh new file mode 100644 index 000000000..c6b487836 --- /dev/null +++ b/backend/go/valkey-store/package.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Script to copy the appropriate libraries based on architecture +# This script is used in the final stage of the Dockerfile + +set -e + +CURDIR=$(dirname "$(realpath $0)") + +mkdir -p $CURDIR/package +cp -avf $CURDIR/valkey-store $CURDIR/package/ +cp -rfv $CURDIR/run.sh $CURDIR/package/ diff --git a/backend/go/valkey-store/run.sh b/backend/go/valkey-store/run.sh new file mode 100644 index 000000000..caf370e2e --- /dev/null +++ b/backend/go/valkey-store/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +CURDIR=$(dirname "$(realpath "$0")") + +exec "$CURDIR"/valkey-store "$@" diff --git a/backend/go/valkey-store/store.go b/backend/go/valkey-store/store.go new file mode 100644 index 000000000..b2b8814a5 --- /dev/null +++ b/backend/go/valkey-store/store.go @@ -0,0 +1,692 @@ +package main + +// Valkey-backed vector store, exposed as a gRPC backend. It mirrors the public +// contract of backend/go/local-store (the four Stores* RPCs + Load) but swaps +// the in-memory sorted slices for Valkey Search (FT.*) so the data persists +// across restarts and can scale beyond an O(N) scan (opt-in HNSW). +// +// Data model — each entry is a Valkey HASH keyed by +// +// prefix + hex(little-endian float32 bytes of the vector) +// +// with two fields: `vec` (the raw float32 bytes, indexed by a lazily-created +// FT VECTOR index of the discovered dimension) and `val` (the opaque value +// bytes). The vector-IS-the-key encoding (see encoding.go) makes Set an +// HSET upsert, Get an HGET, Delete a DEL, and Find an FT.SEARCH KNN. +// +// Similarity — Valkey returns cosine *distance* (0 = identical, 2 = opposite), +// while local-store returns cosine *similarity* (1 = identical, -1 = opposite). +// We convert sim = 1 - distance for COSINE so the values match local-store's +// integration expectations exactly. For L2/IP the raw score is passed through. +// +// Concurrency — base.SingleThread serialises gRPC calls, so the store's +// scalar bookkeeping (keyLen, indexCreated) needs no extra locking. All Valkey +// commands are synchronous via client.Do and bounded by an explicit +// per-request deadline (cfg.RequestTimeout); there is no background event loop. + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "fmt" + "net" + "os" + "strconv" + "strings" + + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/store" + "github.com/mudler/xlog" + valkey "github.com/valkey-io/valkey-go" +) + +const ( + // Hash field names. `vec` is the indexed vector; `val` is the opaque value. + _vecField = "vec" + _valField = "val" + // _scoreField is the KNN distance alias produced by the query and returned + // by FT.SEARCH. Double-underscore avoids colliding with a stored field. + _scoreField = "__score" + + // _keyPrefixPrefix / _indexPrefix namespace the keys and index per model so + // two namespaces (e.g. a 512-d face store and a 192-d voice store) sharing + // one Valkey server never collide. + _keyPrefixPrefix = "vs:" + _indexPrefix = "idx:" + + // _maxTopK bounds a Find so an accidental or abusive huge TopK cannot force + // an unbounded server-side LIMIT / allocation. local-store has no cap, but + // it is in-memory; a networked backend wants a guard. Callers asking for + // more than this get the top _maxTopK results. + _maxTopK = 10000 + + // _maxNsTokenLen bounds the human-readable portion of a namespace token so + // a very long model name cannot produce an unbounded key prefix / index + // name. The appended short hash keeps distinct namespaces collision-free + // even when their sanitized prefixes are truncated to the same value. + _maxNsTokenLen = 64 +) + +// ValkeyStore implements the gRPC store Backend against Valkey Search. +type ValkeyStore struct { + base.SingleThread + + client valkey.Client + cfg Config + + // prefix is the per-namespace key prefix; indexName is the FT index name. + prefix string + indexName string + + // keyLen is the vector dimension, learned from the first Set. -1 means + // "no keys yet" — mirrors local-store so dimension-mismatch errors are + // identical. indexCreated tracks whether FT.CREATE has run (lazy creation). + keyLen int + indexCreated bool +} + +// NewValkeyStore returns a store with an open dimension and no index yet. The +// Valkey client is established in Load once the connection config is known. +func NewValkeyStore() *ValkeyStore { + return &ValkeyStore{keyLen: -1} +} + +// newWithClient builds a store around an already-constructed client for a given +// namespace. It exists so unit tests can inject a mock client without a real +// Valkey server; Load is the production path. +func newWithClient(client valkey.Client, cfg Config, namespace string) *ValkeyStore { + return &ValkeyStore{ + client: client, + cfg: cfg, + prefix: keyPrefix(namespace), + indexName: indexName(namespace), + keyLen: -1, + } +} + +// Load reads the store config from the model config options, connects, and +// verifies the connection. The mandatory ClientName is always set so the +// connection is identifiable via CLIENT LIST. opts.Model is the namespace +// identifier (one process per (backend, model) tuple upstream), so we derive +// an isolated key prefix and index name from it, and opts.Options carries the +// per-store connection/index configuration. +// +// The NamespacePrefix gate mirrors local-store: core's StoreBackend always +// sends the model name with store.NamespacePrefix; anything else is the model +// loader's greedy autoload probing with a real model name, which must be +// refused or the LLM binds to the vector store (the #9287 failure mode). +func (s *ValkeyStore) Load(opts *pb.ModelOptions) error { + if opts == nil { + return fmt.Errorf("valkey-store: refusing to load: nil model options (expected %q prefix)", store.NamespacePrefix) + } + if !strings.HasPrefix(opts.GetModel(), store.NamespacePrefix) { + return fmt.Errorf("valkey-store: refusing to load %q: not a store namespace (expected %q prefix)", opts.GetModel(), store.NamespacePrefix) + } + + cfg, err := loadConfig(opts) + if err != nil { + return err + } + s.cfg = cfg + + namespace := opts.Model + s.prefix = keyPrefix(namespace) + s.indexName = indexName(namespace) + + clientOpt := valkey.ClientOption{ + InitAddress: []string{cfg.Addr}, + Username: cfg.Username, + Password: cfg.Password, + ClientName: cfg.ClientName, + // SelectDB picks a logical Valkey DB (SELECT n) for deployments that use + // numbered DBs for isolation. Defaults to 0; namespace prefixing already + // isolates keyspaces on a shared DB. + SelectDB: cfg.DB, + // Disable client-side caching: values are opaque blobs written once and + // read rarely, so tracking invalidations would only add overhead. + DisableCache: true, + } + if cfg.UseTLS { + tlsCfg, err := buildTLSConfig(cfg) + if err != nil { + return err + } + clientOpt.TLSConfig = tlsCfg + } + + // Close any client from a previous Load so a re-entrant Load does not leak + // the old connection. Not reachable in the one-process-per-namespace model + // today, but keeps Load idempotent. + if s.client != nil { + s.client.Close() + s.client = nil + } + + client, err := valkey.NewClient(clientOpt) + if err != nil { + return fmt.Errorf("valkey-store: connect to %s: %w", cfg.Addr, err) + } + s.client = client + + // Fail fast if the server is unreachable, mirroring how a real vector DB + // backend would refuse to load against a dead endpoint. + ctx, cancel := s.ctx() + defer cancel() + if err := s.client.Do(ctx, s.client.B().Ping().Build()).Error(); err != nil { + s.client.Close() + s.client = nil + return fmt.Errorf("valkey-store: ping %s: %w", cfg.Addr, err) + } + + // A durable Valkey may already hold this namespace's index from a previous + // run (this is the persistence capability local-store lacks). Recover both + // its existence AND its vector dimension so Find works before this fresh + // process issues its first Set, and — critically — so a post-restart Set + // validates the incoming dimension against the real persisted DIM instead + // of silently re-learning a wrong one and dropping mismatched vectors from + // the index (which would return success while making the entry unsearchable). + s.loadIndexState(ctx) + + // Log the sanitized index name (which identifies the namespace) rather than + // the raw model-derived namespace, which could carry control characters. + xlog.Info("valkey-store loaded", "addr", cfg.Addr, "index", s.indexName, "algo", cfg.IndexAlgo, "metric", cfg.DistanceMetric, "indexExists", s.indexCreated, "keyLen", s.keyLen) + return nil +} + +// loadIndexState issues one FT.INFO at Load to recover the persisted index +// state. FT.INFO returns an error for an unknown index, so a successful reply +// means the index exists (indexCreated=true). We then recover the vector +// dimension from the reply and seed keyLen with it: without this, keyLen would +// stay -1 after a restart and the next Set would blindly re-learn whatever +// dimension the caller happened to send, accepting a mismatched vector that +// FT never indexes (silent search-side data loss). If the dimension can't be +// parsed (e.g. an unexpected FT.INFO layout on some server version), keyLen +// is left at -1 and validation degrades to the pre-restart lazy behaviour. +func (s *ValkeyStore) loadIndexState(ctx context.Context) { + msg, err := s.client.Do(ctx, s.client.B().FtInfo().Index(s.indexName).Build()).ToMessage() + if err != nil { + return + } + s.indexCreated = true + if dim, ok := findDimensions(msg); ok && dim > 0 { + s.keyLen = dim + } +} + +// findDimensions walks an FT.INFO reply for the vector field's dimension. In +// Valkey Search the VECTOR attribute nests its parameters under an `index` +// array whose `dimensions` key holds the DIM the index was created with. The +// reply is a nested array (RESP2) or map (RESP3), so we search recursively for +// a `dimensions` key/token and read the value that follows it, tolerating both +// integer and string-encoded values. +func findDimensions(m valkey.ValkeyMessage) (int, bool) { + if m.IsMap() { + mp, err := m.AsMap() + if err != nil { + return 0, false + } + for k, v := range mp { + if strings.EqualFold(k, "dimensions") { + if n, ok := msgToInt(v); ok { + return n, true + } + } + if d, ok := findDimensions(v); ok { + return d, true + } + } + return 0, false + } + if m.IsArray() { + arr, err := m.ToArray() + if err != nil { + return 0, false + } + for i := range arr { + if s, err := arr[i].ToString(); err == nil && strings.EqualFold(s, "dimensions") && i+1 < len(arr) { + if n, ok := msgToInt(arr[i+1]); ok { + return n, true + } + } + if d, ok := findDimensions(arr[i]); ok { + return d, true + } + } + } + return 0, false +} + +// msgToInt reads an integer from a ValkeyMessage that may be an integer reply +// or a string-encoded integer (FT.INFO mixes both across fields/versions). +func msgToInt(m valkey.ValkeyMessage) (int, bool) { + if n, err := m.ToInt64(); err == nil { + return int(n), true + } + if s, err := m.ToString(); err == nil { + if n, err := strconv.Atoi(s); err == nil { + return n, true + } + } + return 0, false +} + +// Free closes the Valkey client. Called by the gRPC server on shutdown. +func (s *ValkeyStore) Free() error { + if s.client != nil { + s.client.Close() + s.client = nil + } + return nil +} + +// buildTLSConfig assembles the tls.Config for a tls=true connection. +// Go only auto-derives ServerName (SNI) from the dial address for hostnames; +// for an IP-addressed endpoint (e.g. 10.0.0.5:6379) SNI is left empty and the +// certificate's SANs won't match the raw IP, so verification fails. We set it +// explicitly from the configured host so both hostname and IP endpoints verify. +// A custom CA bundle (tls_ca_cert) and an explicit insecure-skip escape hatch +// (tls_skip_verify) are supported for enterprise/self-signed setups. +func buildTLSConfig(cfg Config) (*tls.Config, error) { + tlsCfg := &tls.Config{} + if host, _, err := net.SplitHostPort(cfg.Addr); err == nil && host != "" { + tlsCfg.ServerName = host + } + if cfg.TLSSkipVerify { + tlsCfg.InsecureSkipVerify = true + } + if cfg.TLSCACert != "" { + pem, err := os.ReadFile(cfg.TLSCACert) + if err != nil { + return nil, fmt.Errorf("valkey-store: read tls_ca_cert %q: %w", cfg.TLSCACert, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("valkey-store: tls_ca_cert %q: no valid certificate found", cfg.TLSCACert) + } + tlsCfg.RootCAs = pool + } + return tlsCfg, nil +} + +// ctx returns a request-scoped context bounded by the configured timeout. We +// never rely on the client's built-in write timeout because index back-fill +// and large KNN queries can legitimately exceed a short default. +func (s *ValkeyStore) ctx() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), s.cfg.RequestTimeout) +} + +func (s *ValkeyStore) StoresSet(opts *pb.StoresSetOptions) error { + keys := store.UnwrapKeys(opts.Keys) + values := store.UnwrapValues(opts.Values) + if len(keys) == 0 { + return fmt.Errorf("valkey-store: Set: no keys to add") + } + if len(keys) != len(values) { + return fmt.Errorf("valkey-store: Set: len(keys) = %d, len(values) = %d", len(keys), len(values)) + } + + // Learn the dimension from the first key ever set (mirrors local-store's + // keyLen == -1 sentinel), then reject anything that disagrees. checkDims is + // the single source of truth for the per-key length check (shared with + // Get/Delete/Find) so the four RPCs cannot drift apart. + if s.keyLen == -1 { + s.keyLen = len(keys[0]) + } + if err := s.checkDims("Set", keys); err != nil { + return err + } + + // The index needs the dimension up front, but local-store learns it from + // the first Set — so we create it lazily here, once, before writing. + if err := s.ensureIndex(s.keyLen); err != nil { + return err + } + + // Write each entry with an individual round-trip rather than pipelining the + // whole batch via DoMulti. Valkey Search indexes every HSET into the + // FLAT/HNSW index synchronously on the server's main thread; a large + // pipeline of indexed writes can fill the socket buffers while that + // indexing keeps the server from draining them, deadlocking the connection + // (observed as an i/o timeout on high-dimension batches — a single 768-d + // DoMulti of ~20 vectors hangs, while the same writes issued sequentially + // complete in milliseconds). Sequential writes keep each command fully + // round-tripped and stay fast (hundreds of 768-d vectors in a few hundred + // ms). A single failure fails the whole Set — partial writes are surfaced, + // not swallowed. + // + // The request timeout is applied PER command, not once across the whole + // loop: an unbounded SetCols against a remote Valkey would otherwise exhaust + // a single aggregate deadline mid-batch and leave a partial, non-atomic + // write. + for i, k := range keys { + cmd := s.client.B().Hset().Key(encodeKey(s.prefix, k)). + FieldValue(). + FieldValue(_vecField, valkey.BinaryString(vecToBytes(k))). + FieldValue(_valField, valkey.BinaryString(values[i])). + Build() + ctx, cancel := s.ctx() + err := s.client.Do(ctx, cmd).Error() + cancel() + if err != nil { + return fmt.Errorf("valkey-store: Set: HSET key %d: %w", i, err) + } + } + return nil +} + +// StoresGet fetches values for the given keys. Missing keys are omitted from +// the result (not errored), matching local-store; returned slices are aligned. +func (s *ValkeyStore) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error) { + keys := store.UnwrapKeys(opts.Keys) + if len(keys) == 0 { + return pb.StoresGetResult{}, nil + } + if err := s.checkDims("Get", keys); err != nil { + return pb.StoresGetResult{}, err + } + + // Reads pipeline the whole batch via DoMulti under ONE aggregate deadline, + // unlike Set/Delete which use a per-command timeout. That asymmetry is + // deliberate: HGET is non-mutating, so exhausting the deadline mid-batch + // only truncates the result (surfaced as an error) — it can never leave a + // partial write behind, which is the specific hazard the per-command timeout + // guards against for Set/Delete. Pipelining is also safe here because these + // are non-indexed reads (the indexed-DoMulti deadlock only affects writes). + ctx, cancel := s.ctx() + defer cancel() + + cmds := make([]valkey.Completed, len(keys)) + for i, k := range keys { + cmds[i] = s.client.B().Hget().Key(encodeKey(s.prefix, k)).Field(_valField).Build() + } + + var foundKeys [][]float32 + var foundValues [][]byte + for i, res := range s.client.DoMulti(ctx, cmds...) { + v, err := res.ToString() + if err != nil { + // A nil reply means the key/field is absent — omit it, don't error. + if valkey.IsValkeyNil(err) { + continue + } + return pb.StoresGetResult{}, fmt.Errorf("valkey-store: Get: HGET key %d: %w", i, err) + } + // The request vector is exact, so we return it verbatim as the key. + foundKeys = append(foundKeys, keys[i]) + foundValues = append(foundValues, []byte(v)) + } + + return pb.StoresGetResult{ + Keys: store.WrapKeys(foundKeys), + Values: store.WrapValues(foundValues), + }, nil +} + +// StoresDelete removes entries by exact vector. Missing keys are tolerated +// (DEL returns 0), matching local-store. +func (s *ValkeyStore) StoresDelete(opts *pb.StoresDeleteOptions) error { + keys := store.UnwrapKeys(opts.Keys) + if len(keys) == 0 { + return fmt.Errorf("valkey-store: Delete: no keys to delete") + } + if err := s.checkDims("Delete", keys); err != nil { + return err + } + + // Sequential DELs for the same reason StoresSet avoids DoMulti: a DEL of an + // indexed key mutates the search index on the server's main thread, and a + // large pipeline of such mutations can deadlock the connection. Missing + // keys (DEL returns 0) are tolerated, matching local-store. As in Set, the + // timeout is per command so a large DeleteCols cannot exhaust one aggregate + // deadline mid-batch. + for i, k := range keys { + ctx, cancel := s.ctx() + err := s.client.Do(ctx, s.client.B().Del().Key(encodeKey(s.prefix, k)).Build()).Error() + cancel() + if err != nil { + return fmt.Errorf("valkey-store: Delete: DEL key %d: %w", i, err) + } + } + return nil +} + +// StoresFind returns the topK nearest entries by the configured distance +// metric, ordered most-similar first. An empty/uncreated index returns empty +// slices and no error, matching local-store's empty-store behaviour. +func (s *ValkeyStore) StoresFind(opts *pb.StoresFindOptions) (pb.StoresFindResult, error) { + // Guard against a malformed gRPC request with a nil/empty Key before + // dereferencing it — a nil opts.Key would otherwise panic the backend. + if opts.Key == nil || len(opts.Key.Floats) == 0 { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: query key is empty") + } + query := opts.Key.Floats + topK := int(opts.TopK) + if topK < 1 { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: topK = %d, must be >= 1", topK) + } + if topK > _maxTopK { + xlog.Warn("valkey-store: Find topK clamped", "requested", topK, "max", _maxTopK) + topK = _maxTopK + } + // No index yet means nothing has been Set (and none was found at Load) — + // an empty result, not an error. + if !s.indexCreated { + return pb.StoresFindResult{}, nil + } + // Enforce the query dimension against the known keyLen — recovered from + // FT.INFO at Load after a restart, or learned from the first Set — so a + // wrong-dimension query gets the clean local-store-style error. keyLen is + // only -1 in the degraded case where FT.INFO gave no parseable dimension; + // then we let Valkey's own FT.SEARCH validate the query vector. + if s.keyLen != -1 && len(query) != s.keyLen { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: query length %d does not match existing %d", len(query), s.keyLen) + } + + ctx, cancel := s.ctx() + defer cancel() + + // KNN pre-filter query: match everything, rank by vector distance into the + // __score alias. A pure KNN query already returns its topK results ordered + // by distance ascending (nearest-first), so we do NOT add SORTBY __score: + // Valkey Search rejects sorting on the KNN score alias ("Index field + // `__score` does not exist" — it is a query-time computed field, not a + // SORTABLE schema attribute). LIMIT 0 topK caps the result and DIALECT 2 is + // required for the =>[KNN ...] vector syntax. The __score field is still + // returned in each document and read back for the similarity conversion. + // + // Injection-safety: the only caller-controlled value interpolated here is + // topK (an int, already bounded above). _vecField and _scoreField are + // compile-time constants, so this Sprintf cannot be used to inject query + // syntax. Do NOT make those fields operator-configurable without sanitizing + // them first — the KNN query string is otherwise built only from constants. + q := fmt.Sprintf("*=>[KNN %d @%s $q AS %s]", topK, _vecField, _scoreField) + cmd := s.client.B().FtSearch().Index(s.indexName).Query(q). + Return("3").Identifier(_vecField).Identifier(_valField).Identifier(_scoreField). + Limit().OffsetNum(0, int64(topK)). + Params().Nargs(2).NameValue().NameValue("q", valkey.VectorString32(query)). + Dialect(2). + Build() + + _, docs, err := s.client.Do(ctx, cmd).AsFtSearch() + if err != nil { + // The cached indexCreated flag can go stale: an operator runs + // FT.DROPINDEX out of band, or two processes race on a fresh namespace. + // If the index is gone, mirror local-store's empty-store behaviour + // (empty result, no error) and clear the flag so a later Set recreates + // it, rather than surfacing a hard error for what looks like an empty + // store to the caller. + if isNoSuchIndexErr(err) { + s.indexCreated = false + return pb.StoresFindResult{}, nil + } + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: FT.SEARCH: %w", err) + } + + keys := make([][]float32, 0, len(docs)) + values := make([][]byte, 0, len(docs)) + sims := make([]float32, 0, len(docs)) + for _, doc := range docs { + // Decode the key from the returned `vec` bytes rather than the Valkey + // key string: this guarantees the exact original float ordering/values + // without a hex round-trip. + vecBytes := []byte(doc.Doc[_vecField]) + k, err := bytesToVec(vecBytes) + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: decode vec: %w", err) + } + dist, err := strconv.ParseFloat(doc.Doc[_scoreField], 64) + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("valkey-store: Find: parse score %q: %w", doc.Doc[_scoreField], err) + } + keys = append(keys, k) + values = append(values, []byte(doc.Doc[_valField])) + sims = append(sims, distanceToSimilarity(s.cfg.DistanceMetric, dist)) + } + + return pb.StoresFindResult{ + Keys: store.WrapKeys(keys), + Values: store.WrapValues(values), + Similarities: sims, + }, nil +} + +// ensureIndex creates the FT vector index once, lazily, on the first Set. The +// dimension is fixed at creation (a second guard on top of the Go-side keyLen +// check). An "already exists" error is treated as success so a restart against +// a persisted index is a no-op. +func (s *ValkeyStore) ensureIndex(dim int) error { + if s.indexCreated { + return nil + } + + // VECTOR attribute tokens. The count that follows the algorithm name is the + // number of these tokens, so we build the slice and derive the count from + // it — no hand-maintained magic number that drifts when HNSW knobs change. + attrs := []string{"TYPE", "FLOAT32", "DIM", strconv.Itoa(dim), "DISTANCE_METRIC", s.cfg.DistanceMetric} + if s.cfg.IndexAlgo == indexAlgoHNSW { + attrs = append(attrs, + "M", strconv.Itoa(s.cfg.HNSW.M), + "EF_CONSTRUCTION", strconv.Itoa(s.cfg.HNSW.EFConstruction), + "EF_RUNTIME", strconv.Itoa(s.cfg.HNSW.EFRuntime), + ) + } + + args := []string{ + s.indexName, + "ON", "HASH", + "PREFIX", "1", s.prefix, + "SCHEMA", _vecField, "VECTOR", s.cfg.IndexAlgo, strconv.Itoa(len(attrs)), + } + args = append(args, attrs...) + + // FT.CREATE has no typed builder entry point, so we use the Arbitrary escape + // hatch. All tokens are non-key args in standalone mode. + ctx, cancel := s.ctx() + defer cancel() + err := s.client.Do(ctx, s.client.B().Arbitrary("FT.CREATE").Args(args...).Build()).Error() + if err != nil && !isIndexExistsErr(err) { + return fmt.Errorf("valkey-store: FT.CREATE %s: %w", s.indexName, err) + } + + s.indexCreated = true + return nil +} + +// checkDims rejects any key whose dimension disagrees with the learned keyLen. +// When keyLen is still open (-1, nothing set yet) there is nothing to check. +func (s *ValkeyStore) checkDims(op string, keys [][]float32) error { + if s.keyLen == -1 { + return nil + } + for i, k := range keys { + if len(k) != s.keyLen { + return fmt.Errorf("valkey-store: %s: key %d length %d does not match existing %d", op, i, len(k), s.keyLen) + } + } + return nil +} + +// distanceToSimilarity converts a Valkey distance into local-store's similarity +// convention. Only COSINE has a defined [-1, 1] similarity (sim = 1 - dist); +// for L2/IP the raw score is returned as the "similarity" with a documented +// meaning (smaller L2 = closer; larger IP = closer). +func distanceToSimilarity(metric string, dist float64) float32 { + if metric == distanceCosine { + return float32(1 - dist) + } + return float32(dist) +} + +// isIndexExistsErr reports whether an FT.CREATE error is the benign +// "index already exists" case (e.g. after a restart against a persisted index). +func isIndexExistsErr(err error) bool { + return strings.Contains(strings.ToLower(err.Error()), "already exists") +} + +// isNoSuchIndexErr reports whether an FT.SEARCH error means the index no longer +// exists (dropped out of band, or never really created despite a stale cached +// flag). Valkey Search phrases this differently across versions, so we match +// the common variants rather than one exact string. +func isNoSuchIndexErr(err error) bool { + msg := strings.ToLower(err.Error()) + if !strings.Contains(msg, "index") { + return false + } + return strings.Contains(msg, "no such index") || + strings.Contains(msg, "not exist") || + strings.Contains(msg, "not found") || + strings.Contains(msg, "unknown index") +} + +// keyPrefix / indexName derive per-namespace identifiers from the model name so +// entries and indexes never collide across namespaces on a shared server. +func keyPrefix(namespace string) string { + return _keyPrefixPrefix + nsToken(namespace) + ":" +} + +func indexName(namespace string) string { + return _indexPrefix + nsToken(namespace) +} + +// nsToken maps a namespace to a collision-resistant, printable token. sanitize() +// alone is lossy (many distinct characters all fold to '_'), so namespaces like +// "a b", "a/b" and "a:b" would otherwise share one keyspace and FT index — a +// silent data-isolation bug (one store reading/clobbering another). We append a +// short hash of the ORIGINAL namespace so distinct names never collide, while +// the sanitized part keeps the token human-readable. It is deterministic, so a +// persisted index is found again after a restart. +func nsToken(namespace string) string { + sum := sha256.Sum256([]byte(namespace)) + // Cap the human-readable part so a pathologically long model name can't + // produce an unbounded key prefix / index name (which would degrade Valkey + // performance). The 8-char hash suffix below already guarantees collision + // resistance regardless of truncation, so trimming the readable part is safe. + readable := sanitize(namespace) + if len(readable) > _maxNsTokenLen { + readable = readable[:_maxNsTokenLen] + } + return readable + "-" + hex.EncodeToString(sum[:])[:8] +} + +// sanitize maps a namespace to a safe token for keys/index names: alphanumeric, +// '_', '-' and '.' pass through; everything else becomes '_'. An empty +// namespace becomes "default" so the key/index names stay well-formed. +func sanitize(namespace string) string { + if namespace == "" { + return "default" + } + var b strings.Builder + b.Grow(len(namespace)) + for _, r := range namespace { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '-', r == '.': + b.WriteRune(r) + default: + b.WriteRune('_') + } + } + return b.String() +} diff --git a/backend/go/valkey-store/store_suite_test.go b/backend/go/valkey-store/store_suite_test.go new file mode 100644 index 000000000..d15f3fe2a --- /dev/null +++ b/backend/go/valkey-store/store_suite_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestValkeyStore(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "valkey-store test suite") +} diff --git a/backend/go/valkey-store/store_test.go b/backend/go/valkey-store/store_test.go new file mode 100644 index 000000000..c6d563d2b --- /dev/null +++ b/backend/go/valkey-store/store_test.go @@ -0,0 +1,598 @@ +package main + +// Unit tests for the Valkey store, using the valkey-go gomock client so they +// run with no container. They assert the exact commands built for each RPC +// (the wire contract) plus the local-store parity semantics: empty/len/dim +// rejects, omit-missing Get, tolerate-missing Delete, topK<1 reject, the +// sim = 1 - distance conversion, lazy FT.CREATE, and the HNSW arg-shape. + +import ( + "context" + "fmt" + "strconv" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/store" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + valkey "github.com/valkey-io/valkey-go" + "github.com/valkey-io/valkey-go/mock" + "go.uber.org/mock/gomock" +) + +const testNamespace = "test" + +func testCfg() Config { + cfg, err := loadConfig(nil) // reads defaults when no options are set + Expect(err).NotTo(HaveOccurred()) + return cfg +} + +// opts builds a *pb.ModelOptions carrying the given key:value option strings, +// mirroring how core threads a store's model-config `options:` list to the +// backend's LoadModel. +func opts(kv ...string) *pb.ModelOptions { + return &pb.ModelOptions{Options: kv} +} + +func newMockStore(cfg Config) (*ValkeyStore, *mock.Client) { + ctrl := gomock.NewController(GinkgoT()) + DeferCleanup(ctrl.Finish) + c := mock.NewClient(ctrl) + return newWithClient(c, cfg, testNamespace), c +} + +func wrapSet(keys [][]float32, values [][]byte) *pb.StoresSetOptions { + return &pb.StoresSetOptions{Keys: store.WrapKeys(keys), Values: store.WrapValues(values)} +} + +var _ = Describe("loadConfig", func() { + It("uses documented defaults", func() { + cfg, err := loadConfig(nil) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Addr).To(Equal("localhost:6379")) + Expect(cfg.ClientName).To(Equal("localai-valkey-store")) + Expect(cfg.IndexAlgo).To(Equal("FLAT")) + Expect(cfg.DistanceMetric).To(Equal("COSINE")) + Expect(cfg.RequestTimeout.Milliseconds()).To(Equal(int64(5000))) + }) + + It("honours option overrides", func() { + cfg, err := loadConfig(opts( + "addr:valkey.example:6380", + "index_algo:hnsw", + "distance_metric:l2", + "request_timeout_ms:1234", + )) + Expect(err).NotTo(HaveOccurred()) + // addr keeps its embedded colon: strings.Cut splits on the first ':'. + Expect(cfg.Addr).To(Equal("valkey.example:6380")) + Expect(cfg.IndexAlgo).To(Equal("HNSW")) + Expect(cfg.DistanceMetric).To(Equal("L2")) + Expect(cfg.RequestTimeout.Milliseconds()).To(Equal(int64(1234))) + }) + + It("keeps the mandatory client name when blanked", func() { + cfg, err := loadConfig(opts("client_name:")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.ClientName).To(Equal("localai-valkey-store")) + }) + + It("ignores a malformed option without a colon", func() { + cfg, err := loadConfig(opts("addr:valkey.example:6380", "not-a-kv-pair")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Addr).To(Equal("valkey.example:6380")) + }) + + It("rejects an invalid index algo", func() { + _, err := loadConfig(opts("index_algo:bogus")) + Expect(err).To(HaveOccurred()) + }) + + It("rejects an invalid distance metric", func() { + _, err := loadConfig(opts("distance_metric:bogus")) + Expect(err).To(HaveOccurred()) + }) + + It("fails fast on a malformed HNSW integer instead of silently defaulting", func() { + _, err := loadConfig(opts("hnsw_m:1x6")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("hnsw_m")) + }) + + It("honours a valid db override", func() { + cfg, err := loadConfig(opts("db:3")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.DB).To(Equal(3)) + }) + + It("rejects a negative db", func() { + _, err := loadConfig(opts("db:-1")) + Expect(err).To(HaveOccurred()) + }) + + It("resolves username from direct option", func() { + cfg, err := loadConfig(opts("username:admin")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Username).To(Equal("admin")) + }) + + It("resolves password from env indirection via password_env", func() { + GinkgoT().Setenv("TEST_VALKEY_PW_INDIRECT", "s3cret") + cfg, err := loadConfig(opts("password_env:TEST_VALKEY_PW_INDIRECT")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Password).To(Equal("s3cret")) + }) + + It("resolves username from env indirection via username_env", func() { + GinkgoT().Setenv("TEST_VALKEY_USER_INDIRECT", "myuser") + cfg, err := loadConfig(opts("username_env:TEST_VALKEY_USER_INDIRECT")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Username).To(Equal("myuser")) + }) + + It("prefers the direct option over env indirection", func() { + GinkgoT().Setenv("TEST_VALKEY_PW_CLASH", "from-env") + cfg, err := loadConfig(opts("password:direct-value", "password_env:TEST_VALKEY_PW_CLASH")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Password).To(Equal("direct-value")) + }) + + It("returns empty when neither direct nor env indirection is set", func() { + cfg, err := loadConfig(opts("addr:localhost:6379")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.Username).To(BeEmpty()) + Expect(cfg.Password).To(BeEmpty()) + }) +}) + +var _ = Describe("Load namespace gate", func() { + It("accepts prefixed store namespaces", func() { + s := NewValkeyStore() + // Load will fail at the Valkey connect step (no server), but only after + // passing the namespace gate. A network error is acceptable here — it + // means the prefix check passed. + err := s.Load(&pb.ModelOptions{Model: store.NamespacePrefix + "any-namespace", Options: []string{"addr:localhost:1"}}) + Expect(err).NotTo(MatchError(ContainSubstring("not a store namespace"))) + }) + + It("accepts the prefix alone (default store)", func() { + s := NewValkeyStore() + err := s.Load(&pb.ModelOptions{Model: store.NamespacePrefix, Options: []string{"addr:localhost:1"}}) + Expect(err).NotTo(MatchError(ContainSubstring("not a store namespace"))) + }) + + It("refuses model names without the namespace prefix", func() { + s := NewValkeyStore() + err := s.Load(&pb.ModelOptions{Model: "some-llm.gguf"}) + Expect(err).To(MatchError(ContainSubstring("not a store namespace"))) + }) + + It("refuses an empty model name", func() { + s := NewValkeyStore() + err := s.Load(&pb.ModelOptions{}) + Expect(err).To(MatchError(ContainSubstring("not a store namespace"))) + }) + + It("refuses nil opts", func() { + s := NewValkeyStore() + err := s.Load(nil) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("StoresSet", func() { + It("rejects empty input", func() { + s, _ := newMockStore(testCfg()) + Expect(s.StoresSet(&pb.StoresSetOptions{})).NotTo(Succeed()) + }) + + It("rejects key/value length mismatch", func() { + s, _ := newMockStore(testCfg()) + err := s.StoresSet(wrapSet([][]float32{{1, 0, 0}}, [][]byte{[]byte("a"), []byte("b")})) + Expect(err).To(HaveOccurred()) + }) + + It("rejects dimension mismatch on a later add", func() { + s, c := newMockStore(testCfg()) + // First Set issues FT.CREATE then a sequential HSET (both via Do). + c.EXPECT().Do(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, cmd valkey.Completed) valkey.ValkeyResult { + if cmd.Commands()[0] == "FT.CREATE" { + return assertFTCreate(3, "FLAT")(ctx, cmd) + } + return mock.Result(mock.ValkeyInt64(1)) // HSET + }).AnyTimes() + Expect(s.StoresSet(wrapSet([][]float32{{1, 0, 0}}, [][]byte{[]byte("3d")}))).To(Succeed()) + + err := s.StoresSet(wrapSet([][]float32{{1, 0}}, [][]byte{[]byte("2d")})) + Expect(err).To(HaveOccurred()) + }) + + It("rejects dimension mismatch within a batch", func() { + s, _ := newMockStore(testCfg()) + err := s.StoresSet(wrapSet([][]float32{{1, 0, 0}, {1, 0}}, [][]byte{[]byte("3d"), []byte("2d")})) + Expect(err).To(HaveOccurred()) + }) + + It("creates the FLAT index once and HSETs each entry", func() { + s, c := newMockStore(testCfg()) + // FT.CREATE must run exactly once (on the first Set); each entry is then + // written with an individual sequential HSET (Do, not DoMulti — see the + // pipeline-deadlock note in StoresSet). + var ftCreateCount, hsetCount int + c.EXPECT().Do(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, cmd valkey.Completed) valkey.ValkeyResult { + toks := cmd.Commands() + switch toks[0] { + case "FT.CREATE": + ftCreateCount++ + return assertFTCreate(3, "FLAT")(ctx, cmd) + case "HSET": + hsetCount++ + Expect(toks[1]).To(HavePrefix(s.prefix)) + Expect(toks).To(ContainElements("vec", "val")) + return mock.Result(mock.ValkeyInt64(1)) + default: + Fail("unexpected command: " + toks[0]) + return valkey.ValkeyResult{} + } + }).AnyTimes() + Expect(s.StoresSet(wrapSet([][]float32{{1, 0, 0}}, [][]byte{[]byte("a")}))).To(Succeed()) + Expect(s.StoresSet(wrapSet([][]float32{{2, 0, 0}}, [][]byte{[]byte("b")}))).To(Succeed()) + + Expect(ftCreateCount).To(Equal(1)) + Expect(hsetCount).To(Equal(2)) + }) +}) + +var _ = Describe("StoresGet", func() { + It("round-trips values and omits missing keys", func() { + s, c := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + // First key present, second missing (nil reply). + c.EXPECT().DoMulti(gomock.Any(), gomock.Any(), gomock.Any()).Return([]valkey.ValkeyResult{ + mock.Result(mock.ValkeyString("hello")), + mock.Result(mock.ValkeyNil()), + }).Times(1) + + res, err := s.StoresGet(&pb.StoresGetOptions{ + Keys: store.WrapKeys([][]float32{{1, 0, 0}, {9, 0, 0}}), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(HaveLen(1)) + Expect(res.Values).To(HaveLen(1)) + Expect(res.Values[0].Bytes).To(Equal([]byte("hello"))) + }) + + It("rejects dimension mismatch", func() { + s, _ := newMockStore(testCfg()) + s.keyLen = 3 + _, err := s.StoresGet(&pb.StoresGetOptions{Keys: store.WrapKeys([][]float32{{1, 0}})}) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("StoresDelete", func() { + It("issues DEL per key and tolerates missing", func() { + s, c := newMockStore(testCfg()) + s.keyLen = 3 + // DEL of a missing key returns 0 — still a success. DELs are issued + // sequentially (Do, not DoMulti — see the deadlock note in StoresSet). + c.EXPECT().Do(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, cmd valkey.Completed) valkey.ValkeyResult { + Expect(cmd.Commands()[0]).To(Equal("DEL")) + return mock.Result(mock.ValkeyInt64(0)) + }).Times(1) + Expect(s.StoresDelete(&pb.StoresDeleteOptions{ + Keys: store.WrapKeys([][]float32{{9, 0, 0}}), + })).To(Succeed()) + }) + + It("rejects dimension mismatch", func() { + s, _ := newMockStore(testCfg()) + s.keyLen = 3 + err := s.StoresDelete(&pb.StoresDeleteOptions{Keys: store.WrapKeys([][]float32{{1, 0}})}) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("StoresFind", func() { + It("builds the KNN query and converts distance to similarity nearest-first", func() { + s, c := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + + // Distances 0, 1, 2 must map to similarities 1, 0, -1 (COSINE). + docs := []ftDoc{ + {vec: []float32{1, 0, 0}, val: "a", dist: 0}, + {vec: []float32{0, 1, 0}, val: "b", dist: 1}, + {vec: []float32{-1, 0, 0}, val: "c", dist: 2}, + } + c.EXPECT().Do(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, cmd valkey.Completed) valkey.ValkeyResult { + toks := cmd.Commands() + Expect(toks[0]).To(Equal("FT.SEARCH")) + Expect(toks[1]).To(Equal(s.indexName)) + Expect(toks[2]).To(ContainSubstring("KNN 3 @vec $q AS __score")) + Expect(toks).To(ContainElements("PARAMS", "2", "q", "DIALECT", "2")) + return mock.Result(ftSearchReply(s.prefix, docs)) + }).Times(1) + + keys, values, sims, err := findViaRPC(s, []float32{1, 0, 0}, 3) + Expect(err).NotTo(HaveOccurred()) + Expect(sims).To(Equal([]float32{1, 0, -1})) + Expect(values[0]).To(Equal([]byte("a"))) + // Key decoded from returned vec bytes equals the original vector. + Expect(keys[0]).To(Equal([]float32{1, 0, 0})) + }) + + It("rejects topK < 1", func() { + s, _ := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + _, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: []float32{1, 0, 0}}, TopK: 0}) + Expect(err).To(HaveOccurred()) + }) + + It("rejects a nil Key without panicking", func() { + s, _ := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + _, err := s.StoresFind(&pb.StoresFindOptions{TopK: 5}) + Expect(err).To(HaveOccurred()) + }) + + It("rejects an empty query vector", func() { + s, _ := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + _, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: []float32{}}, TopK: 5}) + Expect(err).To(HaveOccurred()) + }) + + It("rejects query dimension mismatch", func() { + s, _ := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + _, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: []float32{1, 0}}, TopK: 1}) + Expect(err).To(HaveOccurred()) + }) + + It("returns empty (no error) when the index was never created", func() { + s, _ := newMockStore(testCfg()) + res, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: []float32{1, 0, 0}}, TopK: 5}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(BeEmpty()) + }) +}) + +var _ = Describe("ensureIndex arg-shape", func() { + It("emits HNSW tuning tokens when the algo is HNSW", func() { + cfg := testCfg() + cfg.IndexAlgo = indexAlgoHNSW + cfg.HNSW = hnswParams{M: 16, EFConstruction: 200, EFRuntime: 10} + s, c := newMockStore(cfg) + c.EXPECT().Do(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, cmd valkey.Completed) valkey.ValkeyResult { + toks := cmd.Commands() + Expect(toks).To(ContainElements("HNSW", "M", "16", "EF_CONSTRUCTION", "200", "EF_RUNTIME", "10")) + return mock.Result(mock.ValkeyString("OK")) + }).Times(1) + Expect(s.ensureIndex(4)).To(Succeed()) + }) + + It("treats an already-exists error as success", func() { + s, c := newMockStore(testCfg()) + c.EXPECT().Do(gomock.Any(), gomock.Any()).Return( + mock.ErrorResult(fmt.Errorf("Index already exists"))).Times(1) + Expect(s.ensureIndex(4)).To(Succeed()) + Expect(s.indexCreated).To(BeTrue()) + }) +}) + +var _ = Describe("distanceToSimilarity", func() { + It("converts cosine distance to similarity", func() { + Expect(distanceToSimilarity(distanceCosine, 0)).To(Equal(float32(1))) + Expect(distanceToSimilarity(distanceCosine, 1)).To(Equal(float32(0))) + Expect(distanceToSimilarity(distanceCosine, 2)).To(Equal(float32(-1))) + }) + + It("passes the raw score through for non-cosine metrics", func() { + Expect(distanceToSimilarity(distanceL2, 0.42)).To(Equal(float32(0.42))) + }) +}) + +var _ = Describe("findDimensions", func() { + It("recovers an integer dimension from a nested FT.INFO reply", func() { + dim, ok := findDimensions(ftInfoReply(mock.ValkeyInt64(768))) + Expect(ok).To(BeTrue()) + Expect(dim).To(Equal(768)) + }) + + It("recovers a string-encoded dimension", func() { + dim, ok := findDimensions(ftInfoReply(mock.ValkeyString("384"))) + Expect(ok).To(BeTrue()) + Expect(dim).To(Equal(384)) + }) + + It("reports not-found when no dimension token is present", func() { + reply := mock.ValkeyArray( + mock.ValkeyString("index_name"), mock.ValkeyString("idx:test"), + mock.ValkeyString("num_docs"), mock.ValkeyInt64(0), + ) + _, ok := findDimensions(reply) + Expect(ok).To(BeFalse()) + }) +}) + +var _ = Describe("loadIndexState", func() { + It("recovers indexCreated and keyLen from a persisted index", func() { + s, c := newMockStore(testCfg()) + c.EXPECT().Do(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, cmd valkey.Completed) valkey.ValkeyResult { + Expect(cmd.Commands()[0]).To(Equal("FT.INFO")) + return mock.Result(ftInfoReply(mock.ValkeyInt64(768))) + }).Times(1) + + ctx, cancel := s.ctx() + defer cancel() + s.loadIndexState(ctx) + Expect(s.indexCreated).To(BeTrue()) + Expect(s.keyLen).To(Equal(768)) + }) + + It("leaves state untouched when the index does not exist", func() { + s, c := newMockStore(testCfg()) + c.EXPECT().Do(gomock.Any(), gomock.Any()).Return( + mock.ErrorResult(fmt.Errorf("Index with name 'idx:test' not found"))).Times(1) + + ctx, cancel := s.ctx() + defer cancel() + s.loadIndexState(ctx) + Expect(s.indexCreated).To(BeFalse()) + Expect(s.keyLen).To(Equal(-1)) + }) + + It("marks the index created but leaves keyLen open when the dim is unparseable", func() { + s, c := newMockStore(testCfg()) + c.EXPECT().Do(gomock.Any(), gomock.Any()).Return( + mock.Result(mock.ValkeyArray(mock.ValkeyString("index_name"), mock.ValkeyString("idx:test")))).Times(1) + + ctx, cancel := s.ctx() + defer cancel() + s.loadIndexState(ctx) + Expect(s.indexCreated).To(BeTrue()) + Expect(s.keyLen).To(Equal(-1)) + }) +}) + +var _ = Describe("StoresFind on a dropped index", func() { + It("returns empty (no error) and clears the stale flag when the index is gone", func() { + s, c := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + c.EXPECT().Do(gomock.Any(), gomock.Any()).Return( + mock.ErrorResult(fmt.Errorf("Index with name 'idx:test' not found"))).Times(1) + + res, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: []float32{1, 0, 0}}, TopK: 5}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Keys).To(BeEmpty()) + Expect(s.indexCreated).To(BeFalse()) + }) + + It("still surfaces a genuine FT.SEARCH error", func() { + s, c := newMockStore(testCfg()) + s.keyLen = 3 + s.indexCreated = true + c.EXPECT().Do(gomock.Any(), gomock.Any()).Return( + mock.ErrorResult(fmt.Errorf("timeout"))).Times(1) + + _, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: []float32{1, 0, 0}}, TopK: 5}) + Expect(err).To(HaveOccurred()) + Expect(s.indexCreated).To(BeTrue()) + }) +}) + +// --- test helpers --- + +type ftDoc struct { + vec []float32 + val string + dist float64 +} + +func findViaRPC(s *ValkeyStore, query []float32, topK int) ([][]float32, [][]byte, []float32, error) { + res, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: query}, TopK: int32(topK)}) + if err != nil { + return nil, nil, nil, err + } + return store.UnwrapKeys(res.Keys), store.UnwrapValues(res.Values), res.Similarities, nil +} + +// assertFTCreate returns a DoAndReturn func that verifies the FT.CREATE command +// carries the expected dimension and algorithm, then replies OK. +func assertFTCreate(dim int, algo string) func(context.Context, valkey.Completed) valkey.ValkeyResult { + return func(_ context.Context, cmd valkey.Completed) valkey.ValkeyResult { + toks := cmd.Commands() + Expect(toks[0]).To(Equal("FT.CREATE")) + Expect(toks).To(ContainElements("VECTOR", algo, "TYPE", "FLOAT32", "DIM", strconv.Itoa(dim), "DISTANCE_METRIC", "COSINE")) + return mock.Result(mock.ValkeyString("OK")) + } +} + +// ftInfoReply builds a RESP2-shaped FT.INFO reply that mirrors Valkey Search's +// nesting: the VECTOR attribute carries its params under an `index` array whose +// `dimensions` key holds the DIM. dimValue is the message the parser must read +// back (integer or string-encoded), so both wire shapes can be exercised. +func ftInfoReply(dimValue valkey.ValkeyMessage) valkey.ValkeyMessage { + vectorAttr := mock.ValkeyArray( + mock.ValkeyString("identifier"), mock.ValkeyString(_vecField), + mock.ValkeyString("attribute"), mock.ValkeyString(_vecField), + mock.ValkeyString("type"), mock.ValkeyString("VECTOR"), + mock.ValkeyString("index"), mock.ValkeyArray( + mock.ValkeyString("capacity"), mock.ValkeyInt64(1000), + mock.ValkeyString("dimensions"), dimValue, + mock.ValkeyString("distance_metric"), mock.ValkeyString("COSINE"), + mock.ValkeyString("data_type"), mock.ValkeyString("FLOAT32"), + ), + ) + return mock.ValkeyArray( + mock.ValkeyString("index_name"), mock.ValkeyString("idx:test"), + mock.ValkeyString("attributes"), mock.ValkeyArray(vectorAttr), + mock.ValkeyString("num_docs"), mock.ValkeyInt64(0), + ) +} + +// ftSearchReply builds a RESP2-shaped FT.SEARCH reply: [total, key, attrs, ...] +// where attrs carries the returned vec/val/__score fields. +func ftSearchReply(prefix string, docs []ftDoc) valkey.ValkeyMessage { + arr := []valkey.ValkeyMessage{mock.ValkeyInt64(int64(len(docs)))} + for _, d := range docs { + arr = append(arr, mock.ValkeyString(encodeKey(prefix, d.vec))) + attrs := mock.ValkeyArray( + mock.ValkeyString(_vecField), mock.ValkeyString(valkey.BinaryString(vecToBytes(d.vec))), + mock.ValkeyString(_valField), mock.ValkeyString(d.val), + mock.ValkeyString(_scoreField), mock.ValkeyString(strconv.FormatFloat(d.dist, 'f', -1, 64)), + ) + arr = append(arr, attrs) + } + return mock.ValkeyArray(arr...) +} + +var _ = Describe("namespace token", func() { + It("is stable for the same namespace (so a persisted index is found again)", func() { + Expect(nsToken("faces")).To(Equal(nsToken("faces"))) + }) + + It("does not collide for namespaces that sanitize to the same token", func() { + // "a b", "a/b" and "a:b" all sanitize to "a_b"; the hash suffix must + // keep them distinct so two logically-distinct stores never share one + // keyspace/index (the data-isolation guarantee). + Expect(sanitize("a b")).To(Equal(sanitize("a/b"))) + Expect(nsToken("a b")).NotTo(Equal(nsToken("a/b"))) + Expect(nsToken("a/b")).NotTo(Equal(nsToken("a:b"))) + }) + + It("keeps the sanitized part human-readable", func() { + Expect(nsToken("faces")).To(HavePrefix("faces-")) + }) + + It("maps an empty namespace to a stable default token", func() { + Expect(nsToken("")).To(HavePrefix("default-")) + Expect(nsToken("")).To(Equal(nsToken(""))) + }) +}) + +var _ = Describe("sanitize", func() { + It("passes through allowed runes and folds the rest to '_'", func() { + Expect(sanitize("Ok_9.-")).To(Equal("Ok_9.-")) + Expect(sanitize("a b/c:d")).To(Equal("a_b_c_d")) + }) + + It("maps empty to 'default'", func() { + Expect(sanitize("")).To(Equal("default")) + }) +}) diff --git a/backend/index.yaml b/backend/index.yaml index 2d002d690..b447d8682 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -425,6 +425,32 @@ nvidia-cuda-12: "cuda12-stablediffusion-ggml" nvidia-l4t-cuda-12: "nvidia-l4t-arm64-stablediffusion-ggml" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-stablediffusion-ggml" +- &trellis2cpp + name: "trellis2cpp" + alias: "trellis2cpp" + license: mit + description: | + TRELLIS.2 image-to-3D generation (GLB meshes with PBR textures) in C++/ggml + urls: + - https://github.com/localai-org/trellis2cpp + - https://github.com/microsoft/TRELLIS.2 + tags: + - image-to-3d + - 3d-generation + - CPU + - GPU + - CUDA + - Metal + capabilities: + default: "cpu-trellis2cpp" + nvidia: "cuda12-trellis2cpp" + vulkan: "vulkan-trellis2cpp" + nvidia-l4t: "nvidia-l4t-arm64-trellis2cpp" + metal: "metal-trellis2cpp" + nvidia-cuda-13: "cuda13-trellis2cpp" + nvidia-cuda-12: "cuda12-trellis2cpp" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-trellis2cpp" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-trellis2cpp" - &rfdetr name: "rfdetr" alias: "rfdetr" @@ -1837,6 +1863,23 @@ capabilities: default: "cpu-cloud-proxy" metal: "metal-cloud-proxy" +- &valkey-store + name: "valkey-store" + urls: + - https://github.com/mudler/LocalAI + description: | + Valkey Store is a Valkey Search (FT.*) backed vector store for LocalAI. It + persists vectors across restarts and supports opt-in HNSW indexing. Requires + a reachable Valkey Search server (valkey/valkey-bundle). + tags: + - vector-database + - valkey + - open-source + - CPU + license: MIT + capabilities: + default: "cpu-valkey-store" + metal: "metal-valkey-store" - &kitten-tts name: "kitten-tts" urls: @@ -1972,6 +2015,18 @@ nvidia-cuda-12: "cuda12-stablediffusion-ggml-development" nvidia-l4t-cuda-12: "nvidia-l4t-arm64-stablediffusion-ggml-development" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-stablediffusion-ggml-development" +- !!merge <<: *trellis2cpp + name: "trellis2cpp-development" + capabilities: + default: "cpu-trellis2cpp-development" + nvidia: "cuda12-trellis2cpp-development" + vulkan: "vulkan-trellis2cpp-development" + nvidia-l4t: "nvidia-l4t-arm64-trellis2cpp-development" + metal: "metal-trellis2cpp-development" + nvidia-cuda-13: "cuda13-trellis2cpp-development" + nvidia-cuda-12: "cuda12-trellis2cpp-development" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-trellis2cpp-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-trellis2cpp-development" - !!merge <<: *neutts name: "cpu-neutts" uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-neutts" @@ -2369,6 +2424,35 @@ uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-cloud-proxy" mirrors: - localai/localai-backends:master-metal-darwin-arm64-cloud-proxy +- !!merge <<: *valkey-store + name: "cpu-valkey-store" + alias: "valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-valkey-store" + mirrors: + - localai/localai-backends:latest-cpu-valkey-store +- !!merge <<: *valkey-store + name: "cpu-valkey-store-development" + alias: "valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-valkey-store" + mirrors: + - localai/localai-backends:master-cpu-valkey-store +- !!merge <<: *valkey-store + name: "valkey-store-development" + alias: "valkey-store" + capabilities: + default: "cpu-valkey-store-development" + metal: "metal-valkey-store-development" +- !!merge <<: *valkey-store + name: "metal-valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-valkey-store" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-valkey-store +- !!merge <<: *valkey-store + name: "metal-valkey-store-development" + alias: "valkey-store" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-valkey-store" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-valkey-store - !!merge <<: *opus name: "cpu-opus" uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-opus" @@ -3688,6 +3772,77 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-stablediffusion-ggml" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-13-stablediffusion-ggml +## trellis2cpp +- !!merge <<: *trellis2cpp + name: "cpu-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-trellis2cpp" + mirrors: + - localai/localai-backends:latest-cpu-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cpu-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-trellis2cpp" + mirrors: + - localai/localai-backends:master-cpu-trellis2cpp +- !!merge <<: *trellis2cpp + name: "metal-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "metal-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "vulkan-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-trellis2cpp" + mirrors: + - localai/localai-backends:latest-gpu-vulkan-trellis2cpp +- !!merge <<: *trellis2cpp + name: "vulkan-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-trellis2cpp" + mirrors: + - localai/localai-backends:master-gpu-vulkan-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda12-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-trellis2cpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda12-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-trellis2cpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-trellis2cpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-trellis2cpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-trellis2cpp +- !!merge <<: *trellis2cpp + name: "nvidia-l4t-arm64-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "nvidia-l4t-arm64-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:master-nvidia-l4t-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-nvidia-l4t-arm64-trellis2cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-trellis2cpp +- !!merge <<: *trellis2cpp + name: "cuda13-nvidia-l4t-arm64-trellis2cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-trellis2cpp" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-trellis2cpp ## privacy-filter - !!merge <<: *privacyfilter name: "cpu-privacy-filter" diff --git a/backend/rust/kokoros/src/service.rs b/backend/rust/kokoros/src/service.rs index 495b0a423..80fd3ceb8 100644 --- a/backend/rust/kokoros/src/service.rs +++ b/backend/rust/kokoros/src/service.rs @@ -334,6 +334,13 @@ impl Backend for KokorosService { Err(Status::unimplemented("Not supported")) } + async fn generate3_d( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not supported")) + } + async fn audio_transcription( &self, _: Request, diff --git a/core/application/application.go b/core/application/application.go index df904c02d..c5853a925 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -165,7 +165,7 @@ func newApplication(appConfig *config.ApplicationConfig) *Application { voiceStoreName = "localai-voice-biometrics" ) faceStoreResolver := func(_ context.Context, storeName string) (pkggrpc.Backend, error) { - return corebackend.StoreBackend(ml, appConfig, storeName, "") + return corebackend.StoreBackend(ml, appConfig, app.backendLoader, storeName, "") } app.faceRegistry = facerecognition.NewStoreRegistry(faceStoreResolver, faceStoreName, faceEmbeddingDim) @@ -173,7 +173,7 @@ func newApplication(appConfig *config.ApplicationConfig) *Application { // namespace so embedding spaces stay isolated (a face vector and a // speaker vector are not comparable and differ in dimensionality). voiceStoreResolver := func(_ context.Context, storeName string) (pkggrpc.Backend, error) { - return corebackend.StoreBackend(ml, appConfig, storeName, "") + return corebackend.StoreBackend(ml, appConfig, app.backendLoader, storeName, "") } app.voiceRegistry = voicerecognition.NewStoreRegistry(voiceStoreResolver, voiceStoreName, voiceEmbeddingDim) diff --git a/core/application/router_factories.go b/core/application/router_factories.go index 2bee523ea..523c4465e 100644 --- a/core/application/router_factories.go +++ b/core/application/router_factories.go @@ -116,5 +116,5 @@ func (l *lazyEmbedder) Embed(ctx context.Context, text string) ([]float32, error // VectorStore takes a store name, not a model name — no adapterConfig, no // staleness to avoid. func (a *Application) VectorStore(storeName string) backend.VectorStore { - return backend.NewVectorStore(a.modelLoader, a.applicationConfig, storeName) + return backend.NewVectorStore(a.modelLoader, a.applicationConfig, a.backendLoader, storeName) } diff --git a/core/backend/model3d.go b/core/backend/model3d.go new file mode 100644 index 000000000..1952501d9 --- /dev/null +++ b/core/backend/model3d.go @@ -0,0 +1,104 @@ +package backend + +import ( + "maps" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/trace" + "github.com/mudler/LocalAI/pkg/grpc/proto" + model "github.com/mudler/LocalAI/pkg/model" +) + +// Model3DGenerationOptions is the backend-neutral request passed to 3D +// generators. Image contains a staged local path by the time it reaches +// this layer. +type Model3DGenerationOptions struct { + Image string + Destination string + Seed int32 + Step int32 + CFGScale float32 + TextureSteps int32 + Quality string + Background string + Params map[string]string +} + +func Model3DGeneration(options Model3DGenerationOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) { + opts := ModelOptions(modelConfig, appConfig) + inferenceModel, err := loader.Load(opts...) + if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) + return nil, err + } + + fn := func() error { + _, err := inferenceModel.Generate3D( + appConfig.Context, + &proto.Generate3DRequest{ + Src: options.Image, + Dst: options.Destination, + Seed: options.Seed, + Step: options.Step, + CfgScale: options.CFGScale, + TextureSteps: options.TextureSteps, + Quality: options.Quality, + Background: options.Background, + Params: maps.Clone(options.Params), + }, + ) + return err + } + + if appConfig.EnableTracing { + trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) + + traceType := trace.BackendTrace3DGeneration + traceSummary := "3d: " + options.Quality + traceData := map[string]any{} + if options.Params["operation"] == "print_remesh" { + traceType = trace.BackendTrace3DRemesh + traceSummary = "3d: remesh" + traceData["detail_percent"] = options.Params["detail_percent"] + traceData["has_mesh"] = options.Image != "" + } else { + traceData = map[string]any{ + "seed": options.Seed, + "step": options.Step, + "cfg_scale": options.CFGScale, + "texture_steps": options.TextureSteps, + "quality": options.Quality, + "background": options.Background, + "has_image": options.Image != "", + } + } + + startTime := time.Now() + originalFn := fn + fn = func() error { + err := originalFn() + duration := time.Since(startTime) + + errStr := "" + if err != nil { + errStr = err.Error() + } + + trace.RecordBackendTrace(trace.BackendTrace{ + Timestamp: startTime, + Duration: duration, + Type: traceType, + ModelName: modelConfig.Name, + Backend: modelConfig.Backend, + Summary: trace.TruncateString(traceSummary, 200), + Error: errStr, + Data: traceData, + }) + + return err + } + } + + return fn, nil +} diff --git a/core/backend/stores.go b/core/backend/stores.go index 480400f42..1c324ab1a 100644 --- a/core/backend/stores.go +++ b/core/backend/stores.go @@ -9,6 +9,7 @@ import ( "github.com/mudler/LocalAI/core/trace" "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/store" ) @@ -23,21 +24,25 @@ type VectorStore interface { // NewVectorStore returns a VectorStore backed by the local-store // gRPC backend, namespaced by storeName so two routers don't collide. -func NewVectorStore(loader *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string) VectorStore { +// cl resolves the per-store model config (backend + options); it may be nil, +// in which case the store falls back to the default backend and its built-in +// defaults. +func NewVectorStore(loader *model.ModelLoader, appConfig *config.ApplicationConfig, cl *config.ModelConfigLoader, storeName string) VectorStore { if storeName == "" { return nil } - return &localVectorStore{loader: loader, appConfig: appConfig, storeName: storeName} + return &localVectorStore{loader: loader, appConfig: appConfig, cl: cl, storeName: storeName} } type localVectorStore struct { loader *model.ModelLoader appConfig *config.ApplicationConfig + cl *config.ModelConfigLoader storeName string } func (s *localVectorStore) backend(_ context.Context) (grpc.Backend, error) { - return StoreBackend(s.loader, s.appConfig, s.storeName, "") + return StoreBackend(s.loader, s.appConfig, s.cl, s.storeName, "") } func (s *localVectorStore) Search(ctx context.Context, vec []float32) (sim float64, payload []byte, ok bool, err error) { @@ -121,7 +126,24 @@ func (s *localVectorStore) recordTrace(start time.Time, op string, vecDim int, s }) } -func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string, backend string) (grpc.Backend, error) { +func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, cl *config.ModelConfigLoader, storeName string, backend string) (grpc.Backend, error) { + // Resolve the per-store model config (keyed by the store namespace, which + // is the model ID for a store). This is the LocalAI-native config surface: + // a store's backend selection and its backend-specific settings live in a + // model YAML's `backend:` and `options:` fields, so different stores can + // point at different servers/indexes. When no config exists for the store, + // we fall back to the default backend and let the backend apply its own + // built-in defaults — preserving the zero-config experience. + var loadOpts []string + if cl != nil { + if cfg, ok := cl.GetModelConfig(storeName); ok { + if backend == "" { + backend = cfg.Backend + } + loadOpts = cfg.Options + } + } + if backend == "" { backend = model.LocalStoreBackend } @@ -145,5 +167,12 @@ func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, st model.WithModel(store.NamespacePrefix + storeName), } + // Thread the store's configured options through to the backend's LoadModel + // via ModelOptions.Options (field 62). The loader clones these opts and + // overrides only Model/ModelFile, so the namespace set above is preserved. + if len(loadOpts) > 0 { + sc = append(sc, model.WithLoadGRPCLoadModelOpts(&pb.ModelOptions{Options: loadOpts})) + } + return sl.Load(sc...) } diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index ad8b4d622..3de119bbb 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -16,6 +16,7 @@ const ( UsecaseTokenize = "tokenize" UsecaseImage = "image" UsecaseVideo = "video" + Usecase3D = "3d" UsecaseTranscript = "transcript" UsecaseTTS = "tts" UsecaseSoundGeneration = "sound_generation" @@ -42,6 +43,7 @@ const ( MethodEmbedding GRPCMethod = "Embedding" MethodGenerateImage GRPCMethod = "GenerateImage" MethodGenerateVideo GRPCMethod = "GenerateVideo" + MethodGenerate3D GRPCMethod = "Generate3D" MethodAudioTranscription GRPCMethod = "AudioTranscription" MethodTTS GRPCMethod = "TTS" MethodTTSStream GRPCMethod = "TTSStream" @@ -124,6 +126,11 @@ var UsecaseInfoMap = map[string]UsecaseInfo{ GRPCMethod: MethodGenerateVideo, Description: "Video generation via the GenerateVideo RPC, with optional image or audio conditioning when supported by the backend.", }, + Usecase3D: { + Flag: FLAG_3D, + GRPCMethod: MethodGenerate3D, + Description: "Image-conditioned 3D asset generation via the Generate3D RPC — a binary glTF (GLB) mesh with optional PBR material (TRELLIS.2).", + }, UsecaseTranscript: { Flag: FLAG_TRANSCRIPT, GRPCMethod: MethodAudioTranscription, @@ -351,6 +358,14 @@ var BackendCapabilities = map[string]BackendCapability{ Description: "Stable Diffusion via GGML quantized models", }, + // --- 3D generation backends --- + "trellis2cpp": { + GRPCMethods: []GRPCMethod{MethodGenerate3D}, + PossibleUsecases: []string{Usecase3D}, + DefaultUsecases: []string{Usecase3D}, + Description: "trellis2.cpp — C++/GGML port of Microsoft TRELLIS.2: single-image to textured 3D mesh (GLB)", + }, + // --- Speech-to-text backends --- "whisper": { GRPCMethods: []GRPCMethod{MethodAudioTranscription, MethodVAD}, diff --git a/core/config/model_capabilities.go b/core/config/model_capabilities.go index 79a117b4a..6b432bddc 100644 --- a/core/config/model_capabilities.go +++ b/core/config/model_capabilities.go @@ -15,9 +15,10 @@ const ( ModalityImage = "image" ModalityAudio = "audio" ModalityVideo = "video" + Modality3D = "3d" ) -var modalityOrder = []string{ModalityText, ModalityImage, ModalityAudio, ModalityVideo} +var modalityOrder = []string{ModalityText, ModalityImage, ModalityAudio, ModalityVideo, Modality3D} func declaredModalities(modalities []string) map[string]bool { declared := make(map[string]bool, len(modalities)) @@ -154,6 +155,7 @@ func (c *ModelConfig) Capabilities() []string { add(c.HasUsecases(FLAG_SOUND_GENERATION), UsecaseSoundGeneration) add(c.HasUsecases(FLAG_IMAGE), UsecaseImage) add(c.HasUsecases(FLAG_VIDEO), UsecaseVideo) + add(c.HasUsecases(FLAG_3D), Usecase3D) add(c.HasUsecases(FLAG_VAD), UsecaseVAD) add(c.HasUsecases(FLAG_DETECTION), UsecaseDetection) add(c.HasUsecases(FLAG_DEPTH), UsecaseDepth) @@ -181,9 +183,10 @@ func (c *ModelConfig) InputModalities() []string { c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) || imageGen || videoGen // Image input via a chat model requires vision (gated on chat, like the - // Ollama surface); detection/depth/face models consume images directly. + // Ollama surface); detection/depth/face/3D models consume images directly. imageIn := (chatish && c.VisionSupported()) || c.LimitMMPerPrompt.LimitImagePerPrompt > 0 || - c.HasUsecases(FLAG_DETECTION) || c.HasUsecases(FLAG_DEPTH) || c.HasUsecases(FLAG_FACE_RECOGNITION) + c.HasUsecases(FLAG_DETECTION) || c.HasUsecases(FLAG_DEPTH) || c.HasUsecases(FLAG_FACE_RECOGNITION) || + c.HasUsecases(FLAG_3D) audioIn := c.AudioInputSupported() || c.HasUsecases(FLAG_TRANSCRIPT) || c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO) || c.HasUsecases(FLAG_VAD) || c.HasUsecases(FLAG_DIARIZATION) || @@ -208,10 +211,12 @@ func (c *ModelConfig) OutputModalities() []string { audioOut := c.HasUsecases(FLAG_TTS) || c.HasUsecases(FLAG_SOUND_GENERATION) || c.HasUsecases(FLAG_AUDIO_TRANSFORM) || c.HasUsecases(FLAG_REALTIME_AUDIO) videoOut := c.HasUsecases(FLAG_VIDEO) + threeDOut := c.HasUsecases(FLAG_3D) modalities[ModalityText] = modalities[ModalityText] || textOut modalities[ModalityImage] = modalities[ModalityImage] || imageOut modalities[ModalityAudio] = modalities[ModalityAudio] || audioOut modalities[ModalityVideo] = modalities[ModalityVideo] || videoOut + modalities[Modality3D] = modalities[Modality3D] || threeDOut return orderedModalities(modalities) } diff --git a/core/config/model_capabilities_test.go b/core/config/model_capabilities_test.go index 545bad50b..84f01af7b 100644 --- a/core/config/model_capabilities_test.go +++ b/core/config/model_capabilities_test.go @@ -109,6 +109,25 @@ var _ = Describe("Model capabilities derivation", func() { Expect(cfg.OutputModalities()).To(Equal([]string{"image"})) }) + It("guesses the 3d usecase from the trellis2cpp backend and only that backend", func() { + cfg := &ModelConfig{Backend: "trellis2cpp"} + Expect(cfg.HasUsecases(FLAG_3D)).To(BeTrue()) + Expect(cfg.Capabilities()).To(ContainElement(Usecase3D)) + + other := &ModelConfig{Backend: "llama-cpp"} + Expect(other.HasUsecases(FLAG_3D)).To(BeFalse()) + }) + + It("a 3D-generation model reads an image and writes a 3D asset", func() { + // Pins the wire strings the UI depends on: capability "3d", + // input modality "image" (no text prompt — TRELLIS.2 is + // image-conditioned only), output modality "3d". + cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_3D), Backend: "trellis2cpp"} + Expect(cfg.Capabilities()).To(Equal([]string{Usecase3D})) + Expect(cfg.InputModalities()).To(Equal([]string{ModalityImage})) + Expect(cfg.OutputModalities()).To(Equal([]string{Modality3D})) + }) + It("conditioned video uses declared modalities without backend-specific inference", func() { cfg := &ModelConfig{ KnownUsecases: usecaseBits(FLAG_VIDEO), diff --git a/core/config/model_config.go b/core/config/model_config.go index 3a32ac2c8..21bc42fac 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -1673,6 +1673,11 @@ const ( // labels via the SoundDetection RPC, e.g. ced). FLAG_SOUND_CLASSIFICATION ModelConfigUsecase = 0b10000000000000000000000 + // Marks a model as wired for the Generate3D gRPC primitive + // (image-conditioned 3D asset generation — a binary glTF mesh with + // optional PBR material, e.g. trellis2cpp). + FLAG_3D ModelConfigUsecase = 0b100000000000000000000000 + // Common Subsets FLAG_LLM ModelConfigUsecase = FLAG_CHAT | FLAG_COMPLETION | FLAG_EDIT ) @@ -1686,7 +1691,7 @@ var ModalityGroups = []ModelConfigUsecase{ FLAG_TRANSCRIPT | FLAG_REALTIME_AUDIO | FLAG_SOUND_CLASSIFICATION, // audio input — realtime_audio is any-to-any, so it counts here too FLAG_TTS | FLAG_SOUND_GENERATION | FLAG_REALTIME_AUDIO, // audio output — and here, so a lone realtime_audio flag still reads as multimodal FLAG_AUDIO_TRANSFORM, // audio in/out transforms - FLAG_IMAGE | FLAG_VIDEO, // visual generation + FLAG_IMAGE | FLAG_VIDEO | FLAG_3D, // visual generation } // IsMultimodal returns true if the given usecases span two or more orthogonal @@ -1733,6 +1738,7 @@ func GetAllModelConfigUsecases() map[string]ModelConfigUsecase { "FLAG_SCORE": FLAG_SCORE, "FLAG_DEPTH": FLAG_DEPTH, "FLAG_TOKEN_CLASSIFY": FLAG_TOKEN_CLASSIFY, + "FLAG_3D": FLAG_3D, } } @@ -1884,6 +1890,13 @@ func (c *ModelConfig) GuessUsecases(u ModelConfigUsecase) bool { } } + if (u & FLAG_3D) == FLAG_3D { + threeDBackends := []string{"trellis2cpp"} + if !slices.Contains(threeDBackends, c.Backend) { + return false + } + } + if (u & FLAG_FACE_RECOGNITION) == FLAG_FACE_RECOGNITION { faceBackends := []string{"insightface"} if !slices.Contains(faceBackends, c.Backend) { diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 8c156144d..a86e86530 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -143,6 +143,11 @@ var defaultImporters = []Importer{ &CoquiImporter{}, // Image/Video (Batch 3) &StableDiffusionGGMLImporter{}, + // Trellis2CppImporter (TRELLIS.2 image-to-3D, native C++/ggml port) must + // run before LlamaCPPImporter so its GGUF sets aren't claimed by the + // generic .gguf importer; matches only trellis-named URIs/repos or the + // distinctive component filenames, so arbitrary GGUFs are never claimed. + &Trellis2CppImporter{}, &ACEStepImporter{}, // LongCat repositories carry generic Diffusers metadata, so this exact // owner/repo matcher must run before DiffuserImporter. diff --git a/core/gallery/importers/trellis2cpp.go b/core/gallery/importers/trellis2cpp.go new file mode 100644 index 000000000..b94bb904d --- /dev/null +++ b/core/gallery/importers/trellis2cpp.go @@ -0,0 +1,170 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "go.yaml.in/yaml/v2" +) + +var _ Importer = &Trellis2CppImporter{} + +// trellis2File describes one component of the TRELLIS.2 GGUF set hosted on +// the LocalAI-io HuggingFace org. The pipeline spans three source repos +// (TRELLIS.2-4B, TRELLIS-image-large for the SS decoder, and a DINOv3 +// mirror), so a single import URI always expands to this full set — no one +// repo can describe it alone. Filenames follow the trellis2cpp converter +// defaults, which the backend resolves without any options. +type trellis2File struct { + filename string + uri string + sha256 string +} + +var trellis2Files = []trellis2File{ + {"dino_f16.gguf", "https://huggingface.co/LocalAI-io/dinov3-vitl16-pretrain-lvd1689m-GGUF/resolve/main/dino_f16.gguf", "385d8186a38a2328ec740fb2ac1f33f9194d8774efc7ccafd4aa2e51cf5f6450"}, + {"ss_flow_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/ss_flow_f16.gguf", "1dded5b74237d24e6876a642a26f90b43742e3554418573860f810e3bbe61e8c"}, + {"ss_dec_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS-image-large-GGUF/resolve/main/ss_dec_f16.gguf", "9c2210b7ed830fdc8286961a8189878ff5bcfd3bfc83ab4eacee005d293d2185"}, + {"slat_flow_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/slat_flow_f16.gguf", "2f94bad7b1c524ad8c01943bc38fcc0c314e7d482ce896f3c6e96eb6e7cec15c"}, + {"slat_flow_1024_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/slat_flow_1024_f16.gguf", "b6a2270131e2e9235e9b6cb525193eb85ae132fa5af3274322aacd39e40a6bc5"}, + {"shape_dec_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/shape_dec_f16.gguf", "6fe53f1d7763dabf7c8d72bc38f4053d87fde6f65bf17a9d378d27edb39d3530"}, + {"shape_enc_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/shape_enc_f16.gguf", "3ec80ff580987fcdb9bc594fc8b6fda890d63101ca442eb2b26f5dc315e8696c"}, + {"tex_dec_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/tex_dec_f16.gguf", "afd304f4dfcb8c94df851b85519b415b99f04070f7d29de1320c50631b1be4e0"}, + {"tex_slat_flow_512_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/tex_slat_flow_512_f16.gguf", "89a081b7f5487a5b31f03d240e4d959a56db0cc2c46c327230097a2554da52ae"}, + {"tex_slat_flow_1024_f16.gguf", "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF/resolve/main/tex_slat_flow_1024_f16.gguf", "bbb55b0910c7929aac5e0612a9bb15113837a2c674cafb9f0f170eda8b5558a8"}, +} + +// trellis2ComponentNames are the distinctive default component filenames. A +// raw .gguf URL with one of these basenames is a strong trellis2 signal. +// dino_f16.gguf is deliberately absent — DINO checkpoints are common enough +// that the bare name would over-claim. +var trellis2ComponentNames = map[string]struct{}{ + "ss_flow_f16.gguf": {}, + "ss_dec_f16.gguf": {}, + "slat_flow_f16.gguf": {}, + "slat_flow_1024_f16.gguf": {}, + "shape_dec_f16.gguf": {}, + "shape_enc_f16.gguf": {}, + "tex_dec_f16.gguf": {}, + "tex_slat_flow_512_f16.gguf": {}, + "tex_slat_flow_1024_f16.gguf": {}, +} + +// Trellis2CppImporter recognises Microsoft TRELLIS.2 image-to-3D GGUF sets +// (the trellis2.cpp converter outputs hosted under LocalAI-io). It must be +// registered BEFORE LlamaCPPImporter so llama-cpp does not steal the .gguf +// match. preferences.backend="trellis2cpp" overrides detection. +type Trellis2CppImporter struct{} + +func (i *Trellis2CppImporter) Name() string { return "trellis2cpp" } +func (i *Trellis2CppImporter) Modality() string { return "3d" } +func (i *Trellis2CppImporter) AutoDetects() bool { return true } + +// containsTrellisToken reports whether s (compared case-insensitively) +// carries a TRELLIS marker ("trellis" covers TRELLIS.2 / trellis2 too). +func containsTrellisToken(s string) bool { + return strings.Contains(strings.ToLower(s), "trellis") +} + +func (i *Trellis2CppImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return false + } + } + + if b, ok := preferencesMap["backend"].(string); ok && b != "" { + return b == "trellis2cpp" + } + + // Raw .gguf URL named after a distinctive pipeline component. + if strings.HasSuffix(strings.ToLower(details.URI), ".gguf") { + base := strings.ToLower(filepath.Base(details.URI)) + if _, ok := trellis2ComponentNames[base]; ok { + return true + } + } + + // A trellis-named URI or HF repo carrying GGUFs. + if containsTrellisToken(details.URI) { + if strings.HasSuffix(strings.ToLower(details.URI), ".gguf") { + return true + } + if details.HuggingFace != nil && hasGGUF(details.HuggingFace.Files) { + return true + } + // HF details may be nil (tree-listing quirk) — decide from the + // owner/repo alone. + if _, repo, ok := HFOwnerRepoFromURI(details.URI); ok && containsTrellisToken(repo) { + return true + } + } + + return false +} + +func (i *Trellis2CppImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return gallery.ModelConfig{}, err + } + } + + name, ok := preferencesMap["name"].(string) + if !ok { + name = "trellis2-4b" + } + + description, ok := preferencesMap["description"].(string) + if !ok { + description = "TRELLIS.2 image-to-3D (GLB with PBR textures) — imported from " + details.URI + } + + cfg := gallery.ModelConfig{ + Name: name, + Description: description, + } + // The full pipeline spans three HF repos, so any trellis URI imports the + // complete known-good set rather than whatever single repo was pasted. + for _, f := range trellis2Files { + cfg.Files = append(cfg.Files, gallery.File{ + URI: f.uri, + Filename: f.filename, + SHA256: f.sha256, + }) + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + Backend: "trellis2cpp", + KnownUsecaseStrings: []string{"FLAG_3D"}, + PredictionOptions: schema.PredictionOptions{ + // ss_flow anchors the GGUF directory; the backend resolves the + // other components from their default filenames next to it. + BasicModelRequest: schema.BasicModelRequest{Model: "ss_flow_f16.gguf"}, + }, + } + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + + cfg.ConfigFile = string(data) + return cfg, nil +} diff --git a/core/gallery/importers/trellis2cpp_test.go b/core/gallery/importers/trellis2cpp_test.go new file mode 100644 index 000000000..00251526f --- /dev/null +++ b/core/gallery/importers/trellis2cpp_test.go @@ -0,0 +1,105 @@ +package importers_test + +import ( + "encoding/json" + "fmt" + + "github.com/mudler/LocalAI/core/gallery/importers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Trellis2CppImporter", func() { + Context("detection from HuggingFace", func() { + // LocalAI-io/TRELLIS.2-4B-GGUF is the canonical GGUF conversion of + // microsoft/TRELLIS.2-4B produced by the trellis2cpp converters. + // Detection must route it to trellis2cpp (and NOT to llama-cpp, + // which otherwise steals every .gguf repo). + It("matches the TRELLIS.2 GGUF repo and imports the full component set", func() { + uri := "https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF" + preferences := json.RawMessage(`{}`) + + modelConfig, err := importers.DiscoverModelConfig(uri, preferences) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: trellis2cpp"), fmt.Sprintf("Model config: %+v", modelConfig)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("known_usecases")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("FLAG_3D")) + // The pipeline spans three repos; the import must carry the whole + // set, anchored on ss_flow. + Expect(modelConfig.Files).To(HaveLen(10)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: ss_flow_f16.gguf")) + }) + + It("matches a raw .gguf URL named after a distinctive pipeline component", func() { + uri := "https://example.com/models/tex_slat_flow_512_f16.gguf" + preferences := json.RawMessage(`{}`) + + modelConfig, err := importers.DiscoverModelConfig(uri, preferences) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: trellis2cpp"), fmt.Sprintf("Model config: %+v", modelConfig)) + }) + }) + + Context("preference override", func() { + It("honours preferences.backend=trellis2cpp for arbitrary URIs", func() { + uri := "https://example.com/some-unrelated-model" + preferences := json.RawMessage(`{"backend": "trellis2cpp"}`) + + modelConfig, err := importers.DiscoverModelConfig(uri, preferences) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: trellis2cpp"), fmt.Sprintf("Model config: %+v", modelConfig)) + }) + + It("does not override a different explicit backend", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/tex_slat_flow_512_f16.gguf", + Preferences: json.RawMessage(`{"backend": "llama-cpp"}`), + }) + + Expect(match).To(BeFalse()) + }) + + It("still auto-detects when the backend preference is empty", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/tex_slat_flow_512_f16.gguf", + Preferences: json.RawMessage(`{"backend": ""}`), + }) + + Expect(match).To(BeTrue()) + }) + }) + + Context("negative detection", func() { + It("does not claim an unrelated raw .gguf URL", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/llama-3-8b-Q4_K.gguf", + Preferences: json.RawMessage(`{}`), + }) + Expect(match).To(BeFalse()) + }) + + It("does not claim a bare dino_f16.gguf (too generic a name)", func() { + imp := &importers.Trellis2CppImporter{} + match := imp.Match(importers.Details{ + URI: "https://example.com/models/dino_f16.gguf", + Preferences: json.RawMessage(`{}`), + }) + Expect(match).To(BeFalse()) + }) + }) + + Context("Importer interface metadata", func() { + It("exposes name/modality/autodetect", func() { + imp := &importers.Trellis2CppImporter{} + Expect(imp.Name()).To(Equal("trellis2cpp")) + Expect(imp.Modality()).To(Equal("3d")) + Expect(imp.AutoDetects()).To(BeTrue()) + }) + }) +}) diff --git a/core/http/app.go b/core/http/app.go index bee47013b..8b08af4ad 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -55,6 +55,12 @@ var quietPaths = []string{"/api/operations", "/api/resources", "/healthz", "/rea // conditional revalidation round-trip. const immutableAssetCacheControl = "public, max-age=31536000, immutable" +func defaultBodyLimitSkipper(c echo.Context) bool { + // Remeshing accepts generated GLBs that routinely exceed the default + // upload limit. The route has its own tighter, format-specific limit. + return c.Request().Method == http.MethodPost && c.Path() == "/3d/remesh" +} + // applyModelLoadCooldown maps a ModelLoadCooldownError anywhere in err's chain // to HTTP 503 with a Retry-After header (whole seconds, floor 1), so a client // polling a model whose load recently failed backs off instead of triggering a @@ -123,7 +129,10 @@ func API(application *application.Application) (*echo.Echo, error) { // Set body limit if application.ApplicationConfig().UploadLimitMB > 0 { - e.Use(middleware.BodyLimit(fmt.Sprintf("%dM", application.ApplicationConfig().UploadLimitMB))) + e.Use(middleware.BodyLimitWithConfig(middleware.BodyLimitConfig{ + Limit: fmt.Sprintf("%dM", application.ApplicationConfig().UploadLimitMB), + Skipper: defaultBodyLimitSkipper, + })) } // SPA fallback handler, set later when React UI is available @@ -305,14 +314,22 @@ func API(application *application.Application) (*echo.Echo, error) { audioPath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "audio") imagePath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "images") videoPath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "videos") + threeDPath := filepath.Join(application.ApplicationConfig().GeneratedContentDir, "3d") os.MkdirAll(audioPath, 0750) os.MkdirAll(imagePath, 0750) os.MkdirAll(videoPath, 0750) + _ = os.MkdirAll(threeDPath, 0750) + + // Go's built-in MIME table has no .glb entry and minimal containers + // ship no /etc/mime.types, so generated GLBs would otherwise be + // served as application/octet-stream. + _ = mime.AddExtensionType(".glb", "model/gltf-binary") e.Static("/generated-audio", audioPath) e.Static("/generated-images", imagePath) e.Static("/generated-videos", videoPath) + e.Static("/generated-3d", threeDPath) } // Usage recording is initialised in application/startup.go and diff --git a/core/http/auth/features.go b/core/http/auth/features.go index a43eef992..063b4d76e 100644 --- a/core/http/auth/features.go +++ b/core/http/auth/features.go @@ -91,6 +91,10 @@ var RouteFeatureRegistry = []RouteFeature{ // Video {"POST", "/video", FeatureVideo}, + // 3D generation + {"POST", "/3d/generations", Feature3D}, + {"POST", "/3d/remesh", Feature3D}, + // Sound generation {"POST", "/v1/sound-generation", FeatureSound}, @@ -182,6 +186,7 @@ func APIFeatureMetas() []FeatureMeta { {FeatureVAD, "Voice Activity Detection", true}, {FeatureDetection, "Detection", true}, {FeatureVideo, "Video Generation", true}, + {Feature3D, "3D Generation", true}, {FeatureEmbeddings, "Embeddings", true}, {FeatureSound, "Sound Generation", true}, {FeatureRealtime, "Realtime", true}, diff --git a/core/http/auth/middleware.go b/core/http/auth/middleware.go index c67954640..dd76bdf09 100644 --- a/core/http/auth/middleware.go +++ b/core/http/auth/middleware.go @@ -584,6 +584,7 @@ func isAPIPath(path string) bool { strings.HasPrefix(path, "/tts") || strings.HasPrefix(path, "/vad") || strings.HasPrefix(path, "/video") || + strings.HasPrefix(path, "/3d/") || strings.HasPrefix(path, "/stores/") || strings.HasPrefix(path, "/system") || strings.HasPrefix(path, "/ws/") || diff --git a/core/http/auth/middleware_test.go b/core/http/auth/middleware_test.go index 5137851e1..94be9336a 100644 --- a/core/http/auth/middleware_test.go +++ b/core/http/auth/middleware_test.go @@ -156,6 +156,16 @@ var _ = Describe("Auth Middleware", func() { Expect(rec.Code).To(Equal(http.StatusUnauthorized)) }) + It("returns 401 for unauthenticated 3D generation requests", func() { + rec := doRequest(app, http.MethodPost, "/3d/generations") + Expect(rec.Code).To(Equal(http.StatusUnauthorized)) + }) + + It("returns 401 for unauthenticated 3D remesh requests", func() { + rec := doRequest(app, http.MethodPost, "/3d/remesh") + Expect(rec.Code).To(Equal(http.StatusUnauthorized)) + }) + It("allows unauthenticated access to non-API paths when no legacy keys", func() { rec := doRequest(app, http.MethodGet, "/app") Expect(rec.Code).To(Equal(http.StatusOK)) diff --git a/core/http/auth/permissions.go b/core/http/auth/permissions.go index 1795792f9..6c01e03a3 100644 --- a/core/http/auth/permissions.go +++ b/core/http/auth/permissions.go @@ -47,6 +47,7 @@ const ( FeatureVAD = "vad" FeatureDetection = "detection" FeatureVideo = "video" + Feature3D = "3d" FeatureEmbeddings = "embeddings" FeatureSound = "sound" FeatureRealtime = "realtime" @@ -73,7 +74,7 @@ var GeneralFeatures = []string{FeatureFineTuning, FeatureQuantization} var APIFeatures = []string{ FeatureChat, FeatureImages, FeatureAudioSpeech, FeatureAudioTranscription, FeatureAudioDiarization, FeatureAudioClassification, - FeatureVAD, FeatureDetection, FeatureVideo, FeatureEmbeddings, FeatureSound, + FeatureVAD, FeatureDetection, FeatureVideo, Feature3D, FeatureEmbeddings, FeatureSound, FeatureRealtime, FeatureRerank, FeatureTokenize, FeatureMCP, FeatureStores, FeatureFaceRecognition, FeatureVoiceRecognition, FeatureAudioTransform, FeaturePIIFilter, diff --git a/core/http/body_limit_test.go b/core/http/body_limit_test.go new file mode 100644 index 000000000..7d398766c --- /dev/null +++ b/core/http/body_limit_test.go @@ -0,0 +1,66 @@ +package http_test + +import ( + "bytes" + "context" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/http" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Request body limits", func() { + It("lets the remesh route apply its larger limit without weakening other routes", func() { + dir := GinkgoT().TempDir() + models := filepath.Join(dir, "models") + backends := filepath.Join(dir, "backends") + Expect(os.Mkdir(models, 0o750)).To(Succeed()) + Expect(os.Mkdir(backends, 0o750)).To(Succeed()) + + state, err := system.GetSystemState( + system.WithModelPath(models), + system.WithBackendPath(backends), + ) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + localApp, err := application.New( + config.WithContext(ctx), + config.WithSystemState(state), + config.WithUploadLimitMB(1), + ) + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = localApp.Shutdown() }() + app, err := API(localApp) + Expect(err).NotTo(HaveOccurred()) + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + Expect(writer.WriteField("model", "missing-model")).To(Succeed()) + part, err := writer.CreateFormFile("mesh", "large.glb") + Expect(err).NotTo(HaveOccurred()) + _, err = part.Write(bytes.Repeat([]byte{'x'}, 2<<20)) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + request := httptest.NewRequest(http.MethodPost, "/3d/remesh", body) + request.Header.Set("Content-Type", writer.FormDataContentType()) + response := httptest.NewRecorder() + app.ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusNotFound), response.Body.String()) + + request = httptest.NewRequest(http.MethodPost, "/3d/generations", bytes.NewReader(make([]byte, 2<<20))) + request.Header.Set("Content-Type", "application/json") + response = httptest.NewRecorder() + app.ServeHTTP(response, request) + Expect(response.Code).To(Equal(http.StatusRequestEntityTooLarge), response.Body.String()) + }) +}) diff --git a/core/http/endpoints/localai/api_instructions.go b/core/http/endpoints/localai/api_instructions.go index 02ff85d1d..62f264cf0 100644 --- a/core/http/endpoints/localai/api_instructions.go +++ b/core/http/endpoints/localai/api_instructions.go @@ -81,6 +81,12 @@ var instructionDefs = []instructionDef{ Tags: []string{"video"}, Intro: "POST /video accepts start_image, end_image, and audio as public URL, base64, or data URI. Backend-specific tuning is passed as string values in params.", }, + { + Name: "3d", + Description: "Image-to-3D asset generation (binary glTF / GLB) via TRELLIS.2", + Tags: []string{"3d"}, + Intro: "POST /3d/generations accepts a conditioning image as public URL, base64, or data URI (no text prompt) and returns one .glb asset as a URL under /generated-3d or as b64_json. quality selects the mesh pipeline (auto|coarse|512|1024); background controls solid-background removal (auto|keep|black|white); step, texture_steps, and cfg_scale tune the flow sampling. POST /3d/remesh accepts multipart model, mesh (GLB), and a single detail percentage to return a watertight print-ready GLB; the enclosing offset is derived automatically.", + }, { Name: "face-recognition", Description: "Face verification (1:1), identification (1:N), embedding, and demographic analysis", diff --git a/core/http/endpoints/localai/api_instructions_test.go b/core/http/endpoints/localai/api_instructions_test.go index 43caa9ee1..9648e674a 100644 --- a/core/http/endpoints/localai/api_instructions_test.go +++ b/core/http/endpoints/localai/api_instructions_test.go @@ -39,7 +39,7 @@ var _ = Describe("API Instructions Endpoints", func() { instructions, ok := resp["instructions"].([]any) Expect(ok).To(BeTrue()) - Expect(instructions).To(HaveLen(17)) + Expect(instructions).To(HaveLen(18)) // Verify each instruction has required fields and correct URL format for _, s := range instructions { @@ -79,6 +79,7 @@ var _ = Describe("API Instructions Endpoints", func() { "middleware-admin", "intelligent-routing", "voice-library", + "3d", )) }) }) @@ -123,6 +124,16 @@ var _ = Describe("API Instructions Endpoints", func() { Expect(string(body)).To(ContainSubstring("stream")) }) + It("should advertise the LocalAI 3D generation path", func() { + req := httptest.NewRequest(http.MethodGet, "/api/instructions/3d", nil) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + + body, _ := io.ReadAll(rec.Body) + Expect(string(body)).To(ContainSubstring("POST /3d/generations")) + Expect(string(body)).NotTo(ContainSubstring("/v1/3d/generations")) + }) + It("should return JSON fragment when format=json", func() { req := httptest.NewRequest(http.MethodGet, "/api/instructions/chat-inference?format=json", nil) rec := httptest.NewRecorder() diff --git a/core/http/endpoints/localai/model3d.go b/core/http/endpoints/localai/model3d.go new file mode 100644 index 000000000..fd514c7db --- /dev/null +++ b/core/http/endpoints/localai/model3d.go @@ -0,0 +1,186 @@ +package localai + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "slices" + "time" + + "github.com/google/uuid" + "github.com/labstack/echo/v4" + + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + + "github.com/mudler/xlog" + + model "github.com/mudler/LocalAI/pkg/model" +) + +// Conditioning images are single frames, so a much tighter cap than the +// video-input limit is enough. +const max3DInputBytes = 32 << 20 + +var ( + valid3DQualities = []string{"", "auto", "coarse", "512", "1024"} + valid3DBackgrounds = []string{"", "auto", "keep", "black", "white"} +) + +// Model3DEndpoint +// @Summary Creates a 3D asset (binary glTF / GLB) from a conditioning image. +// @Tags 3d +// @Param request body schema.Model3DRequest true "query params" +// @Success 200 {object} schema.OpenAIResponse "Response" +// @Router /3d/generations [post] +func Model3DEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { + return func(c echo.Context) error { + input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.Model3DRequest) + if !ok || input.Model == "" { + xlog.Error("3D Endpoint - Invalid Input") + return echo.ErrBadRequest + } + + config, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + if !ok || config == nil { + xlog.Error("3D Endpoint - Invalid Config") + return echo.ErrBadRequest + } + + if input.Image == "" { + return echo.NewHTTPError(http.StatusBadRequest, "image is required: 3D generation is image-conditioned") + } + // Reject unknown enum values here rather than surfacing an opaque + // backend error after a model load. + if !slices.Contains(valid3DQualities, input.Quality) { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid quality %q: must be one of auto, coarse, 512, 1024", input.Quality)) + } + if !slices.Contains(valid3DBackgrounds, input.Background) { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid background %q: must be one of auto, keep, black, white", input.Background)) + } + + src, err := stageVideoMediaWithLimit(c.Request().Context(), appConfig.GeneratedContentDir, input.Image, max3DInputBytes) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid image: %v", err)) + } + defer func() { _ = os.Remove(src) }() + + xlog.Debug("Parameter Config", "config", config) + + if config.Backend == "" { + config.Backend = model.Trellis2CppBackend + } + + step := input.Step + if step == 0 && config.Step != 0 { + step = int32(config.Step) + } + cfgScale := input.CFGScale + if cfgScale == 0 && config.CFGScale != 0 { + cfgScale = config.CFGScale + } + + b64JSON := input.ResponseFormat == "b64_json" + + tempDir := "" + if !b64JSON { + tempDir = filepath.Join(appConfig.GeneratedContentDir, "3d") + if err := os.MkdirAll(tempDir, 0o750); err != nil { + return err + } + } + // Create a temporary file + outputFile, err := os.CreateTemp(tempDir, "b64") + if err != nil { + return err + } + if err := outputFile.Close(); err != nil { + _ = os.Remove(outputFile.Name()) + return err + } + + output := outputFile.Name() + ".glb" + + // Rename the temporary file + err = os.Rename(outputFile.Name(), output) + if err != nil { + _ = os.Remove(outputFile.Name()) + return err + } + preserveOutput := false + defer func() { + if !preserveOutput { + _ = os.Remove(output) + } + }() + + baseURL := middleware.BaseURL(c) + + xlog.Debug("Model3DEndpoint: Calling Model3DGeneration", + "quality", input.Quality, + "background", input.Background, + "cfg_scale", cfgScale, + "step", step, + "texture_steps", input.TextureSteps, + "seed", input.Seed) + + fn, err := backend.Model3DGeneration( + backend.Model3DGenerationOptions{ + Image: src, + Destination: output, + Seed: input.Seed, + Step: step, + CFGScale: cfgScale, + TextureSteps: input.TextureSteps, + Quality: input.Quality, + Background: input.Background, + Params: input.Params, + }, + ml, + *config, + appConfig, + ) + if err != nil { + return mapBackendError(err) + } + if err := fn(); err != nil { + return mapBackendError(err) + } + + item := &schema.Item{} + + if b64JSON { + data, err := os.ReadFile(output) + if err != nil { + return err + } + item.B64JSON = base64.StdEncoding.EncodeToString(data) + } else { + base := filepath.Base(output) + item.URL, err = url.JoinPath(baseURL, "generated-3d", base) + if err != nil { + return err + } + preserveOutput = true + } + + id := uuid.New().String() + created := int(time.Now().Unix()) + resp := &schema.OpenAIResponse{ + ID: id, + Created: created, + Data: []schema.Item{*item}, + } + + jsonResult, _ := json.Marshal(resp) + xlog.Debug("Response", "response", string(jsonResult)) + + return c.JSON(200, resp) + } +} diff --git a/core/http/endpoints/localai/model3d_internal_test.go b/core/http/endpoints/localai/model3d_internal_test.go new file mode 100644 index 000000000..4437a2a3a --- /dev/null +++ b/core/http/endpoints/localai/model3d_internal_test.go @@ -0,0 +1,72 @@ +package localai + +import ( + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" +) + +// The validation branches all return before any model load, so the handler can +// be driven with nil loaders. +var _ = Describe("3D endpoint request validation", func() { + call := func(input *schema.Model3DRequest) error { + appConfig := &config.ApplicationConfig{GeneratedContentDir: GinkgoT().TempDir()} + handler := Model3DEndpoint(nil, nil, appConfig) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/3d/generations", strings.NewReader("{}")) + c := e.NewContext(req, httptest.NewRecorder()) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, input) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{Name: "test-3d"}) + return handler(c) + } + + expectBadRequest := func(err error, substr string) { + var httpErr *echo.HTTPError + Expect(err).To(BeAssignableToTypeOf(httpErr)) + httpErr = err.(*echo.HTTPError) + Expect(httpErr.Code).To(Equal(http.StatusBadRequest)) + Expect(httpErr.Message).To(ContainSubstring(substr)) + } + + It("requires a conditioning image", func() { + err := call(&schema.Model3DRequest{BasicModelRequest: schema.BasicModelRequest{Model: "m"}}) + expectBadRequest(err, "image is required") + }) + + It("rejects unknown quality values", func() { + err := call(&schema.Model3DRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "m"}, + Image: "aGk=", + Quality: "2048", + }) + expectBadRequest(err, "invalid quality") + }) + + It("rejects unknown background values", func() { + err := call(&schema.Model3DRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "m"}, + Image: "aGk=", + Background: "transparent", + }) + expectBadRequest(err, "invalid background") + }) + + It("rejects undecodable image payloads", func() { + err := call(&schema.Model3DRequest{ + BasicModelRequest: schema.BasicModelRequest{Model: "m"}, + Image: "not%%%base64", + Quality: "512", + Background: "auto", + }) + expectBadRequest(err, "invalid image") + }) +}) diff --git a/core/http/endpoints/localai/model3d_remesh.go b/core/http/endpoints/localai/model3d_remesh.go new file mode 100644 index 000000000..a0a2b004e --- /dev/null +++ b/core/http/endpoints/localai/model3d_remesh.go @@ -0,0 +1,163 @@ +package localai + +import ( + "fmt" + "io" + "math" + "net/http" + "os" + "path/filepath" + "strconv" + + "github.com/labstack/echo/v4" + + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + model "github.com/mudler/LocalAI/pkg/model" +) + +const ( + max3DRemeshBytes = 512 << 20 + defaultRemeshDetailPct = float32(0.5) + minRemeshDetailPct = float32(0.35) + maxRemeshDetailPct = float32(2.5) +) + +func normalizedRemeshDetail(detail float32) (float32, error) { + if detail == 0 { + return defaultRemeshDetailPct, nil + } + if math.IsNaN(float64(detail)) || math.IsInf(float64(detail), 0) || detail < minRemeshDetailPct || detail > maxRemeshDetailPct { + return 0, fmt.Errorf("detail must be between %.2f and %.2f percent", minRemeshDetailPct, maxRemeshDetailPct) + } + return detail, nil +} + +func saveRemeshUpload(c echo.Context, dir string) (string, error) { + header, err := c.FormFile("mesh") + if err != nil { + return "", fmt.Errorf("mesh is required") + } + if header.Size > max3DRemeshBytes { + return "", fmt.Errorf("mesh exceeds the 512 MiB limit") + } + source, err := header.Open() + if err != nil { + return "", fmt.Errorf("opening mesh: %w", err) + } + defer func() { _ = source.Close() }() + + if err := os.MkdirAll(dir, 0o750); err != nil { + return "", err + } + temp, err := os.CreateTemp(dir, "remesh-input-*.glb") + if err != nil { + return "", err + } + path := temp.Name() + defer func() { + if err != nil { + _ = os.Remove(path) + } + }() + + written, copyErr := io.Copy(temp, io.LimitReader(source, max3DRemeshBytes+1)) + closeErr := temp.Close() + if copyErr != nil { + err = fmt.Errorf("saving mesh: %w", copyErr) + return "", err + } + if closeErr != nil { + err = closeErr + return "", err + } + if written == 0 { + err = fmt.Errorf("mesh is empty") + return "", err + } + if written > max3DRemeshBytes { + err = fmt.Errorf("mesh exceeds the 512 MiB limit") + return "", err + } + return path, nil +} + +// Model3DRemeshEndpoint rebuilds an existing generated GLB as a watertight mesh. +// @Summary Applies watertight print remeshing to an existing 3D asset. +// @Tags 3d +// @Accept multipart/form-data +// @Produce model/gltf-binary +// @Param model formData string true "3D model name" +// @Param mesh formData file true "Source GLB" +// @Param detail formData number false "Detail size as percent of the source bounding-box diagonal (0.35–2.5; default 0.5)" +// @Success 200 {file} binary "Remeshed GLB" +// @Router /3d/remesh [post] +func Model3DRemeshEndpoint(ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { + return func(c echo.Context) error { + input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.Model3DRemeshRequest) + if !ok || input.Model == "" { + return echo.NewHTTPError(http.StatusBadRequest, "model is required") + } + modelConfig, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) + if !ok || modelConfig == nil { + return echo.ErrBadRequest + } + detail, err := normalizedRemeshDetail(input.Detail) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + + source, err := saveRemeshUpload(c, appConfig.GeneratedContentDir) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } + defer func() { _ = os.Remove(source) }() + + outputFile, err := os.CreateTemp(appConfig.GeneratedContentDir, "remeshed-*.glb") + if err != nil { + return err + } + output := outputFile.Name() + if err := outputFile.Close(); err != nil { + _ = os.Remove(output) + return err + } + defer func() { _ = os.Remove(output) }() + + if modelConfig.Backend == "" { + modelConfig.Backend = model.Trellis2CppBackend + } + fn, err := backend.Model3DGeneration( + backend.Model3DGenerationOptions{ + Image: source, + Destination: output, + Params: map[string]string{ + "operation": "print_remesh", + "alpha_ratio": strconv.FormatFloat(float64(detail/100), 'g', -1, 32), + "detail_percent": strconv.FormatFloat(float64(detail), 'g', -1, 32), + "texture_size": "2048", + }, + }, + ml, + *modelConfig, + appConfig, + ) + if err != nil { + return mapBackendError(err) + } + if err := fn(); err != nil { + return mapBackendError(err) + } + + file, err := os.Open(filepath.Clean(output)) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + c.Response().Header().Set(echo.HeaderContentDisposition, `attachment; filename="remeshed.glb"`) + c.Response().Header().Set(echo.HeaderCacheControl, "no-store") + return c.Stream(http.StatusOK, "model/gltf-binary", file) + } +} diff --git a/core/http/endpoints/localai/model3d_remesh_internal_test.go b/core/http/endpoints/localai/model3d_remesh_internal_test.go new file mode 100644 index 000000000..d3e4108e2 --- /dev/null +++ b/core/http/endpoints/localai/model3d_remesh_internal_test.go @@ -0,0 +1,68 @@ +package localai + +import ( + "bytes" + "math" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("3D print remeshing request handling", func() { + DescribeTable("normalizes the demo detail range", + func(input, expected float32, valid bool) { + value, err := normalizedRemeshDetail(input) + if valid { + Expect(err).NotTo(HaveOccurred()) + Expect(value).To(BeNumerically("~", expected, 1e-6)) + } else { + Expect(err).To(HaveOccurred()) + } + }, + Entry("default", float32(0), float32(0.5), true), + Entry("fine endpoint", float32(0.35), float32(0.35), true), + Entry("coarse endpoint", float32(2.5), float32(2.5), true), + Entry("too fine", float32(0.1), float32(0), false), + Entry("too coarse", float32(3), float32(0), false), + Entry("not a number", float32(math.NaN()), float32(0), false), + ) + + It("streams the multipart GLB to a bounded temporary file", func() { + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + Expect(writer.WriteField("model", "trellis-test-model")).To(Succeed()) + Expect(writer.WriteField("detail", "0.35")).To(Succeed()) + part, err := writer.CreateFormFile("mesh", "source.glb") + Expect(err).NotTo(HaveOccurred()) + _, err = part.Write([]byte("glTF-test")) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/3d/remesh", body) + req.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + ctx := e.NewContext(req, httptest.NewRecorder()) + input := new(schema.Model3DRemeshRequest) + Expect(ctx.Bind(input)).To(Succeed()) + Expect(input.Model).To(Equal("trellis-test-model")) + Expect(input.Detail).To(BeNumerically("~", 0.35, 1e-6)) + path, err := saveRemeshUpload(ctx, GinkgoT().TempDir()) + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.Remove(path) }() + Expect(os.ReadFile(path)).To(Equal([]byte("glTF-test"))) + }) + + It("requires a mesh part", func() { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/3d/remesh", bytes.NewReader(nil)) + ctx := e.NewContext(req, httptest.NewRecorder()) + _, err := saveRemeshUpload(ctx, GinkgoT().TempDir()) + Expect(err).To(MatchError("mesh is required")) + }) +}) diff --git a/core/http/endpoints/localai/stores.go b/core/http/endpoints/localai/stores.go index 8074da9e0..af9cf1569 100644 --- a/core/http/endpoints/localai/stores.go +++ b/core/http/endpoints/localai/stores.go @@ -9,7 +9,7 @@ import ( "github.com/mudler/LocalAI/pkg/store" ) -func StoresSetEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { +func StoresSetEndpoint(sl *model.ModelLoader, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { return func(c echo.Context) error { input := new(schema.StoresSet) @@ -17,7 +17,7 @@ func StoresSetEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfi return err } - sb, err := backend.StoreBackend(sl, appConfig, input.Store, input.Backend) + sb, err := backend.StoreBackend(sl, appConfig, cl, input.Store, input.Backend) if err != nil { return err } @@ -36,7 +36,7 @@ func StoresSetEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfi } } -func StoresDeleteEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { +func StoresDeleteEndpoint(sl *model.ModelLoader, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { return func(c echo.Context) error { input := new(schema.StoresDelete) @@ -44,7 +44,7 @@ func StoresDeleteEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationCo return err } - sb, err := backend.StoreBackend(sl, appConfig, input.Store, input.Backend) + sb, err := backend.StoreBackend(sl, appConfig, cl, input.Store, input.Backend) if err != nil { return err } @@ -57,7 +57,7 @@ func StoresDeleteEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationCo } } -func StoresGetEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { +func StoresGetEndpoint(sl *model.ModelLoader, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { return func(c echo.Context) error { input := new(schema.StoresGet) @@ -65,7 +65,7 @@ func StoresGetEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfi return err } - sb, err := backend.StoreBackend(sl, appConfig, input.Store, input.Backend) + sb, err := backend.StoreBackend(sl, appConfig, cl, input.Store, input.Backend) if err != nil { return err } @@ -88,7 +88,7 @@ func StoresGetEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfi } } -func StoresFindEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { +func StoresFindEndpoint(sl *model.ModelLoader, cl *config.ModelConfigLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc { return func(c echo.Context) error { input := new(schema.StoresFind) @@ -96,7 +96,7 @@ func StoresFindEndpoint(sl *model.ModelLoader, appConfig *config.ApplicationConf return err } - sb, err := backend.StoreBackend(sl, appConfig, input.Store, input.Backend) + sb, err := backend.StoreBackend(sl, appConfig, cl, input.Store, input.Backend) if err != nil { return err } diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index bee3b52ed..32b08b3fd 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -2163,6 +2163,11 @@ type liveResponse struct { output []types.MessageItemUnion usage backend.TokenUsage outcome responseOutcome + // metadata is echoed back on response.created and response.done. It is the + // only thing tying a terminal event to the response.create that asked for + // it, which is what lets a client run an out-of-band response alongside the + // spoken conversation and still recognise its own answer. + metadata map[string]string } func (r *liveResponse) addItem(it types.MessageItemUnion) { r.output = append(r.output, it) } @@ -2192,12 +2197,16 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, // terminals the legacy code emitted (one response.done per turn, with empty // Output/Usage) are gone; tool turns are now internal to this single response. r := &liveResponse{id: generateUniqueID()} + if overrides != nil { + r.metadata = overrides.Metadata + } sendEvent(t, types.ResponseCreatedEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusInProgress, + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusInProgress, + Metadata: r.metadata, }, }) @@ -2208,10 +2217,11 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, sendEvent(t, types.ResponseDoneEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusCancelled, - Output: r.output, + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusCancelled, + Output: r.output, + Metadata: r.metadata, }, }) case outcomeFailed: @@ -2221,11 +2231,12 @@ func triggerResponse(ctx context.Context, session *Session, conv *Conversation, sendEvent(t, types.ResponseDoneEvent{ ServerEventBase: types.ServerEventBase{}, Response: types.Response{ - ID: r.id, - Object: "realtime.response", - Status: types.ResponseStatusCompleted, - Output: r.output, - Usage: responseUsage(r.usage), + ID: r.id, + Object: "realtime.response", + Status: types.ResponseStatusCompleted, + Output: r.output, + Usage: responseUsage(r.usage), + Metadata: r.metadata, }, }) } diff --git a/core/http/endpoints/openai/realtime_stream_test.go b/core/http/endpoints/openai/realtime_stream_test.go index 439f3240e..2d5d7d7a1 100644 --- a/core/http/endpoints/openai/realtime_stream_test.go +++ b/core/http/endpoints/openai/realtime_stream_test.go @@ -263,4 +263,64 @@ var _ = Describe("triggerResponse", func() { Expect(done.Response.Usage.OutputTokens).To(Equal(3)) Expect(done.Response.Usage.TotalTokens).To(Equal(8)) }) + + // response.metadata is the only thing tying a terminal event back to the + // response.create that asked for it. Without the echo, a client running an + // out-of-band response alongside the spoken conversation cannot tell its own + // answer from the conversation's, and blocks until it times out. + It("echoes response.create metadata back on response.created and response.done", func() { + m := &fakeModel{ + cfg: &config.ModelConfig{}, + predictResp: backend.LLMResponse{Response: "Hi there."}, + } + session := &Session{ + OutputSampleRate: 24000, + ModelInterface: m, + ModelConfig: &config.ModelConfig{}, + OutputModalities: []types.Modality{types.ModalityText}, + } + t := &fakeTransport{} + + triggerResponse(context.Background(), session, &Conversation{}, t, &types.ResponseCreateParams{ + Metadata: map[string]string{"client_run": "abc123"}, + }) + + var created *types.ResponseCreatedEvent + var done *types.ResponseDoneEvent + for i := range t.events { + switch e := t.events[i].(type) { + case types.ResponseCreatedEvent: + created = &e + case types.ResponseDoneEvent: + done = &e + } + } + Expect(created).NotTo(BeNil()) + Expect(created.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123")) + Expect(done).NotTo(BeNil()) + Expect(done.Response.Metadata).To(HaveKeyWithValue("client_run", "abc123")) + }) + + // Omitted rather than sent as an empty object, matching the omitempty tag. + It("sends no metadata when response.create carried none", func() { + m := &fakeModel{ + cfg: &config.ModelConfig{}, + predictResp: backend.LLMResponse{Response: "Hi there."}, + } + session := &Session{ + OutputSampleRate: 24000, + ModelInterface: m, + ModelConfig: &config.ModelConfig{}, + OutputModalities: []types.Modality{types.ModalityText}, + } + t := &fakeTransport{} + + triggerResponse(context.Background(), session, &Conversation{}, t, nil) + + for i := range t.events { + if d, ok := t.events[i].(types.ResponseDoneEvent); ok { + Expect(d.Response.Metadata).To(BeEmpty()) + } + } + }) }) diff --git a/core/http/react-ui/e2e/page-render-smoke.spec.js b/core/http/react-ui/e2e/page-render-smoke.spec.js index 27f006b98..3f24752bf 100644 --- a/core/http/react-ui/e2e/page-render-smoke.spec.js +++ b/core/http/react-ui/e2e/page-render-smoke.spec.js @@ -14,6 +14,7 @@ import { test, expect } from './coverage-fixtures.js' // not bounce to a gate redirect (/login or back to /app home). const PAGES = [ ['/app/talk', 'Talk'], + ['/app/3d', '3D Generation'], ['/app/usage', 'Usage'], ['/app/account', 'Account'], ['/app/studio', 'Studio'], diff --git a/core/http/react-ui/e2e/threed-gen.spec.js b/core/http/react-ui/e2e/threed-gen.spec.js new file mode 100644 index 000000000..b10270e19 --- /dev/null +++ b/core/http/react-ui/e2e/threed-gen.spec.js @@ -0,0 +1,319 @@ +import { test, expect } from './coverage-fixtures.js' + +// 3D generation page: mock the capabilities + generation endpoints, feed a +// real (tiny) Form-B GLB through the parser/viewer, and exercise the +// IndexedDB-backed history. All assertions are DOM/text — never pixels — so +// the suite passes with or without working WebGL2 in headless Chromium. + +function mockCapabilities(page) { + return page.route('**/api/models/capabilities', (route) => { + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ data: [{ id: 'trellis-test-model', capabilities: ['FLAG_3D'] }] }), + }) + }) +} + +// A valid one-triangle GLB in trellis2cpp's vertex-PBR form: POSITION/NORMAL +// f32, COLOR_0 u16-normalized VEC4, _METALLIC_ROUGHNESS u8-normalized VEC2, +// u32 indices — the layout mesh_export.cpp's write_vertex_glb emits. +function buildTinyGlb() { + const positions = new Float32Array([0, 0, 0, 1, 0, 0, 0, 1, 0]) + const normals = new Float32Array([0, 0, 1, 0, 0, 1, 0, 0, 1]) + const colors = new Uint16Array([ + 65535, 0, 0, 65535, + 0, 65535, 0, 65535, + 0, 0, 65535, 65535, + ]) + const metalRough = new Uint8Array([0, 153, 0, 153, 0, 153]) + const indices = new Uint32Array([0, 1, 2]) + + const views = [] + let binLength = 0 + const addView = (typed) => { + const byteOffset = binLength + views.push({ buffer: 0, byteOffset, byteLength: typed.byteLength, target: 34962 }) + binLength += typed.byteLength + binLength += (4 - (binLength % 4)) % 4 + return views.length - 1 + } + addView(positions); addView(normals); addView(colors); addView(metalRough) + const idxView = addView(indices) + views[idxView].target = 34963 + + const json = { + asset: { version: '2.0', generator: 'threed-gen.spec' }, + scene: 0, + scenes: [{ nodes: [0] }], + nodes: [{ mesh: 0 }], + meshes: [{ primitives: [{ attributes: { POSITION: 0, NORMAL: 1, COLOR_0: 2, _METALLIC_ROUGHNESS: 3 }, indices: 4, material: 0 }] }], + materials: [{ pbrMetallicRoughness: { baseColorFactor: [1, 1, 1, 1], metallicFactor: 0, roughnessFactor: 0.6 }, doubleSided: true }], + accessors: [ + { bufferView: 0, componentType: 5126, count: 3, type: 'VEC3', min: [0, 0, 0], max: [1, 1, 0] }, + { bufferView: 1, componentType: 5126, count: 3, type: 'VEC3' }, + { bufferView: 2, componentType: 5123, normalized: true, count: 3, type: 'VEC4' }, + { bufferView: 3, componentType: 5121, normalized: true, count: 3, type: 'VEC2' }, + { bufferView: 4, componentType: 5125, count: 3, type: 'SCALAR' }, + ], + bufferViews: views, + buffers: [{ byteLength: binLength }], + } + + const bin = Buffer.alloc(binLength) + const parts = [positions, normals, colors, metalRough, indices] + for (let i = 0; i < parts.length; i++) { + Buffer.from(parts[i].buffer, parts[i].byteOffset, parts[i].byteLength).copy(bin, views[i].byteOffset) + } + + let jsonText = JSON.stringify(json) + while (jsonText.length % 4 !== 0) jsonText += ' ' + const jsonBuf = Buffer.from(jsonText) + + const total = 12 + 8 + jsonBuf.length + 8 + bin.length + const glb = Buffer.alloc(total) + glb.writeUInt32LE(0x46546c67, 0) // magic 'glTF' + glb.writeUInt32LE(2, 4) + glb.writeUInt32LE(total, 8) + glb.writeUInt32LE(jsonBuf.length, 12) + glb.writeUInt32LE(0x4e4f534a, 16) // 'JSON' + jsonBuf.copy(glb, 20) + const binHeader = 20 + jsonBuf.length + glb.writeUInt32LE(bin.length, binHeader) + glb.writeUInt32LE(0x004e4942, binHeader + 4) // 'BIN\0' + bin.copy(glb, binHeader + 8) + return glb +} + +// 1x1 transparent PNG for the conditioning-image upload. +const TINY_PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', + 'base64', +) + +function mockGeneration(page, onRequest) { + return page.route('**/3d/generations', (route) => { + if (route.request().method() !== 'POST') return route.continue() + onRequest?.(route.request().postDataJSON()) + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + created: Math.floor(Date.now() / 1000), + data: [{ url: '/generated-3d/test.glb' }], + }), + }) + }) +} + +function mockGlbDownload(page) { + return page.route('**/generated-3d/test.glb', (route) => { + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'model/gltf-binary' }, + body: buildTinyGlb(), + }) + }) +} + +async function generateOnce(page) { + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'trellis-test-model' })).toBeVisible({ timeout: 10_000 }) + await page.locator('#threed-image-file').setInputFiles({ + name: 'input.png', + mimeType: 'image/png', + buffer: TINY_PNG, + }) + await page.locator('button[type="submit"]').click() +} + +test.describe('3D generation', () => { + test.beforeEach(async ({ page }) => { + await mockCapabilities(page) + await mockGlbDownload(page) + }) + + test('generates, shows mesh stats, and offers a GLB download', async ({ page }) => { + let requestBody = null + await mockGeneration(page, (body) => { requestBody = body }) + + await generateOnce(page) + + // Stats render from the parsed GLB even without working GL. + await expect(page.getByTestId('glb-stats')).toContainText('3', { timeout: 15_000 }) + await expect(page.getByTestId('glb-stats')).toContainText('1') + await expect(page.getByTestId('glb-stats')).toContainText('PBR') + + const download = page.getByTestId('glb-download') + await expect(download).toBeVisible() + await expect(download).toHaveAttribute('href', /^blob:/) + await expect(download).toHaveAttribute('download', /\.glb$/) + + expect(requestBody.model).toBe('trellis-test-model') + expect(requestBody.image).toBeTruthy() + expect(requestBody.quality).toBe('auto') + expect(requestBody.background).toBe('auto') + expect(requestBody.response_format).toBe('url') + }) + + test('advanced settings map to step/texture_steps/cfg_scale/seed', async ({ page }) => { + let requestBody = null + await mockGeneration(page, (body) => { requestBody = body }) + + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'trellis-test-model' })).toBeVisible({ timeout: 10_000 }) + await page.locator('#threed-image-file').setInputFiles({ name: 'input.png', mimeType: 'image/png', buffer: TINY_PNG }) + + await page.locator('select').first().selectOption('512') + await page.getByRole('button', { name: /Advanced Settings/ }).click() + const advanced = page.locator('#threed-advanced-options') + await advanced.locator('input').nth(0).fill('20') // steps + await advanced.locator('input').nth(1).fill('8') // texture steps + await advanced.locator('input').nth(2).fill('5.5') // guidance + await advanced.locator('input').nth(3).fill('42') // seed + await page.locator('button[type="submit"]').click() + + await expect(page.getByTestId('glb-download')).toBeVisible({ timeout: 15_000 }) + expect(requestBody.quality).toBe('512') + expect(requestBody.step).toBe(20) + expect(requestBody.texture_steps).toBe(8) + expect(requestBody.cfg_scale).toBe(5.5) + expect(requestBody.seed).toBe(42) + }) + + test('applies a single-detail watertight remesh and previews it before download', async ({ page }) => { + await mockGeneration(page) + let remeshRequest = null + await page.route('**/3d/remesh', (route) => { + remeshRequest = route.request() + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'model/gltf-binary' }, + body: buildTinyGlb(), + }) + }) + + await generateOnce(page) + await expect(page.getByTestId('glb-remesh')).toBeVisible({ timeout: 15_000 }) + await page.getByLabel('Remesh detail').fill('100') + await page.getByTestId('glb-remesh').click() + + await expect(page.getByTestId('glb-remesh')).toContainText('Show original') + await expect(page.getByTestId('glb-download')).toHaveAttribute('download', /-remeshed\.glb$/) + expect(remeshRequest).not.toBeNull() + expect(remeshRequest.method()).toBe('POST') + expect(remeshRequest.headers()['content-type']).toContain('multipart/form-data') + const multipart = remeshRequest.postDataBuffer().toString() + expect(multipart).toContain('trellis-test-model') + expect(multipart).toContain('0.35') + + await page.getByTestId('glb-remesh').click() + await expect(page.getByTestId('glb-remesh')).toContainText('Apply remeshing') + await expect(page.getByTestId('glb-download')).not.toHaveAttribute('download', /-remeshed\.glb$/) + }) + + test('history entry persists across navigation and reloads into the viewer', async ({ page }) => { + await mockGeneration(page) + await generateOnce(page) + await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 15_000 }) + + // IndexedDB persists within the browser context — navigate away and back. + await page.goto('/app') + await page.goto('/app/3d') + await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 15_000 }) + + // Selecting the entry loads the stored Blob back into the viewer. + await page.getByTestId('media-history-item').click() + await expect(page.getByTestId('glb-stats')).toBeVisible({ timeout: 15_000 }) + await expect(page.getByTestId('glb-download')).toHaveAttribute('href', /^blob:/) + }) + + test('deleting a history entry removes it', async ({ page }) => { + await mockGeneration(page) + await generateOnce(page) + await expect(page.getByTestId('media-history-item')).toHaveCount(1, { timeout: 15_000 }) + + await page.getByTestId('media-history-delete').click() + await expect(page.getByTestId('media-history-item')).toHaveCount(0) + }) + + test('API errors surface through the trace link error box', async ({ page }) => { + await page.route('**/3d/generations', (route) => { + route.fulfill({ + status: 500, + contentType: 'application/json', + body: JSON.stringify({ error: { message: 'trellis2 pipeline load: missing files' } }), + }) + }) + + await generateOnce(page) + await expect(page.locator('.media-result')).toContainText(/missing files|error/i, { timeout: 15_000 }) + }) + + test('rejects oversized conditioning images before making an API request', async ({ page }) => { + let requested = false + await mockGeneration(page, () => { requested = true }) + await page.goto('/app/3d') + await expect(page.getByRole('button', { name: 'trellis-test-model' })).toBeVisible({ timeout: 10_000 }) + + await page.locator('#threed-image-file').setInputFiles({ + name: 'oversized.png', + mimeType: 'image/png', + buffer: Buffer.alloc(32 * 1024 * 1024 + 1), + }) + + await expect(page.locator('.toast')).toContainText('32 MiB limit') + expect(requested).toBe(false) + }) + + test('hides and guards 3D generation when the feature is disabled', async ({ page }) => { + await page.route('**/api/auth/status', (route) => { + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + authEnabled: true, + staticApiKeyRequired: false, + user: { id: 'restricted-user', role: 'user', permissions: { '3d': false } }, + }), + }) + }) + + await page.goto('/app/studio?tab=threed') + await expect(page.getByRole('button', { name: '3D', exact: true })).toHaveCount(0) + await expect(page.locator('.studio-tab', { hasText: 'Images' })).toHaveClass(/studio-tab-active/) + + await page.goto('/app/3d') + await expect(page).toHaveURL(/\/app\/?$/) + }) + + test.describe('touch input', () => { + test.use({ hasTouch: true }) + + test('accepts two-finger touch gestures in the GLB viewer', async ({ page }) => { + await mockGeneration(page) + await generateOnce(page) + const canvas = page.getByTestId('glb-canvas') + await expect(canvas).toBeVisible({ timeout: 15_000 }) + await canvas.scrollIntoViewIfNeeded() + const box = await canvas.boundingBox() + expect(box).not.toBeNull() + + const client = await page.context().newCDPSession(page) + await client.send('Input.dispatchTouchEvent', { + type: 'touchStart', + touchPoints: [ + { id: 1, x: box.x + box.width * 0.35, y: box.y + box.height * 0.5 }, + { id: 2, x: box.x + box.width * 0.65, y: box.y + box.height * 0.5 }, + ], + }) + await client.send('Input.dispatchTouchEvent', { + type: 'touchMove', + touchPoints: [ + { id: 1, x: box.x + box.width * 0.3, y: box.y + box.height * 0.45 }, + { id: 2, x: box.x + box.width * 0.7, y: box.y + box.height * 0.55 }, + ], + }) + await client.send('Input.dispatchTouchEvent', { type: 'touchEnd', touchPoints: [] }) + + await expect(page.getByRole('button', { name: 'Auto-rotate' })).toHaveClass(/btn-secondary/) + }) + }) +}) diff --git a/core/http/react-ui/public/locales/en/media.json b/core/http/react-ui/public/locales/en/media.json index 7beefe10b..d3e9484f2 100644 --- a/core/http/react-ui/public/locales/en/media.json +++ b/core/http/react-ui/public/locales/en/media.json @@ -5,7 +5,8 @@ "video": "Video", "tts": "TTS", "sound": "Sound", - "transform": "Transform" + "transform": "Transform", + "threed": "3D" } }, "image": { @@ -70,6 +71,59 @@ "noResults": "No video generated" } }, + "threed": { + "title": "3D Generation", + "labels": { + "model": "Model", + "image": "Input image", + "quality": "Quality", + "quality_auto": "Auto (best available)", + "quality_coarse": "Coarse preview (fast)", + "quality_512": "512³ fine", + "quality_1024": "1024³ cascade (slow)", + "background": "Background", + "background_auto": "Auto-remove solid background", + "background_keep": "Keep original", + "background_black": "Remove black", + "background_white": "Remove white", + "advanced": "Advanced Settings", + "steps": "Shape steps", + "textureSteps": "Material steps", + "guidance": "Guidance", + "seed": "Seed", + "seedPlaceholder": "Random" + }, + "actions": { + "generate": "Generate", + "generating": "Generating...", + "remesh": "Apply remeshing", + "remeshing": "Applying remeshing...", + "showOriginal": "Show original", + "download": "Download GLB" + }, + "remesh": { + "title": "Watertight print remesh", + "detail": "Remesh detail", + "coarser": "Coarser · faster", + "finer": "Finer · slower", + "hint": "Detail controls the smallest preserved features. The enclosing offset follows it automatically.", + "ready": "Previewing the watertight remeshed model. This is the version that will be downloaded." + }, + "viewer": { + "wireframe": "Wireframe", + "autoRotate": "Auto-rotate", + "noWebgl": "WebGL2 is not available in this browser — download the GLB to view it elsewhere.", + "contextLost": "The 3D view lost the GPU context — reload the page to restore it.", + "stats": "{{verts}} vertices · {{tris}} triangles", + "hint": "drag: rotate · pinch/wheel: zoom · two-finger/right-drag: pan · double-click: reset" + }, + "empty": "Generated 3D model will appear here", + "toasts": { + "noImage": "Please provide an input image", + "noModel": "Please select a model", + "noResults": "No model generated" + } + }, "tts": { "title": "Text to Speech", "labels": { diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index c9a7ab4e9..2e2f21e57 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -29,6 +29,7 @@ "llm": "Chat", "image": "Image", "video": "Video", + "threed": "3D", "multimodal": "Multimodal", "vision": "Vision", "tts": "TTS", diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index 68a7e4d5a..b27ec01a7 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -5166,6 +5166,45 @@ button.collapsible-header:focus-visible { border-radius: var(--radius-md); } +.threed-remesh-controls { + display: flex; + flex-direction: column; + gap: var(--spacing-xs); + padding: var(--spacing-md); + background: var(--color-surface-raised); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-md); +} + +.threed-remesh-heading, +.threed-remesh-scale { + display: flex; + justify-content: space-between; + gap: var(--spacing-md); +} + +.threed-remesh-heading { + color: var(--color-text-primary); + font-size: var(--text-sm); + font-weight: var(--font-weight-medium); +} + +.threed-remesh-heading output, +.threed-remesh-scale, +.threed-remesh-ready { + color: var(--color-text-muted); + font-size: var(--text-xs); +} + +.threed-remesh-controls input[type="range"] { + width: 100%; +} + +.threed-remesh-ready { + margin: var(--spacing-xs) 0 0; + color: var(--color-success); +} + .media-result-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); diff --git a/core/http/react-ui/src/components/GlbViewer.jsx b/core/http/react-ui/src/components/GlbViewer.jsx new file mode 100644 index 000000000..ead0c8a32 --- /dev/null +++ b/core/http/react-ui/src/components/GlbViewer.jsx @@ -0,0 +1,636 @@ +import { useEffect, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { parseGlb } from '../utils/glb' + +/* ── WebGL2 GLB viewer ────────────────────────────────────────────────────── + * Ported from the trellis2cpp demo server's hand-rolled viewer + * (localai-org/trellis2cpp server/web/index.html): quaternion trackball, + * metallic-roughness PBR with a procedural environment, ACES tonemapping, + * hidden-line wireframe. Kept dependency-free — the renderer is tuned for + * the dense unoriented dual-grid meshes TRELLIS.2 produces, which generic + * GLTF viewers shade poorly. + * + * Deltas from the demo: input is a parsed GLB (see utils/glb.js) instead of + * the demo's T2MESH stream; COLOR_0 arrives LINEAR (no gamma decode — the + * demo's pow(2.2) would double-darken it); vertex attributes upload as + * normalized integers straight from the GLB buffers; an optional UV-texture + * path covers the atlas-baked form; the base orientation drops the demo's + * Z-up -> Y-up turn because baked GLBs are already Y-up. + */ + +const VS = `#version 300 es +layout(location=0) in vec3 pos; +layout(location=1) in vec3 nrm; +layout(location=2) in vec4 baseColor; +layout(location=3) in vec2 metalRough; +layout(location=4) in vec2 uv; +uniform mat4 mvp, viewModel; +out vec3 vN; out vec3 vV; out vec4 vCol; out vec2 vMR; out vec2 vUV; +void main() { + gl_Position = mvp * vec4(pos, 1.0); + vec4 vp = viewModel * vec4(pos, 1.0); + vV = -vp.xyz; // view-space: to-camera (camera at origin) + vN = mat3(viewModel) * nrm; // view-space normal (orbits with camera) + vCol = baseColor; + vMR = metalRough; + vUV = uv; +}` + +// Lightweight metallic-roughness PBR in view space. The dual-grid mesh has +// unoriented winding (faithful to TRELLIS.2), so every normal is faced toward +// the camera. No IBL — a procedural environment + one studio key light; +// ACES tonemapping at the end. +const FS = `#version 300 es +precision highp float; +in vec3 vN; in vec3 vV; in vec4 vCol; in vec2 vMR; in vec2 vUV; +uniform int wire; +uniform int colorMode; // 0 = untextured grey, 1 = vertex PBR (linear COLOR_0), 2 = UV textures +uniform sampler2D baseColorTex; +uniform sampler2D mrTex; +uniform float uMetallicFactor, uRoughnessFactor; +out vec4 frag; +// procedural environment radiance in a direction (view space; +y = up) +vec3 envColor(vec3 d) { + vec3 sky = mix(vec3(0.32, 0.40, 0.55), vec3(0.72, 0.80, 0.98), clamp(d.y, 0.0, 1.0)); + vec3 gnd = vec3(0.14, 0.13, 0.12); + return mix(gnd, sky, smoothstep(-0.30, 0.12, d.y)); +} +void main() { + if (wire == 1) { frag = vec4(0.30, 0.64, 1.00, 1.0); return; } + vec3 N = normalize(vN), V = normalize(vV); + if (dot(N, V) < 0.0) N = -N; // face the camera (mesh is unoriented) + vec3 base = vec3(0.62, 0.66, 0.72); + float metal = 0.0, rough = 0.5, opacity = 1.0; + if (colorMode == 1) { + // COLOR_0 is stored linear in the GLB — use it directly. + base = clamp(vCol.rgb, 0.0, 1.0); + metal = clamp(vMR.x, 0.0, 1.0); + rough = clamp(vMR.y, 0.06, 1.0); + opacity = clamp(vCol.a, 0.0, 1.0); + } else if (colorMode == 2) { + vec4 bc = texture(baseColorTex, vUV); + base = pow(clamp(bc.rgb, 0.0, 1.0), vec3(2.2)); // baseColorTexture is sRGB + opacity = bc.a; + vec3 mr = texture(mrTex, vUV).rgb; // G=roughness, B=metallic + metal = clamp(mr.b * uMetallicFactor, 0.0, 1.0); + rough = clamp(mr.g * uRoughnessFactor, 0.06, 1.0); + } + if (opacity < 0.01) discard; + float nv = max(dot(N, V), 1e-3); + + // Schlick Fresnel (grazing reflectance rises to 1 - roughness for metals). + vec3 F0 = mix(vec3(0.04), base, metal); + vec3 F = F0 + (max(vec3(1.0 - rough), F0) - F0) * pow(1.0 - nv, 5.0); + + // diffuse: hemispheric environment irradiance (metals have no diffuse) + vec3 irr = envColor(N) * 0.55 + vec3(0.12); + vec3 diffuse = base * (1.0 - metal) * irr; + + // specular: environment reflection, blurred toward a flat tint by roughness + vec3 refl = mix(envColor(reflect(-V, N)), vec3(0.34, 0.37, 0.44), rough * rough); + vec3 specular = refl * F; + + // one crisp studio key light for a lively highlight + vec3 L = normalize(vec3(0.45, 0.70, 0.55)), H = normalize(L + V); + float shin = mix(8.0, 260.0, pow(1.0 - rough, 2.0)); + float sp = pow(max(dot(N, H), 0.0), shin) * (shin + 2.0) / 6.2831853; + vec3 kc = vec3(1.00, 0.96, 0.88); + float ndl = max(dot(N, L), 0.0); + specular += kc * sp * F * ndl; + diffuse += base * (1.0 - metal) * kc * ndl * 0.28; + + vec3 color = diffuse + specular; + color = (color * (2.51 * color + 0.03)) / (color * (2.43 * color + 0.59) + 0.14); // ACES + frag = vec4(pow(clamp(color, 0.0, 1.0), vec3(1.0/2.2)), opacity); +}` + +/* trackball orientation (quaternion): each drag composes a small rotation + * about the screen axes onto the current orientation — no gimbal lock. */ +const Q = { + axisAngle(x, y, z, a) { const h = a * 0.5, s = Math.sin(h); return [x * s, y * s, z * s, Math.cos(h)] }, + mul(a, b) { // Hamilton product a·b (apply b, then a) + return [ + a[3] * b[0] + a[0] * b[3] + a[1] * b[2] - a[2] * b[1], + a[3] * b[1] - a[0] * b[2] + a[1] * b[3] + a[2] * b[0], + a[3] * b[2] + a[0] * b[1] - a[1] * b[0] + a[2] * b[3], + a[3] * b[3] - a[0] * b[0] - a[1] * b[1] - a[2] * b[2], + ] + }, + norm(q) { const n = Math.hypot(q[0], q[1], q[2], q[3]) || 1; return [q[0] / n, q[1] / n, q[2] / n, q[3] / n] }, + toMat4(q) { // column-major rotation matrix + const [x, y, z, w] = q + const xx = x * x, yy = y * y, zz = z * z, xy = x * y, xz = x * z, yz = y * z, wx = w * x, wy = w * y, wz = w * z + return new Float32Array([ + 1 - 2 * (yy + zz), 2 * (xy + wz), 2 * (xz - wy), 0, + 2 * (xy - wz), 1 - 2 * (xx + zz), 2 * (yz + wx), 0, + 2 * (xz + wy), 2 * (yz - wx), 1 - 2 * (xx + yy), 0, + 0, 0, 0, 1, + ]) + }, +} + +// GLBs are already Y-up (the baker swaps axes on export), so unlike the demo +// there is no Z-up correction here — just a gentle 3/4 default view. +const QBASE = Q.norm(Q.mul(Q.axisAngle(1, 0, 0, -0.30), Q.axisAngle(0, 1, 0, 0.55))) + +/* minimal mat4 helpers (column-major) */ +const M = { + mul(a, b) { + const o = new Float32Array(16) + for (let c = 0; c < 4; ++c) for (let r = 0; r < 4; ++r) + o[c * 4 + r] = a[r] * b[c * 4] + a[4 + r] * b[c * 4 + 1] + a[8 + r] * b[c * 4 + 2] + a[12 + r] * b[c * 4 + 3] + return o + }, + persp(fov, asp, near, far) { + const f = 1 / Math.tan(fov / 2), o = new Float32Array(16) + o[0] = f / asp; o[5] = f + o[10] = (far + near) / (near - far); o[11] = -1 + o[14] = 2 * far * near / (near - far) + return o + }, + trans(x, y, z) { + const o = new Float32Array(16) + o[0] = o[5] = o[10] = o[15] = 1 + o[12] = x; o[13] = y; o[14] = z + return o + }, + scale(s) { + const o = new Float32Array(16) + o[0] = o[5] = o[10] = s; o[15] = 1 + return o + }, +} + +function glType(gl, array) { + if (array instanceof Uint8Array) return gl.UNSIGNED_BYTE + if (array instanceof Uint16Array) return gl.UNSIGNED_SHORT + return gl.FLOAT +} + +export function createGlbViewer(canvas, { onContextLost } = {}) { + const gl = canvas.getContext('webgl2', { antialias: true }) + if (!gl) return null + + // If the GPU context is lost (driver reset / out of memory), preventDefault + // keeps it recoverable and the page shows a notice instead of a dead canvas. + const contextLost = (e) => { + e.preventDefault() + if (onContextLost) onContextLost() + } + canvas.addEventListener('webglcontextlost', contextLost, false) + + function shader(type, src) { + const s = gl.createShader(type) + gl.shaderSource(s, src); gl.compileShader(s) + if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s)) + return s + } + const prog = gl.createProgram() + gl.attachShader(prog, shader(gl.VERTEX_SHADER, VS)) + gl.attachShader(prog, shader(gl.FRAGMENT_SHADER, FS)) + gl.linkProgram(prog) + if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) throw new Error(gl.getProgramInfoLog(prog)) + const uMVP = gl.getUniformLocation(prog, 'mvp') + const uViewModel = gl.getUniformLocation(prog, 'viewModel') + const uWire = gl.getUniformLocation(prog, 'wire') + const uColorMode = gl.getUniformLocation(prog, 'colorMode') + const uBaseColorTex = gl.getUniformLocation(prog, 'baseColorTex') + const uMrTex = gl.getUniformLocation(prog, 'mrTex') + const uMetallicFactor = gl.getUniformLocation(prog, 'uMetallicFactor') + const uRoughnessFactor = gl.getUniformLocation(prog, 'uRoughnessFactor') + + const vao = gl.createVertexArray() + const vbo = gl.createBuffer(), nbo = gl.createBuffer(), cbo = gl.createBuffer() + const mbo = gl.createBuffer(), ubo = gl.createBuffer(), ibo = gl.createBuffer() + const wireIbo = gl.createBuffer() + let nIndices = 0, nWire = 0 + let colorMode = 0 + let baseColorTexture = null, mrTexture = null + let metallicFactor = 1, roughnessFactor = 1 + // fit-to-view: model = rotation * scale * translate(-center) keeps the demo's + // camera constants valid for any GLB extent (trellis meshes are ~unit cube). + let center = [0, 0, 0], fitScale = 1 + + let rot = QBASE.slice(), dist = 1.8, panX = 0, panY = 0 + let wire = false, spin = true + let disposed = false + + function makeTexture(bitmap) { + const tex = gl.createTexture() + gl.bindTexture(gl.TEXTURE_2D, tex) + // Matches the baker's sampler: linear filtering, clamp, no mipmaps. + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, bitmap) + gl.bindTexture(gl.TEXTURE_2D, null) + return tex + } + + function dropTextures() { + if (baseColorTexture) { gl.deleteTexture(baseColorTexture); baseColorTexture = null } + if (mrTexture) { gl.deleteTexture(mrTexture); mrTexture = null } + } + + async function setMesh(mesh) { + gl.bindVertexArray(vao) + gl.bindBuffer(gl.ARRAY_BUFFER, vbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.positions, gl.STATIC_DRAW) + gl.enableVertexAttribArray(0) + gl.vertexAttribPointer(0, 3, gl.FLOAT, false, 0, 0) + if (mesh.normals) { + gl.bindBuffer(gl.ARRAY_BUFFER, nbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.normals, gl.STATIC_DRAW) + gl.enableVertexAttribArray(1) + gl.vertexAttribPointer(1, 3, gl.FLOAT, false, 0, 0) + } else { + gl.disableVertexAttribArray(1) + gl.vertexAttrib3f(1, 0, 1, 0) + } + + dropTextures() + colorMode = 0 + if (mesh.color0) { + // Upload COLOR_0 / _METALLIC_ROUGHNESS as normalized integers straight + // from the GLB buffers — no CPU conversion of multi-million-vertex data. + gl.bindBuffer(gl.ARRAY_BUFFER, cbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.color0.array, gl.STATIC_DRAW) + gl.enableVertexAttribArray(2) + gl.vertexAttribPointer(2, mesh.color0.size, glType(gl, mesh.color0.array), mesh.color0.normalized, 0, 0) + if (mesh.metalRough) { + gl.bindBuffer(gl.ARRAY_BUFFER, mbo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.metalRough.array, gl.STATIC_DRAW) + gl.enableVertexAttribArray(3) + gl.vertexAttribPointer(3, mesh.metalRough.size, glType(gl, mesh.metalRough.array), mesh.metalRough.normalized, 0, 0) + } else { + gl.disableVertexAttribArray(3) + gl.vertexAttrib2f(3, 0, 0.6) + } + gl.disableVertexAttribArray(4) + colorMode = 1 + } else if (mesh.uv && mesh.baseColorPng) { + gl.bindBuffer(gl.ARRAY_BUFFER, ubo) + gl.bufferData(gl.ARRAY_BUFFER, mesh.uv, gl.STATIC_DRAW) + gl.enableVertexAttribArray(4) + gl.vertexAttribPointer(4, 2, gl.FLOAT, false, 0, 0) + gl.disableVertexAttribArray(2) + gl.disableVertexAttribArray(3) + const bitmaps = await Promise.all([ + createImageBitmap(new Blob([mesh.baseColorPng], { type: 'image/png' })), + mesh.metalRoughPng ? createImageBitmap(new Blob([mesh.metalRoughPng], { type: 'image/png' })) : null, + ]) + if (disposed) return + gl.bindVertexArray(vao) + baseColorTexture = makeTexture(bitmaps[0]) + mrTexture = bitmaps[1] ? makeTexture(bitmaps[1]) : makeTexture(bitmaps[0]) + metallicFactor = mesh.material.metallicFactor + roughnessFactor = mesh.material.roughnessFactor + colorMode = 2 + } else { + gl.disableVertexAttribArray(2) + gl.disableVertexAttribArray(3) + gl.disableVertexAttribArray(4) + } + + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo) + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, mesh.indices, gl.STATIC_DRAW) + nIndices = mesh.indices.length + // wireframe index buffer: 3 edges per triangle, bounded to WIRE_BUDGET — + // a full wireframe of a multi-million-triangle mesh would OOM the GPU and + // lose the context (sub-pixel lines also fill in as a solid mass). + const WIRE_BUDGET = 12_000_000 // ~6M segments, ~48 MB + const nTri = nIndices / 3 + const wireStride = Math.max(1, Math.ceil(nTri * 6 / WIRE_BUDGET)) + const wireIdx = new Uint32Array(Math.ceil(nTri / wireStride) * 6) + let o = 0 + const idx = mesh.indices + for (let t = 0; t < nIndices; t += 3 * wireStride) { + wireIdx[o++] = idx[t]; wireIdx[o++] = idx[t + 1] + wireIdx[o++] = idx[t + 1]; wireIdx[o++] = idx[t + 2] + wireIdx[o++] = idx[t + 2]; wireIdx[o++] = idx[t] + } + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, wireIbo) + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, wireIdx.subarray(0, o), gl.STATIC_DRAW) + nWire = o + gl.bindVertexArray(null) + + center = [ + (mesh.bboxMin[0] + mesh.bboxMax[0]) / 2, + (mesh.bboxMin[1] + mesh.bboxMax[1]) / 2, + (mesh.bboxMin[2] + mesh.bboxMax[2]) / 2, + ] + const radius = Math.hypot( + mesh.bboxMax[0] - mesh.bboxMin[0], + mesh.bboxMax[1] - mesh.bboxMin[1], + mesh.bboxMax[2] - mesh.bboxMin[2], + ) / 2 || 1 + fitScale = 0.866 / radius + resetView() + } + + function clear() { + nIndices = 0 + nWire = 0 + dropTextures() + } + + function resetView() { + rot = QBASE.slice(); dist = 1.8; panX = panY = 0 + } + + /* input */ + let dragging = false, panning = false, lx = 0, ly = 0 + let pinchDistance = 0, pinchX = 0, pinchY = 0 + const pointers = new Map() + const pointerPair = () => Array.from(pointers.values()).slice(0, 2) + const beginPinch = () => { + const [a, b] = pointerPair() + if (!a || !b) return + pinchDistance = Math.hypot(b.x - a.x, b.y - a.y) + pinchX = (a.x + b.x) / 2 + pinchY = (a.y + b.y) / 2 + } + const stopSpin = () => { + spin = false + if (onSpinChange) onSpinChange(false) + } + const onPointerDown = (e) => { + e.preventDefault() + canvas.setPointerCapture(e.pointerId) + pointers.set(e.pointerId, { x: e.clientX, y: e.clientY }) + if (pointers.size === 1) { + dragging = true + panning = e.button === 2 || e.shiftKey + lx = e.clientX; ly = e.clientY + } else { + dragging = false + beginPinch() + stopSpin() + } + } + const onPointerUp = (e) => { + pointers.delete(e.pointerId) + if (canvas.hasPointerCapture(e.pointerId)) canvas.releasePointerCapture(e.pointerId) + pinchDistance = 0 + if (pointers.size === 1) { + const remaining = pointers.values().next().value + dragging = true + panning = false + lx = remaining.x; ly = remaining.y + } else { + dragging = false + } + } + const onPointerMove = (e) => { + if (!pointers.has(e.pointerId)) return + pointers.set(e.pointerId, { x: e.clientX, y: e.clientY }) + + if (pointers.size >= 2) { + const [a, b] = pointerPair() + const nextDistance = Math.hypot(b.x - a.x, b.y - a.y) + const nextX = (a.x + b.x) / 2 + const nextY = (a.y + b.y) / 2 + if (pinchDistance > 0 && nextDistance > 0) { + dist *= pinchDistance / nextDistance + dist = Math.max(0.3, Math.min(8, dist)) + panX += (nextX - pinchX) * 0.0015 * dist + panY -= (nextY - pinchY) * 0.0015 * dist + } + pinchDistance = nextDistance + pinchX = nextX + pinchY = nextY + return + } + + if (!dragging) return + const dx = e.clientX - lx, dy = e.clientY - ly + lx = e.clientX; ly = e.clientY + if (panning) { + panX += dx * 0.0015 * dist; panY -= dy * 0.0015 * dist + } else { + // Compose screen-axis turns onto the current orientation (fixed camera + // frame), so rotation stays screen-relative and never locks up. + const k = 0.008 + rot = Q.norm(Q.mul(Q.axisAngle(1, 0, 0, dy * k), Q.mul(Q.axisAngle(0, 1, 0, dx * k), rot))) + stopSpin() + } + } + const onContextMenu = (e) => e.preventDefault() + const onWheel = (e) => { + e.preventDefault() + dist *= Math.exp(e.deltaY * 0.001) + dist = Math.max(0.3, Math.min(8, dist)) + } + const onDblClick = () => resetView() + let onSpinChange = null + + canvas.addEventListener('pointerdown', onPointerDown) + canvas.addEventListener('pointerup', onPointerUp) + canvas.addEventListener('pointercancel', onPointerUp) + canvas.addEventListener('pointermove', onPointerMove) + canvas.addEventListener('contextmenu', onContextMenu) + canvas.addEventListener('wheel', onWheel, { passive: false }) + canvas.addEventListener('dblclick', onDblClick) + + gl.enable(gl.DEPTH_TEST) + gl.enable(gl.BLEND) + gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA) + gl.clearColor(0.063, 0.078, 0.094, 1) + + let rafId = 0 + let last = performance.now() + function frame(now) { + if (disposed) return + const dt = (now - last) / 1000; last = now + // auto-rotate: a slow turn about the screen-vertical axis (turntable feel) + if (spin) rot = Q.norm(Q.mul(Q.axisAngle(0, 1, 0, dt * 0.4), rot)) + + const w = canvas.clientWidth, h = canvas.clientHeight + if (w > 0 && h > 0 && (canvas.width !== w * devicePixelRatio || canvas.height !== h * devicePixelRatio)) { + canvas.width = w * devicePixelRatio; canvas.height = h * devicePixelRatio + } + gl.viewport(0, 0, canvas.width, canvas.height) + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT) + + if (nIndices) { + const rotation = Q.toMat4(rot) + const model = M.mul(rotation, M.mul(M.scale(fitScale), M.trans(-center[0], -center[1], -center[2]))) + const view = M.trans(panX, panY, -dist) + const viewModel = M.mul(view, model) + const proj = M.persp(0.9, (w || 1) / (h || 1), 0.05, 100) + const mvp = M.mul(proj, viewModel) + + gl.useProgram(prog) + gl.uniformMatrix4fv(uMVP, false, mvp) + gl.uniformMatrix4fv(uViewModel, false, viewModel) + gl.uniform1f(uMetallicFactor, metallicFactor) + gl.uniform1f(uRoughnessFactor, roughnessFactor) + if (colorMode === 2) { + gl.activeTexture(gl.TEXTURE0) + gl.bindTexture(gl.TEXTURE_2D, baseColorTexture) + gl.uniform1i(uBaseColorTex, 0) + gl.activeTexture(gl.TEXTURE1) + gl.bindTexture(gl.TEXTURE_2D, mrTexture) + gl.uniform1i(uMrTex, 1) + } + gl.bindVertexArray(vao) + if (wire) { + // hidden-line wireframe: a depth-only prepass (pushed back a hair via + // polygon offset) occludes back-facing edges, so the front surface's + // edges show instead of a see-through blob. + gl.enable(gl.POLYGON_OFFSET_FILL) + gl.polygonOffset(1.0, 1.0) + gl.colorMask(false, false, false, false) + gl.uniform1i(uWire, 0) + gl.uniform1i(uColorMode, 0) + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo) + gl.drawElements(gl.TRIANGLES, nIndices, gl.UNSIGNED_INT, 0) + gl.colorMask(true, true, true, true) + gl.disable(gl.POLYGON_OFFSET_FILL) + gl.uniform1i(uWire, 1) + // Front-surface edges sit at ~equal depth to the offset-back fill, so + // LEQUAL lets them pass while occluded edges (greater depth) fail. + gl.depthFunc(gl.LEQUAL) + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, wireIbo) + gl.drawElements(gl.LINES, nWire, gl.UNSIGNED_INT, 0) + gl.depthFunc(gl.LESS) + } else { + gl.uniform1i(uWire, 0) + gl.uniform1i(uColorMode, colorMode) + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, ibo) + gl.drawElements(gl.TRIANGLES, nIndices, gl.UNSIGNED_INT, 0) + } + gl.bindVertexArray(null) + } + rafId = requestAnimationFrame(frame) + } + rafId = requestAnimationFrame(frame) + + function dispose() { + disposed = true + cancelAnimationFrame(rafId) + canvas.removeEventListener('pointerdown', onPointerDown) + canvas.removeEventListener('pointerup', onPointerUp) + canvas.removeEventListener('pointercancel', onPointerUp) + canvas.removeEventListener('pointermove', onPointerMove) + canvas.removeEventListener('contextmenu', onContextMenu) + canvas.removeEventListener('wheel', onWheel) + canvas.removeEventListener('dblclick', onDblClick) + canvas.removeEventListener('webglcontextlost', contextLost) + dropTextures() + gl.deleteBuffer(vbo); gl.deleteBuffer(nbo); gl.deleteBuffer(cbo) + gl.deleteBuffer(mbo); gl.deleteBuffer(ubo); gl.deleteBuffer(ibo) + gl.deleteBuffer(wireIbo) + gl.deleteVertexArray(vao) + gl.deleteProgram(prog) + gl.getExtension('WEBGL_lose_context')?.loseContext() + } + + return { + setMesh, + clear, + dispose, + resetView, + setWire(v) { wire = v }, + setSpin(v) { spin = v }, + onSpinChanged(fn) { onSpinChange = fn }, + } +} + +export default function GlbViewer({ blob }) { + const { t } = useTranslation('media') + const canvasRef = useRef(null) + const viewerRef = useRef(null) + const [glError, setGlError] = useState(null) // 'no-webgl2' | 'context-lost' | parse error text + const [stats, setStats] = useState(null) + const [wire, setWire] = useState(false) + const [spin, setSpin] = useState(true) + + useEffect(() => { // GL lifecycle — once per mount + let viewer = null + try { + viewer = createGlbViewer(canvasRef.current, { onContextLost: () => setGlError('context-lost') }) + } catch { + viewer = null + } + if (!viewer) { + setGlError('no-webgl2') + return undefined + } + viewer.onSpinChanged(setSpin) + viewerRef.current = viewer + return () => { + viewerRef.current = null + viewer.dispose() + } + }, []) + + useEffect(() => { // (re)load when the blob changes + if (!blob) { + viewerRef.current?.clear() + setStats(null) + return undefined + } + let cancelled = false + ;(async () => { + try { + // Parse before touching GL: stats and parse errors surface even when + // WebGL2 is unavailable (e.g. headless CI), and the download button + // keeps working either way. + const mesh = parseGlb(await blob.arrayBuffer()) + if (cancelled) return + setStats({ + nVerts: mesh.nVerts, + nTris: mesh.nTris, + pbr: !!(mesh.color0 || mesh.baseColorPng), + }) + if (viewerRef.current) await viewerRef.current.setMesh(mesh) + } catch (err) { + if (!cancelled) setGlError(err.message) + } + })() + return () => { cancelled = true } + }, [blob]) + + const toggleWire = () => { + const next = !wire + setWire(next) + viewerRef.current?.setWire(next) + } + const toggleSpin = () => { + const next = !spin + setSpin(next) + viewerRef.current?.setSpin(next) + } + + return ( +
+ +
+ + + {stats && ( + + {t('threed.viewer.stats', { verts: stats.nVerts.toLocaleString(), tris: stats.nTris.toLocaleString() })} + {stats.pbr ? ' · PBR' : ''} + + )} +
+ {glError === 'no-webgl2' &&

{t('threed.viewer.noWebgl')}

} + {glError === 'context-lost' &&

{t('threed.viewer.contextLost')}

} + {glError && glError !== 'no-webgl2' && glError !== 'context-lost' && ( +

{glError}

+ )} +

{t('threed.viewer.hint')}

+
+ ) +} diff --git a/core/http/react-ui/src/components/ThreeDHistory.jsx b/core/http/react-ui/src/components/ThreeDHistory.jsx new file mode 100644 index 000000000..7686669d5 --- /dev/null +++ b/core/http/react-ui/src/components/ThreeDHistory.jsx @@ -0,0 +1,76 @@ +import { memo, useState } from 'react' +import { relativeTime } from '../utils/format' +import { useTranslation } from 'react-i18next' + +// ThreeDHistory — sibling of MediaHistory for IndexedDB-backed 3D entries +// (see use3DHistory). Reuses MediaHistory's markup, CSS classes, and testids +// so styling and e2e helpers carry over; the differences are the entry shape +// (input-image thumbnail, quality subtitle) and the Blob-backed source. +// Deliberately a plain vertical list — no showcase/gallery mode. +export default memo(function ThreeDHistory({ entries, selectedId, onSelect, onDelete, onClearAll }) { + const { t } = useTranslation('media') + const [expanded, setExpanded] = useState(true) + + return ( +
+
setExpanded(!expanded)} + style={{ display: 'flex', alignItems: 'center' }} + > + + {t('history.title')} ({entries.length}) + {entries.length > 0 && ( + + )} +
+ {expanded && ( +
+ {entries.length === 0 ? ( +
{t('history.empty')}
+ ) : ( + entries.map(entry => ( +
onSelect(entry.id)} + data-testid="media-history-item" + > +
+ {entry.inputThumb ? ( + + ) : ( + + )} +
+
+
+ {entry.name || entry.model} + {relativeTime(entry.createdAt)} +
+
+ {entry.params?.quality ? `${entry.params.quality} · ` : ''}{entry.model} +
+
+ +
+ )) + )} +
+ )} +
+ ) +}) diff --git a/core/http/react-ui/src/hooks/use3DHistory.js b/core/http/react-ui/src/hooks/use3DHistory.js new file mode 100644 index 000000000..b15e331ae --- /dev/null +++ b/core/http/react-ui/src/hooks/use3DHistory.js @@ -0,0 +1,132 @@ +import { useState, useEffect, useCallback, useMemo } from 'react' +import { generateId } from '../utils/format' + +// use3DHistory — IndexedDB-backed history of 3D generations. Unlike +// useMediaHistory (localStorage, URL-only), entries here carry the generated +// GLB as a Blob: GLBs are multi-megabyte binaries the server may eventually +// clean up, and IndexedDB is the only browser store that handles blobs of +// that size. Blobs read back from IndexedDB are lazy handles (the bytes are +// not materialized into JS memory until used), so a single store is fine. +// +// Entry: { id, createdAt, name, model, +// params: { seed, steps, textureSteps, guidance, quality, background }, +// inputThumb, // small dataURL of the conditioning image +// glb } // Blob + +const DB_NAME = 'localai-3d-history' +const DB_VERSION = 1 +const STORE = 'generations' +const MAX_ENTRIES = 20 + +function openDb() { + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME, DB_VERSION) + req.onupgradeneeded = () => { + const db = req.result + if (!db.objectStoreNames.contains(STORE)) { + db.createObjectStore(STORE, { keyPath: 'id' }).createIndex('createdAt', 'createdAt') + } + } + req.onsuccess = () => resolve(req.result) + req.onerror = () => reject(req.error) + }) +} + +const txDone = (tx) => new Promise((resolve, reject) => { + tx.oncomplete = resolve + tx.onerror = () => reject(tx.error) + tx.onabort = () => reject(tx.error) +}) + +async function withStore(mode, fn) { + const db = await openDb() + try { + const tx = db.transaction(STORE, mode) + const result = fn(tx.objectStore(STORE)) + await txDone(tx) + return result + } finally { + db.close() + } +} + +async function idbGetAll() { + const req = await withStore('readonly', (store) => store.getAll()) + return (req.result || []).sort((a, b) => b.createdAt - a.createdAt) +} + +// Insert + keep-newest-N eviction in one transaction so a crash between the +// two can't leave the store unbounded. +async function idbPutAndEvict(entry) { + await withStore('readwrite', (store) => { + store.put(entry) + // getAllKeys on the createdAt index yields primary keys oldest-first. + const keysReq = store.index('createdAt').getAllKeys() + keysReq.onsuccess = () => { + const excess = keysReq.result.length - MAX_ENTRIES + for (let i = 0; i < excess; i++) store.delete(keysReq.result[i]) + } + }) +} + +const idbDelete = (id) => withStore('readwrite', (store) => store.delete(id)) +const idbClear = () => withStore('readwrite', (store) => store.clear()) + +export function use3DHistory() { + const [entries, setEntries] = useState([]) + const [selectedId, setSelectedId] = useState(null) + + const refresh = useCallback(async () => { + try { + setEntries(await idbGetAll()) + } catch { + // IndexedDB unavailable (private mode etc.) — degrade to session-only. + setEntries((prev) => prev) + } + }, []) + + useEffect(() => { refresh() }, [refresh]) + + const addEntry = useCallback(async ({ model, params, inputThumb, glb, name }) => { + const entry = { id: generateId(), createdAt: Date.now(), model, params, inputThumb, glb, name } + try { + await idbPutAndEvict(entry) + await refresh() + } catch { + setEntries((prev) => [entry, ...prev].slice(0, MAX_ENTRIES)) + } + return entry + }, [refresh]) + + const deleteEntry = useCallback(async (id) => { + setSelectedId((prev) => (prev === id ? null : prev)) + try { + await idbDelete(id) + await refresh() + } catch { + setEntries((prev) => prev.filter((e) => e.id !== id)) + } + }, [refresh]) + + const clearAll = useCallback(async () => { + setSelectedId(null) + try { + await idbClear() + } catch { + // fall through to the local reset below + } + setEntries([]) + }, []) + + // Toggles: clicking the selected entry deselects it (back to latest result). + const selectEntry = useCallback((id) => { + setSelectedId((prev) => (prev === id ? null : id)) + }, []) + + const selectedEntry = useMemo( + () => entries.find((e) => e.id === selectedId) || null, + [entries, selectedId], + ) + + return { entries, addEntry, deleteEntry, clearAll, selectEntry, selectedId, selectedEntry } +} diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index 53dfccf2f..f7491805a 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -56,6 +56,7 @@ const FILTERS = [ { key: 'chat', labelKey: 'filters.llm', icon: 'fa-brain' }, { key: 'image', labelKey: 'filters.image', icon: 'fa-image' }, { key: 'video', labelKey: 'filters.video', icon: 'fa-video' }, + { key: '3d', labelKey: 'filters.threed', icon: 'fa-cube' }, { key: 'multimodal', labelKey: 'filters.multimodal', icon: 'fa-shapes' }, { key: 'vision', labelKey: 'filters.vision', icon: 'fa-eye' }, { key: 'tts', labelKey: 'filters.tts', icon: 'fa-microphone' }, diff --git a/core/http/react-ui/src/pages/Studio.jsx b/core/http/react-ui/src/pages/Studio.jsx index 5926078c2..43692cea7 100644 --- a/core/http/react-ui/src/pages/Studio.jsx +++ b/core/http/react-ui/src/pages/Studio.jsx @@ -2,6 +2,7 @@ import { useSearchParams } from 'react-router-dom' import { useTranslation } from 'react-i18next' import ImageGen from './ImageGen' import VideoGen from './VideoGen' +import ThreeDGen from './ThreeDGen' import TTS from './TTS' import Sound from './Sound' import AudioTransform from './AudioTransform' @@ -10,6 +11,7 @@ import { useAuth } from '../context/AuthContext' const BASE_TABS = [ { key: 'images', labelKey: 'studio.tabs.images', icon: 'fas fa-image' }, { key: 'video', labelKey: 'studio.tabs.video', icon: 'fas fa-video' }, + { key: 'threed', labelKey: 'studio.tabs.threed', icon: 'fas fa-cube' }, { key: 'tts', labelKey: 'studio.tabs.tts', icon: 'fas fa-headphones' }, { key: 'sound', labelKey: 'studio.tabs.sound', icon: 'fas fa-music' }, ] @@ -19,6 +21,7 @@ const TRANSFORM_TAB = { key: 'transform', labelKey: 'studio.tabs.transform', ico const TAB_COMPONENTS = { images: ImageGen, video: VideoGen, + threed: ThreeDGen, tts: TTS, sound: Sound, transform: AudioTransform, @@ -28,19 +31,23 @@ export default function Studio() { const { t } = useTranslation('media') const { hasFeature } = useAuth() const [searchParams, setSearchParams] = useSearchParams() - const activeTab = searchParams.get('tab') || 'images' + const requestedTab = searchParams.get('tab') || 'images' + const threeDEnabled = hasFeature('3d') + const transformEnabled = hasFeature('audio_transform') + const activeTab = + ((requestedTab === 'threed' && !threeDEnabled) || + (requestedTab === 'transform' && !transformEnabled)) + ? 'images' + : requestedTab - // Transform is a distinct capability; only show its tab when enabled. - const tabs = hasFeature('audio_transform') ? [...BASE_TABS, TRANSFORM_TAB] : BASE_TABS + const enabledTabs = BASE_TABS.filter(tab => tab.key !== 'threed' || threeDEnabled) + const tabs = transformEnabled ? [...enabledTabs, TRANSFORM_TAB] : enabledTabs const setTab = (key) => { setSearchParams({ tab: key }, { replace: true }) } - const ActiveComponent = - (activeTab === 'transform' && !hasFeature('audio_transform')) - ? ImageGen - : (TAB_COMPONENTS[activeTab] || ImageGen) + const ActiveComponent = TAB_COMPONENTS[activeTab] || ImageGen return (
diff --git a/core/http/react-ui/src/pages/ThreeDGen.jsx b/core/http/react-ui/src/pages/ThreeDGen.jsx new file mode 100644 index 000000000..f4a0e3775 --- /dev/null +++ b/core/http/react-ui/src/pages/ThreeDGen.jsx @@ -0,0 +1,285 @@ +import { useState } from 'react' +import { useParams, useOutletContext } from 'react-router-dom' +import { useTranslation } from 'react-i18next' +import ModelSelector from '../components/ModelSelector' +import PageHeader from '../components/PageHeader' +import { CAP_3D } from '../utils/capabilities' +import LoadingSpinner from '../components/LoadingSpinner' +import GenerationProgress from '../components/GenerationProgress' +import ErrorWithTraceLink from '../components/ErrorWithTraceLink' +import ThreeDHistory from '../components/ThreeDHistory' +import GlbViewer from '../components/GlbViewer' +import MediaInput from '../components/biometrics/MediaInput' +import { threeDApi } from '../utils/api' +import { apiUrl } from '../utils/basePath' +import { use3DHistory } from '../hooks/use3DHistory' +import useObjectUrl from '../hooks/useObjectUrl' + +const QUALITIES = ['auto', 'coarse', '512', '1024'] +const BACKGROUNDS = ['auto', 'keep', 'black', 'white'] +const MAX_3D_INPUT_BYTES = 32 * 1024 * 1024 +const REMESH_DETAIL_COARSE = 2.5 +const REMESH_DETAIL_FINE = 0.35 + +function remeshDetail(sliderValue) { + const position = Number(sliderValue) / 100 + return REMESH_DETAIL_COARSE * Math.pow(REMESH_DETAIL_FINE / REMESH_DETAIL_COARSE, position) +} + +function remeshedName(name = '3d-model.glb') { + return `${name.replace(/\.glb$/i, '')}-remeshed.glb` +} + +// Small thumbnail of the conditioning image for the history list — full-size +// data URLs would bloat every IndexedDB entry for no visual gain. +async function makeThumb(dataUrl, size = 96) { + try { + const img = new Image() + await new Promise((resolve, reject) => { + img.onload = resolve + img.onerror = reject + img.src = dataUrl + }) + const scale = size / Math.max(img.width, img.height, 1) + const canvas = document.createElement('canvas') + canvas.width = Math.max(1, Math.round(img.width * scale)) + canvas.height = Math.max(1, Math.round(img.height * scale)) + canvas.getContext('2d').drawImage(img, 0, 0, canvas.width, canvas.height) + return canvas.toDataURL('image/jpeg', 0.7) + } catch { + return null + } +} + +export default function ThreeDGen() { + const { model: urlModel } = useParams() + const { addToast } = useOutletContext() + const { t } = useTranslation('media') + const [model, setModel] = useState(urlModel || '') + const [image, setImage] = useState(null) + const [quality, setQuality] = useState('auto') + const [background, setBackground] = useState('auto') + const [steps, setSteps] = useState('') + const [textureSteps, setTextureSteps] = useState('') + const [guidance, setGuidance] = useState('') + const [seed, setSeed] = useState('') + const [showAdvanced, setShowAdvanced] = useState(false) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + const [result, setResult] = useState(null) // { blob, name, model } + const [remeshSlider, setRemeshSlider] = useState(82) + const [remeshState, setRemeshState] = useState(null) // { sourceBlob, blob?, name?, error? } + const [remeshLoading, setRemeshLoading] = useState(false) + const { entries, addEntry, deleteEntry, clearAll, selectEntry, selectedId, selectedEntry } = use3DHistory() + + const source = selectedEntry + ? { blob: selectedEntry.glb, name: selectedEntry.name, model: selectedEntry.model } + : result + const showingRemesh = !!source && remeshState?.sourceBlob === source.blob && !!remeshState.blob + const active = showingRemesh ? { blob: remeshState.blob, name: remeshState.name } : source + const remeshError = source && remeshState?.sourceBlob === source.blob ? remeshState.error : null + const detail = remeshDetail(remeshSlider) + const downloadUrl = useObjectUrl(active?.blob) + + const handleGenerate = async (e) => { + e.preventDefault() + if (!image?.base64) { addToast(t('threed.toasts.noImage'), 'warning'); return } + if (!model) { addToast(t('threed.toasts.noModel'), 'warning'); return } + + setLoading(true) + setResult(null) + setRemeshState(null) + setError(null) + + const body = { model, image: image.base64, quality, background, response_format: 'url' } + if (steps) body.step = parseInt(steps) + if (textureSteps) body.texture_steps = parseInt(textureSteps) + if (guidance) body.cfg_scale = parseFloat(guidance) + if (seed) body.seed = parseInt(seed) + + try { + const data = await threeDApi.generate(body) + const url = data?.data?.[0]?.url + if (!url) { + addToast(t('threed.toasts.noResults'), 'warning') + return + } + const glbResp = await fetch(apiUrl(url)) + if (!glbResp.ok) throw new Error(`fetching the generated GLB failed: HTTP ${glbResp.status}`) + const glb = await glbResp.blob() + const name = url.split('/').pop() + setResult({ blob: glb, name, model }) + selectEntry(null) + const inputThumb = image.dataUrl ? await makeThumb(image.dataUrl) : null + await addEntry({ + model, + params: { quality, background, steps, textureSteps, guidance, seed }, + inputThumb, + glb, + name, + }) + } catch (err) { + setError(err.message) + } finally { + setLoading(false) + } + } + + const handleRemesh = async () => { + if (!source?.blob || !source.model) return + if (showingRemesh) { + setRemeshState(null) + return + } + + const sourceBlob = source.blob + setRemeshLoading(true) + setRemeshState({ sourceBlob, error: null }) + try { + const blob = await threeDApi.remesh(sourceBlob, source.model, detail) + setRemeshState({ sourceBlob, blob, name: remeshedName(source.name), error: null }) + } catch (err) { + setRemeshState({ sourceBlob, error: err.message }) + } finally { + setRemeshLoading(false) + } + } + + const handleRemeshDetail = (value) => { + setRemeshSlider(value) + if (source && remeshState?.sourceBlob === source.blob) setRemeshState(null) + } + + return ( +
+
+ {t('threed.title')}} /> + +
+
+ + +
+ + addToast(err.message, 'error')} + maxBytes={MAX_3D_INPUT_BYTES} + idPrefix="threed" + /> + +
+
+ + +
+
+ + +
+
+ +