fix(config): only inject llama.cpp serving options on the llama.cpp path (#10822)

SetDefaults injected the llama.cpp server options cache_reuse
(ApplyServingDefaults) and parallel (ApplyHardwareDefaults, re-applied
per selected node by the distributed router) onto every model config
regardless of backend. Every other backend ignores options it does not
understand, so this was harmless until longcat-video, which strictly
validates its options and fails LoadModel with
"unknown model option(s): cache_reuse, parallel".

Gate both injections behind a new UsesLlamaCppServingOptions allow-list
(llama-cpp plus the empty/auto-detect case that resolves to llama.cpp
from a GGUF file, mirroring how llamaCppDefaults is registered). This
follows the existing UsesLlamaSamplerDefaults precedent for llama-only
defaults. The typed NBatch field is deliberately left alone: it is a
proto field every backend simply ignores, which is why batch never
triggered the error.

Also harden the longcat-video backend to warn-and-ignore unknown model
options and request params through a testable select_known_options
helper, matching the other LocalAI Python backends, so a future
server-injected option cannot break loading again.


Assisted-by: Claude:claude-opus-4-8 [Claude Code]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

.agents/adding-backends.md

@@ -251,6 +251,7 @@ After adding a new backend, verify:
 - [ ] No YAML syntax errors (check with linter)
 - [ ] No Makefile syntax errors (check with linter)
 - [ ] Follows the same pattern as similar backends (e.g., if it's a transcription backend, follow `faster-whisper` pattern)
+- [ ] **`Load` validates its input and refuses models it can't serve.** When a model config has no explicit `backend:`, the model loader greedily probes *every* installed backend with the model's name and binds to the first `Load` that succeeds — an accept-anything `Load` will capture arbitrary LLMs (issue #9287). Backends that load a real artefact get this for free (the load fails); backends with no artefact must gate on the name: `opus` accepts only its own name (or none), `local-store` requires the `store.NamespacePrefix` namespace marker sent by `core/backend/stores.go`.
 - [ ] Documented: added to the category list in `docs/content/features/backends.md` (and any new endpoint/realtime capability documented under `docs/content/`)
 - [ ] If it is an in-house native C/C++/GGML engine, added to the maintained-engines table in the top-level `README.md`
 

.github/backend-matrix.yml

@@ -478,6 +478,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -790,6 +803,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-moss-transcribe-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1136,6 +1162,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1344,6 +1383,19 @@ include:
     backend: "vllm-omni"
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
+  - build-type: 'l4t'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-longcat-video'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    ubuntu-version: '2404'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
   - build-type: 'l4t'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1721,6 +1773,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-moss-transcribe-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1760,6 +1825,19 @@ include:
     backend: "parakeet-cpp"
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    skip-drivers: 'false'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp'
+    base-image: "ubuntu:24.04"
+    ubuntu-version: '2404'
+    runs-on: 'ubuntu-24.04-arm'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -3597,6 +3675,115 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  # moss-transcribe-cpp
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-moss-transcribe-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-moss-transcribe-cpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'sycl_f32'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-sycl-f32-moss-transcribe-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'sycl_f16'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-sycl-f16-moss-transcribe-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-moss-transcribe-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-moss-transcribe-cpp'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    skip-drivers: 'false'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-arm64-moss-transcribe-cpp'
+    base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+    runs-on: 'ubuntu-24.04-arm'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2204'
+  - build-type: 'hipblas'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-rocm-hipblas-moss-transcribe-cpp'
+    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+    runs-on: 'ubuntu-latest'
+    skip-drivers: 'false'
+    backend: "moss-transcribe-cpp"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   # ced
   - build-type: 'cublas'
     cuda-major-version: "12"
@@ -5220,6 +5407,10 @@ includeDarwin:
     tag-suffix: "-metal-darwin-arm64-parakeet-cpp"
     build-type: "metal"
     lang: "go"
+  - backend: "moss-transcribe-cpp"
+    tag-suffix: "-metal-darwin-arm64-moss-transcribe-cpp"
+    build-type: "metal"
+    lang: "go"
   - backend: "ced"
     tag-suffix: "-metal-darwin-arm64-ced"
     build-type: "metal"

.github/workflows/backend.yml

@@ -32,16 +32,30 @@ jobs:
     if: github.repository == 'mudler/LocalAI'
     runs-on: ubuntu-latest
     outputs:
-      matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }}
       matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }}
       matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }}
       merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }}
-      merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }}
-      has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }}
       has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }}
       has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }}
       has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }}
-      has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }}
+      # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to
+      # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js).
+      matrix-singlearch-1: ${{ steps.set-matrix.outputs['matrix-singlearch-1'] }}
+      merge-matrix-singlearch-1: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-1'] }}
+      has-backends-singlearch-1: ${{ steps.set-matrix.outputs['has-backends-singlearch-1'] }}
+      has-merges-singlearch-1: ${{ steps.set-matrix.outputs['has-merges-singlearch-1'] }}
+      matrix-singlearch-2: ${{ steps.set-matrix.outputs['matrix-singlearch-2'] }}
+      merge-matrix-singlearch-2: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-2'] }}
+      has-backends-singlearch-2: ${{ steps.set-matrix.outputs['has-backends-singlearch-2'] }}
+      has-merges-singlearch-2: ${{ steps.set-matrix.outputs['has-merges-singlearch-2'] }}
+      matrix-singlearch-3: ${{ steps.set-matrix.outputs['matrix-singlearch-3'] }}
+      merge-matrix-singlearch-3: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-3'] }}
+      has-backends-singlearch-3: ${{ steps.set-matrix.outputs['has-backends-singlearch-3'] }}
+      has-merges-singlearch-3: ${{ steps.set-matrix.outputs['has-merges-singlearch-3'] }}
+      matrix-singlearch-4: ${{ steps.set-matrix.outputs['matrix-singlearch-4'] }}
+      merge-matrix-singlearch-4: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-4'] }}
+      has-backends-singlearch-4: ${{ steps.set-matrix.outputs['has-backends-singlearch-4'] }}
+      has-merges-singlearch-4: ${{ steps.set-matrix.outputs['has-merges-singlearch-4'] }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v7
@@ -109,9 +123,9 @@ jobs:
   # take their full ~6h cold without blocking manifest assembly for the
   # multi-arch backends whose per-arch digests would otherwise sit untagged
   # on quay long enough to be GC'd.
-  backend-jobs-singlearch:
+  backend-jobs-singlearch-1:
     needs: generate-matrix
-    if: needs.generate-matrix.outputs['has-backends-singlearch'] == 'true'
+    if: needs.generate-matrix.outputs['has-backends-singlearch-1'] == 'true'
     uses: ./.github/workflows/backend_build.yml
     with:
       tag-latest: ${{ matrix.tag-latest }}
@@ -138,7 +152,100 @@ jobs:
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-1']) }}
+
+  backend-jobs-singlearch-2:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-2'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-2']) }}
+
+  backend-jobs-singlearch-3:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-3'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-3']) }}
+
+  backend-jobs-singlearch-4:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-4'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-4']) }}
 
   # Apply tags to per-arch digests via `imagetools create`. Split into two
   # jobs that mirror the build split so each merge waits ONLY on its
@@ -174,10 +281,12 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }}
 
-  backend-merge-jobs-singlearch:
-    needs: [generate-matrix, backend-jobs-singlearch]
-    # See note on backend-merge-jobs-multiarch above for !cancelled().
-    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch'] == 'true' }}
+  # One merge shard per build shard: backend-merge-jobs-singlearch-<n> needs only
+  # backend-jobs-singlearch-<n>, preserving the "merge waits only on its own
+  # build" property while staying under the 256-jobs-per-matrix limit.
+  backend-merge-jobs-singlearch-1:
+    needs: [generate-matrix, backend-jobs-singlearch-1]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-1'] == 'true' }}
     uses: ./.github/workflows/backend_merge.yml
     with:
       tag-latest: ${{ matrix.tag-latest }}
@@ -189,7 +298,55 @@ jobs:
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-1']) }}
+
+  backend-merge-jobs-singlearch-2:
+    needs: [generate-matrix, backend-jobs-singlearch-2]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-2'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-2']) }}
+
+  backend-merge-jobs-singlearch-3:
+    needs: [generate-matrix, backend-jobs-singlearch-3]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-3'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-3']) }}
+
+  backend-merge-jobs-singlearch-4:
+    needs: [generate-matrix, backend-jobs-singlearch-4]
+    if: ${{ !cancelled() && needs.generate-matrix.outputs['has-merges-singlearch-4'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      dockerUsername: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerPassword: ${{ secrets.DOCKERHUB_PASSWORD }}
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-4']) }}
 
   backend-jobs-darwin:
     needs: generate-matrix

.github/workflows/backend_pr.yml

@@ -11,16 +11,30 @@ jobs:
   generate-matrix:
     runs-on: ubuntu-latest
     outputs:
-      matrix-singlearch: ${{ steps.set-matrix.outputs['matrix-singlearch'] }}
       matrix-multiarch: ${{ steps.set-matrix.outputs['matrix-multiarch'] }}
       matrix-darwin: ${{ steps.set-matrix.outputs['matrix-darwin'] }}
       merge-matrix-multiarch: ${{ steps.set-matrix.outputs['merge-matrix-multiarch'] }}
-      merge-matrix-singlearch: ${{ steps.set-matrix.outputs['merge-matrix-singlearch'] }}
-      has-backends-singlearch: ${{ steps.set-matrix.outputs['has-backends-singlearch'] }}
       has-backends-multiarch: ${{ steps.set-matrix.outputs['has-backends-multiarch'] }}
       has-backends-darwin: ${{ steps.set-matrix.outputs['has-backends-darwin'] }}
       has-merges-multiarch: ${{ steps.set-matrix.outputs['has-merges-multiarch'] }}
-      has-merges-singlearch: ${{ steps.set-matrix.outputs['has-merges-singlearch'] }}
+      # Single-arch backends are sharded across SINGLEARCH_SHARDS matrix jobs to
+      # stay under GitHub's 256-jobs-per-matrix limit (see changed-backends.js).
+      matrix-singlearch-1: ${{ steps.set-matrix.outputs['matrix-singlearch-1'] }}
+      merge-matrix-singlearch-1: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-1'] }}
+      has-backends-singlearch-1: ${{ steps.set-matrix.outputs['has-backends-singlearch-1'] }}
+      has-merges-singlearch-1: ${{ steps.set-matrix.outputs['has-merges-singlearch-1'] }}
+      matrix-singlearch-2: ${{ steps.set-matrix.outputs['matrix-singlearch-2'] }}
+      merge-matrix-singlearch-2: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-2'] }}
+      has-backends-singlearch-2: ${{ steps.set-matrix.outputs['has-backends-singlearch-2'] }}
+      has-merges-singlearch-2: ${{ steps.set-matrix.outputs['has-merges-singlearch-2'] }}
+      matrix-singlearch-3: ${{ steps.set-matrix.outputs['matrix-singlearch-3'] }}
+      merge-matrix-singlearch-3: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-3'] }}
+      has-backends-singlearch-3: ${{ steps.set-matrix.outputs['has-backends-singlearch-3'] }}
+      has-merges-singlearch-3: ${{ steps.set-matrix.outputs['has-merges-singlearch-3'] }}
+      matrix-singlearch-4: ${{ steps.set-matrix.outputs['matrix-singlearch-4'] }}
+      merge-matrix-singlearch-4: ${{ steps.set-matrix.outputs['merge-matrix-singlearch-4'] }}
+      has-backends-singlearch-4: ${{ steps.set-matrix.outputs['has-backends-singlearch-4'] }}
+      has-merges-singlearch-4: ${{ steps.set-matrix.outputs['has-merges-singlearch-4'] }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v7
@@ -71,10 +85,10 @@ jobs:
       fail-fast: true
       max-parallel: 8
       matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-multiarch']) }}
-  backend-jobs-singlearch:
+  backend-jobs-singlearch-1:
     needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-1'] == 'true'
     uses: ./.github/workflows/backend_build.yml
-    if: needs.generate-matrix.outputs['has-backends-singlearch'] == 'true'
     with:
       tag-latest: ${{ matrix.tag-latest }}
       tag-suffix: ${{ matrix.tag-suffix }}
@@ -98,7 +112,94 @@ jobs:
     strategy:
       fail-fast: true
       max-parallel: 8
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-1']) }}
+
+  backend-jobs-singlearch-2:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-2'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: true
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-2']) }}
+
+  backend-jobs-singlearch-3:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-3'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: true
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-3']) }}
+
+  backend-jobs-singlearch-4:
+    needs: generate-matrix
+    if: needs.generate-matrix.outputs['has-backends-singlearch-4'] == 'true'
+    uses: ./.github/workflows/backend_build.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+      build-type: ${{ matrix.build-type }}
+      cuda-major-version: ${{ matrix.cuda-major-version }}
+      cuda-minor-version: ${{ matrix.cuda-minor-version }}
+      platforms: ${{ matrix.platforms }}
+      platform-tag: ${{ matrix.platform-tag || '' }}
+      runs-on: ${{ matrix.runs-on }}
+      builder-base-image: ${{ matrix.builder-base-image || '' }}
+      base-image: ${{ matrix.base-image }}
+      backend: ${{ matrix.backend }}
+      dockerfile: ${{ matrix.dockerfile }}
+      skip-drivers: ${{ matrix.skip-drivers }}
+      context: ${{ matrix.context }}
+      ubuntu-version: ${{ matrix.ubuntu-version }}
+      amdgpu-targets: ${{ matrix.amdgpu-targets || 'gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: true
+      max-parallel: 8
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['matrix-singlearch-4']) }}
   backend-merge-jobs-multiarch:
     needs: [generate-matrix, backend-jobs-multiarch]
     # backend_merge.yml's push-side steps are all gated on
@@ -118,9 +219,9 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-multiarch']) }}
 
-  backend-merge-jobs-singlearch:
-    needs: [generate-matrix, backend-jobs-singlearch]
-    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch'] == 'true' }}
+  backend-merge-jobs-singlearch-1:
+    needs: [generate-matrix, backend-jobs-singlearch-1]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-1'] == 'true' }}
     uses: ./.github/workflows/backend_merge.yml
     with:
       tag-latest: ${{ matrix.tag-latest }}
@@ -130,7 +231,49 @@ jobs:
       quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch']) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-1']) }}
+
+  backend-merge-jobs-singlearch-2:
+    needs: [generate-matrix, backend-jobs-singlearch-2]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-2'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-2']) }}
+
+  backend-merge-jobs-singlearch-3:
+    needs: [generate-matrix, backend-jobs-singlearch-3]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-3'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-3']) }}
+
+  backend-merge-jobs-singlearch-4:
+    needs: [generate-matrix, backend-jobs-singlearch-4]
+    if: ${{ !cancelled() && github.event_name != 'pull_request' && needs.generate-matrix.outputs['has-merges-singlearch-4'] == 'true' }}
+    uses: ./.github/workflows/backend_merge.yml
+    with:
+      tag-latest: ${{ matrix.tag-latest }}
+      tag-suffix: ${{ matrix.tag-suffix }}
+    secrets:
+      quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }}
+      quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs['merge-matrix-singlearch-4']) }}
   backend-jobs-darwin:
     needs: generate-matrix
     uses: ./.github/workflows/backend_build_darwin.yml

.github/workflows/bump_deps.yaml

@@ -26,6 +26,10 @@ jobs:
             variable: "DS4_VERSION"
             branch: "main"
             file: "backend/cpp/ds4/Makefile"
+          - repository: "meituan-longcat/LongCat-Video"
+            variable: "LONGCAT_VIDEO_VERSION"
+            branch: "main"
+            file: "backend/python/longcat-video/Makefile"
           - repository: "localai-org/privacy-filter.cpp"
             variable: "PRIVACY_FILTER_VERSION"
             branch: "master"
@@ -42,6 +46,10 @@ jobs:
             variable: "PARAKEET_VERSION"
             branch: "master"
             file: "backend/go/parakeet-cpp/Makefile"
+          - repository: "mudler/moss-transcribe.cpp"
+            variable: "MOSS_VERSION"
+            branch: "master"
+            file: "backend/go/moss-transcribe-cpp/Makefile"
           - repository: "mudler/ced.cpp"
             variable: "CED_VERSION"
             branch: "master"

.github/workflows/realtime-conformance.yml

@@ -54,7 +54,7 @@ jobs:
           go-version: ${{ matrix.go-version }}
           cache: false
       - name: Cache FizzBee
-        uses: actions/cache@v4
+        uses: actions/cache@v6
         with:
           path: .tools/fizzbee
           key: fizzbee-v0.5.2-${{ runner.os }}-${{ hashFiles('formal-verification/fizzbee.sha256') }}

.github/workflows/stalebot.yml

@@ -11,7 +11,7 @@ jobs:
     if: github.repository == 'mudler/LocalAI'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v9
+      - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v9
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
           stale-pr-message: 'This PR is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 10 days.'

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -565,6 +565,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/chatterbox
 	$(MAKE) -C backend/python/vllm
 	$(MAKE) -C backend/python/vllm-omni
+	$(MAKE) -C backend/python/longcat-video
 	$(MAKE) -C backend/python/sglang
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/liquid-audio
@@ -594,6 +595,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/chatterbox test
 	$(MAKE) -C backend/python/vllm test
 	$(MAKE) -C backend/python/vllm-omni test
+	$(MAKE) -C backend/python/longcat-video test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/liquid-audio test
 	$(MAKE) -C backend/python/moonshine test
@@ -645,6 +647,9 @@ test-extra: prepare-test-extra
 ## suite against it.
 ##
 BACKEND_TEST_MODEL_URL?=https://huggingface.co/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf
+## Suite timeout for `go test`. Wrappers whose model download alone can eat
+## most of the default (multi-GB models on a slow HF CDN day) override this.
+BACKEND_TEST_TIMEOUT?=30m
 
 ## Generic target — runs the suite against whatever BACKEND_IMAGE points at.
 ## Depends on protogen-go so pkg/grpc/proto is generated before `go test`.
@@ -672,7 +677,7 @@ test-extra-backend: protogen-go
 	BACKEND_TEST_FACE_IMAGE_3_URL="$$BACKEND_TEST_FACE_IMAGE_3_URL" \
 	BACKEND_TEST_FACE_IMAGE_3_FILE="$$BACKEND_TEST_FACE_IMAGE_3_FILE" \
 	BACKEND_TEST_VERIFY_DISTANCE_CEILING="$$BACKEND_TEST_VERIFY_DISTANCE_CEILING" \
-	go test -v -timeout 30m ./tests/e2e-backends/...
+	go test -v -timeout $(BACKEND_TEST_TIMEOUT) ./tests/e2e-backends/...
 
 ## Convenience wrappers: build the image, then exercise it.
 test-extra-backend-llama-cpp: docker-build-llama-cpp
@@ -1012,6 +1017,7 @@ test-extra-backend-vibevoice-cpp-tts: docker-build-vibevoice-cpp
 ## post-image disk budget.
 test-extra-backend-vibevoice-cpp-transcription: docker-build-vibevoice-cpp
 	BACKEND_IMAGE=local-ai-backend:vibevoice-cpp \
+	BACKEND_TEST_TIMEOUT=120m \
 	BACKEND_TEST_MODEL_URL='https://huggingface.co/mudler/vibevoice.cpp-models/resolve/main/vibevoice-asr-q4_k.gguf#vibevoice-asr-q4_k.gguf' \
 	BACKEND_TEST_EXTRA_FILES='https://huggingface.co/mudler/vibevoice.cpp-models/resolve/main/tokenizer.gguf#tokenizer.gguf' \
 	BACKEND_TEST_AUDIO_URL=https://github.com/ggml-org/whisper.cpp/raw/master/samples/jfk.wav \
@@ -1044,6 +1050,18 @@ test-extra-backend-parakeet-cpp-transcription: docker-build-parakeet-cpp
 	BACKEND_TEST_CAPS=health,load,transcription \
 	$(MAKE) test-extra-backend
 
+## Audio transcription wrapper for the moss-transcribe-cpp (moss-transcribe.cpp
+## ggml port) backend. Mirrors test-extra-backend-parakeet-cpp-transcription:
+## drives the AudioTranscription RPC against a published MOSS GGUF using the JFK
+## 11s clip from whisper.cpp's CI samples. Not part of the default test suite -
+## run explicitly once the pinned model URL is reachable.
+test-extra-backend-moss-transcribe-cpp-transcription: docker-build-moss-transcribe-cpp
+	BACKEND_IMAGE=local-ai-backend:moss-transcribe-cpp \
+	BACKEND_TEST_MODEL_URL=https://huggingface.co/mudler/moss-transcribe.cpp-gguf/resolve/main/moss-transcribe-q5_k.gguf \
+	BACKEND_TEST_AUDIO_URL=https://github.com/ggml-org/whisper.cpp/raw/master/samples/jfk.wav \
+	BACKEND_TEST_CAPS=health,load,transcription \
+	$(MAKE) test-extra-backend
+
 ## LocalVQE audio transform (joint AEC + noise suppression + dereverb).
 ## Exercises the audio_transform capability end-to-end: batch transform
 ## of a real WAV fixture and bidi streaming of synthetic silent frames.
@@ -1212,6 +1230,7 @@ BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|tr
 BACKEND_WHISPER = whisper|golang|.|false|true
 BACKEND_CRISPASR = crispasr|golang|.|false|true
 BACKEND_PARAKEET_CPP = parakeet-cpp|golang|.|false|true
+BACKEND_MOSS_TRANSCRIBE_CPP = moss-transcribe-cpp|golang|.|false|true
 BACKEND_DEPTH_ANYTHING_CPP = depth-anything-cpp|golang|.|false|true
 BACKEND_VOXTRAL = voxtral|golang|.|false|true
 BACKEND_ACESTEP_CPP = acestep-cpp|golang|.|false|true
@@ -1237,6 +1256,7 @@ BACKEND_NEUTTS = neutts|python|.|false|true
 BACKEND_KOKORO = kokoro|python|.|false|true
 BACKEND_VLLM = vllm|python|.|false|true
 BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true
+BACKEND_LONGCAT_VIDEO = longcat-video|python|.|--progress=plain|true
 BACKEND_SGLANG = sglang|python|.|false|true
 BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true
 BACKEND_CHATTERBOX = chatterbox|python|.|false|true
@@ -1305,6 +1325,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML)))
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_CRISPASR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PARAKEET_CPP)))
+$(eval $(call generate-docker-build-target,$(BACKEND_MOSS_TRANSCRIBE_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DEPTH_ANYTHING_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VOXTRAL)))
 $(eval $(call generate-docker-build-target,$(BACKEND_OPUS)))
@@ -1321,6 +1342,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KOKORO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI)))
+$(eval $(call generate-docker-build-target,$(BACKEND_LONGCAT_VIDEO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_SGLANG)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX)))
@@ -1357,7 +1379,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-privacy-filter
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
 
 ########################################################
 ### Mock Backend for E2E Tests

README.md

@@ -232,12 +232,16 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native
 | Backend | What it does |
 |---------|-------------|
 | [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet ASR (tdt/ctc/rnnt/hybrid), with cache-aware streaming transcription |
+| [moss-transcribe.cpp](https://github.com/mudler/moss-transcribe.cpp) | C++/GGML port of OpenMOSS MOSS-Transcribe-Diarize: joint long-form transcription, speaker diarization and timestamping in a single pass |
 | [ced.cpp](https://github.com/mudler/ced.cpp) | C++/GGML port of the CED audio-tagging models: sound-event classification (527-class AudioSet) over REST and the realtime API for live recognition |
-| [voxtral.c](https://github.com/mudler/voxtral.c) | Voxtral Realtime 4B speech-to-text in pure C |
+| [voice-detect.cpp](https://github.com/mudler/voice-detect.cpp) | Speaker recognition and voice analysis (ECAPA-TDNN, WeSpeaker, ERes2Net, CAM++, wav2vec2 age/gender/emotion), replacing the Python speaker-recognition backend |
+| [voxtral-tts.c](https://github.com/mudler/voxtral-tts.c) | Voxtral Realtime 4B speech-to-text in pure C |
 | [vibevoice.cpp](https://github.com/mudler/vibevoice.cpp) | Native port of Microsoft VibeVoice for TTS (voice cloning) and long-form ASR with speaker diarization |
 | [rf-detr.cpp](https://github.com/mudler/rf-detr.cpp) | Native RF-DETR object detection and instance segmentation |
 | [locate-anything.cpp](https://github.com/mudler/locate-anything.cpp) | Open-vocabulary object detection and visual grounding (LocateAnything-3B) |
 | [depth-anything.cpp](https://github.com/mudler/depth-anything.cpp) | Depth Anything 3 monocular metric depth + camera pose estimation |
+| [face-detect.cpp](https://github.com/mudler/face-detect.cpp) | Face detection, recognition, demographics and anti-spoofing (SCRFD/ArcFace, YuNet/SFace), replacing the Python insightface backend |
+| [free-splatter.cpp](https://github.com/localai-org/free-splatter.cpp) | Pose-free 3D reconstruction (FreeSplatter): turns a handful of plain photos into 3D Gaussians, no camera poses or GPU required |
 | [privacy-filter.cpp](https://github.com/localai-org/privacy-filter.cpp) | Standalone GGML PII/NER token-classification engine powering LocalAI's PII redaction tier |
 | [LocalVQE](https://github.com/localai-org/LocalVQE) | Joint acoustic echo cancellation, noise suppression, and dereverberation |
 | [local-store](https://github.com/mudler/LocalAI) | Local-first vector database for embeddings (shipped in-tree) |

backend/README.md

@@ -46,6 +46,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **vllm**: High-performance LLM inference
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
+- **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation
 - **Audio**: coqui, faster-whisper, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro

backend/backend.proto

@@ -577,6 +577,10 @@ message GenerateVideoRequest {
   float cfg_scale = 10;    // Classifier-free guidance scale
   int32 step = 11;         // Number of inference steps
   string dst = 12;        // Output path for the generated video
+  string audio = 13;      // Path to staged audio for audio-conditioned video
+  // Backend-specific per-request generation parameters. Values are strings
+  // and are validated/coerced by the selected backend.
+  map<string, string> params = 14;
 }
 
 message TTSRequest {
@@ -1256,4 +1260,3 @@ message ForwardReply {
   repeated ForwardHeader headers = 2;
   bytes body_chunk = 3;
 }
-

backend/cpp/ds4/package.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
-set -e
+set -euo pipefail
 CURDIR=$(dirname "$(realpath "$0")")
 REPO_ROOT="${CURDIR}/../../.."
+PACKAGE_DIR="$CURDIR/package"
 
-mkdir -p "$CURDIR/package/lib"
-cp -avf "$CURDIR/grpc-server" "$CURDIR/package/"
-cp -avf "$CURDIR/ds4-worker"  "$CURDIR/package/"
-cp -rfv "$CURDIR/run.sh"      "$CURDIR/package/"
+rm -rf "$PACKAGE_DIR"
+mkdir -p "$PACKAGE_DIR/lib"
+cp -avf "$CURDIR/grpc-server" "$PACKAGE_DIR/"
+cp -avf "$CURDIR/ds4-worker"  "$PACKAGE_DIR/"
+cp -rfv "$CURDIR/run.sh"      "$PACKAGE_DIR/"
 
 UNAME_S=$(uname -s)
 if [ "$UNAME_S" = "Darwin" ]; then
@@ -16,25 +18,54 @@ if [ "$UNAME_S" = "Darwin" ]; then
 fi
 
 if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
-    cp -arfLv /lib64/ld-linux-x86-64.so.2 "$CURDIR/package/lib/ld.so"
-    LIBDIR=/lib/x86_64-linux-gnu
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 "$PACKAGE_DIR/lib/ld.so"
 elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
-    cp -arfLv /lib/ld-linux-aarch64.so.1 "$CURDIR/package/lib/ld.so"
-    LIBDIR=/lib/aarch64-linux-gnu
+    cp -arfLv /lib/ld-linux-aarch64.so.1 "$PACKAGE_DIR/lib/ld.so"
 else
     echo "package.sh: unknown architecture" >&2; exit 1
 fi
 
-for lib in libc.so.6 libgcc_s.so.1 libstdc++.so.6 libm.so.6 libgomp.so.1 \
-           libdl.so.2 librt.so.1 libpthread.so.0; do
-    cp -arfLv "$LIBDIR/$lib" "$CURDIR/package/lib/$lib"
+# Bundle the complete dependency closure for both executables. In particular,
+# grpc-server links the distro gRPC/protobuf/absl stack; copying only the core
+# C/C++ runtime libraries leaves the scratch image unable to start.
+{
+    ldd "$CURDIR/grpc-server"
+    ldd "$CURDIR/ds4-worker"
+} | awk '$2 == "=>" && $3 ~ /^\// { print $3 }' | sort -u | \
+while read -r so; do
+    cp -arfLv "$so" "$PACKAGE_DIR/lib/"
 done
 
 GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
 if [ -f "$GPU_LIB_SCRIPT" ]; then
-    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    # shellcheck source=/dev/null
+    source "$GPU_LIB_SCRIPT" "$PACKAGE_DIR/lib"
     package_gpu_libs
 fi
 
+# Resolve every dependency through the same loader and library path used by
+# the from-scratch image. The loader can still search host defaults, so reject
+# any absolute dependency path that escapes the package instead of accepting a
+# false-positive validation against a library that scratch will not contain.
+validate_packaged_binary() {
+    local binary="$1"
+    local resolution
+    resolution=$("$PACKAGE_DIR/lib/ld.so" \
+        --library-path "$PACKAGE_DIR/lib" \
+        --list "$PACKAGE_DIR/$binary")
+
+    printf '%s\n' "$resolution" | awk -v prefix="$PACKAGE_DIR/lib/" '
+        $2 == "=>" && $3 ~ /^\// && index($3, prefix) != 1 {
+            print "package.sh: dependency resolved outside package: " $0 > "/dev/stderr"
+            invalid = 1
+        }
+        END { exit invalid }
+    '
+}
+
+for binary in grpc-server ds4-worker; do
+    validate_packaged_binary "$binary"
+done
+
 echo "ds4 package contents:"
-ls -lah "$CURDIR/package/" "$CURDIR/package/lib/"
+ls -lah "$PACKAGE_DIR/" "$PACKAGE_DIR/lib/"

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=bbc7de475178dd0535c16ad85f204a2529806c9d
+IK_LLAMA_VERSION?=6198a356a85ed71534c02a9c1026203389f341e5
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=d4cff114c0084f1fbc9b4c62717eca8fb2ae494a
+LLAMA_VERSION?=6b4dc2116a92c5c8f2782bfe51fabe5ee66fb5ef
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -613,6 +613,24 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
     // starts with '-'. Applied once after the loop via common_params_parse.
     std::vector<std::string> extra_argv;
 
+    auto add_device_options = [&](const std::string & devices) {
+        const std::regex regex{ R"([,]+)" };
+        std::sregex_token_iterator it{ devices.begin(), devices.end(), regex, -1 };
+        std::vector<std::string> split_arg{ it, {} };
+
+        for (std::string device : split_arg) {
+            const auto start = device.find_first_not_of(" \t\n\r");
+            if (start == std::string::npos) {
+                continue;
+            }
+            const auto end = device.find_last_not_of(" \t\n\r");
+            device = device.substr(start, end - start + 1);
+
+            extra_argv.push_back("--device");
+            extra_argv.push_back(device);
+        }
+    };
+
      // decode options. Options are in form optname:optvale, or if booleans only optname.
     for (int i = 0; i < request->options_size(); i++) {
         std::string opt = request->options(i);
@@ -744,6 +762,10 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
             } else if (optval_str == "false" || optval_str == "0" || optval_str == "no" || optval_str == "off" || optval_str == "disabled") {
                 params.no_op_offload = false;
             }
+        } else if (!strcmp(optname, "device") || !strcmp(optname, "devices")) {
+            if (optval != NULL) {
+                add_device_options(optval_str);
+            }
         } else if (!strcmp(optname, "split_mode") || !strcmp(optname, "sm")) {
             // Accepts: none | layer | row | tensor (the latter requires a llama.cpp build
             // that includes ggml-org/llama.cpp#19378, FlashAttention enabled, and KV-cache

backend/go/crispasr/CMakeLists.txt

@@ -17,7 +17,7 @@ target_link_libraries(gocrispasr PRIVATE
     crispasr-lib
     parakeet canary canary_ctc cohere granite_speech granite_nle
     voxtral voxtral4b qwen3_asr qwen3_tts orpheus chatterbox indextts
-    kokoro voxcpm2_tts m2m100 t5_translate wav2vec2-ggml vibevoice
+    kokoro voxcpm2_tts m2m100 t5_translate wav2vec2-ggml vibevoice f5-tts
     silero-lid pyannote-seg funasr paraformer sensevoice
     crisp_audio
     ggml)

backend/go/crispasr/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # CrispASR version (release tag)
 CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
-CRISPASR_VERSION?=f35185b876fc482fcb2053a81a2697936ed5fcc0
+CRISPASR_VERSION?=d76cce027e3b183fc3d8c72e976e69d11f71bc8b
 SO_TARGET?=libgocrispasr.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/crispasr/gocrispasr.go

@@ -530,11 +530,51 @@ func setVoice(voice string) {
 	}
 }
 
+// applyRequestVoice distinguishes named speakers from per-request reference
+// WAVs. The latter are used by F5-TTS and require the exact transcript under
+// the cross-backend params.ref_text contract.
+func applyRequestVoice(req *pb.TTSRequest) error {
+	voice := strings.TrimSpace(req.Voice)
+	if voice == "" {
+		return nil
+	}
+
+	info, statErr := os.Stat(voice)
+	looksLikeFile := filepath.IsAbs(voice) || strings.EqualFold(filepath.Ext(voice), ".wav")
+	if statErr == nil && info.Mode().IsRegular() {
+		refText := ""
+		if req.Params != nil {
+			refText = strings.TrimSpace(req.Params["ref_text"])
+			if refText == "" {
+				refText = strings.TrimSpace(req.Params["voice_text"])
+			}
+		}
+		if refText == "" {
+			return fmt.Errorf("crispasr: params.ref_text is required with a reference voice WAV")
+		}
+		if rc := CppTTSSetVoiceFile(voice, refText); rc < 0 {
+			return fmt.Errorf("crispasr: failed to apply reference voice %q (rc=%d)", voice, rc)
+		}
+		return nil
+	}
+	if looksLikeFile {
+		if statErr != nil {
+			return fmt.Errorf("crispasr: reference voice %q: %w", voice, statErr)
+		}
+		return fmt.Errorf("crispasr: reference voice %q is not a regular file", voice)
+	}
+
+	setVoice(voice)
+	return nil
+}
+
 func (w *CrispASR) TTS(req *pb.TTSRequest) error {
 	if req.Dst == "" {
 		return fmt.Errorf("crispasr: TTS requires a destination path")
 	}
-	setVoice(req.Voice)
+	if err := applyRequestVoice(req); err != nil {
+		return err
+	}
 	pcm, err := w.synthesize(req.Text)
 	if err != nil {
 		return err
@@ -553,7 +593,9 @@ func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error {
 	if req.Text == "" {
 		return fmt.Errorf("crispasr: TTSStream requires text")
 	}
-	setVoice(req.Voice)
+	if err := applyRequestVoice(req); err != nil {
+		return err
+	}
 	pcm, err := w.synthesize(req.Text)
 	if err != nil {
 		return err

backend/go/crispasr/gocrispasr_test.go

@@ -172,6 +172,32 @@ var _ = Describe("CrispASR", func() {
 	})
 
 	Context("TTS", func() {
+		It("applies a per-request reference WAV and transcript", func() {
+			refWAV := filepath.Join(GinkgoT().TempDir(), "reference.wav")
+			Expect(os.WriteFile(refWAV, []byte("fixture"), 0o600)).To(Succeed())
+
+			original := CppTTSSetVoiceFile
+			DeferCleanup(func() { CppTTSSetVoiceFile = original })
+			var gotPath, gotText string
+			CppTTSSetVoiceFile = func(path, refText string) int {
+				gotPath, gotText = path, refText
+				return 0
+			}
+
+			Expect(applyRequestVoice(&pb.TTSRequest{
+				Voice:  refWAV,
+				Params: map[string]string{"ref_text": "The exact words in the clip."},
+			})).To(Succeed())
+			Expect(gotPath).To(Equal(refWAV))
+			Expect(gotText).To(Equal("The exact words in the clip."))
+		})
+
+		It("rejects a reference WAV without a transcript", func() {
+			refWAV := filepath.Join(GinkgoT().TempDir(), "reference.wav")
+			Expect(os.WriteFile(refWAV, []byte("fixture"), 0o600)).To(Succeed())
+			Expect(applyRequestVoice(&pb.TTSRequest{Voice: refWAV})).To(MatchError(ContainSubstring("params.ref_text")))
+		})
+
 		It("synthesizes a non-empty WAV", func() {
 			ttsModel := ttsModelOrSkip()
 			ensureLibLoaded()
@@ -189,5 +215,32 @@ var _ = Describe("CrispASR", func() {
 			Expect(info.Size()).To(BeNumerically(">", 1024),
 				"expected a non-trivial WAV, got %d bytes", info.Size())
 		})
+
+		It("synthesizes with F5-TTS voice cloning (reference WAV + transcript)", func() {
+			// F5-TTS has no baked speaker: it clones from a reference WAV and
+			// its transcript, supplied via the voice/voice_text options. The
+			// spec skips unless all three fixtures are provided.
+			model := os.Getenv("CRISPASR_F5_MODEL_PATH")
+			refWav := os.Getenv("CRISPASR_F5_REF_WAV")
+			refText := os.Getenv("CRISPASR_F5_REF_TEXT")
+			if model == "" || refWav == "" || refText == "" {
+				Skip("set CRISPASR_F5_MODEL_PATH, CRISPASR_F5_REF_WAV and CRISPASR_F5_REF_TEXT to run this spec")
+			}
+			ensureLibLoaded()
+
+			w := &CrispASR{}
+			Expect(w.Load(&pb.ModelOptions{
+				ModelFile: model,
+				Options:   []string{"voice:" + refWav, "voice_text:" + refText},
+			})).To(Succeed())
+
+			dst := filepath.Join(GinkgoT().TempDir(), "f5.wav")
+			Expect(w.TTS(&pb.TTSRequest{Text: "Hello from LocalAI running F5 text to speech.", Dst: dst})).To(Succeed())
+
+			info, err := os.Stat(dst)
+			Expect(err).ToNot(HaveOccurred(), "synthesized WAV should exist at %q", dst)
+			Expect(info.Size()).To(BeNumerically(">", 1024),
+				"expected a non-trivial WAV, got %d bytes", info.Size())
+		})
 	})
 })

backend/go/local-store/store.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"math"
 	"slices"
+	"strings"
 
 	"github.com/mudler/LocalAI/pkg/grpc/base"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
@@ -57,12 +58,18 @@ func NewStore() *Store {
 	}
 }
 
-// Load is a no-op — local-store has no on-disk artefact. opts.Model is
-// just a namespace identifier; isolation is already handled upstream
-// (ModelLoader spawns a fresh local-store process per (backend,
-// model) tuple, so each namespace is its own Store{} instance).
+// Load only validates the namespace — local-store has no on-disk
+// artefact. opts.Model is a namespace identifier, which core's
+// StoreBackend always sends with store.NamespacePrefix; anything else
+// is the model loader's greedy autoload probing with a real model name,
+// which must be refused or the LLM binds to the vector store. Isolation
+// is already handled upstream (ModelLoader spawns a fresh local-store
+// process per (backend, model) tuple, so each namespace is its own
+// Store{} instance).
 func (s *Store) Load(opts *pb.ModelOptions) error {
-	_ = opts
+	if !strings.HasPrefix(opts.GetModel(), store.NamespacePrefix) {
+		return fmt.Errorf("local-store: refusing to load %q: not a store namespace (expected %q prefix)", opts.GetModel(), store.NamespacePrefix)
+	}
 	return nil
 }
 

backend/go/local-store/store_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/LocalAI/pkg/store"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -186,8 +187,18 @@ var _ = Describe("StoresFind", func() {
 })
 
 var _ = Describe("StoresLoad", func() {
-	It("is a no-op", func() {
-		Expect(NewStore().Load(&pb.ModelOptions{Model: "any-namespace"})).To(Succeed())
+	It("accepts prefixed store namespaces", func() {
+		Expect(NewStore().Load(&pb.ModelOptions{Model: store.NamespacePrefix + "any-namespace"})).To(Succeed())
+	})
+
+	It("accepts the prefix alone (default store)", func() {
+		Expect(NewStore().Load(&pb.ModelOptions{Model: store.NamespacePrefix})).To(Succeed())
+	})
+
+	It("refuses model names without the namespace prefix", func() {
+		err := NewStore().Load(&pb.ModelOptions{Model: "some-llm.gguf"})
+		Expect(err).To(MatchError(ContainSubstring("not a store namespace")))
+		Expect(NewStore().Load(&pb.ModelOptions{})).NotTo(Succeed())
 	})
 })
 

backend/go/locate-anything-cpp/Makefile

@@ -10,7 +10,7 @@ JOBS?=$(shell nproc --ignore=1)
 # this on `master` always picks up the latest C-API surface (incl. the
 # per-detection accessor functions used by golocateanythingcpp.go).
 LOCATEANYTHING_REPO?=https://github.com/mudler/locate-anything.cpp.git
-LOCATEANYTHING_VERSION?=92c1682da792c1e8a5dec91acc2be4b02c742ded
+LOCATEANYTHING_VERSION?=ade2634f7f79b56121125e5885628744795a478f
 
 ifeq ($(NATIVE),false)
 	CMAKE_ARGS+=-DGGML_NATIVE=OFF

backend/go/moss-transcribe-cpp/.gitignore

@@ -0,0 +1,12 @@
+.cache/
+sources/
+build/
+package/
+moss-transcribe-cpp-grpc
+# build artifacts staged in-tree by the Makefile (cp from sources/) or
+# symlinked for local dev; the real sources live in moss-transcribe.cpp upstream.
+*.so
+*.so.*
+*.dylib
+moss_transcribe_capi.h
+compile_commands.json

backend/go/moss-transcribe-cpp/Makefile

@@ -0,0 +1,98 @@
+# moss-transcribe-cpp backend Makefile.
+#
+# Upstream pin lives below as MOSS_VERSION?=92a923dca88a41a34e47a364d55ee25731a9a0a2
+# (.github/bump_deps.sh) can find and update it - matches the
+# whisper.cpp / parakeet-cpp / ds4 convention.
+#
+# Local dev shortcut: if you already have an out-of-tree moss-transcribe.cpp
+# build, you can symlink the .so + header into this directory and skip the
+# clone/cmake steps entirely, e.g.:
+#
+#   ln -sf /path/to/moss-transcribe.cpp/build-shared/libmoss-transcribe.so .
+#   ln -sf /path/to/moss-transcribe.cpp/include/moss_transcribe_capi.h .
+#   go build -o moss-transcribe-cpp-grpc .
+#
+# That's what the L0 smoke test uses. The default target below does the proper
+# clone-at-pin + cmake build so CI doesn't need a side-checkout.
+
+MOSS_VERSION?=92a923dca88a41a34e47a364d55ee25731a9a0a2
+MOSS_REPO?=https://github.com/mudler/moss-transcribe.cpp
+
+GOCMD?=go
+GO_TAGS?=
+JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
+
+BUILD_TYPE?=
+NATIVE?=false
+
+# Build ggml statically into libmoss-transcribe.so (PIC) so the shared lib is
+# self-contained: dlopen needs no libggml*.so alongside it, only system libs
+# (libstdc++/libgomp/libc) that the runtime image already provides. MT_SHARED
+# flips ggml to a static PIC build (see the moss-transcribe.cpp CMakeLists).
+CMAKE_ARGS?=-DCMAKE_BUILD_TYPE=Release -DMT_SHARED=ON -DMT_BUILD_CLI=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+
+ifeq ($(NATIVE),false)
+	CMAKE_ARGS+=-DGGML_NATIVE=OFF
+endif
+
+# moss-transcribe.cpp gates its GGML backends behind MT_GGML_* options and does
+# set(GGML_<be> ${MT_GGML_<be>} CACHE BOOL "" FORCE), so a bare -DGGML_CUDA=ON is
+# overwritten back to OFF and the build silently falls back to CPU. Forward the
+# MT_GGML_* options instead (openblas is not gated, so -DGGML_BLAS passes through).
+ifeq ($(BUILD_TYPE),cublas)
+	CMAKE_ARGS+=-DMT_GGML_CUDA=ON -DGGML_CUDA_GRAPHS=ON
+else ifeq ($(BUILD_TYPE),openblas)
+	CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
+else ifeq ($(BUILD_TYPE),hipblas)
+	CMAKE_ARGS+=-DMT_GGML_HIP=ON
+else ifeq ($(BUILD_TYPE),vulkan)
+	CMAKE_ARGS+=-DMT_GGML_VULKAN=ON
+else ifeq ($(BUILD_TYPE),metal)
+	CMAKE_ARGS+=-DMT_GGML_METAL=ON
+endif
+
+.PHONY: moss-transcribe-cpp-grpc package build clean purge test all
+
+all: moss-transcribe-cpp-grpc
+
+# Clone the upstream moss-transcribe.cpp source at the pinned commit. Directory
+# acts as the target so make only re-clones when missing. After a MOSS_VERSION
+# bump, run 'make purge && make' to refetch.
+sources/moss-transcribe.cpp:
+	mkdir -p sources/moss-transcribe.cpp
+	cd sources/moss-transcribe.cpp && \
+	git init -q && \
+	git remote add origin $(MOSS_REPO) && \
+	git fetch --depth 1 origin $(MOSS_VERSION) && \
+	git checkout FETCH_HEAD && \
+	git submodule update --init --recursive --depth 1 --single-branch
+
+# Build the shared lib + header out-of-tree, then stage them next to the Go
+# sources so purego.Dlopen("libmoss-transcribe.so") and the cgo-less build both
+# pick them up.
+libmoss-transcribe.so: sources/moss-transcribe.cpp
+	cmake -B sources/moss-transcribe.cpp/build-shared -S sources/moss-transcribe.cpp $(CMAKE_ARGS)
+	cmake --build sources/moss-transcribe.cpp/build-shared --config Release -j$(JOBS)
+	cp -fv sources/moss-transcribe.cpp/build-shared/libmoss-transcribe.so* ./ 2>/dev/null || true
+	cp -fv sources/moss-transcribe.cpp/build-shared/libmoss-transcribe.dylib ./ 2>/dev/null || true
+	cp -fv sources/moss-transcribe.cpp/include/moss_transcribe_capi.h ./
+
+moss-transcribe-cpp-grpc: libmoss-transcribe.so main.go gomosstranscribecpp.go segments.go
+	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o moss-transcribe-cpp-grpc .
+
+package: moss-transcribe-cpp-grpc
+	bash package.sh
+
+build: package
+
+# Test target. The model-backed smoke test is gated on
+# MOSS_BACKEND_TEST_MODEL + MOSS_BACKEND_TEST_WAV; without them that spec
+# auto-skips, leaving the pure-Go parser/unit tests.
+test:
+	LD_LIBRARY_PATH=$(CURDIR):$$LD_LIBRARY_PATH $(GOCMD) test ./... -count=1
+
+clean: purge
+	rm -rf libmoss-transcribe.so* moss_transcribe_capi.h package moss-transcribe-cpp-grpc
+
+purge:
+	rm -rf sources/moss-transcribe.cpp

backend/go/moss-transcribe-cpp/gomosstranscribecpp.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"unsafe"
+
+	"github.com/mudler/LocalAI/pkg/grpc/base"
+	"github.com/mudler/LocalAI/pkg/grpc/grpcerrors"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/LocalAI/pkg/utils"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// purego-bound entry points from libmoss-transcribe.so. Names match
+// moss_transcribe_capi.h exactly so a `nm libmoss-transcribe.so | grep
+// moss_transcribe_capi` is enough to spot drift.
+//
+// The transcribe_* functions return char* declared here as uintptr so we can
+// call moss_transcribe_capi_free_string on the same pointer after copying: the
+// C-API contract is "caller owns and must free the returned buffer".
+var (
+	CppAbiVersion     func() int32
+	CppLoad           func(ggufPath string) uintptr
+	CppFree           func(ctx uintptr)
+	CppTranscribePath func(ctx uintptr, wavPath string, maxNew int32) uintptr
+	CppTranscribePcm  func(ctx uintptr, samples []float32, nSamples int32, sampleRate int32, maxNew int32) uintptr
+	CppFreeString     func(s uintptr)
+	CppLastError      func(ctx uintptr) string
+)
+
+// MossTranscribeCpp owns a single loaded moss_transcribe_ctx. MOSS is an
+// offline transcription + diarization + timestamping engine: one model, one
+// context, no streaming. The C engine holds a single mutable context and is
+// not reentrant, so we embed base.SingleThread — LocalAI serialises every RPC
+// through the server-level lock, and only one transcription touches the engine
+// at a time.
+type MossTranscribeCpp struct {
+	base.SingleThread
+	ctxPtr uintptr
+	maxNew int32
+}
+
+// Load is the LocalAI gRPC entry point for LoadModel: it calls
+// moss_transcribe_capi_load with the GGUF path and stashes the resulting
+// opaque context pointer for AudioTranscription.
+func (m *MossTranscribeCpp) Load(opts *pb.ModelOptions) error {
+	if opts.ModelFile == "" {
+		return errors.New("moss-transcribe-cpp: ModelFile is required")
+	}
+
+	// max_new_tokens caps the generated tokens per transcription; <=0 uses the
+	// GGUF's default_max_new_tokens (the C-API's own default). Exposed as a
+	// model YAML option: (key:value form, like the sibling ggml backends).
+	m.maxNew = int32(optInt(opts, "max_new_tokens", 0))
+
+	ctx := CppLoad(opts.ModelFile)
+	if ctx == 0 {
+		// No ctx to ask for last_error (the C-API's last-error buffer lives on
+		// the ctx that was never returned). Surface the path so the operator at
+		// least knows which load failed.
+		return fmt.Errorf("moss-transcribe-cpp: moss_transcribe_capi_load failed for %q", opts.ModelFile)
+	}
+	m.ctxPtr = ctx
+	return nil
+}
+
+// optInt reads an integer model option (key:value form) from ModelOptions,
+// returning def when absent or unparseable. The options array carries the
+// model YAML's options: entries (see core/config; siblings such as parakeet-cpp
+// parse the same key:value form via strings.Cut on ":").
+func optInt(opts *pb.ModelOptions, key string, def int) int {
+	for _, o := range opts.GetOptions() {
+		k, v, ok := strings.Cut(o, ":")
+		if ok && strings.TrimSpace(k) == key {
+			if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
+				return n
+			}
+		}
+	}
+	return def
+}
+
+// AudioTranscription converts the audio at opts.Dst to a 16 kHz mono WAV and
+// hands the path to moss_transcribe_capi_transcribe_path. The model emits its
+// own speaker-labelled, time-aligned transcript in the compact
+// "[start][Sxx]text[end]..." format (seconds); we parse it into LocalAI
+// TranscriptSegments carrying int64-nanosecond timestamps and the per-segment
+// speaker label.
+//
+// MOSS does joint transcription + diarization + timestamps in one pass, so
+// translate/language/prompt/temperature/threads are not applicable and are
+// ignored. Streaming is not supported (offline model).
+func (m *MossTranscribeCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) {
+	if m.ctxPtr == 0 {
+		return pb.TranscriptResult{}, grpcerrors.ModelNotLoaded("moss-transcribe-cpp")
+	}
+	if opts.Dst == "" {
+		return pb.TranscriptResult{}, errors.New("moss-transcribe-cpp: TranscriptRequest.dst (audio path) is required")
+	}
+	if err := ctx.Err(); err != nil {
+		return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled")
+	}
+
+	// The C loader understands WAV; convert any input (MP3, etc.) to 16 kHz
+	// mono WAV first - the same normalisation every other audio backend
+	// (whisper, parakeet-cpp) does via utils.AudioToWav before handing the file
+	// to the engine.
+	converted, cleanup, err := convertToWavMono16k(opts.Dst)
+	if err != nil {
+		return pb.TranscriptResult{}, err
+	}
+	defer cleanup()
+
+	cstr := CppTranscribePath(m.ctxPtr, converted, m.maxNew)
+	if cstr == 0 {
+		return pb.TranscriptResult{}, fmt.Errorf("moss-transcribe-cpp: transcribe_path failed: %s", CppLastError(m.ctxPtr))
+	}
+	raw := goStringFromCPtr(cstr)
+	CppFreeString(cstr)
+
+	return transcriptResultFromRaw(raw), nil
+}
+
+// Free releases the underlying moss_transcribe_ctx. Called by LocalAI when the
+// model is unloaded.
+func (m *MossTranscribeCpp) Free() error {
+	if m.ctxPtr != 0 {
+		CppFree(m.ctxPtr)
+		m.ctxPtr = 0
+	}
+	return nil
+}
+
+// convertToWavMono16k converts an arbitrary audio file to a 16 kHz mono WAV in
+// a fresh temp dir and returns the path together with a cleanup func the caller
+// must defer. WAV inputs already at 16 kHz/mono/16-bit are passed through by
+// utils.AudioToWav (hardlink/copy), everything else is transcoded via ffmpeg.
+func convertToWavMono16k(path string) (string, func(), error) {
+	dir, err := os.MkdirTemp("", "moss-transcribe")
+	if err != nil {
+		return "", func() {}, err
+	}
+	cleanup := func() { _ = os.RemoveAll(dir) }
+
+	converted := filepath.Join(dir, "converted.wav")
+	if err := utils.AudioToWav(path, converted); err != nil {
+		cleanup()
+		return "", func() {}, err
+	}
+	return converted, cleanup, nil
+}
+
+// goStringFromCPtr copies a NUL-terminated C string into Go memory. cptr is the
+// raw pointer returned by purego from the C-API (a malloc'd buffer the caller
+// owns); callers must free it via CppFreeString after the copy lands.
+//
+// The uintptr->unsafe.Pointer conversion below trips go vet's unsafeptr check,
+// which can't distinguish a C-owned heap pointer from Go-managed memory. It is
+// safe here: the pointer addresses a malloc'd C buffer the Go GC neither tracks
+// nor moves, and we dereference it immediately to copy the bytes out (the same
+// pattern the whisper / parakeet-cpp backends use).
+func goStringFromCPtr(cptr uintptr) string {
+	if cptr == 0 {
+		return ""
+	}
+	p := unsafe.Pointer(cptr) //nolint:govet // C-owned malloc'd buffer, not Go-GC memory (see doc above)
+	n := 0
+	for *(*byte)(unsafe.Add(p, n)) != 0 {
+		n++
+	}
+	return string(unsafe.Slice((*byte)(p), n))
+}

backend/go/moss-transcribe-cpp/gomosstranscribecpp_test.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/ebitengine/purego"
+	"github.com/go-audio/audio"
+	"github.com/go-audio/wav"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMossTranscribeCpp(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "moss-transcribe-cpp Backend Suite")
+}
+
+var (
+	libLoadOnce sync.Once
+	libLoadErr  error
+)
+
+// ensureLibLoaded mirrors main.go's bootstrap so a Go test can drive the C-API
+// bridge without spinning up the gRPC server. Skips the current spec when
+// libmoss-transcribe.so isn't loadable from cwd ($LD_LIBRARY_PATH or a symlink
+// in ./).
+func ensureLibLoaded() {
+	libLoadOnce.Do(func() {
+		libName := os.Getenv("MOSS_TRANSCRIBE_LIBRARY")
+		if libName == "" {
+			libName = "libmoss-transcribe.so"
+		}
+		lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+		if err != nil {
+			libLoadErr = err
+			return
+		}
+		purego.RegisterLibFunc(&CppAbiVersion, lib, "moss_transcribe_capi_abi_version")
+		purego.RegisterLibFunc(&CppLoad, lib, "moss_transcribe_capi_load")
+		purego.RegisterLibFunc(&CppFree, lib, "moss_transcribe_capi_free")
+		purego.RegisterLibFunc(&CppTranscribePath, lib, "moss_transcribe_capi_transcribe_path")
+		purego.RegisterLibFunc(&CppTranscribePcm, lib, "moss_transcribe_capi_transcribe_pcm")
+		purego.RegisterLibFunc(&CppFreeString, lib, "moss_transcribe_capi_free_string")
+		purego.RegisterLibFunc(&CppLastError, lib, "moss_transcribe_capi_last_error")
+	})
+	if libLoadErr != nil {
+		Skip("libmoss-transcribe.so not loadable: " + libLoadErr.Error())
+	}
+}
+
+// fixturesOrSkip returns the model + audio paths or skips the spec if either
+// env var is unset. The smoke test never runs in default CI; it needs a real
+// MOSS GGUF and a WAV on disk.
+func fixturesOrSkip() (string, string) {
+	modelPath := os.Getenv("MOSS_BACKEND_TEST_MODEL")
+	audioPath := os.Getenv("MOSS_BACKEND_TEST_WAV")
+	if modelPath == "" || audioPath == "" {
+		Skip("set MOSS_BACKEND_TEST_MODEL and MOSS_BACKEND_TEST_WAV to run this spec")
+	}
+	return modelPath, audioPath
+}
+
+// writeMono16kWav writes `samples` frames of 16 kHz mono 16-bit silence to
+// path. The result is already in AudioToWav's target format, so the conversion
+// helper copies it through without invoking ffmpeg.
+func writeMono16kWav(path string, samples int) {
+	GinkgoHelper()
+	f, err := os.Create(path)
+	Expect(err).ToNot(HaveOccurred())
+	enc := wav.NewEncoder(f, 16000, 16, 1, 1)
+	buf := &audio.IntBuffer{
+		Format:         &audio.Format{NumChannels: 1, SampleRate: 16000},
+		SourceBitDepth: 16,
+		Data:           make([]int, samples),
+	}
+	Expect(enc.Write(buf)).To(Succeed())
+	Expect(enc.Close()).To(Succeed())
+	Expect(f.Close()).To(Succeed())
+}
+
+var _ = Describe("MossTranscribeCpp", func() {
+	Context("ABI / load smoke (needs libmoss-transcribe.so)", func() {
+		It("reports a positive ABI version", func() {
+			ensureLibLoaded()
+			Expect(CppAbiVersion()).To(BeNumerically(">=", 1))
+		})
+
+		It("transcribes a WAV into speaker-labelled segments", func() {
+			modelPath, audioPath := fixturesOrSkip()
+			ensureLibLoaded()
+
+			m := &MossTranscribeCpp{}
+			Expect(m.Load(&pb.ModelOptions{ModelFile: modelPath})).To(Succeed())
+			defer func() { _ = m.Free() }()
+
+			res, err := m.AudioTranscription(context.Background(), &pb.TranscriptRequest{
+				Dst: audioPath,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.TrimSpace(res.Text)).ToNot(BeEmpty(),
+				"expected non-empty transcript for %s", audioPath)
+			Expect(res.Segments).ToNot(BeEmpty(), "expected at least one segment")
+			var prevEnd int64
+			for i, seg := range res.Segments {
+				Expect(strings.TrimSpace(seg.Text)).ToNot(BeEmpty(),
+					"segment %d must have text", i)
+				Expect(seg.End).To(BeNumerically(">=", seg.Start),
+					"segment %d end must not precede its start", i)
+				Expect(seg.Start).To(BeNumerically(">=", prevEnd),
+					"segments must be in time order")
+				prevEnd = seg.End
+			}
+		})
+	})
+
+	Context("Load validation", func() {
+		It("rejects an empty ModelFile", func() {
+			m := &MossTranscribeCpp{}
+			Expect(m.Load(&pb.ModelOptions{})).To(HaveOccurred())
+		})
+	})
+
+	Context("AudioTranscription guards (no C library required)", func() {
+		It("returns ModelNotLoaded when no context is loaded", func() {
+			m := &MossTranscribeCpp{}
+			_, err := m.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: "x.wav"})
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("errors when the audio path is empty", func() {
+			m := &MossTranscribeCpp{ctxPtr: 1}
+			_, err := m.AudioTranscription(context.Background(), &pb.TranscriptRequest{})
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Context("convertToWavMono16k", func() {
+		It("returns a decodable 16kHz mono WAV copy and cleans it up", func() {
+			dir := GinkgoT().TempDir()
+			src := filepath.Join(dir, "input.wav")
+			writeMono16kWav(src, 16000) // 1s of silence at 16 kHz
+
+			converted, cleanup, err := convertToWavMono16k(src)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(converted).ToNot(Equal(src))
+			Expect(converted).To(BeAnExistingFile())
+
+			cleanup()
+			Expect(converted).ToNot(BeAnExistingFile(), "cleanup removes the temp dir")
+		})
+
+		It("errors on a non-existent input rather than passing the path through", func() {
+			_, _, err := convertToWavMono16k(filepath.Join(GinkgoT().TempDir(), "missing.mp3"))
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

backend/go/moss-transcribe-cpp/main.go

@@ -0,0 +1,70 @@
+package main
+
+// Started internally by LocalAI - one gRPC server per loaded model.
+//
+// Loads the moss-transcribe shared library via purego and registers the flat
+// C-API entry points declared in moss_transcribe_capi.h. The library name can
+// be overridden with MOSS_TRANSCRIBE_LIBRARY (mirrors the WHISPER_LIBRARY /
+// PARAKEET_LIBRARY convention in the sibling backends); the default looks next
+// to this binary for libmoss-transcribe.so on Linux and
+// libmoss-transcribe.dylib on macOS.
+import (
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/ebitengine/purego"
+	grpc "github.com/mudler/LocalAI/pkg/grpc"
+)
+
+var (
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+)
+
+type LibFuncs struct {
+	FuncPtr any
+	Name    string
+}
+
+func main() {
+	libName := os.Getenv("MOSS_TRANSCRIBE_LIBRARY")
+	if libName == "" {
+		if runtime.GOOS == "darwin" {
+			libName = "libmoss-transcribe.dylib"
+		} else {
+			libName = "libmoss-transcribe.so"
+		}
+	}
+
+	lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+	if err != nil {
+		panic(fmt.Errorf("moss-transcribe-cpp: dlopen %q: %w", libName, err))
+	}
+
+	// Bound 1:1 to moss_transcribe_capi.h. The transcribe_* entry points return
+	// a malloc'd char* the caller owns; we register those as uintptr so we get
+	// the raw pointer back and can call moss_transcribe_capi_free_string on it
+	// (purego's string return would copy and forget the original pointer,
+	// leaking it on every call).
+	libFuncs := []LibFuncs{
+		{&CppAbiVersion, "moss_transcribe_capi_abi_version"},
+		{&CppLoad, "moss_transcribe_capi_load"},
+		{&CppFree, "moss_transcribe_capi_free"},
+		{&CppTranscribePath, "moss_transcribe_capi_transcribe_path"},
+		{&CppTranscribePcm, "moss_transcribe_capi_transcribe_pcm"},
+		{&CppFreeString, "moss_transcribe_capi_free_string"},
+		{&CppLastError, "moss_transcribe_capi_last_error"},
+	}
+	for _, lf := range libFuncs {
+		purego.RegisterLibFunc(lf.FuncPtr, lib, lf.Name)
+	}
+
+	fmt.Fprintf(os.Stderr, "[moss-transcribe-cpp] ABI=%d\n", CppAbiVersion())
+
+	flag.Parse()
+
+	if err := grpc.StartServer(*addr, &MossTranscribeCpp{}); err != nil {
+		panic(err)
+	}
+}

backend/go/moss-transcribe-cpp/package.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Bundle the moss-transcribe-cpp-grpc binary, libmoss-transcribe.so, the core
+# runtime libs (libc/libstdc++/libgomp + ld.so) and the GPU runtime for the
+# active BUILD_TYPE so the package is self-contained. Mirrors
+# backend/go/parakeet-cpp/package.sh; run.sh routes the (CGO_ENABLED=0) binary
+# through lib/ld.so so the packaged libc is used instead of the host's.
+
+set -e
+
+CURDIR=$(dirname "$(realpath "$0")")
+REPO_ROOT="${CURDIR}/../../.."
+
+mkdir -p "$CURDIR/package/lib"
+
+cp -avf "$CURDIR/moss-transcribe-cpp-grpc" "$CURDIR/package/"
+cp -avf "$CURDIR/run.sh" "$CURDIR/package/"
+
+# libmoss-transcribe shared lib + any soname symlinks. On Linux this is
+# libmoss-transcribe.so[.X.Y]; on macOS it is libmoss-transcribe.dylib.
+# purego.Dlopen resolves it via the *_LIBRARY_PATH that run.sh points at lib/.
+cp -avf "$CURDIR"/libmoss-transcribe.so* "$CURDIR/package/lib/" 2>/dev/null || true
+cp -avf "$CURDIR"/libmoss-transcribe.dylib "$CURDIR/package/lib/" 2>/dev/null || true
+if ! ls "$CURDIR"/package/lib/libmoss-transcribe.* >/dev/null 2>&1; then
+	echo "ERROR: libmoss-transcribe shared library not found in $CURDIR, run 'make' first" >&2
+	exit 1
+fi
+
+# Detect architecture and copy the core runtime libs libmoss-transcribe.so links
+# against, plus the matching dynamic loader as lib/ld.so.
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 "$CURDIR/package/lib/ld.so"
+    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 "$CURDIR/package/lib/libc.so.6"
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 "$CURDIR/package/lib/libgcc_s.so.1"
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 "$CURDIR/package/lib/libstdc++.so.6"
+    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 "$CURDIR/package/lib/libm.so.6"
+    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 "$CURDIR/package/lib/libgomp.so.1"
+    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 "$CURDIR/package/lib/libdl.so.2"
+    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 "$CURDIR/package/lib/librt.so.1"
+    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 "$CURDIR/package/lib/libpthread.so.0"
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp -arfLv /lib/ld-linux-aarch64.so.1 "$CURDIR/package/lib/ld.so"
+    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 "$CURDIR/package/lib/libc.so.6"
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 "$CURDIR/package/lib/libgcc_s.so.1"
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 "$CURDIR/package/lib/libstdc++.so.6"
+    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 "$CURDIR/package/lib/libm.so.6"
+    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 "$CURDIR/package/lib/libgomp.so.1"
+    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 "$CURDIR/package/lib/libdl.so.2"
+    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 "$CURDIR/package/lib/librt.so.1"
+    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 "$CURDIR/package/lib/libpthread.so.0"
+elif [ "$(uname -s)" = "Darwin" ]; then
+    echo "Detected Darwin — system frameworks linked dynamically, no bundled libs needed"
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+# Package GPU libraries (CUDA/ROCm/Intel/Vulkan loader + ICDs + drivers) based
+# on BUILD_TYPE so the backend can reach the GPU without the runtime base image
+# shipping those drivers.
+GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
+if [ -f "$GPU_LIB_SCRIPT" ]; then
+    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
+    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    package_gpu_libs
+fi
+
+echo "Packaging completed successfully"
+ls -liah "$CURDIR/package/" "$CURDIR/package/lib/"

backend/go/moss-transcribe-cpp/run.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+CURDIR=$(dirname "$(realpath "$0")")
+
+if [ "$(uname)" = "Darwin" ]; then
+	export DYLD_LIBRARY_PATH="$CURDIR/lib:"$CURDIR":${DYLD_LIBRARY_PATH:-}"
+	export MOSS_TRANSCRIBE_LIBRARY="$CURDIR/lib/libmoss-transcribe.dylib"
+else
+	export LD_LIBRARY_PATH="$CURDIR/lib:"$CURDIR":${LD_LIBRARY_PATH:-}"
+	export MOSS_TRANSCRIBE_LIBRARY="$CURDIR/lib/libmoss-transcribe.so"
+fi
+
+# If a self-contained ld.so was packaged, route through it so the packaged libc
+# / libstdc++ are used instead of the host's (matches the whisper / parakeet-cpp
+# backends' runtime layout). Linux only.
+if [ -f "$CURDIR/lib/ld.so" ]; then
+	echo "Using lib/ld.so"
+	exec "$CURDIR/lib/ld.so" "$CURDIR/moss-transcribe-cpp-grpc" "$@"
+fi
+
+exec "$CURDIR/moss-transcribe-cpp-grpc" "$@"

backend/go/moss-transcribe-cpp/segments.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+)
+
+// mossSegment is one parsed unit of the MOSS transcript: a speaker-labelled
+// span with fractional-second start/end times, straight from the model's own
+// output. Timestamps stay in seconds here; secondsToNanos converts them at the
+// TranscriptSegment boundary.
+type mossSegment struct {
+	Start   float64
+	End     float64
+	Speaker string
+	Text    string
+}
+
+// bracketRe matches one "[...]" token (timestamp or speaker tag). The MOSS
+// transcript is a concatenation of "[start][Sxx]text[end]" segments, e.g.
+//
+//	[0.28][S01] And so, my fellow Americans,[7.71][8.12][S02] ask ...[10.59]
+//
+// so the bracketed tokens carry all the structure and the free text lives
+// between a speaker tag and the following (end) timestamp.
+var bracketRe = regexp.MustCompile(`\[([^\]]*)\]`)
+
+// speakerRe matches a speaker tag: an 'S' (any case) followed by one or more
+// digits, e.g. "S01", "S12". Anything else in brackets that isn't a speaker tag
+// is treated as a timestamp candidate.
+var speakerRe = regexp.MustCompile(`^[Ss][0-9]+$`)
+
+// bracketToken is a "[...]" token plus the free text that follows it up to the
+// next token (or end of string). For a speaker tag that trailing text is the
+// segment's transcript.
+type bracketToken struct {
+	content   string
+	textAfter string
+}
+
+// parseTranscript parses the compact "[start][Sxx]text[end]..." MOSS transcript
+// into structured segments. It walks the bracket tokens looking for a
+// start-timestamp, a speaker tag, and an end-timestamp, taking the text that
+// follows the speaker tag as the segment transcript. Tokens that don't fit the
+// grammar are skipped rather than aborting the parse, so a slightly malformed
+// stream still yields the segments it can.
+func parseTranscript(raw string) []mossSegment {
+	locs := bracketRe.FindAllStringSubmatchIndex(raw, -1)
+	toks := make([]bracketToken, len(locs))
+	for i, m := range locs {
+		// m[2]:m[3] is the capture group (the content inside the brackets);
+		// m[1] is the byte just past the closing ']'.
+		nextStart := len(raw)
+		if i+1 < len(locs) {
+			nextStart = locs[i+1][0]
+		}
+		toks[i] = bracketToken{
+			content:   raw[m[2]:m[3]],
+			textAfter: raw[m[1]:nextStart],
+		}
+	}
+
+	var segs []mossSegment
+	i := 0
+	for i < len(toks) {
+		start, ok := parseTimestamp(toks[i].content)
+		if !ok {
+			i++
+			continue
+		}
+		// A start timestamp must be followed by a speaker tag; otherwise this
+		// isn't a segment head, so skip it.
+		if i+1 >= len(toks) || !isSpeakerTag(toks[i+1].content) {
+			i++
+			continue
+		}
+		speaker := strings.ToUpper(strings.TrimSpace(toks[i+1].content))
+		text := strings.TrimSpace(toks[i+1].textAfter)
+
+		// The end timestamp is the next token if present and numeric; a segment
+		// that runs to the end of the stream without a closing timestamp falls
+		// back to start==end.
+		end := start
+		consumed := 2
+		if i+2 < len(toks) {
+			if e, ok := parseTimestamp(toks[i+2].content); ok {
+				end = e
+				consumed = 3
+			}
+		}
+
+		segs = append(segs, mossSegment{Start: start, End: end, Speaker: speaker, Text: text})
+		i += consumed
+	}
+	return segs
+}
+
+// isSpeakerTag reports whether the bracket content is a MOSS speaker tag ("Sxx").
+func isSpeakerTag(content string) bool {
+	return speakerRe.MatchString(strings.TrimSpace(content))
+}
+
+// parseTimestamp parses a bracket content as a fractional-second timestamp. A
+// speaker tag ("S01") never parses as a float, so this cleanly distinguishes
+// the two token kinds.
+func parseTimestamp(content string) (float64, bool) {
+	f, err := strconv.ParseFloat(strings.TrimSpace(content), 64)
+	if err != nil {
+		return 0, false
+	}
+	return f, true
+}
+
+// secondsToNanos converts the transcript's fractional-second timestamps into
+// the int64 nanoseconds LocalAI carries on TranscriptSegment, the same
+// nanosecond convention the whisper / parakeet-cpp backends use.
+func secondsToNanos(sec float64) int64 {
+	return int64(sec * 1e9)
+}
+
+// transcriptResultFromRaw parses the raw MOSS transcript and shapes it into a
+// TranscriptResult. Each parsed segment becomes a TranscriptSegment with
+// nanosecond start/end and the model's own speaker label; Text is the segments
+// joined with single spaces. When nothing parses (no bracket structure) it
+// falls back to a single whole-clip text segment so callers always get a
+// transcript.
+func transcriptResultFromRaw(raw string) pb.TranscriptResult {
+	segs := parseTranscript(raw)
+	if len(segs) == 0 {
+		text := strings.TrimSpace(raw)
+		return pb.TranscriptResult{
+			Text:     text,
+			Segments: []*pb.TranscriptSegment{{Id: 0, Text: text}},
+		}
+	}
+
+	var full strings.Builder
+	pbSegs := make([]*pb.TranscriptSegment, 0, len(segs))
+	for id, s := range segs {
+		if id > 0 && s.Text != "" {
+			full.WriteString(" ")
+		}
+		full.WriteString(s.Text)
+		pbSegs = append(pbSegs, &pb.TranscriptSegment{
+			Id:      int32(id),
+			Start:   secondsToNanos(s.Start),
+			End:     secondsToNanos(s.End),
+			Text:    s.Text,
+			Speaker: s.Speaker,
+		})
+	}
+	return pb.TranscriptResult{Text: strings.TrimSpace(full.String()), Segments: pbSegs}
+}

backend/go/moss-transcribe-cpp/segments_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("parseTranscript ([start][Sxx]text[end] grammar)", func() {
+	It("parses the README two-speaker example into ordered segments", func() {
+		raw := "[0.28][S01] And so, my fellow Americans, ask not what your country can do for you,[7.71][8.12][S02] ask what you can do for your country.[10.59]"
+		segs := parseTranscript(raw)
+		Expect(segs).To(HaveLen(2))
+
+		Expect(segs[0].Speaker).To(Equal("S01"))
+		Expect(segs[0].Start).To(BeNumerically("~", 0.28, 1e-9))
+		Expect(segs[0].End).To(BeNumerically("~", 7.71, 1e-9))
+		Expect(segs[0].Text).To(Equal("And so, my fellow Americans, ask not what your country can do for you,"))
+
+		Expect(segs[1].Speaker).To(Equal("S02"))
+		Expect(segs[1].Start).To(BeNumerically("~", 8.12, 1e-9))
+		Expect(segs[1].End).To(BeNumerically("~", 10.59, 1e-9))
+		Expect(segs[1].Text).To(Equal("ask what you can do for your country."))
+	})
+
+	It("parses a single segment", func() {
+		segs := parseTranscript("[1.50][S01] hello world[3.25]")
+		Expect(segs).To(HaveLen(1))
+		Expect(segs[0].Start).To(BeNumerically("~", 1.50, 1e-9))
+		Expect(segs[0].End).To(BeNumerically("~", 3.25, 1e-9))
+		Expect(segs[0].Speaker).To(Equal("S01"))
+		Expect(segs[0].Text).To(Equal("hello world"))
+	})
+
+	It("uppercases a lowercase speaker tag", func() {
+		segs := parseTranscript("[0.00][s03] hi[0.90]")
+		Expect(segs).To(HaveLen(1))
+		Expect(segs[0].Speaker).To(Equal("S03"))
+	})
+
+	It("falls back to start==end when the closing timestamp is missing", func() {
+		segs := parseTranscript("[2.00][S01] trailing words with no end")
+		Expect(segs).To(HaveLen(1))
+		Expect(segs[0].Start).To(BeNumerically("~", 2.00, 1e-9))
+		Expect(segs[0].End).To(Equal(segs[0].Start))
+		Expect(segs[0].Text).To(Equal("trailing words with no end"))
+	})
+
+	It("handles multi-digit speaker ids and higher speaker counts", func() {
+		segs := parseTranscript("[0.0][S01] a[1.0][1.0][S12] b[2.0]")
+		Expect(segs).To(HaveLen(2))
+		Expect(segs[0].Speaker).To(Equal("S01"))
+		Expect(segs[1].Speaker).To(Equal("S12"))
+	})
+
+	It("returns no segments for text without bracket structure", func() {
+		Expect(parseTranscript("just some plain text")).To(BeEmpty())
+		Expect(parseTranscript("")).To(BeEmpty())
+	})
+
+	It("skips a start timestamp not followed by a speaker tag", func() {
+		// A stray timestamp pair without a speaker tag is not a segment head.
+		segs := parseTranscript("[1.0][2.0][S01] real[3.0]")
+		Expect(segs).To(HaveLen(1))
+		Expect(segs[0].Speaker).To(Equal("S01"))
+		Expect(segs[0].Text).To(Equal("real"))
+	})
+})
+
+var _ = Describe("transcriptResultFromRaw", func() {
+	It("shapes parsed segments into a TranscriptResult with nanosecond times and speakers", func() {
+		raw := "[0.28][S01] first part,[7.71][8.12][S02] second part.[10.59]"
+		res := transcriptResultFromRaw(raw)
+
+		Expect(res.Segments).To(HaveLen(2))
+		Expect(res.Segments[0].Id).To(Equal(int32(0)))
+		Expect(res.Segments[0].Start).To(Equal(secondsToNanos(0.28)))
+		Expect(res.Segments[0].End).To(Equal(secondsToNanos(7.71)))
+		Expect(res.Segments[0].Speaker).To(Equal("S01"))
+		Expect(res.Segments[0].Text).To(Equal("first part,"))
+
+		Expect(res.Segments[1].Id).To(Equal(int32(1)))
+		Expect(res.Segments[1].Speaker).To(Equal("S02"))
+		Expect(res.Segments[1].Start).To(Equal(secondsToNanos(8.12)))
+
+		// Full text is the segments joined with single spaces.
+		Expect(res.Text).To(Equal("first part, second part."))
+	})
+
+	It("falls back to a single whole-clip text segment when nothing parses", func() {
+		res := transcriptResultFromRaw("plain transcript with no markers")
+		Expect(res.Segments).To(HaveLen(1))
+		Expect(res.Segments[0].Text).To(Equal("plain transcript with no markers"))
+		Expect(res.Segments[0].Start).To(Equal(int64(0)))
+		Expect(res.Text).To(Equal("plain transcript with no markers"))
+	})
+})
+
+var _ = Describe("secondsToNanos", func() {
+	It("converts fractional seconds to int64 nanoseconds", func() {
+		Expect(secondsToNanos(0)).To(Equal(int64(0)))
+		Expect(secondsToNanos(1)).To(Equal(int64(1e9)))
+		Expect(secondsToNanos(0.28)).To(Equal(int64(0.28 * 1e9)))
+	})
+})

backend/go/omnivoice-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # omnivoice.cpp version
 OMNIVOICE_REPO?=https://github.com/ServeurpersoCom/omnivoice.cpp
-OMNIVOICE_VERSION?=0f37401bebe9b20c0160a888e592108fc1d17607
+OMNIVOICE_VERSION?=bbdf4be03aa5bc3c188b5db778b6f3fd63ceff6c
 SO_TARGET?=libgomnivoicecpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/opus/opus.go

@@ -34,7 +34,15 @@ type Opus struct {
 	decoders   map[string]*cachedDecoder
 }
 
+// Load accepts only the codec's own name (what the realtime WebRTC path
+// sends) or no name at all — there is no model artefact here, so without
+// this check the model loader's greedy autoload (which probes every
+// installed backend with real model names) would happily bind an LLM to
+// the audio codec.
 func (o *Opus) Load(opts *pb.ModelOptions) error {
+	if m := opts.GetModel(); m != "" && m != "opus" {
+		return fmt.Errorf("opus: refusing to load %q: opus is an audio codec, not a model backend", m)
+	}
 	o.decoders = make(map[string]*cachedDecoder)
 	go o.evictLoop()
 	return Init()

backend/go/opus/opus_test.go

@@ -304,6 +304,17 @@ func computeTHD(samples []int16, fundamentalHz float64, sampleRate, numHarmonics
 
 // --- Opus specs ---
 
+var _ = Describe("Opus Load", func() {
+	It("accepts its own name (the realtime WebRTC path)", func() {
+		Expect((&Opus{}).Load(&pb.ModelOptions{Model: "opus"})).To(Succeed())
+	})
+
+	It("refuses foreign model names from the greedy autoload", func() {
+		err := (&Opus{}).Load(&pb.ModelOptions{Model: "some-llm.gguf"})
+		Expect(err).To(MatchError(ContainSubstring("audio codec")))
+	})
+})
+
 var _ = Describe("Opus", func() {
 	var o *Opus
 

backend/go/qwen3-tts-cpp/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # qwentts.cpp version
 QWEN3TTS_REPO?=https://github.com/ServeurpersoCom/qwentts.cpp
-QWEN3TTS_CPP_VERSION?=9dbe7ea26a01b30fccb117ae5e86807c1dc23d42
+QWEN3TTS_CPP_VERSION?=d17c33d4ee2f56d15f9ca8a1bb82f7389305f838
 SO_TARGET?=libgoqwen3ttscpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/qwen3-tts-cpp/README.md

@@ -0,0 +1,85 @@
+# Qwen3-TTS C++ backend
+
+This backend runs Qwen3-TTS GGUF models through
+[qwentts.cpp](https://github.com/ServeurpersoCom/qwentts.cpp). It supports
+24 kHz speech generation, streaming, named speakers, voice design, and
+reference-audio cloning depending on the model variant.
+
+## Gallery models
+
+The following Base models accept LocalAI Voice Library profiles:
+
+- `qwen3-tts-cpp`
+- `qwen3-tts-cpp-0.6b-base-q4`
+- `qwen3-tts-cpp-1.7b-base`
+- `qwen3-tts-cpp-1.7b-base-q4`
+
+Gallery models containing `customvoice` or `voicedesign` implement those Qwen
+modes instead and are not advertised as raw reference-audio models.
+
+Install a Base model with:
+
+```bash
+local-ai models install qwen3-tts-cpp
+```
+
+## Model configuration
+
+Base filenames are detected automatically. Set `tts.voice_cloning` only when a
+verified private conversion has a name that does not identify it as a Base or
+VoiceClone model:
+
+```yaml
+name: private-qwen-voice
+backend: qwen3-tts-cpp
+parameters:
+  model: qwen-private/talker.gguf
+known_usecases:
+  - tts
+tts:
+  voice_cloning: true
+  audio_path: voices/default-reference.wav  # optional model-wide fallback
+```
+
+The tokenizer GGUF is auto-discovered when its filename contains `tokenizer`
+and it is stored beside the talker. Otherwise set
+`options: ["tokenizer:qwen-private/tokenizer.gguf"]`.
+
+`tts.voice_cloning: false` removes a model from Voice Library compatibility
+results and rejects saved `localai://voice-profiles/...` references. It does
+not disable Qwen's named-speaker or VoiceDesign modes. Setting it to `true`
+cannot add cloning to a backend that lacks LocalAI's reference-audio contract.
+
+Request precedence is: a request `voice`, then `tts.voice`, then
+`tts.audio_path`. A saved profile supplies its private WAV and exact transcript
+for that request without changing the model YAML.
+
+## API example
+
+Create or select a profile in **Operate → Voice Library**, then pass its stable
+URI to either speech endpoint:
+
+```bash
+curl http://localhost:8080/v1/audio/speech \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "qwen3-tts-cpp",
+    "input": "This request uses a saved reference voice.",
+    "voice": "localai://voice-profiles/PROFILE_ID"
+  }' \
+  --output speech.wav
+```
+
+## Native end-to-end test
+
+The labeled test loads real GGUFs, synthesizes speech, streams audio, and
+exercises cloning with a generated 24 kHz reference WAV:
+
+```bash
+make -C backend/go/qwen3-tts-cpp qwen3-tts-cpp
+
+QWEN3TTS_MODEL=/path/to/qwen-talker-0.6b-base-Q8_0.gguf \
+QWEN3TTS_CODEC=/path/to/qwen-tokenizer-12hz-Q8_0.gguf \
+QWEN3TTS_LIBRARY=backend/go/qwen3-tts-cpp/libgoqwen3ttscpp-fallback.so \
+  go test ./backend/go/qwen3-tts-cpp -ginkgo.label-filter=e2e
+```

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=2574f5936571645f784b77623e1f09bad97d948a
+STABLEDIFFUSION_GGML_VERSION?=b5d812008eb7082a238fc589444544b3278187ae
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/stablediffusion-ggml/cpp/gosd.cpp

@@ -660,9 +660,16 @@ int load_model(const char *model, char *model_path, char* options[], int threads
     ctx_params.diffusion_conv_direct = diffusion_conv_direct;
     ctx_params.vae_conv_direct = vae_conv_direct;
     ctx_params.force_sdxl_vae_conv_scale = force_sdxl_vae_conv_scale;
-    ctx_params.chroma_use_dit_mask = chroma_use_dit_mask;
-    ctx_params.chroma_use_t5_mask = chroma_use_t5_mask;
-    ctx_params.chroma_t5_mask_pad = chroma_t5_mask_pad;
+    // Chroma knobs: upstream dropped the dedicated chroma_use_dit_mask /
+    // chroma_use_t5_mask / chroma_t5_mask_pad struct fields and now reads them
+    // from the generic model_args key=value spec (parse_key_value_args). Emit
+    // them there so the existing chroma options keep working. This string must
+    // outlive new_sd_ctx() below.
+    std::string model_args_spec =
+        "chroma_use_dit_mask=" + std::string(chroma_use_dit_mask ? "true" : "false") +
+        ",chroma_use_t5_mask=" + std::string(chroma_use_t5_mask ? "true" : "false") +
+        ",chroma_t5_mask_pad=" + std::to_string(chroma_t5_mask_pad);
+    ctx_params.model_args = model_args_spec.c_str();
     sd_ctx_t* sd_ctx = new_sd_ctx(&ctx_params);
 
     if (sd_ctx == NULL) {

backend/go/vibevoice-cpp/Makefile

@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1)
 # already do for ik_llama.cpp / llama.cpp / whisper.cpp). Floating on
 # `master` led to silent ABI breaks reaching CI — pin it.
 VIBEVOICE_REPO?=https://github.com/mudler/vibevoice.cpp
-VIBEVOICE_CPP_VERSION?=ad856bda6b1311b7f3d7c4a667be43eeb8a8249a
+VIBEVOICE_CPP_VERSION?=000e37282bc5bb09edc20f7047a47924122ba3a0
 SO_TARGET?=libgovibevoicecpp.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/vibevoice-cpp/govibevoicecpp.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,7 +11,9 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"unsafe"
 
+	"github.com/ebitengine/purego"
 	laudio "github.com/mudler/LocalAI/pkg/audio"
 	"github.com/mudler/LocalAI/pkg/grpc/base"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
@@ -113,12 +116,76 @@ var (
 		refAudioPaths []*byte, nRefAudioPaths int32,
 		dstWav string,
 		nSteps int32, cfgScale float32, maxSpeechFrames int32, seed uint32) int32
+	// CppTTSStream drives vv_capi_tts_stream: it synthesizes `text` and
+	// invokes the C callback `cb` once per decoded PCM window instead of
+	// writing a file. `cb` is the address of a purego callback (see
+	// streamCB); `user` is an opaque pointer handed back to every
+	// callback invocation - we route via the package-level activeStream
+	// instead, so it is always nil here.
+	CppTTSStream func(text, voicePath string,
+		nSteps int32, cfgScale float32, maxFrames int32, seed uint32,
+		cb uintptr, user unsafe.Pointer) int32
 	CppASR func(srcWav string, outJSON []byte, capacity uint64,
 		maxNewTokens int32) int32
 	CppUnload  func()
 	CppVersion func() string
 )
 
+// streamState carries the destination channel for one in-flight
+// TTSStream call. vibevoice's engine is a single process-global, and
+// backend calls are serialized through base.SingleThread, so a single
+// package-level pointer is safe: only one TTSStream runs at a time.
+type streamState struct {
+	results chan []byte
+}
+
+// activeStream points at the streamState for the currently-running
+// TTSStream. The C callback (streamCB) and the deliverPCMForTest hook
+// read it to find the channel. Guarded by base.SingleThread
+// serialization; TTSStream sets it and clears it in a defer.
+var activeStream *streamState
+
+// pushPCM copies a transient int16 PCM window into a fresh little-endian
+// []byte and pushes it onto the active stream. The C buffer handed to
+// the callback is only valid for the duration of the call, so we must
+// copy before returning. A nil/empty input or a missing active stream
+// is a no-op.
+func pushPCM(pcm []int16) {
+	s := activeStream
+	if s == nil || len(pcm) == 0 {
+		return
+	}
+	buf := make([]byte, len(pcm)*2)
+	for i, v := range pcm {
+		binary.LittleEndian.PutUint16(buf[i*2:], uint16(v))
+	}
+	s.results <- buf
+}
+
+// streamCB is the ONE reusable purego callback bound to the C ABI's
+// vv_pcm_cb. purego cannot free callbacks and enforces a process-global
+// limit, so we create exactly one at package init and reuse it for every
+// TTSStream call - the per-call state lives in activeStream, not here.
+// purego marshals the C `const int16_t*` first argument straight into a
+// Go *int16, so we can unsafe.Slice it without a uintptr round-trip
+// (keeps go vet clean); pushPCM copies the transient buffer out and
+// returns 0 to keep synthesizing.
+var streamCB = purego.NewCallback(func(samples *int16, n int32, _ uintptr) uintptr {
+	if activeStream == nil || samples == nil || n <= 0 {
+		return 0
+	}
+	pcm := unsafe.Slice(samples, int(n))
+	pushPCM(pcm)
+	return 0
+})
+
+// deliverPCMForTest exercises the exact copy-and-push path streamCB runs
+// against activeStream, but from a Go []int16 - so unit tests can
+// validate the callback -> channel framing without the C library.
+func deliverPCMForTest(samples []int16) {
+	pushPCM(samples)
+}
+
 // VibevoiceCpp speaks gRPC against vibevoice.cpp's flat C ABI. The
 // engine is a single global, so we serialize calls through SingleThread.
 type VibevoiceCpp struct {
@@ -404,14 +471,16 @@ func (v *VibevoiceCpp) callASR(srcWav string, maxNewTokens int32) (string, error
 	return string(buf[:rc]), nil
 }
 
-// TTSStream is the streaming counterpart to TTS. vibevoice's C ABI is
-// file-only (vv_capi_tts writes a complete WAV), so we synthesize to
-// a tempfile, then emit a streaming-WAV header followed by the PCM
-// body in chunks. The main reason this exists at all is the gRPC
-// server wrapper (pkg/grpc/server.go:TTSStream) blocks on a channel
-// that only this method can close - if we leave the default Base
-// stub in place, every TTSStream call hangs until the client
-// deadline.
+// TTSStream is the streaming counterpart to TTS. It drives
+// vv_capi_tts_stream, which synthesizes `text` and invokes our C
+// callback (streamCB) once per decoded PCM window instead of writing a
+// file - so the client starts receiving audio while the model is still
+// generating. We first emit a streaming-WAV header, install the results
+// channel as the active stream, then let the callback push each PCM
+// window (copied to little-endian bytes) onto that channel. The gRPC
+// server wrapper (pkg/grpc/server.go:TTSStream) blocks on the channel
+// until this method closes it, so `defer close(results)` is mandatory
+// even on the error paths.
 func (v *VibevoiceCpp) TTSStream(req *pb.TTSRequest, results chan []byte) error {
 	defer close(results)
 	if v.ttsModel == "" {
@@ -421,28 +490,6 @@ func (v *VibevoiceCpp) TTSStream(req *pb.TTSRequest, results chan []byte) error
 		return fmt.Errorf("vibevoice-cpp: TTSStream requires text")
 	}
 
-	tmp, err := os.CreateTemp("", "vibevoice-cpp-stream-*.wav")
-	if err != nil {
-		return fmt.Errorf("vibevoice-cpp: tempfile: %w", err)
-	}
-	dst := tmp.Name()
-	_ = tmp.Close()
-	defer func() { _ = os.Remove(dst) }()
-
-	if err := v.TTS(&pb.TTSRequest{
-		Text:     req.Text,
-		Voice:    req.Voice,
-		Dst:      dst,
-		Language: req.Language,
-	}); err != nil {
-		return err
-	}
-
-	wav, err := os.ReadFile(dst)
-	if err != nil {
-		return fmt.Errorf("vibevoice-cpp: read tempfile: %w", err)
-	}
-
 	// Streaming WAV header: declare 0xFFFFFFFF for chunk sizes so HTTP
 	// clients can start playback before they see the full PCM.
 	const streamingSize = 0xFFFFFFFF
@@ -455,18 +502,38 @@ func (v *VibevoiceCpp) TTSStream(req *pb.TTSRequest, results chan []byte) error
 	}
 	results <- hdrBuf
 
-	// PCM body: send in ~64 KB slices so the client gets multiple
-	// reply chunks (e2e harness asserts >=2 frames).
-	pcm := laudio.StripWAVHeader(wav)
-	const chunkBytes = 64 * 1024
-	for off := 0; off < len(pcm); off += chunkBytes {
-		end := off + chunkBytes
-		if end > len(pcm) {
-			end = len(pcm)
-		}
-		chunk := make([]byte, end-off)
-		copy(chunk, pcm[off:end])
-		results <- chunk
+	// vv_capi_tts_stream takes a single voice_path (realtime-0.5B path);
+	// unlike vv_capi_tts it has no ref_audio array. Resolve the per-call
+	// override when it names a voice gguf, otherwise fall back to the
+	// load-time default that already went to vv_capi_load.
+	voice := v.voice
+	if reqVoice := strings.TrimSpace(req.Voice); reqVoice != "" && !isRefAudioOverride(reqVoice) {
+		voice = resolvePath(reqVoice, v.modelRoot)
+	}
+
+	if req.Language != nil && *req.Language != "" {
+		fmt.Fprintf(os.Stderr,
+			"[vibevoice-cpp] note: TTSRequest.language=%q ignored - vibevoice picks language from the voice prompt\n",
+			*req.Language)
+	}
+
+	const (
+		defaultSteps     = 20
+		defaultMaxFrames = 200
+	)
+	defaultCfg := float32(1.3)
+
+	// Serialized by base.SingleThread, so a single package-level
+	// activeStream is race-free: exactly one TTSStream runs at a time.
+	// The callback reads it to find `results`; clear it on the way out.
+	activeStream = &streamState{results: results}
+	defer func() { activeStream = nil }()
+
+	rc := CppTTSStream(req.Text, voice,
+		int32(defaultSteps), defaultCfg, int32(defaultMaxFrames), 0,
+		streamCB, nil)
+	if rc != 0 {
+		return fmt.Errorf("vibevoice-cpp: vv_capi_tts_stream failed (rc=%d)", rc)
 	}
 	return nil
 }

backend/go/vibevoice-cpp/main.go

@@ -37,6 +37,7 @@ func main() {
 	libFuncs := []LibFuncs{
 		{&CppLoad, "vv_capi_load"},
 		{&CppTTS, "vv_capi_tts"},
+		{&CppTTSStream, "vv_capi_tts_stream"},
 		{&CppASR, "vv_capi_asr"},
 		{&CppUnload, "vv_capi_unload"},
 		{&CppVersion, "vv_capi_version"},

backend/go/vibevoice-cpp/vibevoicecpp_integration_test.go

@@ -0,0 +1,262 @@
+package main
+
+// Real end-to-end streaming integration test for the vibevoice-cpp
+// backend. It drives the actual Go -> purego -> C path against the real
+// model, proving that TTSStream delivers audio incrementally (measurable
+// time-to-first-audio) while TTS only yields anything after full
+// synthesis. Gated behind VIBEVOICE_IT=1 so normal `go test` and CI stay
+// unaffected - it needs the built engine .so and ~1.7 GB of model files.
+//
+// Run:
+//
+//	VIBEVOICE_IT=1 \
+//	VIBEVOICECPP_LIBRARY=<abs path to libgovibevoicecpp-fallback.so> \
+//	  go test ./... -run TestVibevoiceCpp -v -timeout 600s
+//
+// Optional overrides (default to the staged bundle under
+// ~/_git/vibevoice-models):
+//
+//	VIBEVOICE_IT_MODEL      vibevoice-realtime-0.5B-q8_0.gguf (abs path)
+//	VIBEVOICE_IT_TOKENIZER  tokenizer.gguf (abs path)
+//	VIBEVOICE_IT_VOICE      voice-en-Carter_man.gguf (abs path)
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/ebitengine/purego"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// itDefaultModelDir is where Task 6 staged the model bundle. Individual
+// files can be overridden via the VIBEVOICE_IT_* env vars.
+const itDefaultModelDir = "/home/mudler/_git/vibevoice-models"
+
+// itLibLoaded guards the one-time purego Dlopen + RegisterLibFunc into
+// the package-level Cpp* vars. purego cannot free a loaded library and
+// the Cpp* symbols are process-global, so we bind exactly once.
+var itLibLoaded sync.Once
+
+// integrationOrSkip enforces the VIBEVOICE_IT=1 gate and binds the C ABI
+// symbols into the package vars the backend calls (mirrors main.go's
+// libFuncs list). It Skip()s the spec when the gate is off or the .so /
+// model files are missing, so the suite stays green in every other run.
+func integrationOrSkip() (model, tokenizer, voice string) {
+	if os.Getenv("VIBEVOICE_IT") != "1" {
+		Skip("VIBEVOICE_IT!=1, skipping real-model streaming integration test")
+	}
+
+	lib := os.Getenv("VIBEVOICECPP_LIBRARY")
+	if lib == "" {
+		Skip("VIBEVOICECPP_LIBRARY not set, cannot dlopen the engine .so")
+	}
+	if _, err := os.Stat(lib); err != nil {
+		Skip("engine .so not found at " + lib)
+	}
+
+	model = itEnvOr("VIBEVOICE_IT_MODEL", filepath.Join(itDefaultModelDir, "vibevoice-realtime-0.5B-q8_0.gguf"))
+	tokenizer = itEnvOr("VIBEVOICE_IT_TOKENIZER", filepath.Join(itDefaultModelDir, "tokenizer.gguf"))
+	voice = itEnvOr("VIBEVOICE_IT_VOICE", filepath.Join(itDefaultModelDir, "voice-en-Carter_man.gguf"))
+	for _, p := range []string{model, tokenizer, voice} {
+		if _, err := os.Stat(p); err != nil {
+			Skip("model file missing: " + p)
+		}
+	}
+
+	itLibLoaded.Do(func() {
+		handle, err := purego.Dlopen(lib, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+		Expect(err).ToNot(HaveOccurred(), "dlopen %s", lib)
+		// Mirror the libFuncs list in main.go verbatim so the test binds
+		// the exact same symbols the production backend binary does.
+		purego.RegisterLibFunc(&CppLoad, handle, "vv_capi_load")
+		purego.RegisterLibFunc(&CppTTS, handle, "vv_capi_tts")
+		purego.RegisterLibFunc(&CppTTSStream, handle, "vv_capi_tts_stream")
+		purego.RegisterLibFunc(&CppASR, handle, "vv_capi_asr")
+		purego.RegisterLibFunc(&CppUnload, handle, "vv_capi_unload")
+		purego.RegisterLibFunc(&CppVersion, handle, "vv_capi_version")
+	})
+	return model, tokenizer, voice
+}
+
+func itEnvOr(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+// itParseWavPCM returns the int16 PCM samples from a standard WAV file by
+// locating the "data" sub-chunk, so it works regardless of how many
+// bytes of header/metadata the engine wrote.
+func itParseWavPCM(b []byte) []int16 {
+	// Find the "data" sub-chunk id, then its 4-byte little-endian size.
+	idx := -1
+	for i := 12; i+8 <= len(b); i += 2 {
+		if string(b[i:i+4]) == "data" {
+			idx = i
+			break
+		}
+	}
+	Expect(idx).To(BeNumerically(">=", 0), "no data chunk in WAV")
+	pcmStart := idx + 8
+	size := int(b[idx+4]) | int(b[idx+5])<<8 | int(b[idx+6])<<16 | int(b[idx+7])<<24
+	if size <= 0 || pcmStart+size > len(b) {
+		size = len(b) - pcmStart
+	}
+	n := size / 2
+	out := make([]int16, n)
+	for i := 0; i < n; i++ {
+		out[i] = int16(uint16(b[pcmStart+i*2]) | uint16(b[pcmStart+i*2+1])<<8)
+	}
+	return out
+}
+
+// itRMS computes the root-mean-square of int16 PCM as a sanity signal:
+// > 0 means non-silent, and NaN/Inf-free means well-formed samples.
+func itRMS(pcm []int16) float64 {
+	if len(pcm) == 0 {
+		return 0
+	}
+	var sumSq float64
+	for _, s := range pcm {
+		v := float64(s)
+		sumSq += v * v
+	}
+	return sqrt(sumSq / float64(len(pcm)))
+}
+
+// sqrt via Newton's method to avoid dragging math into the file's tiny
+// need (keeps the dependency surface of the test minimal).
+func sqrt(x float64) float64 {
+	if x <= 0 {
+		return 0
+	}
+	z := x
+	for i := 0; i < 40; i++ {
+		z = 0.5 * (z + x/z)
+	}
+	return z
+}
+
+var _ = Describe("VibeVoice-cpp real-model streaming (VIBEVOICE_IT=1)", Ordered, func() {
+	// The paragraph is deliberately long (multiple sentences) so the
+	// engine decodes several streaming windows and TTFA is meaningfully
+	// earlier than full synthesis.
+	const paragraph = "The quick brown fox jumps over the lazy dog near the riverbank. " +
+		"A gentle breeze carried the sound of distant bells across the quiet valley. " +
+		"Streaming synthesis lets you hear the very first words while the rest is still being generated."
+
+	var v *VibevoiceCpp
+
+	BeforeAll(func() {
+		model, tokenizer, voice := integrationOrSkip()
+
+		v = &VibevoiceCpp{}
+		err := v.Load(&pb.ModelOptions{
+			ModelFile: model,
+			ModelPath: filepath.Dir(model),
+			Options: []string{
+				"tokenizer=" + tokenizer,
+				"voice=" + voice,
+			},
+			Threads: 4,
+		})
+		Expect(err).ToNot(HaveOccurred(), "Load must succeed with the real model")
+	})
+
+	It("streams audio incrementally and beats the batch path to first audio", func() {
+		// ---- Streaming run -------------------------------------------
+		results := make(chan []byte, 256)
+		streamErr := make(chan error, 1)
+
+		start := time.Now()
+		go func() {
+			streamErr <- v.TTSStream(&pb.TTSRequest{Text: paragraph}, results)
+		}()
+
+		var (
+			header      []byte
+			ttfa        time.Duration
+			totalStream time.Duration
+			chunkCount  int
+			pcmBytes    int
+			firstPCM    bool
+		)
+		for buf := range results {
+			if header == nil {
+				header = buf
+				continue
+			}
+			if len(buf) == 0 {
+				continue
+			}
+			if !firstPCM {
+				ttfa = time.Since(start)
+				firstPCM = true
+			}
+			chunkCount++
+			pcmBytes += len(buf)
+		}
+		totalStream = time.Since(start)
+		Expect(<-streamErr).ToNot(HaveOccurred(), "TTSStream returned an error")
+
+		// First message must be a 44-byte streaming WAV header.
+		Expect(header).To(HaveLen(44), "first stream message must be the WAV header")
+		Expect(string(header[0:4])).To(Equal("RIFF"))
+		Expect(string(header[8:12])).To(Equal("WAVE"))
+
+		// Streaming invariants: multiple windows, real audio, early
+		// delivery (first audio strictly before the stream completes).
+		Expect(chunkCount).To(BeNumerically(">=", 2), "expected multiple streamed PCM windows")
+		Expect(pcmBytes).To(BeNumerically(">", 0), "no PCM bytes streamed")
+		Expect(firstPCM).To(BeTrue(), "never received a PCM chunk")
+		Expect(ttfa).To(BeNumerically("<", totalStream), "TTFA must precede stream completion")
+
+		// ---- Batch baseline ------------------------------------------
+		tmp, err := os.MkdirTemp("", "vv-it-batch-*")
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(func() { _ = os.RemoveAll(tmp) })
+		dst := filepath.Join(tmp, "batch.wav")
+
+		batchStart := time.Now()
+		Expect(v.TTS(&pb.TTSRequest{Text: paragraph, Dst: dst})).To(Succeed())
+		totalBatch := time.Since(batchStart)
+
+		wav, err := os.ReadFile(dst)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(len(wav)).To(BeNumerically(">", 44), "batch wav has no PCM payload")
+		Expect(string(wav[0:4])).To(Equal("RIFF"))
+		Expect(string(wav[8:12])).To(Equal("WAVE"))
+		batchPCM := itParseWavPCM(wav)
+		Expect(len(batchPCM)).To(BeNumerically(">", 0), "batch produced no samples")
+		rms := itRMS(batchPCM)
+		Expect(rms).To(BeNumerically(">", 0), "batch audio is silent")
+		Expect(rms).To(BeNumerically("<", 40000), "batch rms out of int16 range (corrupt samples)")
+
+		// ---- Headline numbers ----------------------------------------
+		// Emit to both GinkgoWriter and stderr: GinkgoWriter needs
+		// -ginkgo.v to surface, stderr is always captured so the headline
+		// TTFA vs batch numbers are never lost in an unattended run.
+		ratio := float64(ttfa) / float64(totalBatch)
+		report := fmt.Sprintf("\n================ vibevoice-cpp streaming TTFA ================\n"+
+			"input words        : ~%d\n"+
+			"TTFA (first audio) : %v\n"+
+			"total_stream       : %v\n"+
+			"total_batch        : %v\n"+
+			"stream chunks      : %d\n"+
+			"stream PCM bytes   : %d\n"+
+			"batch samples/rms  : %d / %.1f\n"+
+			"TTFA / total_batch : %.3f (first audio in this fraction of batch's deliver time)\n"+
+			"==============================================================\n\n",
+			len(paragraph)/6, ttfa, totalStream, totalBatch, chunkCount, pcmBytes, len(batchPCM), rms, ratio)
+		for _, out := range []io.Writer{GinkgoWriter, os.Stderr} {
+			_, _ = io.WriteString(out, report)
+		}
+	})
+})

backend/go/vibevoice-cpp/vibevoicecpp_stream_test.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"encoding/binary"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("TTSStream callback framing", func() {
+	// The real C callback copies a transient int16 PCM buffer out of the
+	// engine into a fresh []byte and pushes it onto the active stream's
+	// results channel as little-endian bytes. deliverPCMForTest runs that
+	// exact copy-and-push path against a []int16 so we can validate the
+	// framing without the C library (full audio e2e is a later task).
+	It("copies int16 PCM from the C callback into the results channel as LE bytes", func() {
+		samples := []int16{0, 1, -1, 32767, -32768, 1234, -4321}
+
+		prev := activeStream
+		DeferCleanup(func() { activeStream = prev })
+
+		s := &streamState{results: make(chan []byte, 1)}
+		activeStream = s
+
+		deliverPCMForTest(samples)
+
+		var got []byte
+		Eventually(s.results).Should(Receive(&got))
+		Expect(got).To(HaveLen(len(samples) * 2))
+
+		want := make([]byte, len(samples)*2)
+		for i, v := range samples {
+			binary.LittleEndian.PutUint16(want[i*2:], uint16(v))
+		}
+		Expect(got).To(Equal(want))
+	})
+
+	It("is a no-op when there is no active stream", func() {
+		prev := activeStream
+		DeferCleanup(func() { activeStream = prev })
+		activeStream = nil
+
+		// Must not panic when no stream is installed.
+		Expect(func() { deliverPCMForTest([]int16{1, 2, 3}) }).ToNot(Panic())
+	})
+})

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=6fc7c33b4c3a2cec83e4b65abd5e96a890480375
+WHISPER_CPP_VERSION?=080bbbe85230f624f0b52127f1ae1218247989f9
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/index.yaml

@@ -178,6 +178,35 @@
     nvidia-cuda-12: "cuda12-parakeet-cpp"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-parakeet-cpp"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-parakeet-cpp"
+- &mosstranscribecpp
+  name: "moss-transcribe-cpp"
+  alias: "moss-transcribe-cpp"
+  license: mit
+  icon: https://avatars.githubusercontent.com/u/95302084
+  description: |
+    moss-transcribe.cpp is a C++/ggml port of OpenMOSS MOSS-Transcribe-Diarize, an end-to-end audio understanding model.
+    It does joint long-form transcription, speaker diarization and timestamping in a single pass, emitting a speaker-labelled,
+    time-aligned transcript, and runs on CPU, NVIDIA CUDA, AMD ROCm/HIP, Intel SYCL, Vulkan, Apple Metal and NVIDIA Jetson (L4T) targets.
+  urls:
+    - https://github.com/mudler/moss-transcribe.cpp
+  tags:
+    - audio-transcription
+    - CPU
+    - GPU
+    - CUDA
+    - HIP
+  capabilities:
+    default: "cpu-moss-transcribe-cpp"
+    nvidia: "cuda12-moss-transcribe-cpp"
+    intel: "intel-sycl-f16-moss-transcribe-cpp"
+    metal: "metal-moss-transcribe-cpp"
+    amd: "rocm-moss-transcribe-cpp"
+    vulkan: "vulkan-moss-transcribe-cpp"
+    nvidia-l4t: "nvidia-l4t-arm64-moss-transcribe-cpp"
+    nvidia-cuda-13: "cuda13-moss-transcribe-cpp"
+    nvidia-cuda-12: "cuda12-moss-transcribe-cpp"
+    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-moss-transcribe-cpp"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp"
 - &ced
   name: "ced"
   alias: "ced"
@@ -795,6 +824,30 @@
     nvidia-cuda-12: "cuda12-vllm-omni"
     nvidia-cuda-13: "cuda13-vllm-omni"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-vllm-omni"
+- &longcat-video
+  name: "longcat-video"
+  alias: "longcat-video"
+  license: mit
+  urls:
+    - https://github.com/meituan-longcat/LongCat-Video
+  tags:
+    - text-to-video
+    - image-to-video
+    - audio-to-video
+    - avatar-generation
+    - video-generation
+    - CUDA
+  icon: https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/longcat-video_logo.svg
+  description: |
+    LongCat-Video generation for text, image, and audio-conditioned avatars.
+    Supports LongCat-Video and LongCat-Video-Avatar-1.5, including multi-segment
+    talking-head continuation and an SDPA path for NVIDIA Blackwell ARM64 systems.
+    Requires Linux with an NVIDIA CUDA GPU; CPU, ROCm, and macOS are unsupported.
+  capabilities:
+    nvidia: "cuda12-longcat-video"
+    nvidia-cuda-12: "cuda12-longcat-video"
+    nvidia-cuda-13: "cuda13-longcat-video"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video"
 - &mlx
   name: "mlx"
   icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
@@ -2784,6 +2837,121 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-parakeet-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-parakeet-cpp
+## moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "moss-transcribe-cpp-development"
+  capabilities:
+    default: "cpu-moss-transcribe-cpp-development"
+    nvidia: "cuda12-moss-transcribe-cpp-development"
+    intel: "intel-sycl-f16-moss-transcribe-cpp-development"
+    metal: "metal-moss-transcribe-cpp-development"
+    amd: "rocm-moss-transcribe-cpp-development"
+    vulkan: "vulkan-moss-transcribe-cpp-development"
+    nvidia-l4t: "nvidia-l4t-arm64-moss-transcribe-cpp-development"
+    nvidia-cuda-13: "cuda13-moss-transcribe-cpp-development"
+    nvidia-cuda-12: "cuda12-moss-transcribe-cpp-development"
+    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-moss-transcribe-cpp-development"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp-development"
+- !!merge <<: *mosstranscribecpp
+  name: "nvidia-l4t-arm64-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-arm64-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "nvidia-l4t-arm64-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-arm64-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cpu-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-cpu-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cpu-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-cpu-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "metal-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "metal-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cuda12-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cuda12-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "rocm-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-rocm-hipblas-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "rocm-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-rocm-hipblas-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "intel-sycl-f32-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-intel-sycl-f32-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "intel-sycl-f32-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-intel-sycl-f32-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "intel-sycl-f16-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-intel-sycl-f16-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "intel-sycl-f16-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-intel-sycl-f16-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "vulkan-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-vulkan-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "vulkan-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-vulkan-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cuda13-moss-transcribe-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-moss-transcribe-cpp
+- !!merge <<: *mosstranscribecpp
+  name: "cuda13-moss-transcribe-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-moss-transcribe-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-moss-transcribe-cpp
 ## ced
 - !!merge <<: *ced
   name: "ced-development"
@@ -3461,6 +3629,44 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-vllm-omni"
   mirrors:
     - localai/localai-backends:master-gpu-rocm-hipblas-vllm-omni
+# longcat-video
+- !!merge <<: *longcat-video
+  name: "longcat-video-development"
+  capabilities:
+    nvidia: "cuda12-longcat-video-development"
+    nvidia-cuda-12: "cuda12-longcat-video-development"
+    nvidia-cuda-13: "cuda13-longcat-video-development"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-longcat-video-development"
+- !!merge <<: *longcat-video
+  name: "cuda12-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-nvidia-l4t-arm64-longcat-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda12-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-longcat-video
+- !!merge <<: *longcat-video
+  name: "cuda13-nvidia-l4t-arm64-longcat-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-longcat-video
 # rfdetr
 - !!merge <<: *rfdetr
   name: "rfdetr-development"

backend/python/README.md

@@ -27,6 +27,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro
 
 ### Computer Vision
 - **diffusers** - Stable Diffusion and image generation
+- **longcat-video** - CUDA video and speech-driven avatar generation with LongCat-Video
 - **mlx-vlm** - Vision-language models for Apple Silicon
 - **rfdetr** - Object detection models
 

backend/python/chatterbox/backend.py

@@ -199,7 +199,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
             if "language" in self.options:
                 kwargs["language_id"] = self.options["language"]
-            if self.AudioPath is not None:
+            if request.voice and os.path.isfile(request.voice):
+                kwargs["audio_prompt_path"] = request.voice
+            elif self.AudioPath is not None:
                 kwargs["audio_prompt_path"] = self.AudioPath
 
             # add options to kwargs
@@ -211,6 +213,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # strings on the wire and are coerced to float/int/bool.
             if hasattr(request, "params") and request.params:
                 for key, value in request.params.items():
+                    if key == "ref_text":
+                        continue
                     kwargs[key] = coerce_param_value(value)
 
             # Check if text exceeds 250 characters

backend/python/common/libbackend.sh

@@ -397,10 +397,29 @@ function ensureVenv() {
 
 function runProtogen() {
     ensureVenv
+
+    # Match grpcio-tools to the grpcio already installed by the backend's
+    # requirements. grpcio and grpcio-tools are released in lockstep, and the
+    # protoc that grpcio-tools bundles stamps a Protobuf "gencode" version into
+    # backend_pb2.py. Left unpinned, `uv pip install grpcio-tools` pulls the
+    # newest release, whose newer gencode (e.g. 7.35.0) trips Protobuf's
+    # runtime >= gencode guarantee at import time when a backend caps the
+    # protobuf runtime lower (vLLM pins it to 6.33.6), crashing the backend with
+    # "grpc service not ready" before it ever loads a model. Pinning
+    # grpcio-tools to the installed grpcio version keeps the gencode in step with
+    # the runtime. Falls back to unpinned when grpcio isn't installed yet.
+    # See mudler/LocalAI#10718.
+    local grpcio_tools_spec="grpcio-tools"
+    local grpcio_version
+    grpcio_version="$(python -c 'import importlib.metadata as m; print(m.version("grpcio"))' 2>/dev/null || true)"
+    if [ -n "${grpcio_version}" ]; then
+        grpcio_tools_spec="grpcio-tools==${grpcio_version}"
+    fi
+
     if [ "x${USE_PIP}" == "xtrue" ]; then
-        pip install grpcio-tools
+        pip install "${grpcio_tools_spec}"
     else
-        uv pip install grpcio-tools
+        uv pip install "${grpcio_tools_spec}"
     fi
     pushd "${EDIR}" >/dev/null
         # use the venv python (ensures correct interpreter & sys.path)

backend/python/coqui/backend.py

@@ -79,14 +79,19 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             if self.tts.is_multi_lingual and lang is None:
                return backend_pb2.Result(success=False, message=f"Model is multi-lingual, but no language was provided")
 
-            # if model is multi-speaker, use speaker_wav or the speaker_id from request.voice
-            if self.tts.is_multi_speaker and self.AudioPath is None and request.voice is None:
+            # A path-shaped per-request voice is a cloning reference; otherwise
+            # preserve Coqui's named-speaker behavior.
+            request_voice = request.voice if request.voice else ""
+            speaker_wav = request_voice if os.path.isfile(request_voice) else self.AudioPath
+            if self.tts.is_multi_speaker and speaker_wav is None and not request_voice:
                 return backend_pb2.Result(success=False, message=f"Model is multi-speaker, but no speaker was provided")
 
-            if self.tts.is_multi_speaker and request.voice is not None:
-               self.tts.tts_to_file(text=request.text, speaker=request.voice, language=lang, file_path=request.dst)
+            if speaker_wav is not None:
+                self.tts.tts_to_file(text=request.text, speaker_wav=speaker_wav, language=lang, file_path=request.dst)
+            elif self.tts.is_multi_speaker and request_voice:
+               self.tts.tts_to_file(text=request.text, speaker=request_voice, language=lang, file_path=request.dst)
             else:
-                self.tts.tts_to_file(text=request.text, speaker_wav=self.AudioPath, language=lang, file_path=request.dst)
+                self.tts.tts_to_file(text=request.text, language=lang, file_path=request.dst)
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
         return backend_pb2.Result(success=True)

backend/python/diffusers/backend.py

@@ -864,8 +864,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 **kwargs
             ).images[0]
 
-        # save the result
-        image.save(request.dst)
+        # save the result. Save as PNG explicitly instead of letting Pillow
+        # infer the encoder from the extension: the core passes an absolute
+        # staging path ending in .tmp (e.g. /staging/localai-output-*.tmp),
+        # which Pillow can't map to a format and would raise
+        # "unknown file extension: .tmp". LocalAI serves generated images as
+        # PNG regardless of the temp path.
+        image.save(request.dst, format="PNG")
 
         return backend_pb2.Result(message="Media generated", success=True)
 

backend/python/faster-qwen3-tts/backend.py

@@ -83,20 +83,23 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         return backend_pb2.Result(message="Model loaded successfully", success=True)
 
     def _get_ref_audio_path(self, request):
-        if not self.audio_path:
+        # A per-request voice path is the canonical LocalAI voice-profile
+        # contract. Keep AudioPath as the backwards-compatible YAML fallback.
+        audio_path = request.voice if hasattr(request, "voice") and request.voice else self.audio_path
+        if not audio_path:
             return None
-        if os.path.isabs(self.audio_path):
-            return self.audio_path
+        if os.path.isabs(audio_path):
+            return audio_path
         if self.model_file:
             model_file_base = os.path.dirname(self.model_file)
-            ref_path = os.path.join(model_file_base, self.audio_path)
+            ref_path = os.path.join(model_file_base, audio_path)
             if os.path.exists(ref_path):
                 return ref_path
         if self.model_path:
-            ref_path = os.path.join(self.model_path, self.audio_path)
+            ref_path = os.path.join(self.model_path, audio_path)
             if os.path.exists(ref_path):
                 return ref_path
-        return self.audio_path
+        return audio_path
 
     def TTS(self, request, context):
         try:
@@ -122,13 +125,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     success=False,
                     message="AudioPath is required for voice clone (set in LoadModel)"
                 )
-            ref_text = self.options.get("ref_text")
-            if not ref_text and hasattr(request, 'ref_text') and request.ref_text:
-                ref_text = request.ref_text
+            ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+            if not ref_text:
+                ref_text = self.options.get("ref_text")
             if not ref_text:
                 return backend_pb2.Result(
                     success=False,
-                    message="ref_text is required for voice clone (set via LoadModel Options, e.g. ref_text:Your reference transcript)"
+                    message="ref_text is required for voice clone (set in request.params or LoadModel options)"
                 )
 
             chunk_size = self.options.get("chunk_size")

backend/python/fish-speech/backend.py

@@ -267,6 +267,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
     def _get_ref_audio_path(self, voice_name=None):
         """Get reference audio path from voices dict or stored AudioPath."""
+        if voice_name and os.path.isfile(voice_name):
+            return voice_name
         if voice_name and voice_name in self.voices:
             audio_path = self.voices[voice_name]["audio"]
 
@@ -332,7 +334,19 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             references = []
             voice_name = request.voice if request.voice else None
 
-            if voice_name and voice_name in self.voices:
+            if voice_name and os.path.isfile(voice_name):
+                ref_audio_path = self._get_ref_audio_path(voice_name)
+                with open(ref_audio_path, "rb") as f:
+                    audio_bytes = f.read()
+                ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
+                references.append(
+                    ServeReferenceAudio(audio=audio_bytes, text=ref_text)
+                )
+                print(
+                    f"[INFO] Using per-request reference audio: {ref_audio_path}",
+                    file=sys.stderr,
+                )
+            elif voice_name and voice_name in self.voices:
                 ref_audio_path = self._get_ref_audio_path(voice_name)
                 if ref_audio_path and os.path.exists(ref_audio_path):
                     with open(ref_audio_path, "rb") as f:
@@ -350,7 +364,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 if ref_audio_path and os.path.exists(ref_audio_path):
                     with open(ref_audio_path, "rb") as f:
                         audio_bytes = f.read()
-                    ref_text = self.options.get("ref_text", "")
+                    ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
+                    if not ref_text:
+                        ref_text = self.options.get("ref_text", "")
                     references.append(
                         ServeReferenceAudio(audio=audio_bytes, text=ref_text)
                     )

backend/python/longcat-video/.gitignore

@@ -0,0 +1,6 @@
+backend_pb2.py
+backend_pb2_grpc.py
+lib/
+python/
+sources/
+venv/

backend/python/longcat-video/Makefile

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: MIT
+
+LONGCAT_VIDEO_VERSION?=6b3f4b8582a8bc3f20f795735f5383716c4ba794
+LONGCAT_VIDEO_REPO?=https://github.com/meituan-longcat/LongCat-Video
+LONGCAT_SOURCE_STAMP=sources/LongCat-Video/.localai-$(LONGCAT_VIDEO_VERSION)
+
+.PHONY: all
+all: $(LONGCAT_SOURCE_STAMP)
+	bash install.sh
+
+$(LONGCAT_SOURCE_STAMP): patches/0001-sdpa-attention-fallback.patch
+	rm -rf sources/LongCat-Video
+	mkdir -p sources/LongCat-Video
+	cd sources/LongCat-Video && git init -q && \
+		git remote add origin $(LONGCAT_VIDEO_REPO) && \
+		git fetch --depth 1 origin $(LONGCAT_VIDEO_VERSION) && \
+		git checkout --detach FETCH_HEAD && \
+		git apply ../../patches/0001-sdpa-attention-fallback.patch && \
+		rm -rf .git && \
+		touch .localai-$(LONGCAT_VIDEO_VERSION)
+
+.PHONY: run
+run: all
+	bash run.sh
+
+.PHONY: test
+test: all
+	bash test.sh
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2.py backend_pb2_grpc.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf __pycache__ lib python sources venv

backend/python/longcat-video/README.md

@@ -0,0 +1,44 @@
+# LongCat Video backend
+
+This backend serves Meituan's `LongCat-Video` and
+`LongCat-Video-Avatar-1.5` checkpoints through LocalAI's `GenerateVideo`
+RPC. It supports:
+
+- text-to-video and image-to-video with `LongCat-Video`;
+- audio + text-to-avatar and portrait + audio-to-avatar with Avatar 1.5;
+- multi-segment avatar continuation for speech longer than one segment;
+- PyTorch SDPA when FlashAttention is unavailable, including CUDA 13 ARM64
+  systems such as NVIDIA DGX Spark.
+
+Install the `longcat-video` or `longcat-video-avatar-1.5` recipe from the
+LocalAI Model Gallery. See the [LongCat user guide](../../../docs/content/features/longcat-video.md)
+for Studio and API examples, hardware requirements, and manual configuration.
+
+The upstream source is pinned in `Makefile` and patched at build time. The
+patch adds only the missing SDPA attention branches; model and source licenses
+remain MIT.
+
+## Model options
+
+| Option | Default | Description |
+| --- | --- | --- |
+| `attention_backend` | `sdpa` | `sdpa`, `auto`, `flash2`, `flash3`, or `xformers`. The packaged backend guarantees only `sdpa`. |
+| `use_distill` | `true` for Avatar, `false` for base | Loads the checkpoint's fast distillation LoRA. |
+| `use_int8` | `false` | Loads Avatar 1.5's INT8 DiT. BF16 has a lower load-time peak on unified-memory systems. |
+| `base_model` | `meituan-longcat/LongCat-Video` | Base components used by Avatar 1.5. |
+| `max_segments` | `8` | Maximum avatar continuation segments accepted per request. |
+| `resolution` | `480p` | Image-conditioned generation resolution (`480p` or `720p`). |
+
+Per-request `params` may set `num_segments`, `audio_guidance_scale`,
+`offload_kv_cache`, `ref_img_index`, `mask_frame_range`, and `resolution`.
+
+Gallery and imported configs declare `known_input_modalities` and
+`known_output_modalities`. Keep those declarations in manual configs as well;
+they let model discovery distinguish base image-conditioned video from Avatar
+audio conditioning without inspecting the backend or checkpoint name.
+
+LongCat is CUDA-only and very large. Avatar 1.5 also loads tokenizer,
+text-encoder, and VAE components from the base checkpoint. Keep ample unified
+memory and storage available; no CPU or macOS backend image is published. The
+initial backend supports one GPU per process; tensor parallel sizes above one
+are rejected explicitly.

backend/python/longcat-video/backend.py

@@ -0,0 +1,916 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+
+import argparse
+import datetime
+import gc
+import math
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import traceback
+from concurrent import futures
+
+import grpc
+
+import backend_pb2
+import backend_pb2_grpc
+
+from longcat_utils import (
+    BASE_MODEL_ID,
+    MODEL_KIND_AVATAR,
+    MODEL_KIND_BASE,
+    attention_overrides,
+    avatar_segments_for_duration,
+    avatar_segments_for_frames,
+    classify_model,
+    normalize_model_source,
+    normalize_num_frames,
+    parse_options,
+    require_bool,
+    require_float,
+    require_int,
+    select_known_options,
+    validate_dimensions,
+)
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "sources", "LongCat-Video"))
+
+from grpc_auth import get_auth_interceptors
+
+
+MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
+
+DEFAULT_NEGATIVE_PROMPT = (
+    "Close-up, bright tones, overexposed, static, blurred details, subtitles, "
+    "paintings, low quality, JPEG compression residue, ugly, incomplete, extra "
+    "fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, "
+    "misshapen limbs, fused fingers, still picture, messy background, three legs, "
+    "many people in the background, walking backwards"
+)
+
+LOAD_OPTIONS = {
+    "attention_backend",
+    "base_model",
+    "max_segments",
+    "resolution",
+    "use_distill",
+    "use_int8",
+}
+
+REQUEST_PARAMS = {
+    "audio_guidance_scale",
+    "mask_frame_range",
+    "num_segments",
+    "offload_kv_cache",
+    "ref_img_index",
+    "resolution",
+}
+
+BASE_CHECKPOINT_PATTERNS = [
+    "config.json",
+    "model_index.json",
+    "dit/**",
+    "lora/cfg_step_lora.safetensors",
+    "scheduler/**",
+    "text_encoder/**",
+    "tokenizer/**",
+    "vae/**",
+]
+
+AVATAR_BASE_PATTERNS = [
+    "config.json",
+    "model_index.json",
+    "text_encoder/**",
+    "tokenizer/**",
+    "vae/**",
+]
+
+AVATAR_COMMON_PATTERNS = [
+    "config.json",
+    "lora/dmd_lora.safetensors",
+    "model_index.json",
+    "scheduler/**",
+    "whisper-large-v3/config.json",
+    "whisper-large-v3/model.safetensors",
+    "whisper-large-v3/preprocessor_config.json",
+]
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def __init__(self):
+        self.model_kind = None
+        self.pipeline = None
+        self.options = {}
+        self.device_index = 0
+        self.cp_split_hw = None
+        self._dist_store_dir = None
+
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=b"OK")
+
+    def LoadModel(self, request, context):
+        model = request.Model
+        if request.ModelFile and os.path.isdir(request.ModelFile):
+            model = request.ModelFile
+
+        model_kind = classify_model(model)
+        if model_kind is None:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "longcat-video only accepts LongCat-Video or LongCat-Video-Avatar-1.5 checkpoints",
+            )
+
+        try:
+            options, ignored = select_known_options(
+                parse_options(request.Options), LOAD_OPTIONS
+            )
+            if ignored:
+                # The server injects llama.cpp serving defaults (cache_reuse,
+                # parallel) onto every config; ignore what we don't understand
+                # rather than fail to load.
+                print(
+                    f"longcat-video ignoring unknown model option(s): {', '.join(ignored)}",
+                    file=sys.stderr,
+                )
+
+            self._import_torch()
+            if not self.torch.cuda.is_available():
+                return self._fail(
+                    context,
+                    grpc.StatusCode.FAILED_PRECONDITION,
+                    "longcat-video requires an NVIDIA CUDA GPU",
+                )
+            if request.TensorParallelSize > 1:
+                return self._fail(
+                    context,
+                    grpc.StatusCode.UNIMPLEMENTED,
+                    "longcat-video currently supports one GPU per backend process",
+                )
+            self._import_runtime()
+
+            attention_name = str(options.get("attention_backend", "sdpa")).lower()
+            attention_overrides(attention_name)
+            resolution = str(options.get("resolution", "480p")).lower()
+            if resolution not in {"480p", "720p"}:
+                raise ValueError("resolution must be 480p or 720p")
+
+            use_distill_default = model_kind == MODEL_KIND_AVATAR
+            use_distill = require_bool(
+                options.get("use_distill", use_distill_default),
+                "use_distill",
+            )
+            use_int8 = require_bool(options.get("use_int8", False), "use_int8")
+            if model_kind == MODEL_KIND_BASE and use_int8:
+                raise ValueError(
+                    "use_int8 is supported only by LongCat-Video-Avatar-1.5"
+                )
+
+            self.options = {
+                **options,
+                "attention_backend": attention_name,
+                "resolution": resolution,
+                "use_distill": use_distill,
+                "use_int8": use_int8,
+                "max_segments": require_int(
+                    options.get("max_segments", 8),
+                    "max_segments",
+                    minimum=1,
+                    maximum=64,
+                ),
+            }
+
+            self._release_model()
+            self._ensure_distributed()
+            if model_kind == MODEL_KIND_BASE:
+                self._load_base_model(model)
+            else:
+                self._load_avatar_model(model)
+            self.model_kind = model_kind
+            print(
+                f"Loaded {normalize_model_source(model)} as {model_kind} "
+                f"with attention_backend={attention_name}",
+                file=sys.stderr,
+            )
+            return backend_pb2.Result(message="Model loaded successfully", success=True)
+        except ValueError as err:
+            self._release_model()
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err))
+        except Exception as err:
+            self._release_model()
+            print(f"Error loading LongCat model: {err}", file=sys.stderr)
+            traceback.print_exc()
+            return self._fail(
+                context,
+                grpc.StatusCode.INTERNAL,
+                f"failed to load LongCat model: {err}",
+            )
+
+    def Free(self, request, context):
+        self._release_model()
+        return backend_pb2.Result(message="Model released", success=True)
+
+    def GenerateVideo(self, request, context):
+        if self.pipeline is None or self.model_kind is None:
+            return self._fail(
+                context,
+                grpc.StatusCode.FAILED_PRECONDITION,
+                "model is not loaded",
+            )
+        if not request.prompt.strip():
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "prompt is required",
+            )
+        if not request.dst:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "output destination is required",
+            )
+        if request.end_image:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "longcat-video does not support end_image conditioning",
+            )
+
+        request_state = {"finished": False}
+
+        def interrupt_if_cancelled():
+            if not request_state["finished"] and self.pipeline is not None:
+                self.pipeline._interrupt = True
+
+        try:
+            params, ignored_params = select_known_options(
+                dict(request.params), REQUEST_PARAMS
+            )
+            if ignored_params:
+                print(
+                    f"longcat-video ignoring unknown request param(s): {', '.join(ignored_params)}",
+                    file=sys.stderr,
+                )
+
+            os.makedirs(os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True)
+            if hasattr(context, "add_callback"):
+                context.add_callback(interrupt_if_cancelled)
+
+            if request.start_image and not os.path.isfile(request.start_image):
+                raise ValueError("start_image is not a readable staged file")
+            if request.num_frames < 0:
+                raise ValueError("num_frames must not be negative")
+
+            if self.model_kind == MODEL_KIND_BASE:
+                if request.audio:
+                    raise ValueError(
+                        "audio input requires a LongCat-Video-Avatar-1.5 model"
+                    )
+                self._generate_base(request, params)
+            else:
+                self._generate_avatar(request, params, context)
+
+            return backend_pb2.Result(
+                message="Video generated successfully", success=True
+            )
+        except ValueError as err:
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err))
+        except Exception as err:
+            print(f"Error generating LongCat video: {err}", file=sys.stderr)
+            traceback.print_exc()
+            return self._fail(
+                context,
+                grpc.StatusCode.INTERNAL,
+                f"LongCat video generation failed: {err}",
+            )
+        finally:
+            request_state["finished"] = True
+            if self.pipeline is not None:
+                self.pipeline._interrupt = False
+
+    def _import_torch(self):
+        if hasattr(self, "torch"):
+            return
+
+        import torch
+
+        self.torch = torch
+
+    def _import_runtime(self):
+        if hasattr(self, "LongCatVideoPipeline"):
+            return
+
+        import imageio.v2 as imageio
+        import imageio_ffmpeg
+        import librosa
+        import numpy as np
+        import torch.distributed as dist
+        from diffusers.utils import load_image
+        from huggingface_hub import snapshot_download
+        from PIL import Image
+        from transformers import AutoTokenizer, UMT5EncoderModel
+
+        from longcat_video.audio_process import (
+            get_audio_encoder,
+            get_audio_feature_extractor,
+        )
+        from longcat_video.context_parallel import context_parallel_util
+        from longcat_video.modules.autoencoder_kl_wan import AutoencoderKLWan
+        from longcat_video.modules.avatar.longcat_video_dit_avatar import (
+            LongCatVideoAvatarTransformer3DModel,
+        )
+        from longcat_video.modules.longcat_video_dit import (
+            LongCatVideoTransformer3DModel,
+        )
+        from longcat_video.modules.quantization import load_quantized_dit
+        from longcat_video.modules.scheduling_flow_match_euler_discrete import (
+            FlowMatchEulerDiscreteScheduler,
+        )
+        from longcat_video.pipeline_longcat_video import LongCatVideoPipeline
+        from longcat_video.pipeline_longcat_video_avatar import (
+            LongCatVideoAvatarPipeline,
+        )
+
+        self.imageio = imageio
+        self.imageio_ffmpeg = imageio_ffmpeg
+        self.librosa = librosa
+        self.np = np
+        self.dist = dist
+        self.load_image = load_image
+        self.snapshot_download = snapshot_download
+        self.Image = Image
+        self.AutoTokenizer = AutoTokenizer
+        self.UMT5EncoderModel = UMT5EncoderModel
+        self.get_audio_encoder = get_audio_encoder
+        self.get_audio_feature_extractor = get_audio_feature_extractor
+        self.context_parallel_util = context_parallel_util
+        self.AutoencoderKLWan = AutoencoderKLWan
+        self.LongCatVideoAvatarTransformer3DModel = LongCatVideoAvatarTransformer3DModel
+        self.LongCatVideoTransformer3DModel = LongCatVideoTransformer3DModel
+        self.load_quantized_dit = load_quantized_dit
+        self.FlowMatchEulerDiscreteScheduler = FlowMatchEulerDiscreteScheduler
+        self.LongCatVideoPipeline = LongCatVideoPipeline
+        self.LongCatVideoAvatarPipeline = LongCatVideoAvatarPipeline
+
+    def _ensure_distributed(self):
+        self.torch.cuda.set_device(self.device_index)
+        if not self.dist.is_initialized():
+            self._dist_store_dir = tempfile.mkdtemp(prefix="localai-longcat-dist-")
+            init_file = os.path.join(self._dist_store_dir, "store")
+            self.dist.init_process_group(
+                backend="nccl",
+                init_method=f"file://{init_file}",
+                rank=0,
+                world_size=1,
+                timeout=datetime.timedelta(hours=24),
+            )
+            self.context_parallel_util.init_context_parallel(
+                context_parallel_size=1,
+                global_rank=0,
+                world_size=1,
+            )
+            self.cp_split_hw = self.context_parallel_util.get_optimal_split(1)
+
+    def _resolve_checkpoint(self, model, patterns):
+        source = normalize_model_source(model)
+        if os.path.isdir(source):
+            return source
+        print(f"Downloading required files for {source}", file=sys.stderr)
+        return self.snapshot_download(repo_id=source, allow_patterns=patterns)
+
+    def _load_base_model(self, model):
+        checkpoint = self._resolve_checkpoint(model, BASE_CHECKPOINT_PATTERNS)
+        dtype = self.torch.bfloat16
+        overrides = attention_overrides(self.options["attention_backend"])
+
+        tokenizer = self.AutoTokenizer.from_pretrained(
+            checkpoint,
+            subfolder="tokenizer",
+        )
+        text_encoder = self.UMT5EncoderModel.from_pretrained(
+            checkpoint,
+            subfolder="text_encoder",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        vae = self.AutoencoderKLWan.from_pretrained(
+            checkpoint,
+            subfolder="vae",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained(
+            checkpoint,
+            subfolder="scheduler",
+        )
+        dit = self.LongCatVideoTransformer3DModel.from_pretrained(
+            checkpoint,
+            subfolder="dit",
+            cp_split_hw=self.cp_split_hw,
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+            **overrides,
+        )
+        if self.options["use_distill"]:
+            dit.load_lora(
+                os.path.join(checkpoint, "lora", "cfg_step_lora.safetensors"),
+                "cfg_step_lora",
+            )
+            dit.enable_loras(["cfg_step_lora"])
+
+        self.pipeline = self.LongCatVideoPipeline(
+            tokenizer=tokenizer,
+            text_encoder=text_encoder,
+            vae=vae,
+            scheduler=scheduler,
+            dit=dit,
+        )
+        self.pipeline.to(self.device_index)
+
+    def _load_avatar_model(self, model):
+        avatar_patterns = list(AVATAR_COMMON_PATTERNS)
+        model_subfolder = (
+            "base_model_int8" if self.options["use_int8"] else "base_model"
+        )
+        avatar_patterns.append(f"{model_subfolder}/**")
+        checkpoint = self._resolve_checkpoint(model, avatar_patterns)
+
+        base_model = self.options.get("base_model")
+        if not base_model and os.path.isdir(normalize_model_source(model)):
+            sibling = os.path.join(
+                os.path.dirname(normalize_model_source(model)), "LongCat-Video"
+            )
+            if os.path.isdir(sibling):
+                base_model = sibling
+        base_model = base_model or BASE_MODEL_ID
+        if classify_model(str(base_model)) != MODEL_KIND_BASE:
+            raise ValueError("base_model must point to a LongCat-Video checkpoint")
+        base_checkpoint = self._resolve_checkpoint(base_model, AVATAR_BASE_PATTERNS)
+
+        dtype = self.torch.bfloat16
+        overrides = attention_overrides(self.options["attention_backend"])
+        tokenizer = self.AutoTokenizer.from_pretrained(
+            base_checkpoint,
+            subfolder="tokenizer",
+        )
+        text_encoder = self.UMT5EncoderModel.from_pretrained(
+            base_checkpoint,
+            subfolder="text_encoder",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        vae = self.AutoencoderKLWan.from_pretrained(
+            base_checkpoint,
+            subfolder="vae",
+            torch_dtype=dtype,
+            low_cpu_mem_usage=True,
+        )
+        scheduler = self.FlowMatchEulerDiscreteScheduler.from_pretrained(
+            checkpoint,
+            subfolder="scheduler",
+        )
+
+        if self.options["use_int8"]:
+            previous_dtype = self.torch.get_default_dtype()
+            self.torch.set_default_dtype(dtype)
+            try:
+                dit = self.load_quantized_dit(
+                    checkpoint,
+                    subfolder="base_model_int8",
+                    cp_split_hw=self.cp_split_hw,
+                    **overrides,
+                )
+            finally:
+                self.torch.set_default_dtype(previous_dtype)
+        else:
+            dit = self.LongCatVideoAvatarTransformer3DModel.from_pretrained(
+                checkpoint,
+                subfolder="base_model",
+                cp_split_hw=self.cp_split_hw,
+                torch_dtype=dtype,
+                low_cpu_mem_usage=True,
+                **overrides,
+            )
+
+        if self.options["use_distill"]:
+            dit.load_lora(
+                os.path.join(checkpoint, "lora", "dmd_lora.safetensors"),
+                "dmd",
+                multiplier=1.0,
+                lora_network_dim=128,
+                lora_network_alpha=64,
+            )
+            dit.enable_loras(["dmd"])
+
+        audio_checkpoint = os.path.join(checkpoint, "whisper-large-v3")
+        audio_encoder = self.get_audio_encoder(
+            audio_checkpoint,
+            MODEL_KIND_AVATAR + "-v1.5",
+        ).to(self.device_index)
+        audio_feature_extractor = self.get_audio_feature_extractor(
+            audio_checkpoint,
+            MODEL_KIND_AVATAR + "-v1.5",
+        )
+        self.pipeline = self.LongCatVideoAvatarPipeline(
+            tokenizer=tokenizer,
+            text_encoder=text_encoder,
+            vae=vae,
+            scheduler=scheduler,
+            dit=dit,
+            audio_encoder=audio_encoder,
+            audio_feature_extractor=audio_feature_extractor,
+            model_type="avatar-v1.5",
+        )
+        self.pipeline.to(self.device_index)
+
+    def _generate_base(self, request, params):
+        use_distill = self.options["use_distill"]
+        frames = normalize_num_frames(request.num_frames)
+        steps = (
+            16
+            if use_distill
+            else require_int(
+                request.step or 50,
+                "step",
+                minimum=1,
+                maximum=200,
+            )
+        )
+        guidance_scale = (
+            1.0
+            if use_distill
+            else require_float(
+                request.cfg_scale or 4.0,
+                "cfg_scale",
+                minimum=0.0,
+                maximum=30.0,
+            )
+        )
+        fps = require_int(request.fps or 15, "fps", minimum=1, maximum=60)
+        seed = request.seed if request.seed > 0 else 42
+        negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT
+        generator = self.torch.Generator(device=self.device_index).manual_seed(seed)
+
+        if request.start_image:
+            resolution = self._resolution(params)
+            image = self.load_image(request.start_image)
+            output = self.pipeline.generate_i2v(
+                image=image,
+                prompt=request.prompt,
+                negative_prompt=negative_prompt,
+                resolution=resolution,
+                num_frames=frames,
+                num_inference_steps=steps,
+                use_distill=use_distill,
+                guidance_scale=guidance_scale,
+                generator=generator,
+            )[0]
+        else:
+            width, height = validate_dimensions(request.width, request.height)
+            output = self.pipeline.generate_t2v(
+                prompt=request.prompt,
+                negative_prompt=negative_prompt,
+                height=height,
+                width=width,
+                num_frames=frames,
+                num_inference_steps=steps,
+                use_distill=use_distill,
+                guidance_scale=guidance_scale,
+                generator=generator,
+            )[0]
+
+        self._save_video(output, request.dst, fps)
+
+    def _generate_avatar(self, request, params, context):
+        if not request.audio:
+            raise ValueError("audio is required for LongCat-Video-Avatar-1.5")
+        if not os.path.isfile(request.audio):
+            raise ValueError("audio input is not a readable staged file")
+
+        use_distill = self.options["use_distill"]
+        steps = (
+            8
+            if use_distill
+            else require_int(
+                request.step or 50,
+                "step",
+                minimum=1,
+                maximum=200,
+            )
+        )
+        text_guidance = (
+            1.0
+            if use_distill
+            else require_float(
+                request.cfg_scale or 4.0,
+                "cfg_scale",
+                minimum=0.0,
+                maximum=30.0,
+            )
+        )
+        audio_guidance = (
+            1.0
+            if use_distill
+            else require_float(
+                params.get("audio_guidance_scale", 4.0),
+                "audio_guidance_scale",
+                minimum=0.0,
+                maximum=20.0,
+            )
+        )
+        seed = request.seed if request.seed > 0 else 42
+        generator = self.torch.Generator(device=self.device_index).manual_seed(seed)
+        negative_prompt = request.negative_prompt or DEFAULT_NEGATIVE_PROMPT
+        resolution = self._resolution(params)
+
+        speech, sample_rate = self.librosa.load(request.audio, sr=16000, mono=True)
+        if speech.size == 0:
+            raise ValueError("audio contains no samples")
+        audio_duration = len(speech) / sample_rate
+        segments = self._avatar_segments(request, params, audio_duration)
+
+        segment_frames = 93
+        conditioning_frames = 13
+        avatar_fps = 25
+        generated_duration = (
+            segment_frames + (segments - 1) * (segment_frames - conditioning_frames)
+        ) / avatar_fps
+        pad_samples = max(
+            0, math.ceil((generated_duration - audio_duration) * sample_rate)
+        )
+        if pad_samples:
+            speech = self.np.pad(speech, (0, pad_samples))
+
+        full_audio_embedding = self.pipeline.get_audio_embedding(
+            speech,
+            fps=avatar_fps,
+            device=self.device_index,
+            sample_rate=sample_rate,
+            model_type="avatar-v1.5",
+        )
+        if not self.torch.isfinite(full_audio_embedding).all():
+            raise ValueError("audio encoder returned non-finite values")
+
+        indices = self.torch.arange(5) - 2
+
+        def audio_window(start_index):
+            centers = self.torch.arange(
+                start_index,
+                start_index + segment_frames,
+            ).unsqueeze(1) + indices.unsqueeze(0)
+            centers = self.torch.clamp(
+                centers,
+                min=0,
+                max=full_audio_embedding.shape[0] - 1,
+            )
+            return full_audio_embedding[centers][None, ...].to(self.device_index)
+
+        audio_start = 0
+        common = {
+            "prompt": request.prompt,
+            "negative_prompt": negative_prompt,
+            "num_frames": segment_frames,
+            "num_inference_steps": steps,
+            "text_guidance_scale": text_guidance,
+            "audio_guidance_scale": audio_guidance,
+            "output_type": "both",
+            "generator": generator,
+            "audio_emb": audio_window(audio_start),
+            "use_distill": use_distill,
+        }
+
+        if request.start_image:
+            output, latent = self.pipeline.generate_ai2v(
+                image=self.load_image(request.start_image),
+                resolution=resolution,
+                **common,
+            )
+        else:
+            width, height = validate_dimensions(request.width, request.height)
+            output, latent = self.pipeline.generate_at2v(
+                height=height,
+                width=width,
+                **common,
+            )
+
+        video = self._frames_to_pil(output[0])
+        width, height = video[0].size
+        current_video = video
+        reference_latent = latent[:, :, :1].clone()
+        all_frames = list(video)
+
+        for segment in range(1, segments):
+            if hasattr(context, "is_active") and not context.is_active():
+                raise RuntimeError("request was cancelled")
+            print(
+                f"Generating avatar segment {segment + 1}/{segments}", file=sys.stderr
+            )
+            audio_start += segment_frames - conditioning_frames
+            output, latent = self.pipeline.generate_avc(
+                video=current_video,
+                video_latent=latent,
+                prompt=request.prompt,
+                negative_prompt=negative_prompt,
+                height=height,
+                width=width,
+                num_frames=segment_frames,
+                num_cond_frames=conditioning_frames,
+                num_inference_steps=steps,
+                text_guidance_scale=text_guidance,
+                audio_guidance_scale=audio_guidance,
+                generator=generator,
+                output_type="both",
+                use_kv_cache=True,
+                offload_kv_cache=require_bool(
+                    params.get("offload_kv_cache", False),
+                    "offload_kv_cache",
+                ),
+                enhance_hf=not use_distill,
+                audio_emb=audio_window(audio_start),
+                ref_latent=reference_latent,
+                ref_img_index=require_int(
+                    params.get("ref_img_index", 10),
+                    "ref_img_index",
+                    minimum=-30,
+                    maximum=30,
+                ),
+                mask_frame_range=require_int(
+                    params.get("mask_frame_range", 3),
+                    "mask_frame_range",
+                    minimum=0,
+                    maximum=32,
+                ),
+                use_distill=use_distill,
+            )
+            current_video = self._frames_to_pil(output[0])
+            all_frames.extend(current_video[conditioning_frames:])
+
+        self._save_avatar_video(all_frames, request.audio, request.dst, avatar_fps)
+
+    def _avatar_segments(self, request, params, audio_duration):
+        if "num_segments" in params:
+            segments = require_int(
+                params["num_segments"],
+                "num_segments",
+                minimum=1,
+            )
+        elif request.num_frames > 0:
+            segments = avatar_segments_for_frames(request.num_frames)
+        else:
+            segments = avatar_segments_for_duration(audio_duration)
+
+        max_segments = self.options["max_segments"]
+        if segments > max_segments:
+            raise ValueError(
+                f"request needs {segments} avatar segments, but max_segments is {max_segments}; "
+                "trim the audio or raise the model's max_segments option"
+            )
+        return segments
+
+    def _resolution(self, params):
+        resolution = str(params.get("resolution", self.options["resolution"])).lower()
+        if resolution not in {"480p", "720p"}:
+            raise ValueError("resolution must be 480p or 720p")
+        return resolution
+
+    def _frames_to_pil(self, frames):
+        images = []
+        for frame in frames:
+            array = self.np.asarray(frame)
+            if self.np.issubdtype(array.dtype, self.np.floating):
+                array = self.np.clip(array, 0.0, 1.0) * 255
+            images.append(self.Image.fromarray(array.astype(self.np.uint8)))
+        return images
+
+    def _save_video(self, frames, path, fps):
+        writer = self.imageio.get_writer(
+            path,
+            format="FFMPEG",
+            mode="I",
+            fps=fps,
+            codec="libx264",
+            macro_block_size=1,
+            ffmpeg_params=[
+                "-crf",
+                "18",
+                "-pix_fmt",
+                "yuv420p",
+                "-movflags",
+                "+faststart",
+                "-f",
+                "mp4",
+            ],
+        )
+        try:
+            for frame in frames:
+                array = self.np.asarray(frame)
+                if self.np.issubdtype(array.dtype, self.np.floating):
+                    array = self.np.clip(array, 0.0, 1.0) * 255
+                writer.append_data(array.astype(self.np.uint8))
+        finally:
+            writer.close()
+
+    def _save_avatar_video(self, frames, audio_path, dst, fps):
+        output_dir = os.path.dirname(dst) or "."
+        handle, silent_path = tempfile.mkstemp(
+            prefix="longcat-silent-",
+            suffix=".mp4",
+            dir=output_dir,
+        )
+        os.close(handle)
+        try:
+            self._save_video(frames, silent_path, fps)
+            command = [
+                self.imageio_ffmpeg.get_ffmpeg_exe(),
+                "-y",
+                "-i",
+                silent_path,
+                "-i",
+                audio_path,
+                "-map",
+                "0:v:0",
+                "-map",
+                "1:a:0",
+                "-c:v",
+                "copy",
+                "-c:a",
+                "aac",
+                "-b:a",
+                "192k",
+                "-shortest",
+                "-movflags",
+                "+faststart",
+                "-f",
+                "mp4",
+                dst,
+            ]
+            subprocess.run(
+                command,
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+        except subprocess.CalledProcessError as err:
+            details = (err.stderr or "ffmpeg failed")[-2000:]
+            raise RuntimeError(f"failed to mux avatar audio: {details}") from err
+        finally:
+            try:
+                os.remove(silent_path)
+            except FileNotFoundError:
+                pass
+
+    def _release_model(self):
+        self.pipeline = None
+        self.model_kind = None
+        gc.collect()
+        if hasattr(self, "torch") and self.torch.cuda.is_available():
+            self.torch.cuda.empty_cache()
+            self.torch.cuda.ipc_collect()
+
+    @staticmethod
+    def _fail(context, code, message):
+        if context is not None:
+            context.set_code(code)
+            context.set_details(message)
+        return backend_pb2.Result(message=message, success=False)
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ("grpc.max_message_length", 64 * 1024 * 1024),
+            ("grpc.max_send_message_length", 64 * 1024 * 1024),
+            ("grpc.max_receive_message_length", 64 * 1024 * 1024),
+        ],
+        interceptors=get_auth_interceptors(),
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print(f"LongCat Video backend listening on {address}", file=sys.stderr)
+
+    def stop_server(signum, frame):
+        del signum, frame
+        server.stop(0)
+
+    signal.signal(signal.SIGINT, stop_server)
+    signal.signal(signal.SIGTERM, stop_server)
+    server.wait_for_termination()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the LongCat Video gRPC backend")
+    parser.add_argument(
+        "--addr",
+        default="localhost:50051",
+        help="address on which to serve the backend",
+    )
+    arguments = parser.parse_args()
+    serve(arguments.addr)

backend/python/longcat-video/install.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+PYTHON_VERSION="3.12"
+PYTHON_PATCH="12"
+PY_STANDALONE_TAG="20251120"
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+
+installRequirements

backend/python/longcat-video/longcat_utils.py

@@ -0,0 +1,199 @@
+# SPDX-License-Identifier: MIT
+
+import json
+import math
+import os
+from urllib.parse import urlparse
+
+
+BASE_MODEL_ID = "meituan-longcat/LongCat-Video"
+AVATAR_MODEL_ID = "meituan-longcat/LongCat-Video-Avatar-1.5"
+MODEL_KIND_BASE = "base"
+MODEL_KIND_AVATAR = "avatar"
+
+ATTENTION_OVERRIDES = {
+    "auto": {},
+    "sdpa": {
+        "enable_flashattn2": False,
+        "enable_flashattn3": False,
+        "enable_xformers": False,
+    },
+    "flash2": {
+        "enable_flashattn2": True,
+        "enable_flashattn3": False,
+        "enable_xformers": False,
+    },
+    "flash3": {
+        "enable_flashattn2": False,
+        "enable_flashattn3": True,
+        "enable_xformers": False,
+    },
+    "xformers": {
+        "enable_flashattn2": False,
+        "enable_flashattn3": False,
+        "enable_xformers": True,
+    },
+}
+
+
+def parse_options(values):
+    options = {}
+    for raw in values:
+        if ":" not in raw:
+            options[raw.strip()] = True
+            continue
+        key, value = raw.split(":", 1)
+        key = key.strip()
+        value = value.strip()
+        if not key:
+            continue
+        lower = value.lower()
+        if lower in {"true", "false"}:
+            options[key] = lower == "true"
+            continue
+        try:
+            options[key] = int(value)
+            continue
+        except ValueError:
+            pass
+        try:
+            options[key] = float(value)
+            continue
+        except ValueError:
+            pass
+        options[key] = value
+    return options
+
+
+def select_known_options(options, known):
+    """Split parsed options into the subset this backend understands and the
+    unknown keys to ignore.
+
+    LocalAI injects serving defaults (e.g. the llama.cpp cache_reuse / parallel
+    options) onto every model config regardless of backend. A backend should
+    tolerate options it does not understand rather than refuse to load, matching
+    the other LocalAI Python backends; the caller logs the ignored keys.
+
+    Returns (kept, ignored) where kept preserves the known entries and ignored is
+    the sorted list of dropped keys.
+    """
+    ignored = sorted(key for key in options if key not in known)
+    kept = {key: value for key, value in options.items() if key in known}
+    return kept, ignored
+
+
+def require_bool(value, name):
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str) and value.lower() in {"true", "false"}:
+        return value.lower() == "true"
+    raise ValueError(f"{name} must be true or false")
+
+
+def require_int(value, name, minimum=None, maximum=None):
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as err:
+        raise ValueError(f"{name} must be an integer") from err
+    if minimum is not None and parsed < minimum:
+        raise ValueError(f"{name} must be at least {minimum}")
+    if maximum is not None and parsed > maximum:
+        raise ValueError(f"{name} must be at most {maximum}")
+    return parsed
+
+
+def require_float(value, name, minimum=None, maximum=None):
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError) as err:
+        raise ValueError(f"{name} must be a number") from err
+    if minimum is not None and parsed < minimum:
+        raise ValueError(f"{name} must be at least {minimum}")
+    if maximum is not None and parsed > maximum:
+        raise ValueError(f"{name} must be at most {maximum}")
+    return parsed
+
+
+def attention_overrides(name):
+    try:
+        return dict(ATTENTION_OVERRIDES[name])
+    except KeyError as err:
+        choices = ", ".join(ATTENTION_OVERRIDES)
+        raise ValueError(f"attention_backend must be one of: {choices}") from err
+
+
+def _model_name_from_directory(path):
+    for filename in ("model_index.json", "config.json"):
+        config_path = os.path.join(path, filename)
+        try:
+            with open(config_path, "r", encoding="utf-8") as config_file:
+                model_name = json.load(config_file).get("model_name", "")
+        except (FileNotFoundError, OSError, ValueError, TypeError):
+            continue
+        if model_name:
+            return model_name
+    return ""
+
+
+def normalize_model_source(model):
+    value = model.rstrip("/")
+    for prefix in ("huggingface://", "hf://"):
+        if value.startswith(prefix):
+            return value[len(prefix) :]
+    parsed = urlparse(value)
+    if parsed.scheme in {"http", "https"} and parsed.netloc.lower() == "huggingface.co":
+        parts = [part for part in parsed.path.split("/") if part]
+        if len(parts) >= 2:
+            return "/".join(parts[:2])
+    return value
+
+
+def classify_model(model):
+    if not model:
+        return None
+    normalized = normalize_model_source(model)
+    if os.path.isdir(normalized):
+        name = _model_name_from_directory(normalized).lower()
+        if name == "longcat-video":
+            return MODEL_KIND_BASE
+        if name == "longcat-video-avatar-1.5":
+            return MODEL_KIND_AVATAR
+        return None
+
+    normalized = normalized.lower()
+    if normalized == BASE_MODEL_ID.lower():
+        return MODEL_KIND_BASE
+    if normalized == AVATAR_MODEL_ID.lower():
+        return MODEL_KIND_AVATAR
+    return None
+
+
+def normalize_num_frames(value, default=93):
+    frames = default if not value or value < 1 else value
+    return max(1, ((frames - 1) // 4) * 4 + 1)
+
+
+def avatar_segments_for_frames(frames):
+    if not frames or frames <= 93:
+        return 1
+    return 1 + math.ceil((frames - 93) / 80)
+
+
+def avatar_segments_for_duration(duration_seconds, fps=25):
+    if duration_seconds <= 0:
+        return 1
+    return avatar_segments_for_frames(math.ceil(duration_seconds * fps))
+
+
+def validate_dimensions(width, height):
+    width = width or 832
+    height = height or 480
+    if width < 256 or height < 256:
+        raise ValueError("width and height must each be at least 256")
+    if width > 1280 or height > 768:
+        raise ValueError("width and height must not exceed 1280x768")
+    if width % 16 != 0 or height % 16 != 0:
+        raise ValueError("width and height must be divisible by 16")
+    if width * height > 1280 * 768:
+        raise ValueError("requested video dimensions exceed the 1280x768 pixel limit")
+    return width, height

backend/python/longcat-video/patches/0001-sdpa-attention-fallback.patch

@@ -0,0 +1,75 @@
+diff --git a/longcat_video/modules/attention.py b/longcat_video/modules/attention.py
+index bb5630f..9b9f3cc 100644
+--- a/longcat_video/modules/attention.py
++++ b/longcat_video/modules/attention.py
+@@ -2,6 +2,7 @@ from typing import List, Optional
+
+ import torch
+ import torch.nn as nn
++import torch.nn.functional as F
+
+ from einops import rearrange
+
+@@ -100,7 +101,8 @@ class Attention(nn.Module):
+             x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,)
+             x = rearrange(x, "B M H K -> B H M K")
+         else:
+-            raise RuntimeError("Unsupported attention operations.")
++            # Keep a dependency-free path for systems without optional kernels.
++            x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
+
+         return x
+
+@@ -245,8 +247,22 @@ class MultiHeadCrossAttention(nn.Module):
+             attn_bias = xformers.ops.fmha.attn_bias.BlockDiagonalMask.from_seqlens([N] * B, kv_seqlen)
+             x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+         else:
+-            raise RuntimeError("Unsupported attention operations.")
+-
++            # Preserve the variable-length block boundaries without materializing
++            # a dense attention mask.
++            blocks = []
++            offset = 0
++            for batch_index, key_count in enumerate(kv_seqlen):
++                query = q[0][batch_index * N:(batch_index + 1) * N]
++                key = k[0][offset:offset + key_count]
++                value = v[0][offset:offset + key_count]
++                output = F.scaled_dot_product_attention(
++                    query.transpose(0, 1),
++                    key.transpose(0, 1),
++                    value.transpose(0, 1),
++                )
++                blocks.append(output.transpose(0, 1))
++                offset += key_count
++            x = torch.cat(blocks, dim=0)
+
+         x = x.view(B, -1, C)
+         x = self.proj(x)
+diff --git a/longcat_video/modules/avatar/attention.py b/longcat_video/modules/avatar/attention.py
+index a169a7a..df9a469 100644
+--- a/longcat_video/modules/avatar/attention.py
++++ b/longcat_video/modules/avatar/attention.py
+@@ -2,6 +2,7 @@ from typing import List, Optional
+
+ import torch
+ import torch.nn as nn
++import torch.nn.functional as F
+
+ from einops import rearrange
+
+@@ -111,7 +112,8 @@ class Attention(nn.Module):
+             x = xformers.ops.memory_efficient_attention(q, k, v, attn_bias=None, op=None,)
+             x = rearrange(x, "B M H K -> B H M K")
+         else:
+-            raise RuntimeError("Unsupported attention operations.")
++            # Keep a dependency-free path for systems without optional kernels.
++            x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
+
+         return x
+
+@@ -429,2 +431,5 @@ class SingleStreamAttention(nn.Module):
++        else:
++            # This branch uses the native PyTorch kernel when optional kernels are off.
++            x = F.scaled_dot_product_attention(q, encoder_k, encoder_v, scale=self.scale)
+
+         # linear transform

backend/python/longcat-video/requirements-after.txt

@@ -0,0 +1 @@
+accelerate

backend/python/longcat-video/requirements-cpu.txt

@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cpu
+torch==2.12.1
+torchvision==0.27.1

backend/python/longcat-video/requirements-cublas12.txt

@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cu126
+torch==2.12.1
+torchvision==0.27.1

backend/python/longcat-video/requirements-cublas13.txt

@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cu130
+torch==2.12.1
+torchvision==0.27.1

backend/python/longcat-video/requirements-l4t13.txt

@@ -0,0 +1,3 @@
+--index-url https://download.pytorch.org/whl/cu130
+torch==2.12.1
+torchvision==0.27.1

backend/python/longcat-video/requirements.txt

@@ -0,0 +1,23 @@
+certifi
+diffusers==0.35.1
+einops==0.8.0
+ftfy==6.2.0
+grpcio==1.76.0
+huggingface-hub>=0.23,<1.0
+imageio==2.37.0
+imageio-ffmpeg==0.6.0
+librosa==0.11.0
+loguru==0.7.2
+numpy==1.26.4
+packaging
+pillow
+protobuf
+pyloudnorm==0.1.1
+regex
+safetensors
+scipy==1.15.3
+sentencepiece
+soundfile==0.13.1
+soxr==0.5.0.post1
+tqdm
+transformers==4.41.0

backend/python/longcat-video/run.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+
+startBackend "$@"

backend/python/longcat-video/test.py

@@ -0,0 +1,239 @@
+# SPDX-License-Identifier: MIT
+
+import importlib.util
+import json
+import os
+import sys
+import tempfile
+import unittest
+
+BACKEND_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, BACKEND_DIR)
+
+# longcat-video is a backend directory, not an importable Python package name.
+from longcat_utils import (  # noqa: E402
+    MODEL_KIND_AVATAR,
+    MODEL_KIND_BASE,
+    attention_overrides,
+    avatar_segments_for_duration,
+    avatar_segments_for_frames,
+    classify_model,
+    normalize_model_source,
+    normalize_num_frames,
+    parse_options,
+    select_known_options,
+    validate_dimensions,
+)
+
+
+SOURCE_DIR = os.path.join(BACKEND_DIR, "sources", "LongCat-Video")
+try:
+    import torch
+
+    sys.path.insert(0, SOURCE_DIR)
+    ATTENTION_TESTS_AVAILABLE = (
+        os.path.isdir(SOURCE_DIR) and importlib.util.find_spec("triton") is not None
+    )
+except ImportError:
+    torch = None
+    ATTENTION_TESTS_AVAILABLE = False
+
+AVATAR_ATTENTION_TESTS_AVAILABLE = ATTENTION_TESTS_AVAILABLE and all(
+    importlib.util.find_spec(module) is not None
+    for module in ("pyloudnorm", "scipy", "torchvision")
+)
+
+
+class LongCatUtilsTest(unittest.TestCase):
+    def test_parse_options_preserves_colons_and_coerces_scalars(self):
+        options = parse_options(
+            [
+                "use_distill:true",
+                "max_segments:4",
+                "audio_guidance_scale:3.5",
+                "source:https://example.com/model",
+                "flag",
+            ]
+        )
+
+        self.assertEqual(options["use_distill"], True)
+        self.assertEqual(options["max_segments"], 4)
+        self.assertEqual(options["audio_guidance_scale"], 3.5)
+        self.assertEqual(options["source"], "https://example.com/model")
+        self.assertEqual(options["flag"], True)
+
+    def test_select_known_options_keeps_known_and_reports_unknown(self):
+        # The server injects llama.cpp serving defaults (cache_reuse, parallel)
+        # onto every model config. longcat-video must ignore what it does not
+        # understand rather than refuse to load - matching every other backend.
+        options = {
+            "use_distill": True,
+            "max_segments": 4,
+            "cache_reuse": 256,
+            "parallel": 1,
+        }
+        kept, ignored = select_known_options(options, {"use_distill", "max_segments"})
+
+        self.assertEqual(kept, {"use_distill": True, "max_segments": 4})
+        self.assertEqual(ignored, ["cache_reuse", "parallel"])
+
+    def test_select_known_options_reports_nothing_when_all_known(self):
+        kept, ignored = select_known_options({"use_distill": True}, {"use_distill"})
+        self.assertEqual(kept, {"use_distill": True})
+        self.assertEqual(ignored, [])
+
+    def test_classify_model_accepts_only_supported_longcat_models(self):
+        cases = {
+            "meituan-longcat/LongCat-Video": MODEL_KIND_BASE,
+            "https://huggingface.co/meituan-longcat/LongCat-Video": MODEL_KIND_BASE,
+            "hf://meituan-longcat/LongCat-Video-Avatar-1.5": MODEL_KIND_AVATAR,
+            "other-org/LongCat-Video": None,
+            "meituan-longcat/LongCat-Video-Avatar": None,
+            "some-org/unrelated-model": None,
+        }
+
+        for model, expected in cases.items():
+            with self.subTest(model=model):
+                self.assertEqual(classify_model(model), expected)
+
+    def test_classify_model_reads_local_checkpoint_metadata(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with open(
+                os.path.join(directory, "model_index.json"),
+                "w",
+                encoding="utf-8",
+            ) as config_file:
+                json.dump({"model_name": "LongCat-Video-Avatar-1.5"}, config_file)
+
+            self.assertEqual(classify_model(directory), MODEL_KIND_AVATAR)
+
+    def test_normalize_model_source_handles_huggingface_uri_forms(self):
+        self.assertEqual(
+            normalize_model_source(
+                "https://huggingface.co/meituan-longcat/LongCat-Video/tree/main"
+            ),
+            "meituan-longcat/LongCat-Video",
+        )
+        self.assertEqual(
+            normalize_model_source("huggingface://meituan-longcat/LongCat-Video"),
+            "meituan-longcat/LongCat-Video",
+        )
+
+    def test_frame_and_segment_rounding_matches_longcat_temporal_shape(self):
+        self.assertEqual(normalize_num_frames(94), 93)
+        self.assertEqual(normalize_num_frames(0), 93)
+        self.assertEqual(avatar_segments_for_frames(93), 1)
+        self.assertEqual(avatar_segments_for_frames(94), 2)
+        self.assertEqual(avatar_segments_for_frames(173), 2)
+        self.assertEqual(avatar_segments_for_frames(174), 3)
+        self.assertEqual(avatar_segments_for_duration(10.0), 3)
+
+    def test_dimensions_are_bounded_and_aligned(self):
+        self.assertEqual(validate_dimensions(0, 0), (832, 480))
+        self.assertEqual(validate_dimensions(512, 512), (512, 512))
+        with self.assertRaisesRegex(ValueError, "divisible by 16"):
+            validate_dimensions(513, 512)
+        with self.assertRaisesRegex(ValueError, "must not exceed"):
+            validate_dimensions(1920, 1080)
+
+    def test_attention_backend_validation(self):
+        self.assertEqual(
+            attention_overrides("sdpa"),
+            {
+                "enable_flashattn2": False,
+                "enable_flashattn3": False,
+                "enable_xformers": False,
+            },
+        )
+        with self.assertRaisesRegex(ValueError, "attention_backend"):
+            attention_overrides("unknown")
+
+
+@unittest.skipUnless(
+    ATTENTION_TESTS_AVAILABLE,
+    "patched LongCat source and torch are required for attention tests",
+)
+class SDPAFallbackTest(unittest.TestCase):
+    def test_base_self_attention_matches_reference(self):
+        from longcat_video.modules.attention import Attention
+
+        dim, heads, sequence = 64, 4, 32
+        attention = Attention(
+            dim,
+            heads,
+            enable_flashattn2=False,
+            enable_flashattn3=False,
+            enable_xformers=False,
+            enable_bsa=False,
+        ).float()
+        query = torch.randn(2, heads, sequence, dim // heads)
+        key = torch.randn_like(query)
+        value = torch.randn_like(query)
+
+        output = attention._process_attn(query, key, value, shape=(1, 1, sequence))
+        reference = (
+            torch.softmax(
+                (query @ key.transpose(-1, -2)) * attention.scale,
+                dim=-1,
+            )
+            @ value
+        )
+
+        self.assertLess((output - reference).abs().max().item(), 1e-4)
+
+    @unittest.skipUnless(
+        AVATAR_ATTENTION_TESTS_AVAILABLE,
+        "avatar audio dependencies are required for the avatar attention test",
+    )
+    def test_avatar_self_attention_matches_reference(self):
+        from longcat_video.modules.avatar.attention import Attention
+
+        dim, heads, sequence = 64, 4, 16
+        attention = Attention(
+            dim,
+            heads,
+            enable_flashattn2=False,
+            enable_flashattn3=False,
+            enable_xformers=False,
+        ).float()
+        query = torch.randn(1, heads, sequence, dim // heads)
+        key = torch.randn_like(query)
+        value = torch.randn_like(query)
+
+        output = attention._process_attn(query, key, value, shape=(1, 1, sequence))
+        reference = (
+            torch.softmax(
+                (query @ key.transpose(-1, -2)) * attention.scale,
+                dim=-1,
+            )
+            @ value
+        )
+
+        self.assertLess((output - reference).abs().max().item(), 1e-4)
+
+    def test_base_cross_attention_remains_block_diagonal(self):
+        from longcat_video.modules.attention import MultiHeadCrossAttention
+
+        dim, heads = 64, 4
+        attention = MultiHeadCrossAttention(
+            dim,
+            heads,
+            enable_flashattn2=False,
+            enable_flashattn3=False,
+            enable_xformers=False,
+        ).float()
+        query = torch.randn(2, 8, dim)
+        key_lengths = [5, 7]
+        condition = torch.randn(1, sum(key_lengths), dim)
+
+        first = attention._process_cross_attn(query, condition, key_lengths)
+        changed = condition.clone()
+        changed[:, key_lengths[0] :] = torch.randn_like(changed[:, key_lengths[0] :])
+        second = attention._process_cross_attn(query, changed, key_lengths)
+
+        self.assertLess((first[0] - second[0]).abs().max().item(), 1e-5)
+        self.assertGreater((first[1] - second[1]).abs().max().item(), 1e-3)
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/longcat-video/test.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+
+runUnittests

backend/python/neutts/backend.py

@@ -119,9 +119,18 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # add options to kwargs
             kwargs.update(self.options)
 
-            ref_codes = self.model.encode_reference(self.AudioPath)
+            ref_audio = request.voice if request.voice else self.AudioPath
+            if not ref_audio:
+                return backend_pb2.Result(success=False, message="reference audio is required")
+            ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+            if not ref_text:
+                ref_text = self.ref_text
+            if not ref_text:
+                return backend_pb2.Result(success=False, message="ref_text is required")
 
-            wav = self.model.infer(request.text, ref_codes, self.ref_text)
+            ref_codes = self.model.encode_reference(ref_audio)
+
+            wav = self.model.infer(request.text, ref_codes, ref_text)
 
             sf.write(request.dst, wav, 24000)            
         except Exception as err:

backend/python/qwen-tts/backend.py

@@ -362,7 +362,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         # model_type explicitly set
         if self.model_type == "CustomVoice":
             return "CustomVoice"
-        if self.model_type == "VoiceClone":
+        if self.model_type in ("VoiceClone", "Base"):
             return "VoiceClone"
         if self.model_type == "VoiceDesign":
             return "VoiceDesign"
@@ -380,6 +380,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
     def _get_ref_audio_path(self, request, voice_name=None):
         """Get reference audio path from stored AudioPath or from voices dict."""
+        if hasattr(request, "voice") and request.voice and os.path.isfile(request.voice):
+            return request.voice
         # If voice_name is provided and exists in voices dict, use that
         if voice_name and voice_name in self.voices:
             audio_path = self.voices[voice_name]["audio"]
@@ -735,6 +737,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # model receives the types it expects. These override YAML-derived kwargs.
             if hasattr(request, "params") and request.params:
                 for key, value in request.params.items():
+                    if key == "ref_text":
+                        continue
                     generation_kwargs[key] = coerce_param_value(value)
 
             # Generate audio based on mode
@@ -743,7 +747,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
                 # Check if multi-voice mode is active (voices dict is populated)
                 voice_name = None
-                if self.voices:
+                request_voice_path = request.voice if request.voice and os.path.isfile(request.voice) else None
+                if self.voices and request_voice_path is None:
                     # Get voice from request (priority) or options
                     voice_name = request.voice if request.voice else None
                     if not voice_name:
@@ -775,11 +780,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 if voice_name and voice_name in self.voices:
                     ref_text_source = self.voices[voice_name]["ref_text"]
                 else:
-                    ref_text_source = self.options.get("ref_text", None)
+                    ref_text_source = request.params.get("ref_text") if hasattr(request, "params") else None
                     if not ref_text_source:
-                        # Try to get from request if available
-                        if hasattr(request, "ref_text") and request.ref_text:
-                            ref_text_source = request.ref_text
+                        ref_text_source = self.options.get("ref_text", None)
 
                 if not ref_text_source:
                     # x_vector_only_mode doesn't require ref_text

backend/python/tinygrad/backend.py

@@ -670,7 +670,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             arr = img_tensor.numpy()
             image = Image.fromarray(arr)
             dst = request.dst or os.path.join(tempfile.gettempdir(), "tinygrad_image.png")
-            image.save(dst)
+            # Force PNG rather than letting Pillow guess from the extension: the
+            # core passes a staging path ending in .tmp, which Pillow can't map
+            # to a format ("unknown file extension: .tmp").
+            image.save(dst, format="PNG")
             return backend_pb2.Result(success=True, message=dst)
         except Exception as exc:
             import traceback

backend/python/vllm-omni/backend.py

@@ -337,9 +337,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             if not images or len(images) == 0:
                 return backend_pb2.Result(success=False, message="Empty images list")
 
-            # Save image
+            # Save image. Force PNG rather than letting Pillow guess from the
+            # extension: the core passes a staging path ending in .tmp, which
+            # Pillow can't map to a format ("unknown file extension: .tmp").
             output_image = images[0]
-            output_image.save(request.dst)
+            output_image.save(request.dst, format="PNG")
             return backend_pb2.Result(message="Image generated successfully", success=True)
 
         except Exception as err:
@@ -698,10 +700,17 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 inputs["additional_information"]["non_streaming_mode"] = [True]
             elif task_type == "Base":
                 # Voice cloning requires ref_audio and ref_text
-                if "ref_audio" in self.options:
-                    inputs["additional_information"]["ref_audio"] = [self.options["ref_audio"]]
-                if "ref_text" in self.options:
-                    inputs["additional_information"]["ref_text"] = [self.options["ref_text"]]
+                ref_audio = voice if voice and os.path.isfile(voice) else self.options.get("ref_audio")
+                ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+                if not ref_text:
+                    ref_text = self.options.get("ref_text")
+                if not ref_audio or not ref_text:
+                    return backend_pb2.Result(
+                        success=False,
+                        message="Base TTS voice cloning requires request.voice/ref_audio and params.ref_text",
+                    )
+                inputs["additional_information"]["ref_audio"] = [ref_audio]
+                inputs["additional_information"]["ref_text"] = [ref_text]
                 if "x_vector_only_mode" in self.options:
                     inputs["additional_information"]["x_vector_only_mode"] = [self.options["x_vector_only_mode"]]
 

backend/python/vllm/install.sh

@@ -119,7 +119,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
     # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux
     # vllm pin (requirements-cublas13-after.txt, bumped independently against
     # vllm/vllm) until vllm-metal supports a newer vLLM.
-    VLLM_METAL_VERSION="v0.3.0.dev20260701212152"
+    VLLM_METAL_VERSION="v0.3.0.dev20260713103604"
 
     # The coupled vLLM source version is whatever this vllm-metal release builds
     # against -- it declares it in its own installer as `vllm_v=`. Derive it from

backend/python/vllm/requirements-cublas13-after.txt

@@ -3,8 +3,8 @@
 # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index
 # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match
 # so uv consults this index alongside PyPI.
---extra-index-url https://wheels.vllm.ai/0.24.0/cu130
+--extra-index-url https://wheels.vllm.ai/0.25.0/cu130
 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh),
 # which pins this exact vLLM version. Bumping vllm here means coordinating with a
 # vllm-metal release that supports the new version, or macOS/Metal builds break.
-vllm==0.24.0
+vllm==0.25.0

backend/python/vllm/requirements-l4t13-after.txt

@@ -1,4 +1,12 @@
 # vLLM 0.20+ ships an aarch64 manylinux wheel on PyPI whose Requires-Dist pins
 # torch==2.11.0 / torchvision==0.26.0 / torchaudio==2.11.0, locking an ABI-
 # consistent set with the cu130 torch wheel installed above.
-vllm
+#
+# Pinned to match the cublas13 build (requirements-cublas13-after.txt) and to
+# stay deterministic on GB10 / DGX Spark: 0.24.0 carries vllm-project/vllm#45179
+# ("release cached device memory under pressure on UMA GPUs during weight
+# loading"), without which cold model loads on the Grace Blackwell unified-
+# memory architecture crash deterministically with an empty "Engine core init
+# failed" set (mudler/LocalAI#10722). Leaving this unpinned let the L4T image
+# drift onto whatever wheel was latest at build time.
+vllm==0.25.0

backend/python/vllm/requirements.txt

@@ -1,7 +1,7 @@
-grpcio==1.81.1
+grpcio==1.82.1
 protobuf
 certifi
 setuptools
 pillow
-charset-normalizer>=3.4.7
+charset-normalizer>=3.4.9
 chardet

backend/python/voxcpm/backend.py

@@ -144,7 +144,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     if os.path.exists(potential_path):
                         prompt_wav_path = potential_path
 
-            if hasattr(request, 'AudioPath') and request.AudioPath:
+            if prompt_wav_path is None and hasattr(request, 'AudioPath') and request.AudioPath:
                 if os.path.isabs(request.AudioPath):
                     prompt_wav_path = request.AudioPath
                 elif hasattr(request, 'ModelFile') and request.ModelFile:
@@ -155,8 +155,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 else:
                     prompt_wav_path = request.AudioPath
 
-            # Get prompt_text from options if available
-            if "prompt_text" in self.options:
+            # Per-request profile transcript takes precedence over YAML.
+            if hasattr(request, "params") and request.params.get("ref_text"):
+                prompt_text = request.params["ref_text"]
+            elif "prompt_text" in self.options:
                 prompt_text = self.options["prompt_text"]
 
             # Prepare text
@@ -241,7 +243,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     if os.path.exists(potential_path):
                         prompt_wav_path = potential_path
 
-            if hasattr(request, 'AudioPath') and request.AudioPath:
+            if prompt_wav_path is None and hasattr(request, 'AudioPath') and request.AudioPath:
                 if os.path.isabs(request.AudioPath):
                     prompt_wav_path = request.AudioPath
                 elif hasattr(request, 'ModelFile') and request.ModelFile:
@@ -252,8 +254,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 else:
                     prompt_wav_path = request.AudioPath
 
-            # Get prompt_text from options if available
-            if "prompt_text" in self.options:
+            # Per-request profile transcript takes precedence over YAML.
+            if hasattr(request, "params") and request.params.get("ref_text"):
+                prompt_text = request.params["ref_text"]
+            elif "prompt_text" in self.options:
                 prompt_text = self.options["prompt_text"]
 
             # Prepare text

cmd/launcher/internal/launcher.go

@@ -207,21 +207,7 @@ func (l *Launcher) StartLocalAI() error {
 	}
 
 	// Build command arguments
-	dataPath := l.GetDataPath()
-	args := []string{
-		"run",
-		"--models-path", l.config.ModelsPath,
-		"--backends-path", l.config.BackendsPath,
-		"--address", l.config.Address,
-		"--log-level", l.config.LogLevel,
-		// Keep persistent data and dynamic config under the launcher's data
-		// directory (~/.localai) rather than letting the server resolve them
-		// to ${basepath}/{data,configuration}. ${basepath} expands to the
-		// launcher process's CWD (often the user's home root), which puts
-		// ~/data and ~/configuration outside ~/.localai. See #10610.
-		"--data-path", filepath.Join(dataPath, "data"),
-		"--localai-config-dir", filepath.Join(dataPath, "configuration"),
-	}
+	args := l.BuildRunArgs()
 
 	l.localaiCmd = exec.CommandContext(l.ctx, binaryPath, args...)
 
@@ -406,6 +392,32 @@ func (l *Launcher) GetWebUIURL() string {
 	return address
 }
 
+// BuildRunArgs assembles the argument list passed to `local-ai run`.
+//
+// Storage paths are anchored to the launcher's data directory instead of the
+// server's own defaults. The server resolves data/config to ${basepath}
+// (the launcher process CWD, often the user's home root) and generated-content
+// /uploads to shared /tmp paths. On a shared /tmp (macOS routes /tmp to
+// /private/tmp for every user) the first account to run LocalAI creates
+// /tmp/generated with 0750 perms, so any other account then fails startup with
+// "mkdir /tmp/generated/content: permission denied". Keeping every writable
+// path under the per-user data directory avoids both the misplacement (#10610)
+// and the cross-user /tmp collision.
+func (l *Launcher) BuildRunArgs() []string {
+	dataPath := l.GetDataPath()
+	return []string{
+		"run",
+		"--models-path", l.config.ModelsPath,
+		"--backends-path", l.config.BackendsPath,
+		"--address", l.config.Address,
+		"--log-level", l.config.LogLevel,
+		"--data-path", filepath.Join(dataPath, "data"),
+		"--localai-config-dir", filepath.Join(dataPath, "configuration"),
+		"--generated-content-path", filepath.Join(dataPath, "generated"),
+		"--upload-path", filepath.Join(dataPath, "uploads"),
+	}
+}
+
 // GetDataPath returns the path where LocalAI data and logs are stored
 func (l *Launcher) GetDataPath() string {
 	// LocalAI typically stores data in the current working directory or a models directory

cmd/launcher/internal/launcher_test.go

@@ -149,6 +149,41 @@ var _ = Describe("Launcher", func() {
 		})
 	})
 
+	Describe("BuildRunArgs", func() {
+		// Regression for the macOS "mkdir /tmp/generated/content: permission denied"
+		// startup failure: the launcher must redirect generated-content and upload
+		// paths under its own data directory instead of letting the server fall back
+		// to the shared /tmp defaults, which collide across users on a shared /tmp.
+		It("should keep generated-content and upload paths under the data directory", func() {
+			config := launcherInstance.GetConfig()
+			config.ModelsPath = filepath.Join(tempDir, "models")
+			launcherInstance.SetConfig(config)
+
+			dataPath := launcherInstance.GetDataPath()
+			args := launcherInstance.BuildRunArgs()
+
+			assertFlagValue := func(flag, expected string) {
+				idx := -1
+				for i, a := range args {
+					if a == flag {
+						idx = i
+						break
+					}
+				}
+				Expect(idx).To(BeNumerically(">=", 0), "expected %s to be present in run args", flag)
+				Expect(idx+1).To(BeNumerically("<", len(args)), "expected a value after %s", flag)
+				Expect(args[idx+1]).To(Equal(expected))
+			}
+
+			assertFlagValue("--generated-content-path", filepath.Join(dataPath, "generated"))
+			assertFlagValue("--upload-path", filepath.Join(dataPath, "uploads"))
+			// The bug was the server resolving these to shared /tmp paths.
+			for _, a := range args {
+				Expect(a).ToNot(HavePrefix("/tmp/"), "run args must not reference shared /tmp paths, got %s", a)
+			}
+		})
+	})
+
 	Describe("Logs", func() {
 		It("should return empty logs initially", func() {
 			logs := launcherInstance.GetLogs()

cmd/local-ai/main.go

@@ -57,10 +57,17 @@ For documentation and support:
 		),
 		kong.UsageOnError(),
 		kong.Vars{
-			"basepath":  kong.ExpandPath("."),
-			"galleries": `[{"name":"localai", "url":"github:mudler/LocalAI/gallery/index.yaml@master"}]`,
-			"backends":  `[{"name":"localai", "url":"github:mudler/LocalAI/backend/index.yaml@master"}]`,
-			"version":   internal.PrintableVersion(),
+			"basepath": kong.ExpandPath("."),
+			// Per-user temp locations for ephemeral writable content. A fixed
+			// shared name under /tmp collides across accounts on multi-user hosts
+			// (notably macOS, where /tmp is the shared /private/tmp for everyone),
+			// failing startup with "mkdir /tmp/generated/content: permission
+			// denied". See cli.DefaultGeneratedContentPath.
+			"generatedcontentpath": cli.DefaultGeneratedContentPath(),
+			"uploadpath":           cli.DefaultUploadPath(),
+			"galleries":            `[{"name":"localai", "url":"github:mudler/LocalAI/gallery/index.yaml@master"}]`,
+			"backends":             `[{"name":"localai", "url":"github:mudler/LocalAI/backend/index.yaml@master"}]`,
+			"version":              internal.PrintableVersion(),
 		},
 	)
 	ctx, err := k.Parse(os.Args[1:])

core/application/application.go

@@ -22,6 +22,7 @@ import (
 	"github.com/mudler/LocalAI/core/services/routing/pii"
 	"github.com/mudler/LocalAI/core/services/routing/piidetector"
 	"github.com/mudler/LocalAI/core/services/routing/router"
+	"github.com/mudler/LocalAI/core/services/voiceprofile"
 	"github.com/mudler/LocalAI/core/services/voicerecognition"
 	"github.com/mudler/LocalAI/core/templates"
 	pkggrpc "github.com/mudler/LocalAI/pkg/grpc"
@@ -58,6 +59,7 @@ type Application struct {
 	agentPoolService   atomic.Pointer[agentpool.AgentPoolService]
 	faceRegistry       facerecognition.Registry
 	voiceRegistry      voicerecognition.Registry
+	voiceProfileStore  *voiceprofile.Store
 	authDB             *gorm.DB
 	metricsService     *monitoring.LocalAIMetricsService
 	statsRecorder      *billing.Recorder
@@ -98,6 +100,11 @@ type Application struct {
 func newApplication(appConfig *config.ApplicationConfig) *Application {
 	ml := model.NewModelLoader(appConfig.SystemState)
 
+	// Apply the per-model load-failure cooldown (0 disables). Set here rather
+	// than in the watchdog block so it takes effect regardless of whether the
+	// watchdog/LRU limiter is enabled.
+	ml.SetLoadFailureCooldown(appConfig.ModelLoadFailureCooldown, 0)
+
 	// Close MCP sessions when a model is unloaded (watchdog eviction, manual shutdown, etc.)
 	ml.OnModelUnload(func(modelName string) {
 		mcpTools.CloseMCPSessions(modelName)
@@ -113,6 +120,7 @@ func newApplication(appConfig *config.ApplicationConfig) *Application {
 		modelLoader:        ml,
 		applicationConfig:  appConfig,
 		templatesEvaluator: templates.NewEvaluator(appConfig.SystemState.Model.ModelsPath),
+		voiceProfileStore:  voiceprofile.NewStore(appConfig.DataPath),
 	}
 
 	// Face-recognition registry backed by LocalAI's built-in vector store.
@@ -211,6 +219,13 @@ func (a *Application) VoiceRegistry() voicerecognition.Registry {
 	return a.voiceRegistry
 }
 
+// VoiceProfileStore returns the persistent library of reusable voice-cloning
+// references. It is distinct from VoiceRegistry, which stores speaker
+// recognition embeddings rather than synthesis reference audio.
+func (a *Application) VoiceProfileStore() *voiceprofile.Store {
+	return a.voiceProfileStore
+}
+
 // AuthDB returns the auth database connection, or nil if auth is not enabled.
 func (a *Application) AuthDB() *gorm.DB {
 	return a.authDB
@@ -456,6 +471,11 @@ func (a *Application) Shutdown() error {
 		if a.modelLoader != nil {
 			err = a.modelLoader.StopAllGRPC()
 		}
+		if a.voiceProfileStore != nil {
+			if closeErr := a.voiceProfileStore.Close(); err == nil {
+				err = closeErr
+			}
+		}
 	})
 	return err
 }
@@ -520,6 +540,7 @@ func (a *Application) start() error {
 		// "unavailable" error if startup ran with --disable-stats.
 		assistantClient.StatsRecorder = a.statsRecorder
 		assistantClient.FallbackUser = a.fallbackUser
+		assistantClient.VoiceProfiles = a.voiceProfileStore
 		// PII filter — same nil-or-real wiring.
 		assistantClient.PIIRedactor = a.piiRedactor
 		assistantClient.PIIEvents = a.piiEvents

core/application/runtime_settings_branding_test.go

@@ -261,4 +261,35 @@ var _ = Describe("loadRuntimeSettingsFromFile", func() {
 			Expect(cfg.AgentPool.AgentHubURL).To(Equal("https://hub.acme.io"))
 		})
 	})
+
+	// Backend logging capture. Worker/distributed mode force-enables it
+	// (core/services/worker.SetBackendLoggingEnabled(true)); single mode used
+	// to leave it off by default with no CLI flag, so the UI "Backend Logs"
+	// page was silently empty unless the operator found the Settings toggle.
+	// It now defaults on in single mode too. Because the default is on, the
+	// loader must let a persisted enable_backend_logging=false (the UI
+	// toggle-off) win over the default - the sticky "only flip false->true"
+	// merge used for env-backed flags would otherwise ignore it and revert
+	// the toggle on every restart.
+	Describe("backend logging capture", func() {
+		It("captures backend logs by default in single mode", func() {
+			cfg := config.NewApplicationConfig()
+			Expect(cfg.EnableBackendLogging).To(BeTrue(),
+				"single mode should capture backend logs out of the box, matching worker mode")
+		})
+
+		It("honors a persisted enable_backend_logging=false across restart (toggle-off wins over default-on)", func() {
+			cfg := config.NewApplicationConfig() // default-on boot state
+			cfg.DynamicConfigsDir = seedSettings(`{"enable_backend_logging": false}`)
+			loadRuntimeSettingsFromFile(cfg)
+			Expect(cfg.EnableBackendLogging).To(BeFalse(),
+				"a UI toggle-off persisted to runtime_settings.json must survive a restart")
+		})
+
+		It("loads a persisted enable_backend_logging=true", func() {
+			cfg := &config.ApplicationConfig{DynamicConfigsDir: seedSettings(`{"enable_backend_logging": true}`)}
+			loadRuntimeSettingsFromFile(cfg)
+			Expect(cfg.EnableBackendLogging).To(BeTrue())
+		})
+	})
 })

core/application/startup.go

@@ -698,10 +698,12 @@ func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
 		}
 	}
 
+	// Backend logging defaults on in single mode (NewApplicationConfig), so
+	// the usual "only flip false->true" merge would ignore a persisted false
+	// and revert the UI toggle-off on every restart. There is no env var/CLI
+	// flag for it, so an explicit persisted value is authoritative here.
 	if settings.EnableBackendLogging != nil {
-		if !options.EnableBackendLogging {
-			options.EnableBackendLogging = *settings.EnableBackendLogging
-		}
+		options.EnableBackendLogging = *settings.EnableBackendLogging
 	}
 
 	// Tracing settings

core/backend/llm.go

@@ -47,6 +47,28 @@ func needsThinkingProbe(c *config.ModelConfig) bool {
 			c.ReasoningConfig.DisableReasoningTagPrefill == nil)
 }
 
+// persistProbedReasoning writes the post-probe reasoning slots (and media
+// marker) from probed back into the loader's persisted config for modelName,
+// skipping any reasoning slot the probe was not actually allowed to fill.
+// persistDisableReasoning/persistDisableTagPrefill must be snapshotted from
+// probed's reasoning slots *before* the probe ran: a slot that already
+// carried a value at that point was populated by request-time
+// ApplyReasoningEffort, not by backend detection, and persisting it would
+// masquerade as an operator's explicit reasoning.disable (see #10622).
+func persistProbedReasoning(cl *config.ModelConfigLoader, modelName string, probed *config.ModelConfig, persistDisableReasoning, persistDisableTagPrefill bool) {
+	cl.UpdateModelConfig(modelName, func(cfg *config.ModelConfig) {
+		if persistDisableReasoning {
+			cfg.ReasoningConfig.DisableReasoning = probed.ReasoningConfig.DisableReasoning
+		}
+		if persistDisableTagPrefill {
+			cfg.ReasoningConfig.DisableReasoningTagPrefill = probed.ReasoningConfig.DisableReasoningTagPrefill
+		}
+		if probed.MediaMarker != "" {
+			cfg.MediaMarker = probed.MediaMarker
+		}
+	})
+}
+
 // HasChatDeltaContent returns true if any chat delta carries content or reasoning text.
 // Used to decide whether to prefer C++ autoparser deltas over Go-side tag extraction.
 func (t TokenUsage) HasChatDeltaContent() bool {
@@ -127,15 +149,19 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
 	needsMarkerProbe := c.MediaMarker == ""
 	if shouldProbeThinking || needsMarkerProbe {
 		modelOpts := grpcModelOpts(*c, o.SystemState.Model.ModelsPath)
+		// DetectThinkingSupportFromBackend only fills reasoning slots that are
+		// still nil, so a slot that already carries a value here was populated by
+		// request-time ApplyReasoningEffort (e.g. a `reasoning_effort: none`
+		// default), not by backend detection. Persisting such a request-scoped
+		// value would masquerade as an operator's explicit reasoning.disable and
+		// permanently defeat future per-request reasoning_effort overrides
+		// (see #10622). Only persist the slots the probe is actually allowed to
+		// fill.
+		persistDisableReasoning := c.ReasoningConfig.DisableReasoning == nil
+		persistDisableTagPrefill := c.ReasoningConfig.DisableReasoningTagPrefill == nil
 		config.DetectThinkingSupportFromBackend(ctx, c, inferenceModel, modelOpts)
 		// Update the config in the loader so it persists for future requests
-		cl.UpdateModelConfig(c.Name, func(cfg *config.ModelConfig) {
-			cfg.ReasoningConfig.DisableReasoning = c.ReasoningConfig.DisableReasoning
-			cfg.ReasoningConfig.DisableReasoningTagPrefill = c.ReasoningConfig.DisableReasoningTagPrefill
-			if c.MediaMarker != "" {
-				cfg.MediaMarker = c.MediaMarker
-			}
-		})
+		persistProbedReasoning(cl, c.Name, c, persistDisableReasoning, persistDisableTagPrefill)
 	}
 
 	var protoMessages []*proto.Message

core/backend/llm_probe_test.go

@@ -1,6 +1,8 @@
 package backend
 
 import (
+	"os"
+
 	"github.com/mudler/LocalAI/core/config"
 
 	"github.com/gpustack/gguf-parser-go/util/ptr"
@@ -27,3 +29,90 @@ var _ = Describe("thinking probe gating", func() {
 		Expect(needsThinkingProbe(cfg)).To(BeFalse())
 	})
 })
+
+var _ = Describe("persistProbedReasoning", func() {
+	const modelName = "probe-test"
+
+	// newLoaderWithConfig seeds a ModelConfigLoader with a single model config
+	// parsed from yamlBody, mirroring how the loader is populated from disk.
+	newLoaderWithConfig := func(yamlBody string) *config.ModelConfigLoader {
+		tmp, err := os.CreateTemp("", "persist-probed-reasoning-*.yaml")
+		Expect(err).ToNot(HaveOccurred())
+		defer func() { _ = os.Remove(tmp.Name()) }()
+
+		_, err = tmp.WriteString(yamlBody)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(tmp.Close()).To(Succeed())
+
+		cl := config.NewModelConfigLoader("")
+		Expect(cl.ReadModelConfig(tmp.Name())).To(Succeed())
+		return cl
+	}
+
+	It("persists a reasoning slot the probe was allowed to fill (was nil beforehand)", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		probed.ReasoningConfig.DisableReasoning = ptr.To(false) // backend detected: supports thinking
+		probed.ReasoningConfig.DisableReasoningTagPrefill = ptr.To(true)
+
+		persistProbedReasoning(cl, modelName, probed, true, true)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoning).To(BeFalse())
+		Expect(cfg.ReasoningConfig.DisableReasoningTagPrefill).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoningTagPrefill).To(BeTrue())
+	})
+
+	It("does not persist a slot that already carried a request-scoped value before the probe ran", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		// Simulates ApplyReasoningEffort("none") having set this on the
+		// request-scoped copy before the probe ran - not a genuine backend
+		// detection, so it must never reach the persisted config (#10622).
+		probed.ReasoningConfig.DisableReasoning = ptr.To(true)
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).To(BeNil())
+		Expect(cfg.ReasoningConfig.DisableReasoningTagPrefill).To(BeNil())
+	})
+
+	It("preserves an operator's explicit persisted disable when the guard is false", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\nreasoning:\n  disable: true\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		// Even if the request-scoped copy ends up holding a different value,
+		// persistDisableReasoning=false must keep the operator's own setting.
+		probed.ReasoningConfig.DisableReasoning = ptr.To(false)
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.ReasoningConfig.DisableReasoning).ToNot(BeNil())
+		Expect(*cfg.ReasoningConfig.DisableReasoning).To(BeTrue())
+	})
+
+	It("persists the media marker regardless of the reasoning guards", func() {
+		cl := newLoaderWithConfig("name: probe-test\nbackend: llama-cpp\n")
+
+		probed := &config.ModelConfig{}
+		probed.Name = modelName
+		probed.MediaMarker = "<__media__>"
+
+		persistProbedReasoning(cl, modelName, probed, false, false)
+
+		cfg, ok := cl.GetModelConfig(modelName)
+		Expect(ok).To(BeTrue())
+		Expect(cfg.MediaMarker).To(Equal("<__media__>"))
+	})
+})

core/backend/options.go

@@ -215,21 +215,35 @@ const (
 )
 
 // EffectiveContextSize is the context window the backend will run with: the
-// configured value, or DefaultContextSize when unset.
+// configured value, or DefaultContextSize when unset. A negative value (the
+// context_size: -1 auto-max sentinel) that survived config resolution, e.g. on
+// a backend that never ran the GGUF resolver, is clamped here so a negative
+// n_ctx never reaches a backend.
 func EffectiveContextSize(c config.ModelConfig) int {
-	if c.ContextSize != nil {
+	if c.ContextSize != nil && *c.ContextSize > 0 {
 		return *c.ContextSize
 	}
 	return DefaultContextSize
 }
 
+// localGPU resolves the device that will run the model, for single-pass batch
+// sizing. It is a package var so tests inject a deterministic device; production
+// reads config.LocalGPU, whose detection is sync.Once-cached in xsysinfo — so the
+// per-request call from the router's prompt trimmer (modelTokenTrim) stays cheap.
+var localGPU = config.LocalGPU
+
 // EffectiveBatchSize is the single-decode batch the backend will run with.
 // Score, embedding and rerank all process the whole input in one pass: score
 // decodes prompt+candidate (asserts n_tokens <= n_batch), and embedding/rerank
-// pool over the full sequence in one physical batch (n_ubatch). So the batch
-// is sized to the context — anything that fits the context fits one pass,
+// pool over the full sequence in one physical batch (n_ubatch). Ideally the batch
+// covers the whole context so any input that fits the context fits one pass,
 // avoiding both the GGML_ASSERT crash and the "input is too large to process"
-// error. Explicit `batch:` always wins.
+// error — BUT a full ctx-sized n_ubatch makes the per-device CUDA compute buffer
+// multi-GiB (it scales ~ n_ubatch * n_ctx and can't be split across GPUs), so a
+// large-context embedding model aborts on load with free VRAM to spare (#10485).
+// So we cap the batch to the largest that fits the per-device VRAM headroom; an
+// input longer than that cap is the accepted tradeoff (it can't be pooled in one
+// pass, but the load no longer OOMs). Explicit `batch:` always wins.
 func EffectiveBatchSize(c config.ModelConfig) int {
 	if c.Batch != 0 {
 		return c.Batch
@@ -238,7 +252,7 @@ func EffectiveBatchSize(c config.ModelConfig) int {
 		c.HasUsecases(config.FLAG_EMBEDDINGS) ||
 		c.HasUsecases(config.FLAG_RERANK)
 	if ctx := EffectiveContextSize(c); singlePass && ctx > DefaultBatchSize {
-		return ctx
+		return config.SinglePassBatchForContext(localGPU(), ctx)
 	}
 	return DefaultBatchSize
 }

core/backend/options_internal_test.go

@@ -103,6 +103,19 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 	threads := 1
 	ctx := 4096
 
+	// The single-pass batch is now VRAM-aware, so inject a deterministic GPU with
+	// ample per-device VRAM: at these small contexts the compute buffer fits
+	// easily, so EffectiveBatchSize returns the full context (the pre-#10485
+	// behaviour these cases assert). Without injection the value would depend on
+	// the CI host's real (often unknown) VRAM.
+	const gib = uint64(1) << 30
+	var origLocalGPU func() config.GPU
+	BeforeEach(func() {
+		origLocalGPU = localGPU
+		localGPU = func() config.GPU { return config.GPU{VRAM: 119 * gib} }
+	})
+	AfterEach(func() { localGPU = origLocalGPU })
+
 	It("defaults to 512 for an ordinary model", func() {
 		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
 		opts := grpcModelOpts(cfg, "/tmp/models")
@@ -162,6 +175,61 @@ var _ = Describe("grpcModelOpts NBatch", func() {
 	})
 })
 
+// Guards the VRAM-aware cap on the single-pass (embedding/score/rerank) batch:
+// a large context must not turn n_ubatch into a multi-GiB compute buffer that
+// aborts the load on a device with free VRAM (issue #10485). The GPU is injected
+// via the localGPU package var so the cap is deterministic without a real device.
+var _ = Describe("EffectiveBatchSize VRAM cap", func() {
+	const gib = uint64(1) << 30
+	embeddings := config.FLAG_EMBEDDINGS
+	threads := 1
+
+	var origLocalGPU func() config.GPU
+	BeforeEach(func() { origLocalGPU = localGPU })
+	AfterEach(func() { localGPU = origLocalGPU })
+
+	singlePassCfg := func(ctx int) config.ModelConfig {
+		return config.ModelConfig{
+			Threads:       &threads,
+			LLMConfig:     config.LLMConfig{ContextSize: &ctx},
+			KnownUsecases: &embeddings,
+		}
+	}
+
+	It("caps a large embedding context to a batch below the context but at least the default", func() {
+		// Reproduces qwen3-embedding-4b: context 40960 on a modest 20 GiB card.
+		// Full-context n_ubatch=40960 aborts; the cap must fit the VRAM headroom.
+		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
+		batch := EffectiveBatchSize(singlePassCfg(40960))
+		Expect(batch).To(BeNumerically(">=", DefaultBatchSize))
+		Expect(batch).To(BeNumerically("<", 40960))
+	})
+
+	It("keeps an explicit batch even with a large context and small VRAM", func() {
+		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
+		cfg := singlePassCfg(40960)
+		cfg.Batch = 512
+		Expect(EffectiveBatchSize(cfg)).To(Equal(512))
+	})
+
+	It("returns the full context when per-device VRAM is unknown", func() {
+		// Unknown VRAM (CPU / detection gap) preserves the original single-pass
+		// behavior: batch follows context. The VRAM cap is a downward safety that
+		// only engages when the per-device ceiling is known — clamping here would
+		// re-break single-pass pooling and over-trim inputs, with no OOM benefit on
+		// CPU where the compute buffer lives in system RAM.
+		localGPU = func() config.GPU { return config.GPU{VRAM: 0} }
+		Expect(EffectiveBatchSize(singlePassCfg(40960))).To(Equal(40960))
+	})
+
+	It("returns the default batch for a non-single-pass model regardless of VRAM", func() {
+		localGPU = func() config.GPU { return config.GPU{VRAM: 20 * gib} }
+		ctx := 40960
+		cfg := config.ModelConfig{Threads: &threads, LLMConfig: config.LLMConfig{ContextSize: &ctx}}
+		Expect(EffectiveBatchSize(cfg)).To(Equal(DefaultBatchSize))
+	})
+})
+
 // Guards the generic chat_template_kwargs forwarding: the model config map plus any
 // per-request metadata overrides are merged, coerced, and serialised into the
 // backend metadata blob that llama.cpp reads. Client metadata also overrides the
@@ -225,3 +293,24 @@ var _ = Describe("gRPCPredictOpts chat_template_kwargs metadata", func() {
 		Expect(opts.Metadata).ToNot(HaveKey("chat_template_kwargs"))
 	})
 })
+
+var _ = Describe("EffectiveContextSize", func() {
+	Context("EffectiveContextSize", func() {
+		It("clamps a negative (auto-max sentinel) context size to the default", func() {
+			neg := -1
+			cfg := config.ModelConfig{LLMConfig: config.LLMConfig{ContextSize: &neg}}
+			Expect(EffectiveContextSize(cfg)).To(Equal(DefaultContextSize))
+		})
+
+		It("returns an explicit positive context size unchanged", func() {
+			ctx := 8192
+			cfg := config.ModelConfig{LLMConfig: config.LLMConfig{ContextSize: &ctx}}
+			Expect(EffectiveContextSize(cfg)).To(Equal(8192))
+		})
+
+		It("falls back to the default when context size is unset", func() {
+			cfg := config.ModelConfig{}
+			Expect(EffectiveContextSize(cfg)).To(Equal(DefaultContextSize))
+		})
+	})
+})

core/backend/stores.go

@@ -133,10 +133,16 @@ func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, st
 	//   Try to add key with length N when existing length is M
 	// Use the store namespace as modelID so each namespace gets its own
 	// process instance and its own in-memory Store{}.
+	//
+	// The model name sent over gRPC carries store.NamespacePrefix so the
+	// backend can tell a genuine store load from the greedy autoload
+	// probing it with LLM model names; local-store refuses names without
+	// the prefix (core and backend ship from the same release, so the
+	// convention upgrades in lockstep).
 	sc := []model.Option{
 		model.WithBackendString(backend),
 		model.WithModelID(storeName),
-		model.WithModel(storeName),
+		model.WithModel(store.NamespacePrefix + storeName),
 	}
 
 	return sl.Load(sc...)

core/backend/video.go

@@ -1,21 +1,37 @@
 package backend
 
 import (
+	"maps"
 	"time"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/trace"
-
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
 	model "github.com/mudler/LocalAI/pkg/model"
 )
 
-func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, endImage, dst string, numFrames, fps, seed int32, cfgScale float32, step int32, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) {
+// VideoGenerationOptions is the backend-neutral request passed to video generators.
+// Media fields contain staged local paths by the time they reach this layer.
+type VideoGenerationOptions struct {
+	Height         int32
+	Width          int32
+	Prompt         string
+	NegativePrompt string
+	StartImage     string
+	EndImage       string
+	Audio          string
+	Destination    string
+	NumFrames      int32
+	FPS            int32
+	Seed           int32
+	CFGScale       float32
+	Step           int32
+	Params         map[string]string
+}
 
+func VideoGeneration(options VideoGenerationOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func() error, error) {
 	opts := ModelOptions(modelConfig, appConfig)
-	inferenceModel, err := loader.Load(
-		opts...,
-	)
+	inferenceModel, err := loader.Load(opts...)
 	if err != nil {
 		recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil)
 		return nil, err
@@ -25,19 +41,22 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
 		_, err := inferenceModel.GenerateVideo(
 			appConfig.Context,
 			&proto.GenerateVideoRequest{
-				Height:         height,
-				Width:          width,
-				Prompt:         prompt,
-				NegativePrompt: negativePrompt,
-				StartImage:     startImage,
-				EndImage:       endImage,
-				NumFrames:      numFrames,
-				Fps:            fps,
-				Seed:           seed,
-				CfgScale:       cfgScale,
-				Step:           step,
-				Dst:            dst,
-			})
+				Height:         options.Height,
+				Width:          options.Width,
+				Prompt:         options.Prompt,
+				NegativePrompt: options.NegativePrompt,
+				StartImage:     options.StartImage,
+				EndImage:       options.EndImage,
+				Audio:          options.Audio,
+				NumFrames:      options.NumFrames,
+				Fps:            options.FPS,
+				Seed:           options.Seed,
+				CfgScale:       options.CFGScale,
+				Step:           options.Step,
+				Dst:            options.Destination,
+				Params:         maps.Clone(options.Params),
+			},
+		)
 		return err
 	}
 
@@ -45,15 +64,18 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
 		trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes)
 
 		traceData := map[string]any{
-			"prompt":          prompt,
-			"negative_prompt": negativePrompt,
-			"height":          height,
-			"width":           width,
-			"num_frames":      numFrames,
-			"fps":             fps,
-			"seed":            seed,
-			"cfg_scale":       cfgScale,
-			"step":            step,
+			"prompt":          options.Prompt,
+			"negative_prompt": options.NegativePrompt,
+			"height":          options.Height,
+			"width":           options.Width,
+			"num_frames":      options.NumFrames,
+			"fps":             options.FPS,
+			"seed":            options.Seed,
+			"cfg_scale":       options.CFGScale,
+			"step":            options.Step,
+			"has_start_image": options.StartImage != "",
+			"has_end_image":   options.EndImage != "",
+			"has_audio":       options.Audio != "",
 		}
 
 		startTime := time.Now()
@@ -73,7 +95,7 @@ func VideoGeneration(height, width int32, prompt, negativePrompt, startImage, en
 				Type:      trace.BackendTraceVideoGeneration,
 				ModelName: modelConfig.Name,
 				Backend:   modelConfig.Backend,
-				Summary:   trace.TruncateString(prompt, 200),
+				Summary:   trace.TruncateString(options.Prompt, 200),
 				Error:     errStr,
 				Data:      traceData,
 			})

core/cli/chat/session.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"slices"
 	"strings"
 )
 
@@ -58,7 +59,7 @@ func (s *chatSession) Clear() {
 }
 
 func (s *chatSession) SwitchModel(model string) error {
-	if !modelExists(s.models, model) {
+	if !slices.Contains(s.models, model) {
 		return fmt.Errorf("model %q is not available. Use /models to see installed models", model)
 	}
 	s.model = model
@@ -103,18 +104,9 @@ Then start a chat session:
 		b.WriteString("multiple models are available; choose one with --model:\n")
 		b.WriteString(formatChatModelList(models, ""))
 		return "", errors.New(b.String())
-	case !modelExists(models, requested):
+	case !slices.Contains(models, requested):
 		return "", fmt.Errorf("model %q is not available. Use `local-ai models list` and `local-ai models install <model>`, or pass an installed model with --model", requested)
 	default:
 		return requested, nil
 	}
 }
-
-func modelExists(models []string, name string) bool {
-	for _, model := range models {
-		if model == name {
-			return true
-		}
-	}
-	return false
-}

core/cli/run.go

@@ -35,8 +35,8 @@ type RunCMD struct {
 	BackendsPath                 string        `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
 	BackendsSystemPath           string        `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/var/lib/local-ai/backends" help:"Path containing system backends used for inferencing" group:"backends"`
 	ModelsPath                   string        `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
-	GeneratedContentPath         string        `env:"LOCALAI_GENERATED_CONTENT_PATH,GENERATED_CONTENT_PATH" type:"path" default:"/tmp/generated/content" help:"Location for generated content (e.g. images, audio, videos)" group:"storage"`
-	UploadPath                   string        `env:"LOCALAI_UPLOAD_PATH,UPLOAD_PATH" type:"path" default:"/tmp/localai/upload" help:"Path to store uploads from files api" group:"storage"`
+	GeneratedContentPath         string        `env:"LOCALAI_GENERATED_CONTENT_PATH,GENERATED_CONTENT_PATH" type:"path" default:"${generatedcontentpath}" help:"Location for generated content (e.g. images, audio, videos)" group:"storage"`
+	UploadPath                   string        `env:"LOCALAI_UPLOAD_PATH,UPLOAD_PATH" type:"path" default:"${uploadpath}" help:"Path to store uploads from files api" group:"storage"`
 	DataPath                     string        `env:"LOCALAI_DATA_PATH" type:"path" default:"${basepath}/data" help:"Path for persistent data (collectiondb, agent state, tasks, jobs). Separates mutable data from configuration" group:"storage"`
 	LocalaiConfigDir             string        `env:"LOCALAI_CONFIG_DIR" type:"path" default:"${basepath}/configuration" help:"Directory for dynamic loading of certain configuration files (currently api_keys.json and external_backends.json)" group:"storage"`
 	LocalaiConfigDirPollInterval time.Duration `env:"LOCALAI_CONFIG_DIR_POLL_INTERVAL" help:"Typically the config path picks up changes automatically, but if your system has broken fsnotify events, set this to an interval to poll the LocalAI Config Dir (example: 1m)" group:"storage"`
@@ -96,6 +96,7 @@ type RunCMD struct {
 	SizeAwareEviction                  bool     `env:"LOCALAI_SIZE_AWARE_EVICTION,SIZE_AWARE_EVICTION" default:"false" help:"Evict the largest loaded model first rather than the least-recently-used one, keeping small utility models resident and maximizing freed memory per eviction" group:"backends"`
 	LRUEvictionMaxRetries              int      `env:"LOCALAI_LRU_EVICTION_MAX_RETRIES,LRU_EVICTION_MAX_RETRIES" default:"30" help:"Maximum number of retries when waiting for busy models to become idle before eviction (default: 30)" group:"backends"`
 	LRUEvictionRetryInterval           string   `env:"LOCALAI_LRU_EVICTION_RETRY_INTERVAL,LRU_EVICTION_RETRY_INTERVAL" default:"1s" help:"Interval between retries when waiting for busy models to become idle (e.g., 1s, 2s) (default: 1s)" group:"backends"`
+	ModelLoadFailureCooldown           string   `env:"LOCALAI_MODEL_LOAD_FAILURE_COOLDOWN,MODEL_LOAD_FAILURE_COOLDOWN" default:"10s" help:"After a model load fails, refuse new load attempts for that model for this long (returned as HTTP 503 + Retry-After) so a client polling a broken model doesn't respawn a crashing backend every request. Doubles per consecutive failure up to 5m; reset on success. Set to 0 to disable (e.g., 10s, 30s)" group:"backends"`
 	Federated                          bool     `env:"LOCALAI_FEDERATED,FEDERATED" help:"Enable federated instance" group:"federated"`
 	DisableGalleryEndpoint             bool     `env:"LOCALAI_DISABLE_GALLERY_ENDPOINT,DISABLE_GALLERY_ENDPOINT" help:"Disable the gallery endpoints" group:"api"`
 	DisableMCP                         bool     `env:"LOCALAI_DISABLE_MCP,DISABLE_MCP" help:"Disable MCP (Model Context Protocol) support" group:"api" default:"false"`
@@ -186,6 +187,31 @@ type RunCMD struct {
 	PIIDefaultDetectors []string `env:"LOCALAI_PII_DEFAULT_DETECTORS" help:"Instance-wide default PII/secret detector model names applied to any PII-enabled model (chiefly cloud-proxy / MITM models) that names no pii.detectors of its own. Comma-separated, e.g. privacy-filter-nemotron,secret-filter. Takes precedence over the value persisted via the Middleware UI." group:"middleware"`
 }
 
+// userScopedTempDir returns a temp directory namespaced to the current user.
+//
+// The generated-content and upload directories are ephemeral, so they live
+// under the OS temp dir - but a fixed shared name like /tmp/generated is a trap
+// on any multi-user host. macOS routes /tmp to the shared /private/tmp for every
+// account, so whichever user starts LocalAI first creates the parent with 0750
+// perms and every other account then fails startup with
+// "mkdir /tmp/generated/content: permission denied" (the same happens on Linux
+// once a stale root-owned /tmp/generated is left behind). Scoping to the current
+// UID gives each account its own tree so they never collide.
+func userScopedTempDir() string {
+	return filepath.Join(os.TempDir(), fmt.Sprintf("localai-%d", os.Getuid()))
+}
+
+// DefaultGeneratedContentPath returns the default location for backend-generated
+// content (images, audio, videos).
+func DefaultGeneratedContentPath() string {
+	return filepath.Join(userScopedTempDir(), "generated", "content")
+}
+
+// DefaultUploadPath returns the default location for uploads from the files API.
+func DefaultUploadPath() string {
+	return filepath.Join(userScopedTempDir(), "upload")
+}
+
 func (r *RunCMD) Run(ctx *cliContext.Context) error {
 	warnDeprecatedFlags()
 
@@ -589,6 +615,13 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		}
 		opts = append(opts, config.WithLRUEvictionRetryInterval(dur))
 	}
+	if r.ModelLoadFailureCooldown != "" {
+		dur, err := time.ParseDuration(r.ModelLoadFailureCooldown)
+		if err != nil {
+			return fmt.Errorf("invalid model load failure cooldown: %w", err)
+		}
+		opts = append(opts, config.WithModelLoadFailureCooldown(dur))
+	}
 
 	// Handle Open Responses store TTL
 	if r.OpenResponsesStoreTTL != "" && r.OpenResponsesStoreTTL != "0" {

core/cli/run_paths_test.go

@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Regression for the startup failure observed when a second OS account (or a
+// leftover root-owned directory) already created the shared /tmp locations:
+//
+//	unable to create ImageDir: "mkdir /tmp/generated/content: permission denied"
+//
+// The historical defaults (/tmp/generated/content and /tmp/localai/upload) are
+// shared across every user of a machine. On macOS /tmp is routed to the shared
+// /private/tmp for all accounts, so the first account to run LocalAI creates the
+// parent with 0750 perms and locks everyone else out. The defaults must instead
+// be scoped to the current user so unrelated accounts never collide.
+var _ = Describe("default writable paths", func() {
+	userScope := fmt.Sprintf("localai-%d", os.Getuid())
+
+	Describe("DefaultGeneratedContentPath", func() {
+		It("is scoped to the current user under the OS temp dir", func() {
+			p := DefaultGeneratedContentPath()
+			Expect(p).To(HavePrefix(os.TempDir()))
+			Expect(p).To(ContainSubstring(userScope))
+			Expect(p).To(HaveSuffix(filepath.Join("generated", "content")))
+		})
+
+		It("is not the historical shared path", func() {
+			Expect(DefaultGeneratedContentPath()).ToNot(Equal("/tmp/generated/content"))
+		})
+	})
+
+	Describe("DefaultUploadPath", func() {
+		It("is scoped to the current user under the OS temp dir", func() {
+			p := DefaultUploadPath()
+			Expect(p).To(HavePrefix(os.TempDir()))
+			Expect(p).To(ContainSubstring(userScope))
+			Expect(p).To(HaveSuffix("upload"))
+		})
+
+		It("is not the historical shared path", func() {
+			Expect(DefaultUploadPath()).ToNot(Equal("/tmp/localai/upload"))
+		})
+	})
+})

core/config/application_config.go

@@ -131,6 +131,13 @@ type ApplicationConfig struct {
 	LRUEvictionMaxRetries    int           // Maximum number of retries when waiting for busy models to become idle (default: 30)
 	LRUEvictionRetryInterval time.Duration // Interval between retries when waiting for busy models (default: 1s)
 
+	// ModelLoadFailureCooldown is the base cooldown applied after a model load
+	// fails: new load attempts for that model are refused (HTTP 503 + Retry-After)
+	// until it elapses, doubling per consecutive failure up to a 5m cap and reset
+	// on success. Prevents a client polling a broken model from respawning a
+	// crashing backend on every request. 0 disables it. Default: 10s.
+	ModelLoadFailureCooldown time.Duration
+
 	ModelsURL []string
 
 	WatchDogBusyTimeout, WatchDogIdleTimeout time.Duration
@@ -239,12 +246,20 @@ type AppOption func(*ApplicationConfig)
 
 func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
 	opt := &ApplicationConfig{
-		Context:                  context.Background(),
-		UploadLimitMB:            15,
-		Debug:                    true,
+		Context:       context.Background(),
+		UploadLimitMB: 15,
+		Debug:         true,
+		// Capture backend process stdout/stderr into the per-model
+		// BackendLogStore by default so the UI "Backend Logs" page works out
+		// of the box in single mode, matching worker/distributed mode (which
+		// force-enables it). It's a small in-memory ring buffer; the Settings
+		// toggle can still turn it off (a persisted false wins - see
+		// loadRuntimeSettingsFromFile).
+		EnableBackendLogging:     true,
 		AgentJobRetentionDays:    30,              // Default: 30 days
 		LRUEvictionMaxRetries:    30,              // Default: 30 retries
 		LRUEvictionRetryInterval: 1 * time.Second, // Default: 1 second
+		ModelLoadFailureCooldown: 10 * time.Second, // Default: 10s base cooldown after a failed load
 		// WatchDogInterval is intentionally left at the zero value here.
 		// The startup loader applies a persisted runtime_settings.json value
 		// only when the interval is still 0 (its "not set by env var"
@@ -531,6 +546,17 @@ func WithLRUEvictionRetryInterval(interval time.Duration) AppOption {
 	}
 }
 
+// WithModelLoadFailureCooldown sets the base cooldown applied after a failed
+// model load. 0 disables the cooldown, so unlike most options it accepts any
+// non-negative value.
+func WithModelLoadFailureCooldown(cooldown time.Duration) AppOption {
+	return func(o *ApplicationConfig) {
+		if cooldown >= 0 {
+			o.ModelLoadFailureCooldown = cooldown
+		}
+	}
+}
+
 var EnableGalleriesAutoload = func(o *ApplicationConfig) {
 	o.AutoloadGalleries = true
 }

core/config/backend_capabilities.go

@@ -120,7 +120,7 @@ var UsecaseInfoMap = map[string]UsecaseInfo{
 	UsecaseVideo: {
 		Flag:        FLAG_VIDEO,
 		GRPCMethod:  MethodGenerateVideo,
-		Description: "Video generation via the GenerateVideo RPC.",
+		Description: "Video generation via the GenerateVideo RPC, with optional image or audio conditioning when supported by the backend.",
 	},
 	UsecaseTranscript: {
 		Flag:        FLAG_TRANSCRIPT,
@@ -209,10 +209,29 @@ type BackendCapability struct {
 	AcceptsVideos bool
 	// AcceptsAudios indicates multimodal audio input in Predict.
 	AcceptsAudios bool
+	// VoiceCloning describes the backend's per-request reference-audio
+	// contract. Model variants that share a backend may narrow this further;
+	// use VoiceCloningForModel for UI/API decisions.
+	VoiceCloning *VoiceCloningCapability
 	// Description is a human-readable summary of the backend.
 	Description string
 }
 
+// VoiceCloningCapability is the model-facing contract for reusable reference
+// voices. The first release intentionally accepts only browser-normalizable
+// PCM WAV so every advertised backend sees the same input shape.
+type VoiceCloningCapability struct {
+	ReferenceTranscriptRequired bool     `json:"reference_transcript_required"`
+	AcceptedAudioFormats        []string `json:"accepted_audio_formats"`
+}
+
+func referenceVoiceCloning() *VoiceCloningCapability {
+	return &VoiceCloningCapability{
+		ReferenceTranscriptRequired: true,
+		AcceptedAudioFormats:        []string{"audio/wav"},
+	}
+}
+
 // BackendCapabilities maps each backend name (as used in model configs and gallery
 // entries) to its verified capabilities. This is the single source of truth for
 // what each backend supports.
@@ -261,6 +280,7 @@ var BackendCapabilities = map[string]BackendCapability{
 		AcceptsImages:    true,
 		AcceptsVideos:    true,
 		AcceptsAudios:    true,
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "vLLM omni-modal — supports text, image, video generation and TTS",
 	},
 	"transformers": {
@@ -303,6 +323,14 @@ var BackendCapabilities = map[string]BackendCapability{
 		DefaultUsecases:  []string{UsecaseImage},
 		Description:      "HuggingFace diffusers — Stable Diffusion, Flux, video generation",
 	},
+	"longcat-video": {
+		GRPCMethods:      []GRPCMethod{MethodGenerateVideo},
+		PossibleUsecases: []string{UsecaseVideo},
+		DefaultUsecases:  []string{UsecaseVideo},
+		AcceptsImages:    true,
+		AcceptsAudios:    true,
+		Description:      "LongCat-Video — text, image, and audio-conditioned avatar video generation on NVIDIA CUDA",
+	},
 	"stablediffusion": {
 		GRPCMethods:      []GRPCMethod{MethodGenerateImage},
 		PossibleUsecases: []string{UsecaseImage},
@@ -375,6 +403,7 @@ var BackendCapabilities = map[string]BackendCapability{
 		GRPCMethods:      []GRPCMethod{MethodAudioTranscription, MethodTTS, MethodTTSStream},
 		PossibleUsecases: []string{UsecaseTranscript, UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTranscript, UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "VibeVoice C++ — bidirectional speech, C++ backend with streaming TTS",
 	},
 	"sherpa-onnx": {
@@ -401,6 +430,7 @@ var BackendCapabilities = map[string]BackendCapability{
 		GRPCMethods:      []GRPCMethod{MethodTTS},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "Coqui TTS — multi-speaker neural synthesis",
 	},
 	"kitten-tts": {
@@ -419,50 +449,72 @@ var BackendCapabilities = map[string]BackendCapability{
 		GRPCMethods:      []GRPCMethod{MethodTTS},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "Pocket TTS — lightweight text-to-speech",
 	},
 	"qwen-tts": {
 		GRPCMethods:      []GRPCMethod{MethodTTS},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "Qwen TTS",
 	},
 	"qwen3-tts-cpp": {
 		GRPCMethods:      []GRPCMethod{MethodTTS, MethodTTSStream},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "Qwen3 TTS C++ - text-to-speech with streaming, named speakers, voice design and cloning (qwentts.cpp / GGML)",
 	},
 	"faster-qwen3-tts": {
 		GRPCMethods:      []GRPCMethod{MethodTTS},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "Faster Qwen3 TTS — accelerated Qwen TTS",
 	},
 	"fish-speech": {
 		GRPCMethods:      []GRPCMethod{MethodTTS},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "Fish Speech TTS",
 	},
 	"neutts": {
 		GRPCMethods:      []GRPCMethod{MethodTTS},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "NeuTTS — neural text-to-speech",
 	},
 	"chatterbox": {
 		GRPCMethods:      []GRPCMethod{MethodTTS},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "Chatterbox TTS",
 	},
 	"voxcpm": {
 		GRPCMethods:      []GRPCMethod{MethodTTS, MethodTTSStream},
 		PossibleUsecases: []string{UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
 		Description:      "VoxCPM TTS with streaming support",
 	},
+	"omnivoice-cpp": {
+		GRPCMethods:      []GRPCMethod{MethodTTS, MethodTTSStream},
+		PossibleUsecases: []string{UsecaseTTS},
+		DefaultUsecases:  []string{UsecaseTTS},
+		VoiceCloning:     referenceVoiceCloning(),
+		Description:      "OmniVoice C++ — multilingual TTS with streaming voice cloning and voice design",
+	},
+	"crispasr": {
+		GRPCMethods:      []GRPCMethod{MethodAudioTranscription, MethodTTS, MethodTTSStream, MethodVAD},
+		PossibleUsecases: []string{UsecaseTranscript, UsecaseTTS, UsecaseVAD},
+		DefaultUsecases:  []string{UsecaseTranscript},
+		VoiceCloning:     referenceVoiceCloning(),
+		Description:      "CrispASR GGUF runtime — speech recognition, VAD, and model-dependent TTS",
+	},
 
 	// --- Sound generation backends ---
 	"ace-step": {
@@ -596,6 +648,27 @@ func UsesLlamaSamplerDefaults(backend string) bool {
 	return !isNonLlama
 }
 
+// UsesLlamaCppServingOptions reports whether a backend understands llama.cpp's
+// serving-tuning model options - the free-form option strings cache_reuse /
+// n_cache_reuse (cross-request KV-prefix reuse) and parallel / n_parallel
+// (concurrent slots). These are llama.cpp server flags; LocalAI injects them as
+// defaults, but a backend that strictly validates its options (e.g.
+// longcat-video) rejects an unknown one with "unknown model option(s)" at
+// LoadModel. Only the llama.cpp backend - and the empty/auto-detect case, which
+// resolves to llama.cpp from a GGUF file, mirroring how llamaCppDefaults is
+// registered - should receive them.
+//
+// This is an allow-list on purpose (unlike UsesLlamaSamplerDefaults's
+// deny-list): these options are meaningful to no other backend, so a new
+// backend defaults to NOT getting them rather than breaking the same way.
+func UsesLlamaCppServingOptions(backend string) bool {
+	switch NormalizeBackendName(backend) {
+	case "", "llama-cpp":
+		return true
+	}
+	return false
+}
+
 // GetBackendCapability returns the capability info for a backend, or nil if unknown.
 // Handles backend name normalization.
 func GetBackendCapability(backend string) *BackendCapability {
@@ -605,6 +678,82 @@ func GetBackendCapability(backend string) *BackendCapability {
 	return nil
 }
 
+// VoiceCloningForModel returns the reference-audio contract only when the
+// installed model variant can honor it. Several backends serve both Base
+// (voice cloning) and CustomVoice/VoiceDesign models, so backend name alone is
+// deliberately insufficient. Operators with custom filenames can opt in or
+// out explicitly with tts.voice_cloning; the model option spelling remains a
+// compatibility fallback for configurations created before the typed field.
+func VoiceCloningForModel(cfg *ModelConfig) *VoiceCloningCapability {
+	if cfg == nil {
+		return nil
+	}
+	backend := NormalizeBackendName(cfg.Backend)
+	capability := GetBackendCapability(backend)
+	if capability == nil || capability.VoiceCloning == nil {
+		return nil
+	}
+	if cfg.VoiceCloning != nil {
+		if !*cfg.VoiceCloning {
+			return nil
+		}
+		return cloneVoiceCloningCapability(capability.VoiceCloning)
+	}
+
+	if enabled, explicit := voiceCloningOverride(cfg.Options); explicit {
+		if !enabled {
+			return nil
+		}
+		return cloneVoiceCloningCapability(capability.VoiceCloning)
+	}
+
+	identity := strings.ToLower(strings.Join([]string{cfg.Name, cfg.Model, strings.Join(cfg.Options, " ")}, " "))
+	supported := false
+	switch backend {
+	case "qwen3-tts-cpp", "qwen-tts", "vllm-omni":
+		supported = strings.Contains(identity, "base") || strings.Contains(identity, "voiceclone") || strings.Contains(identity, "voice_clone")
+	case "vibevoice-cpp":
+		// Realtime 0.5B consumes a precomputed .gguf voice prompt; the 1.5B
+		// path consumes raw WAV references per request.
+		supported = strings.Contains(identity, "1.5b")
+	case "coqui":
+		supported = strings.Contains(identity, "xtts") || strings.Contains(identity, "your_tts")
+	case "crispasr":
+		supported = strings.Contains(identity, "f5-tts") || strings.Contains(identity, "f5_tts")
+	default:
+		supported = true
+	}
+	if !supported {
+		return nil
+	}
+	return cloneVoiceCloningCapability(capability.VoiceCloning)
+}
+
+func voiceCloningOverride(options []string) (enabled, explicit bool) {
+	for _, option := range options {
+		parts := strings.FieldsFunc(option, func(r rune) bool { return r == ':' || r == '=' })
+		if len(parts) != 2 || !strings.EqualFold(strings.TrimSpace(parts[0]), "voice_cloning") {
+			continue
+		}
+		switch strings.ToLower(strings.TrimSpace(parts[1])) {
+		case "true", "1", "yes", "on":
+			return true, true
+		case "false", "0", "no", "off":
+			return false, true
+		}
+	}
+	return false, false
+}
+
+func cloneVoiceCloningCapability(capability *VoiceCloningCapability) *VoiceCloningCapability {
+	if capability == nil {
+		return nil
+	}
+	clone := *capability
+	clone.AcceptedAudioFormats = slices.Clone(capability.AcceptedAudioFormats)
+	return &clone
+}
+
 // PossibleUsecasesForBackend returns all usecases a backend can support.
 // Returns nil if the backend is unknown.
 func PossibleUsecasesForBackend(backend string) []string {

core/config/backend_capabilities_test.go

@@ -72,6 +72,29 @@ var _ = Describe("GetBackendCapability", func() {
 	})
 })
 
+var _ = Describe("VoiceCloningForModel", func() {
+	voiceCloningSetting := func(enabled bool) *bool { return &enabled }
+
+	DescribeTable("advertises only compatible model variants",
+		func(cfg ModelConfig, expected bool) {
+			Expect(VoiceCloningForModel(&cfg) != nil).To(Equal(expected))
+		},
+		Entry("Qwen C++ Base", ModelConfig{Name: "qwen3-tts-cpp-0.6b-base", Backend: "qwen3-tts-cpp"}, true),
+		Entry("Qwen C++ CustomVoice", ModelConfig{Name: "qwen3-tts-cpp-customvoice", Backend: "qwen3-tts-cpp"}, false),
+		Entry("VibeVoice realtime 0.5B", ModelConfig{Name: "vibevoice-cpp-0.5b", Backend: "vibevoice-cpp"}, false),
+		Entry("VibeVoice 1.5B", ModelConfig{Name: "vibevoice-1.5b", Backend: "vibevoice-cpp"}, true),
+		Entry("F5 through CrispASR", ModelConfig{Name: "f5-tts-crispasr", Backend: "crispasr"}, true),
+		Entry("ASR through CrispASR", ModelConfig{Name: "parakeet-crispasr", Backend: "crispasr"}, false),
+		Entry("VoxCPM", ModelConfig{Name: "voxcpm-1.5", Backend: "voxcpm"}, true),
+		Entry("unsupported Piper", ModelConfig{Name: "piper", Backend: "piper"}, false),
+		Entry("typed custom opt-in", ModelConfig{Name: "private-build", Backend: "qwen3-tts-cpp", TTSConfig: TTSConfig{VoiceCloning: voiceCloningSetting(true)}}, true),
+		Entry("typed opt-out", ModelConfig{Name: "voxcpm-1.5", Backend: "voxcpm", TTSConfig: TTSConfig{VoiceCloning: voiceCloningSetting(false)}}, false),
+		Entry("typed setting wins over compatibility option", ModelConfig{Name: "private-build", Backend: "qwen3-tts-cpp", TTSConfig: TTSConfig{VoiceCloning: voiceCloningSetting(true)}, Options: []string{"voice_cloning:false"}}, true),
+		Entry("legacy option custom opt-in", ModelConfig{Name: "private-build", Backend: "qwen3-tts-cpp", Options: []string{"voice_cloning:true"}}, true),
+		Entry("legacy option opt-out", ModelConfig{Name: "voxcpm-1.5", Backend: "voxcpm", Options: []string{"voice_cloning=false"}}, false),
+	)
+})
+
 var _ = Describe("IsValidUsecaseForBackend", func() {
 	It("accepts a backend's declared usecases", func() {
 		Expect(IsValidUsecaseForBackend("piper", "tts")).To(BeTrue())

core/config/context_fit.go

@@ -0,0 +1,144 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/LocalAI/pkg/xsysinfo"
+	"github.com/mudler/xlog"
+)
+
+// contextFitHeadroomDivisor reserves a slice of per-device VRAM as headroom when
+// deciding whether an auto-derived context fits. The gguf-parser footprint
+// already covers weights + KV + compute buffer, but a live load also pays for
+// allocator fragmentation, the CUDA/HIP context, and whatever else shares the
+// card, so we require the estimate to leave at least 1/divisor of the device
+// free. /5 (~20% headroom) mirrors the SWA full-cache gate's margin.
+const contextFitHeadroomDivisor = 5
+
+// contextFitCandidates is the descending set of context windows tried when the
+// DefaultAutoContextSize cap itself does not fit per-device VRAM. Only the rare
+// big-model-on-tiny-card case reaches this walk; it is capped at the base
+// choice and floored at DefaultContextSize, and returns the first (largest)
+// candidate that fits.
+var contextFitCandidates = []int{8192, 6144, 4096}
+
+// perDeviceVRAM reports the smallest per-GPU VRAM ceiling in bytes (0 = unknown
+// or no GPU). It is a package var so tests can inject a deterministic value —
+// detection does a live GPU probe. Per-device (not summed) is the right budget:
+// with all layers offloaded to a single device the whole footprint must fit that
+// one card, and a multi-GPU host is bounded by its smallest card. This mirrors
+// localGPU's use of MinPerGPUVRAM in hardware_defaults.go.
+var perDeviceVRAM = func() uint64 {
+	v, _ := xsysinfo.MinPerGPUVRAM()
+	return v
+}
+
+// estimateContextVRAM returns the estimated per-device VRAM footprint (bytes) of
+// running f fully offloaded at ctx tokens — weights + KV cache + compute buffer.
+// It returns 0 when it cannot produce an estimate (nil file, no tensors, or a
+// parser panic), which the caller treats as "cannot confirm a smaller fit" and
+// so keeps the conservative cap rather than clamping on a bogus number. It is a
+// package var so tests can stub it (a fabricated GGUF carries no tensors and
+// estimates to ~0).
+var estimateContextVRAM = func(f *gguf.GGUFFile, ctx int) (footprint uint64) {
+	if f == nil {
+		return 0
+	}
+	if ctx <= 0 {
+		ctx = DefaultContextSize
+	}
+	// The gguf-parser estimator panics on degenerate / partially-parsed GGUFs;
+	// treat any failure as "unknown" so config loading never crashes on a model
+	// the parser mis-handles.
+	defer func() {
+		if r := recover(); r != nil {
+			xlog.Debug("[context_fit] per-device VRAM estimate failed; treating as unknown", "error", r)
+			footprint = 0
+		}
+	}()
+	// Offload all layers (LocalAI's DefaultNGPULayers default; the estimator
+	// clamps to the model's block count) so the estimate reflects a fully
+	// GPU-resident model. NonUMA is the discrete-GPU figure (larger than the UMA
+	// one), which keeps the fit check conservative on unified-memory hosts — they
+	// have ample memory to clear it anyway.
+	est := f.EstimateLLaMACppRun(
+		gguf.WithLLaMACppContextSize(int32(ctx)),
+		gguf.WithLLaMACppOffloadLayers(uint64(DefaultNGPULayers)),
+	)
+	sum := est.Summarize(true, 0, 0)
+	if len(sum.Items) == 0 {
+		return 0
+	}
+	var total uint64
+	for _, v := range sum.Items[0].VRAMs {
+		total += uint64(v.NonUMA)
+	}
+	return total
+}
+
+// contextFitsVRAM reports whether an estimated footprint fits a per-device VRAM
+// ceiling with headroom (VRAM must exceed the footprint by ~1/divisor). Unknown
+// inputs (0) are treated as "cannot confirm" so a detection or estimate gap does
+// not clamp the context.
+func contextFitsVRAM(footprint, vram uint64) bool {
+	if footprint == 0 || vram == 0 {
+		return false
+	}
+	return vram >= footprint+footprint/contextFitHeadroomDivisor
+}
+
+// autoContextSize picks the default context to use for f when the user did not
+// set context_size. The choice is deliberately conservative, NOT
+// VRAM-maximizing:
+//
+//  1. Base cap: min(trainedMax, DefaultAutoContextSize). A small model keeps its
+//     trained window; a long-context model (128k / 256k / 1M) is capped so its
+//     KV cache does not default to a size no consumer GPU can hold. This applies
+//     always, including CPU / unknown-VRAM hosts.
+//  2. VRAM is only a downward safety: when a per-device VRAM ceiling IS detected
+//     and even the base cap would not fit it (with headroom), step down through
+//     contextFitCandidates to the largest window that fits, floored at
+//     DefaultContextSize. When VRAM is unknown we skip this — the base cap is
+//     already safe and we must not regress CPU / detection-gap hosts.
+//
+// trainedMax <= 0 means the estimate yielded nothing usable; the caller keeps
+// its existing DefaultContextSize fallback in that case, so this is only called
+// with a positive trainedMax.
+func autoContextSize(f *gguf.GGUFFile, trainedMax int) int {
+	chosen := trainedMax
+	if chosen > DefaultAutoContextSize {
+		chosen = DefaultAutoContextSize
+	}
+
+	vram := perDeviceVRAM()
+	if vram == 0 {
+		// No per-device VRAM detected (CPU-only, unified memory reporting nothing,
+		// or a detection gap). The bug is GPU OOM-on-load, so with no GPU budget to
+		// reason about we must not clamp — the base cap already bounds long-context
+		// models.
+		return chosen
+	}
+
+	if contextFitsVRAM(estimateContextVRAM(f, chosen), vram) {
+		return chosen
+	}
+
+	// The base cap does not fit this card. Walk candidates downward and take the
+	// largest that fits, never below DefaultContextSize.
+	for _, cand := range contextFitCandidates {
+		if cand > chosen || cand < DefaultContextSize {
+			continue
+		}
+		if contextFitsVRAM(estimateContextVRAM(f, cand), vram) {
+			xlog.Debug("[context_fit] capped auto context to fit per-device VRAM",
+				"context", cand, "base_cap", chosen, "vram_gib", vram>>30)
+			return cand
+		}
+	}
+
+	// Nothing fit (an unusually large model on a tiny card): fall back to the
+	// floor. The backend still clamps n_gpu_layers to what fits, so a partial
+	// offload can keep the model loadable rather than aborting outright.
+	xlog.Debug("[context_fit] no candidate context fit per-device VRAM; using floor",
+		"context", DefaultContextSize, "base_cap", chosen, "vram_gib", vram>>30)
+	return DefaultContextSize
+}

core/config/context_fit_internal_test.go

@@ -0,0 +1,101 @@
+package config
+
+import (
+	gguf "github.com/gpustack/gguf-parser-go"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// These specs exercise the auto-derived default context. The detection seams
+// (perDeviceVRAM, estimateContextVRAM) are package vars so a deterministic VRAM
+// ceiling and footprint can be injected without a real GPU or model file — the
+// same pattern hardware_defaults_internal_test.go uses for localGPU.
+var _ = Describe("Auto-derived default context (VRAM-aware cap)", func() {
+	const gib = uint64(1) << 30
+
+	var (
+		origVRAM     func() uint64
+		origEstimate func(f *gguf.GGUFFile, ctx int) uint64
+	)
+
+	BeforeEach(func() {
+		origVRAM = perDeviceVRAM
+		origEstimate = estimateContextVRAM
+	})
+	AfterEach(func() {
+		perDeviceVRAM = origVRAM
+		estimateContextVRAM = origEstimate
+	})
+
+	Context("autoContextSize", func() {
+		It("caps a long-context model at DefaultAutoContextSize when VRAM is ample", func() {
+			// 1M-context model on an 80 GiB card: we do NOT chase the trained max,
+			// we keep the conservative 8k cap (users opt into more via context_size).
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib } // trivially fits
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+
+		It("keeps a small model's trained window instead of inflating it", func() {
+			// trained 4096 < 8192: min() keeps 4096, it is not raised to the cap.
+			perDeviceVRAM = func() uint64 { return 80 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return gib }
+			Expect(autoContextSize(nil, 4096)).To(Equal(4096))
+		})
+
+		It("steps below the cap when even 8k would not fit a tiny card", func() {
+			// A large model on a 2 GiB card where the 8k footprint overflows but a
+			// smaller context fits: choose the largest that fits, never below the
+			// floor. Footprint grows with context so the walk finds a fit.
+			perDeviceVRAM = func() uint64 { return 2 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, ctx int) uint64 {
+				return gib + uint64(ctx)*100000
+			}
+			chosen := autoContextSize(nil, 1048576)
+			Expect(chosen).To(BeNumerically("<", DefaultAutoContextSize))
+			Expect(chosen).To(BeNumerically(">=", DefaultContextSize))
+			// The chosen context's footprint must actually fit the card with headroom.
+			Expect(contextFitsVRAM(estimateContextVRAM(nil, chosen), 2*gib)).To(BeTrue())
+		})
+
+		It("falls back to the floor when nothing fits", func() {
+			// Even DefaultContextSize does not fit: return the floor and let the
+			// backend clamp n_gpu_layers to what it can (partial offload) rather
+			// than defaulting to a window guaranteed to abort.
+			perDeviceVRAM = func() uint64 { return 1 * gib }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 100 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultContextSize))
+		})
+
+		It("does not clamp when per-device VRAM is unknown", func() {
+			// CPU-only / detection gap: no GPU budget to reason about, so we must
+			// not regress — keep the conservative base cap regardless of estimate.
+			perDeviceVRAM = func() uint64 { return 0 }
+			estimateContextVRAM = func(_ *gguf.GGUFFile, _ int) uint64 { return 999 * gib }
+			Expect(autoContextSize(nil, 1048576)).To(Equal(DefaultAutoContextSize))
+		})
+	})
+
+	Context("guessGGUFFromFile", func() {
+		It("never overrides an explicitly configured context_size", func() {
+			// A fabricated GGUF is enough: the context branch is skipped entirely
+			// when the user pinned context_size, so the estimate is never consulted.
+			explicit := 262144
+			cfg := &ModelConfig{LLMConfig: LLMConfig{ContextSize: &explicit}}
+			f := &gguf.GGUFFile{
+				Header: gguf.GGUFHeader{
+					MetadataKV: gguf.GGUFMetadataKVs{
+						{
+							Key:       "general.architecture",
+							ValueType: gguf.GGUFMetadataValueTypeString,
+							Value:     "llama",
+						},
+					},
+				},
+			}
+			guessGGUFFromFile(cfg, f, 0)
+			Expect(cfg.ContextSize).ToNot(BeNil())
+			Expect(*cfg.ContextSize).To(Equal(262144))
+		})
+	})
+})

core/config/defaults.go

@@ -18,6 +18,18 @@ const (
 	// safe default beats a tiny, surprising window that truncates real prompts.
 	DefaultContextSize = 4096
 
+	// DefaultAutoContextSize caps the context we auto-derive from a GGUF when the
+	// user did not set context_size. The GGUF importer used to default a model's
+	// context to its full trained window (n_ctx_train). For long-context models
+	// (128k / 256k / 1M) that KV cache cannot fit a consumer GPU and the backend
+	// aborts on load (exitCode=-1) even though the model file is fine. So instead
+	// of shooting for the trained max, we keep a modest default: a small model
+	// (trained < this) keeps its trained window, while a long-context model caps
+	// here. Users who want the full window raise context_size explicitly. This is
+	// a conservative default, not a VRAM-maximizing one — VRAM is only used to
+	// step further DOWN when even this cap would not fit (see context_fit.go).
+	DefaultAutoContextSize = 8192
+
 	// DefaultNGPULayers means "offload all layers"; the backend (fit_params)
 	// clamps to what actually fits in device memory.
 	DefaultNGPULayers = 99999999

core/config/gguf.go

@@ -27,10 +27,33 @@ func reservedNonChatModel(cfg *ModelConfig) bool {
 }
 
 func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
+	// Explicit opt-in: a negative context_size (canonically -1) means "use the
+	// model's full trained context (n_ctx_train) from GGUF metadata". Unlike the
+	// silent unset path below, this overrides an already-present value and warns
+	// when the resolved window will not fit detected VRAM.
+	if cfg.ContextSize != nil && *cfg.ContextSize < 0 {
+		if maxCtx := int(f.Architecture().MaximumContextLength); maxCtx > 0 {
+			cfg.ContextSize = &maxCtx
+			warnIfContextExceedsVRAM(f, maxCtx, f.Metadata().Name)
+		} else {
+			// No usable trained max in metadata: degrade to the safe default
+			// rather than leak a negative n_ctx downstream.
+			d := DefaultContextSize
+			cfg.ContextSize = &d
+			xlog.Warn("[gguf] context_size=-1 requested but GGUF exposes no trained max; using default",
+				"default", d, "model", f.Metadata().Name)
+		}
+	}
+
 	if defaultCtx == 0 && cfg.ContextSize == nil {
-		ctxSize := f.EstimateLLaMACppRun().ContextSize
-		if ctxSize > 0 {
-			cSize := int(ctxSize)
+		// trainedMax is the model's full trained context window (n_ctx_train).
+		// Defaulting a model to it unbounded is what OOMs long-context models at
+		// load: a 128k / 256k / 1M KV cache cannot fit a consumer GPU and the
+		// backend aborts (exitCode=-1). autoContextSize instead caps to a modest
+		// default and only steps below it when detected per-device VRAM demands.
+		trainedMax := int(f.EstimateLLaMACppRun().ContextSize)
+		if trainedMax > 0 {
+			cSize := autoContextSize(f, trainedMax)
 			cfg.ContextSize = &cSize
 		} else {
 			defaultCtx = DefaultContextSize
@@ -220,3 +243,35 @@ func applyDetectedThinkingConfig(cfg *ModelConfig, metadata *pb.ModelMetadataRes
 
 	xlog.Debug("[gguf] DetectThinkingSupportFromBackend: preserving explicit reasoning config", "supports_thinking", metadata.SupportsThinking, "disable_reasoning", *cfg.ReasoningConfig.DisableReasoning, "disable_reasoning_tag_prefill", *cfg.ReasoningConfig.DisableReasoningTagPrefill)
 }
+
+// warnIfContextExceedsVRAM logs a best-effort warning when running the model at
+// the given context would not fit detected VRAM. It never blocks load: any
+// detection or estimation gap (no GPU, unknown VRAM, estimate failure) silently
+// skips the warning. Used by the context_size=-1 auto-max path, where the raw
+// trained max can be far larger than a consumer card holds.
+func warnIfContextExceedsVRAM(f *gguf.GGUFFile, ctx int, name string) {
+	defer func() { _ = recover() }() // the run estimate can panic on unusual headers
+
+	if !xsysinfo.HasGPU("nvidia") && !xsysinfo.HasGPU("amd") {
+		return // no VRAM to compare against
+	}
+	vram, err := xsysinfo.TotalAvailableVRAM()
+	if err != nil || vram == 0 {
+		return
+	}
+
+	sum := f.EstimateLLaMACppRun(gguf.WithLLaMACppContextSize(int32(ctx))).Summarize(true, 0, 0)
+	if len(sum.Items) == 0 {
+		return
+	}
+	var used uint64
+	for _, v := range sum.Items[0].VRAMs {
+		used += uint64(v.NonUMA)
+	}
+	if used == 0 || used <= vram {
+		return
+	}
+	xlog.Warn("[gguf] context_size=-1 resolved to the model's trained max; estimated VRAM may exceed available - expect OOM, or set an explicit context_size",
+		"model", name, "context", ctx,
+		"estimated_vram_gib", used>>30, "available_vram_gib", vram>>30)
+}

core/config/hardware_defaults.go

@@ -149,6 +149,51 @@ func largeContextForDevice(g GPU, ctx int) bool {
 	return extra > g.VRAM/blackwellBatchHeadroomDivisor
 }
 
+// SinglePassBatchForContext caps the physical batch (n_batch / n_ubatch) for a
+// single-pass load — embedding, score and rerank all decode/pool the whole input
+// in ONE physical batch, so they want a batch >= the input length to avoid the
+// GGML_ASSERT(n_tokens <= n_batch) abort and the "input is too large to process"
+// error. The naive choice is batch == context, but n_ubatch == context turns the
+// per-device CUDA compute buffer (which scales ~ n_ubatch * n_ctx and is NOT
+// split across GPUs) into multi-GiB of scratch that must fit on a SINGLE card, so
+// a large-context embedding model aborts on load (exitCode=-1) even with plenty
+// of free VRAM — the same #10485 root cause the Blackwell batch boost guards
+// against, which the single-pass path previously bypassed entirely.
+//
+// So instead of the full context we return the LARGEST batch whose compute buffer
+// fits the per-device VRAM headroom (VRAM / blackwellBatchHeadroomDivisor),
+// clamped to [DefaultPhysicalBatch, ctx]. The tradeoff: an input longer than the
+// returned cap can no longer be pooled in a single pass — but a batch that OOMs
+// the device processes nothing at all.
+//
+// g.VRAM must be the PER-DEVICE ceiling (smallest device on a multi-GPU host).
+// VRAM 0 (unknown — CPU-only or a detection gap) returns the full context,
+// preserving the original single-pass behavior (batch follows context): the cap
+// is a DOWNWARD safety that only engages when the per-device ceiling is known.
+// Returning a smaller batch on unknown VRAM would re-break single-pass pooling
+// (n_tokens > n_batch) and over-trim score/embed/rerank inputs, with no OOM
+// benefit on CPU where the buffer lives in system RAM.
+func SinglePassBatchForContext(g GPU, ctx int) int {
+	if ctx <= DefaultPhysicalBatch {
+		return DefaultPhysicalBatch
+	}
+	if g.VRAM == 0 {
+		return ctx
+	}
+	perBatchCell := uint64(ctx) * computeBufferBytesPerCell
+	if perBatchCell == 0 {
+		return DefaultPhysicalBatch
+	}
+	batchCap := int(g.VRAM / blackwellBatchHeadroomDivisor / perBatchCell)
+	if batchCap < DefaultPhysicalBatch {
+		return DefaultPhysicalBatch
+	}
+	if batchCap > ctx {
+		return ctx
+	}
+	return batchCap
+}
+
 // IsManagedPhysicalBatch reports whether n is a value PhysicalBatch assigns.
 // Callers that re-tune a value chosen by an upstream host (the distributed
 // router correcting the frontend's guess) use this to avoid clobbering an
@@ -254,6 +299,14 @@ var localGPU = func() GPU {
 	}
 }
 
+// LocalGPU exposes the locally-detected device descriptor to other packages
+// (e.g. core/backend's single-pass batch sizing) so they resolve the same
+// per-device VRAM this package's heuristics reason about. It goes through the
+// injectable localGPU var, so a config-package test seam also affects callers.
+func LocalGPU() GPU {
+	return localGPU()
+}
+
 // ApplyHardwareDefaults fills ModelConfig values that depend on the target GPU
 // and were left unset by the user. Currently: a larger physical batch on
 // Blackwell. Explicit config always wins (we only touch zero values).
@@ -283,7 +336,12 @@ func ApplyHardwareDefaults(cfg *ModelConfig, gpu GPU) {
 	// context budget, but a context large enough to fill a single device leaves
 	// no room for the per-slot scratch, so the slot count is gated on per-device
 	// headroom too (issue #10485). Explicit parallel/n_parallel always wins.
-	if before := len(cfg.Options); true {
+	//
+	// parallel is a llama.cpp option string; skip it for backends that would
+	// reject an unknown option (e.g. longcat-video). The typed batch above is a
+	// proto field every backend simply ignores, so it needs no such gate.
+	if UsesLlamaCppServingOptions(cfg.Backend) {
+		before := len(cfg.Options)
 		cfg.Options = EnsureParallelOptionForContext(cfg.Options, gpu, ctx)
 		if len(cfg.Options) > before {
 			xlog.Debug("[hardware_defaults] defaulting parallel slots for concurrent serving",