ci(vllm): disable tests-vllm-grpc job (heterogeneous runners)

Both ubuntu-latest and bigger-runner have inconsistent CPU baselines:
some instances support the AVX-512 VNNI/BF16 instructions the prebuilt
vllm 0.14.1+cpu wheel was compiled with, others SIGILL on import of
vllm.model_executor.models.registry. The libnuma packaging fix doesn't
help when the wheel itself can't be loaded.

FROM_SOURCE=true compiles vllm against the actual host CPU and works
everywhere, but takes 30-50 minutes per run — too slow for a smoke
test on every PR.

Comment out the job for now. The test itself is intact and passes
locally; run it via 'make test-extra-backend-vllm' on a host with the
required SIMD baseline. Re-enable when:
  - we have a self-hosted runner label with guaranteed AVX-512 VNNI/BF16, or
  - vllm publishes a CPU wheel with a wider baseline, or
  - we set up a docker layer cache that makes FROM_SOURCE acceptable

The detect-changes vllm output, the test harness changes (tests/
e2e-backends + tools cap), the make target (test-extra-backend-vllm),
the package.sh and the Dockerfile/install.sh plumbing all stay in
place.

.agents/adding-backends.md

@@ -28,7 +28,7 @@ Add build matrix entries for each platform/GPU type you want to support. Look at
 - CUDA 13 builds: Add after other CUDA 13 builds (e.g., after `gpu-nvidia-cuda-13-chatterbox`)
 
 **Additional build types you may need:**
-- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:6.4.4"`
+- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.2.1"`
 - Intel/SYCL: Use `build-type: 'intel'` or `build-type: 'sycl_f16'`/`sycl_f32` with `base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"`
 - L4T (ARM): Use `build-type: 'l4t'` with `platforms: 'linux/arm64'` and `runs-on: 'ubuntu-24.04-arm'`
 

.agents/building-and-testing.md

@@ -10,7 +10,7 @@ Let's say the user wants to build a particular backend for a given platform. For
 - At a minimum we need to set the BUILD_TYPE, BASE_IMAGE build-args
   - Use .github/workflows/backend.yml as a reference it lists the needed args in the `include` job strategy matrix
   - l4t and cublas also requires the CUDA major and minor version
-- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:6.4.4 make docker-build-coqui`
+- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.2.1 make docker-build-coqui`
 - Unless the user specifies that they want you to run the command, then just print it because not all agent frontends handle long running jobs well and the output may overflow your context
 - The user may say they want to build AMD or ROCM instead of hipblas, or Intel instead of SYCL or NVIDIA insted of l4t or cublas. Ask for confirmation if there is ambiguity.
 - Sometimes the user may need extra parameters to be added to `docker build` (e.g. `--platform` for cross-platform builds or `--progress` to view the full logs), in which case you can generate the `docker build` command directly.

.github/workflows/backend.yml

@@ -53,6 +53,19 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2204'
+          - build-type: ''
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-cpu-vllm'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'true'
+            backend: "vllm"
+            dockerfile: "./backend/Dockerfile.python"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: ''
             cuda-major-version: ""
             cuda-minor-version: ""
@@ -613,6 +626,19 @@ jobs:
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "12"
+            cuda-minor-version: "8"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-12-qwen3-tts-cpp'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "8"
@@ -1225,6 +1251,19 @@ jobs:
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-nvidia-cuda-13-qwen3-tts-cpp'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "13"
             cuda-minor-version: "0"
@@ -1238,6 +1277,19 @@ jobs:
             backend: "acestep-cpp"
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
+          - build-type: 'cublas'
+            cuda-major-version: "13"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            skip-drivers: 'false'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp'
+            base-image: "ubuntu:24.04"
+            ubuntu-version: '2404'
+            runs-on: 'ubuntu-24.04-arm'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
           - build-type: 'cublas'
             cuda-major-version: "13"
             cuda-minor-version: "0"
@@ -1259,7 +1311,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-rerankers'
             runs-on: 'ubuntu-latest'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "rerankers"
             dockerfile: "./backend/Dockerfile.python"
@@ -1272,7 +1324,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-llama-cpp'
             runs-on: 'ubuntu-latest'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "llama-cpp"
             dockerfile: "./backend/Dockerfile.llama-cpp"
@@ -1285,7 +1337,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-vllm'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "vllm"
             dockerfile: "./backend/Dockerfile.python"
@@ -1298,7 +1350,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-vllm-omni'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "vllm-omni"
             dockerfile: "./backend/Dockerfile.python"
@@ -1311,7 +1363,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-transformers'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "transformers"
             dockerfile: "./backend/Dockerfile.python"
@@ -1324,7 +1376,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-diffusers'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "diffusers"
             dockerfile: "./backend/Dockerfile.python"
@@ -1337,7 +1389,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-ace-step'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "ace-step"
             dockerfile: "./backend/Dockerfile.python"
@@ -1351,7 +1403,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-kokoro'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "kokoro"
             dockerfile: "./backend/Dockerfile.python"
@@ -1364,7 +1416,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-vibevoice'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "vibevoice"
             dockerfile: "./backend/Dockerfile.python"
@@ -1377,7 +1429,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-qwen-asr'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "qwen-asr"
             dockerfile: "./backend/Dockerfile.python"
@@ -1390,7 +1442,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-nemo'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "nemo"
             dockerfile: "./backend/Dockerfile.python"
@@ -1403,7 +1455,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-qwen-tts'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "qwen-tts"
             dockerfile: "./backend/Dockerfile.python"
@@ -1416,7 +1468,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-fish-speech'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "fish-speech"
             dockerfile: "./backend/Dockerfile.python"
@@ -1429,7 +1481,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-voxcpm'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "voxcpm"
             dockerfile: "./backend/Dockerfile.python"
@@ -1442,7 +1494,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-pocket-tts'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "pocket-tts"
             dockerfile: "./backend/Dockerfile.python"
@@ -1455,7 +1507,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-faster-whisper'
             runs-on: 'bigger-runner'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "faster-whisper"
             dockerfile: "./backend/Dockerfile.python"
@@ -1468,7 +1520,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-whisperx'
             runs-on: 'bigger-runner'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "whisperx"
             dockerfile: "./backend/Dockerfile.python"
@@ -1481,7 +1533,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-coqui'
             runs-on: 'bigger-runner'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "coqui"
             dockerfile: "./backend/Dockerfile.python"
@@ -1906,6 +1958,19 @@ jobs:
             dockerfile: "./backend/Dockerfile.llama-cpp"
             context: "./"
             ubuntu-version: '2404'
+          - build-type: ''
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-cpu-ik-llama-cpp'
+            runs-on: 'bigger-runner'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "ik-llama-cpp"
+            dockerfile: "./backend/Dockerfile.ik-llama-cpp"
+            context: "./"
+            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "0"
@@ -2136,7 +2201,7 @@ jobs:
             platforms: 'linux/amd64'
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-whisper'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             runs-on: 'ubuntu-latest'
             skip-drivers: 'false'
             backend: "whisper"
@@ -2215,13 +2280,92 @@ jobs:
             platforms: 'linux/amd64'
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-acestep-cpp'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             runs-on: 'ubuntu-latest'
             skip-drivers: 'false'
             backend: "acestep-cpp"
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
             ubuntu-version: '2404'
+          # qwen3-tts-cpp
+          - build-type: ''
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64,linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-cpu-qwen3-tts-cpp'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'sycl_f32'
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-intel-sycl-f32-qwen3-tts-cpp'
+            runs-on: 'ubuntu-latest'
+            base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+            skip-drivers: 'false'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'sycl_f16'
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-intel-sycl-f16-qwen3-tts-cpp'
+            runs-on: 'ubuntu-latest'
+            base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+            skip-drivers: 'false'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'vulkan'
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64,linux/arm64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-vulkan-qwen3-tts-cpp'
+            runs-on: 'ubuntu-latest'
+            base-image: "ubuntu:24.04"
+            skip-drivers: 'false'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2404'
+          - build-type: 'cublas'
+            cuda-major-version: "12"
+            cuda-minor-version: "0"
+            platforms: 'linux/arm64'
+            skip-drivers: 'false'
+            tag-latest: 'auto'
+            tag-suffix: '-nvidia-l4t-arm64-qwen3-tts-cpp'
+            base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+            runs-on: 'ubuntu-24.04-arm'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2204'
+          - build-type: 'hipblas'
+            cuda-major-version: ""
+            cuda-minor-version: ""
+            platforms: 'linux/amd64'
+            tag-latest: 'auto'
+            tag-suffix: '-gpu-rocm-hipblas-qwen3-tts-cpp'
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            runs-on: 'ubuntu-latest'
+            skip-drivers: 'false'
+            backend: "qwen3-tts-cpp"
+            dockerfile: "./backend/Dockerfile.golang"
+            context: "./"
+            ubuntu-version: '2404'
           # voxtral
           - build-type: ''
             cuda-major-version: ""
@@ -2338,7 +2482,7 @@ jobs:
           #   platforms: 'linux/amd64'
           #   tag-latest: 'auto'
           #   tag-suffix: '-gpu-hipblas-rfdetr'
-          #   base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+          #   base-image: "rocm/dev-ubuntu-24.04:7.2.1"
           #   runs-on: 'ubuntu-latest'
           #   skip-drivers: 'false'
           #   backend: "rfdetr"
@@ -2379,7 +2523,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-neutts'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
             skip-drivers: 'false'
             backend: "neutts"
             dockerfile: "./backend/Dockerfile.python"
@@ -2527,6 +2671,10 @@ jobs:
             tag-suffix: "-metal-darwin-arm64-acestep-cpp"
             build-type: "metal"
             lang: "go"
+          - backend: "qwen3-tts-cpp"
+            tag-suffix: "-metal-darwin-arm64-qwen3-tts-cpp"
+            build-type: "metal"
+            lang: "go"
           - backend: "voxtral"
             tag-suffix: "-metal-darwin-arm64-voxtral"
             build-type: "metal"

.github/workflows/bump_deps.yaml

@@ -14,6 +14,10 @@ jobs:
             variable: "LLAMA_VERSION"
             branch: "master"
             file: "backend/cpp/llama-cpp/Makefile"
+          - repository: "ikawrakow/ik_llama.cpp"
+            variable: "IK_LLAMA_VERSION"
+            branch: "main"
+            file: "backend/cpp/ik-llama-cpp/Makefile"
           - repository: "ggml-org/whisper.cpp"
             variable: "WHISPER_CPP_VERSION"
             branch: "master"
@@ -38,6 +42,10 @@ jobs:
             variable: "SAM3_VERSION"
             branch: "main"
             file: "backend/go/sam3-cpp/Makefile"
+          - repository: "predict-woo/qwen3-tts.cpp"
+            variable: "QWEN3TTS_CPP_VERSION"
+            branch: "main"
+            file: "backend/go/qwen3-tts-cpp/Makefile"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6

.github/workflows/image-pr.yml

@@ -59,7 +59,7 @@
               platforms: 'linux/amd64'
               tag-latest: 'false'
               tag-suffix: '-hipblas'
-              base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+              base-image: "rocm/dev-ubuntu-24.04:7.2.1"
               grpc-base-image: "ubuntu:24.04"
               runs-on: 'ubuntu-latest'
               makeflags: "--jobs=3 --output-sync=target"

.github/workflows/image.yml

@@ -41,7 +41,7 @@
               platforms: 'linux/amd64'
               tag-latest: 'auto'
               tag-suffix: '-gpu-hipblas'
-              base-image: "rocm/dev-ubuntu-24.04:6.4.4"
+              base-image: "rocm/dev-ubuntu-24.04:7.2.1"
               grpc-base-image: "ubuntu:24.04"
               runs-on: 'ubuntu-latest'
               makeflags: "--jobs=3 --output-sync=target"

.github/workflows/test-extra.yml

@@ -29,7 +29,11 @@ jobs:
       nemo: ${{ steps.detect.outputs.nemo }}
       voxcpm: ${{ steps.detect.outputs.voxcpm }}
       llama-cpp-quantization: ${{ steps.detect.outputs.llama-cpp-quantization }}
+      llama-cpp: ${{ steps.detect.outputs.llama-cpp }}
+      ik-llama-cpp: ${{ steps.detect.outputs.ik-llama-cpp }}
+      vllm: ${{ steps.detect.outputs.vllm }}
       acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
+      qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }}
       voxtral: ${{ steps.detect.outputs.voxtral }}
       kokoros: ${{ steps.detect.outputs.kokoros }}
     steps:
@@ -464,6 +468,86 @@ jobs:
       - name: Test llama-cpp-quantization
         run: |
           make --jobs=5 --output-sync=target -C backend/python/llama-cpp-quantization test
+  tests-llama-cpp-grpc:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.llama-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.25.4'
+      - name: Build llama-cpp backend image and run gRPC e2e tests
+        run: |
+          make test-extra-backend-llama-cpp
+  tests-ik-llama-cpp-grpc:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.ik-llama-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.25.4'
+      - name: Build ik-llama-cpp backend image and run gRPC e2e tests
+        run: |
+          make test-extra-backend-ik-llama-cpp
+  # tests-vllm-grpc is currently disabled in CI.
+  #
+  # The prebuilt vllm CPU wheel is compiled with AVX-512 VNNI/BF16
+  # instructions, and neither ubuntu-latest nor the bigger-runner pool
+  # offers a stable CPU baseline that supports them — runners come
+  # back with different hardware between runs and SIGILL on import of
+  # vllm.model_executor.models.registry. Compiling vllm from source
+  # via FROM_SOURCE=true works on any CPU but takes 30-50 minutes per
+  # run, which is too slow for a smoke test.
+  #
+  # The test itself (tests/e2e-backends + make test-extra-backend-vllm)
+  # is fully working and validated locally on a host with the right
+  # SIMD baseline. Run it manually with:
+  #
+  #   make test-extra-backend-vllm
+  #
+  # Re-enable this job once we have a self-hosted runner label with
+  # guaranteed AVX-512 VNNI/BF16 support, or once the vllm project
+  # publishes a CPU wheel with a wider baseline.
+  #
+  # tests-vllm-grpc:
+  #   needs: detect-changes
+  #   if: needs.detect-changes.outputs.vllm == 'true' || needs.detect-changes.outputs.run-all == 'true'
+  #   runs-on: bigger-runner
+  #   timeout-minutes: 90
+  #   steps:
+  #     - name: Clone
+  #       uses: actions/checkout@v6
+  #       with:
+  #         submodules: true
+  #     - name: Dependencies
+  #       run: |
+  #         sudo apt-get update
+  #         sudo apt-get install -y --no-install-recommends \
+  #             make build-essential curl unzip ca-certificates git tar
+  #     - name: Setup Go
+  #       uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '1.25.4'
+  #     - name: Free disk space
+  #       run: |
+  #         sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL || true
+  #         df -h
+  #     - name: Build vllm (cpu) backend image and run gRPC e2e tests
+  #       run: |
+  #         make test-extra-backend-vllm
   tests-acestep-cpp:
     needs: detect-changes
     if: needs.detect-changes.outputs.acestep-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
@@ -496,6 +580,38 @@ jobs:
       - name: Test acestep-cpp
         run: |
           make --jobs=5 --output-sync=target -C backend/go/acestep-cpp test
+  tests-qwen3-tts-cpp:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.qwen3-tts-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake curl libopenblas-dev ffmpeg
+      - name: Setup Go
+        uses: actions/setup-go@v5
+      - name: Display Go version
+        run: go version
+      - name: Proto Dependencies
+        run: |
+          # Install protoc
+          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
+          unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
+          rm protoc.zip
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
+          PATH="$PATH:$HOME/go/bin" make protogen-go
+      - name: Build qwen3-tts-cpp
+        run: |
+          make --jobs=5 --output-sync=target -C backend/go/qwen3-tts-cpp
+      - name: Test qwen3-tts-cpp
+        run: |
+          make --jobs=5 --output-sync=target -C backend/go/qwen3-tts-cpp test
   tests-voxtral:
     needs: detect-changes
     if: needs.detect-changes.outputs.voxtral == 'true' || needs.detect-changes.outputs.run-all == 'true'

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -456,6 +456,69 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/trl test
 	$(MAKE) -C backend/rust/kokoros test
 
+##
+## End-to-end gRPC tests that exercise a built backend container image.
+##
+## The test suite in tests/e2e-backends is backend-agnostic. You drive it via env
+## vars (see tests/e2e-backends/backend_test.go for the full list) and the
+## capability-driven harness picks which gRPC RPCs to exercise:
+##
+##   BACKEND_IMAGE            Required. Docker image to test, e.g. local-ai-backend:llama-cpp.
+##   BACKEND_TEST_MODEL_URL   URL of a model file to download and load.
+##   BACKEND_TEST_MODEL_FILE  Path to an already-downloaded model (skips download).
+##   BACKEND_TEST_MODEL_NAME  HuggingFace repo id (e.g. Qwen/Qwen2.5-0.5B-Instruct).
+##                            Use this instead of MODEL_URL for backends that
+##                            resolve HF model ids natively (vllm, vllm-omni).
+##   BACKEND_TEST_CAPS        Comma-separated capabilities, default "health,load,predict,stream".
+##                            Adds "tools" to exercise ChatDelta tool call extraction.
+##   BACKEND_TEST_PROMPT      Override the prompt used in predict/stream specs.
+##   BACKEND_TEST_OPTIONS     Comma-separated Options[] entries forwarded to LoadModel,
+##                            e.g. "tool_parser:hermes,reasoning_parser:qwen3".
+##
+## Direct usage (image already built, no docker-build-* dependency):
+##
+##   make test-extra-backend BACKEND_IMAGE=local-ai-backend:llama-cpp \
+##       BACKEND_TEST_MODEL_URL=https://.../model.gguf
+##
+## Convenience wrappers below build a specific backend image first, then run the
+## suite against it.
+##
+BACKEND_TEST_MODEL_URL?=https://huggingface.co/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf
+
+## Generic target — runs the suite against whatever BACKEND_IMAGE points at.
+## Depends on protogen-go so pkg/grpc/proto is generated before `go test`.
+test-extra-backend: protogen-go
+	@test -n "$$BACKEND_IMAGE" || { echo "BACKEND_IMAGE must be set" >&2; exit 1; }
+	BACKEND_IMAGE="$$BACKEND_IMAGE" \
+	BACKEND_TEST_MODEL_URL="$${BACKEND_TEST_MODEL_URL:-$(BACKEND_TEST_MODEL_URL)}" \
+	BACKEND_TEST_MODEL_FILE="$$BACKEND_TEST_MODEL_FILE" \
+	BACKEND_TEST_MODEL_NAME="$$BACKEND_TEST_MODEL_NAME" \
+	BACKEND_TEST_CAPS="$$BACKEND_TEST_CAPS" \
+	BACKEND_TEST_PROMPT="$$BACKEND_TEST_PROMPT" \
+	BACKEND_TEST_OPTIONS="$$BACKEND_TEST_OPTIONS" \
+	BACKEND_TEST_TOOL_PROMPT="$$BACKEND_TEST_TOOL_PROMPT" \
+	BACKEND_TEST_TOOL_NAME="$$BACKEND_TEST_TOOL_NAME" \
+	go test -v -timeout 30m ./tests/e2e-backends/...
+
+## Convenience wrappers: build the image, then exercise it.
+test-extra-backend-llama-cpp: docker-build-llama-cpp
+	BACKEND_IMAGE=local-ai-backend:llama-cpp $(MAKE) test-extra-backend
+
+test-extra-backend-ik-llama-cpp: docker-build-ik-llama-cpp
+	BACKEND_IMAGE=local-ai-backend:ik-llama-cpp $(MAKE) test-extra-backend
+
+## vllm is resolved from a HuggingFace model id (no file download) and
+## exercises Predict + streaming + tool-call extraction via the hermes parser.
+## Requires a host CPU with the SIMD instructions the prebuilt vllm CPU
+## wheel was compiled against (AVX-512 VNNI/BF16); older CPUs will SIGILL
+## on import — on CI this means using the bigger-runner label.
+test-extra-backend-vllm: docker-build-vllm
+	BACKEND_IMAGE=local-ai-backend:vllm \
+	BACKEND_TEST_MODEL_NAME=Qwen/Qwen2.5-0.5B-Instruct \
+	BACKEND_TEST_CAPS=health,load,predict,stream,tools \
+	BACKEND_TEST_OPTIONS=tool_parser:hermes \
+	$(MAKE) test-extra-backend
+
 DOCKER_IMAGE?=local-ai
 IMAGE_TYPE?=core
 BASE_IMAGE?=ubuntu:24.04
@@ -549,6 +612,8 @@ backend-images:
 # Backend metadata: BACKEND_NAME | DOCKERFILE_TYPE | BUILD_CONTEXT | PROGRESS_FLAG | NEEDS_BACKEND_ARG
 # llama-cpp is special - uses llama-cpp Dockerfile and doesn't need BACKEND arg
 BACKEND_LLAMA_CPP = llama-cpp|llama-cpp|.|false|false
+# ik-llama-cpp is a fork of llama.cpp with superior CPU performance
+BACKEND_IK_LLAMA_CPP = ik-llama-cpp|ik-llama-cpp|.|false|false
 
 # Golang backends
 BACKEND_PIPER = piper|golang|.|false|true
@@ -559,6 +624,7 @@ BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|tr
 BACKEND_WHISPER = whisper|golang|.|false|true
 BACKEND_VOXTRAL = voxtral|golang|.|false|true
 BACKEND_ACESTEP_CPP = acestep-cpp|golang|.|false|true
+BACKEND_QWEN3_TTS_CPP = qwen3-tts-cpp|golang|.|false|true
 BACKEND_OPUS = opus|golang|.|false|true
 
 # Python backends with root context
@@ -606,6 +672,7 @@ define docker-build-backend
 		--build-arg CUDA_MINOR_VERSION=$(CUDA_MINOR_VERSION) \
 		--build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) \
 		--build-arg UBUNTU_CODENAME=$(UBUNTU_CODENAME) \
+		$(if $(FROM_SOURCE),--build-arg FROM_SOURCE=$(FROM_SOURCE)) \
 		$(if $(filter true,$(5)),--build-arg BACKEND=$(1)) \
 		-t local-ai-backend:$(1) -f backend/Dockerfile.$(2) $(3)
 endef
@@ -618,6 +685,7 @@ endef
 
 # Generate all docker-build targets
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
+$(eval $(call generate-docker-build-target,$(BACKEND_IK_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PIPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE)))
@@ -651,6 +719,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
 $(eval $(call generate-docker-build-target,$(BACKEND_ACE_STEP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_ACESTEP_CPP)))
+$(eval $(call generate-docker-build-target,$(BACKEND_QWEN3_TTS_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_MLX_DISTRIBUTED)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRL)))
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP_QUANTIZATION)))
@@ -661,7 +730,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SAM3_CPP)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp
 
 ########################################################
 ### Mock Backend for E2E Tests

README.md

@@ -32,7 +32,7 @@
 **LocalAI** is the open-source AI engine. Run any model - LLMs, vision, voice, image, video - on any hardware. No GPU required.
 
 - **Drop-in API compatibility** — OpenAI, Anthropic, ElevenLabs APIs
-- **35+ backends** — llama.cpp, vLLM, transformers, whisper, diffusers, MLX...
+- **36+ backends** — llama.cpp, vLLM, transformers, whisper, diffusers, MLX...
 - **Any hardware** — NVIDIA, AMD, Intel, Apple Silicon, Vulkan, or CPU-only
 - **Multi-user ready** — API key auth, user quotas, role-based access
 - **Built-in AI agents** — autonomous agents with tool use, RAG, MCP, and skills
@@ -185,7 +185,7 @@ For older news and full release notes, see [GitHub Releases](https://github.com/
 
 ## Supported Backends & Acceleration
 
-LocalAI supports **35+ backends** including llama.cpp, vLLM, transformers, whisper.cpp, diffusers, MLX, MLX-VLM, and many more. Hardware acceleration is available for **NVIDIA** (CUDA 12/13), **AMD** (ROCm), **Intel** (oneAPI/SYCL), **Apple Silicon** (Metal), **Vulkan**, and **NVIDIA Jetson** (L4T). All backends can be installed on-the-fly from the [Backend Gallery](https://localai.io/backends/).
+LocalAI supports **36+ backends** including llama.cpp, vLLM, transformers, whisper.cpp, diffusers, MLX, MLX-VLM, and many more. Hardware acceleration is available for **NVIDIA** (CUDA 12/13), **AMD** (ROCm), **Intel** (oneAPI/SYCL), **Apple Silicon** (Metal), **Vulkan**, and **NVIDIA Jetson** (L4T). All backends can be installed on-the-fly from the [Backend Gallery](https://localai.io/backends/).
 
 See the full [Backend & Model Compatibility Table](https://localai.io/model-compatibility/) and [GPU Acceleration guide](https://localai.io/features/gpu-acceleration/).
 

backend/Dockerfile.ik-llama-cpp

@@ -0,0 +1,281 @@
+ARG BASE_IMAGE=ubuntu:24.04
+ARG GRPC_BASE_IMAGE=${BASE_IMAGE}
+
+
+# The grpc target does one thing, it builds and installs GRPC.  This is in it's own layer so that it can be effectively cached by CI.
+# You probably don't need to change anything here, and if you do, make sure that CI is adjusted so that the cache continues to work.
+FROM ${GRPC_BASE_IMAGE} AS grpc
+
+# This is a bit of a hack, but it's required in order to be able to effectively cache this layer in CI
+ARG GRPC_MAKEFLAGS="-j4 -Otarget"
+ARG GRPC_VERSION=v1.65.0
+ARG CMAKE_FROM_SOURCE=false
+# CUDA Toolkit 13.x compatibility: CMake 3.31.9+ fixes toolchain detection/arch table issues
+ARG CMAKE_VERSION=3.31.10
+
+ENV MAKEFLAGS=${GRPC_MAKEFLAGS}
+
+WORKDIR /build
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        ca-certificates \
+        build-essential curl libssl-dev \
+        git wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install CMake (the version in 22.04 is too old)
+RUN <<EOT bash
+    if [ "${CMAKE_FROM_SOURCE}" = "true" ]; then
+        curl -L -s https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -o cmake.tar.gz && tar xvf cmake.tar.gz && cd cmake-${CMAKE_VERSION} && ./configure && make && make install
+    else
+        apt-get update && \
+        apt-get install -y \
+            cmake && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+    fi
+EOT
+
+# We install GRPC to a different prefix here so that we can copy in only the build artifacts later
+# saves several hundred MB on the final docker image size vs copying in the entire GRPC source tree
+# and running make install in the target container
+RUN git clone --recurse-submodules --jobs 4 -b ${GRPC_VERSION} --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
+    mkdir -p /build/grpc/cmake/build && \
+    cd /build/grpc/cmake/build && \
+    sed -i "216i\  TESTONLY" "../../third_party/abseil-cpp/absl/container/CMakeLists.txt" && \
+    cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX:PATH=/opt/grpc ../.. && \
+    make && \
+    make install && \
+    rm -rf /build
+
+FROM ${BASE_IMAGE} AS builder
+ARG CMAKE_FROM_SOURCE=false
+ARG CMAKE_VERSION=3.31.10
+# We can target specific CUDA ARCHITECTURES like --build-arg CUDA_DOCKER_ARCH='75;86;89;120'
+ARG CUDA_DOCKER_ARCH
+ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
+ARG CMAKE_ARGS
+ENV CMAKE_ARGS=${CMAKE_ARGS}
+ARG BACKEND=rerankers
+ARG BUILD_TYPE
+ENV BUILD_TYPE=${BUILD_TYPE}
+ARG CUDA_MAJOR_VERSION
+ARG CUDA_MINOR_VERSION
+ARG SKIP_DRIVERS=false
+ENV CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION}
+ENV CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION}
+ENV DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG GO_VERSION=1.25.4
+ARG UBUNTU_VERSION=2404
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        ccache git \
+        ca-certificates \
+        make \
+        pkg-config libcurl4-openssl-dev \
+        curl unzip \
+        libssl-dev wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Cuda
+ENV PATH=/usr/local/cuda/bin:${PATH}
+
+# HipBLAS requirements
+ENV PATH=/opt/rocm/bin:${PATH}
+
+
+# Vulkan requirements
+RUN <<EOT bash
+    if [ "${BUILD_TYPE}" = "vulkan" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
+        apt-get update && \
+        apt-get install -y  --no-install-recommends \
+            software-properties-common pciutils wget gpg-agent && \
+        apt-get install -y libglm-dev cmake libxcb-dri3-0 libxcb-present0 libpciaccess0 \
+            libpng-dev libxcb-keysyms1-dev libxcb-dri3-dev libx11-dev g++ gcc \
+            libwayland-dev libxrandr-dev libxcb-randr0-dev libxcb-ewmh-dev \
+            git python-is-python3 bison libx11-xcb-dev liblz4-dev libzstd-dev \
+            ocaml-core ninja-build pkg-config libxml2-dev wayland-protocols python3-jsonschema \
+            clang-format qtbase5-dev qt6-base-dev libxcb-glx0-dev sudo xz-utils
+        if [ "amd64" = "$TARGETARCH" ]; then
+            wget "https://sdk.lunarg.com/sdk/download/1.4.335.0/linux/vulkansdk-linux-x86_64-1.4.335.0.tar.xz" && \
+            tar -xf vulkansdk-linux-x86_64-1.4.335.0.tar.xz && \
+            rm vulkansdk-linux-x86_64-1.4.335.0.tar.xz && \
+            mkdir -p /opt/vulkan-sdk && \
+            mv 1.4.335.0 /opt/vulkan-sdk/ && \
+            cd /opt/vulkan-sdk/1.4.335.0 && \
+            ./vulkansdk --no-deps --maxjobs \
+                vulkan-loader \
+                vulkan-validationlayers \
+                vulkan-extensionlayer \
+                vulkan-tools \
+                shaderc && \
+            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/bin/* /usr/bin/ && \
+            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/lib/* /usr/lib/x86_64-linux-gnu/ && \
+            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/include/* /usr/include/ && \
+            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/share/* /usr/share/ && \
+            rm -rf /opt/vulkan-sdk
+        fi
+        if [ "arm64" = "$TARGETARCH" ]; then
+            mkdir vulkan && cd vulkan && \
+            curl -L -o vulkan-sdk.tar.xz https://github.com/mudler/vulkan-sdk-arm/releases/download/1.4.335.0/vulkansdk-ubuntu-24.04-arm-1.4.335.0.tar.xz && \
+            tar -xvf vulkan-sdk.tar.xz && \
+            rm vulkan-sdk.tar.xz && \
+            cd 1.4.335.0 && \
+            cp -rfv aarch64/bin/* /usr/bin/ && \
+            cp -rfv aarch64/lib/* /usr/lib/aarch64-linux-gnu/ && \
+            cp -rfv aarch64/include/* /usr/include/ && \
+            cp -rfv aarch64/share/* /usr/share/ && \
+            cd ../.. && \
+            rm -rf vulkan
+        fi
+        ldconfig && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+    fi
+EOT
+
+# CuBLAS requirements
+RUN <<EOT bash
+    if ( [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "l4t" ] ) && [ "${SKIP_DRIVERS}" = "false" ]; then
+        apt-get update && \
+        apt-get install -y  --no-install-recommends \
+            software-properties-common pciutils
+        if [ "amd64" = "$TARGETARCH" ]; then
+            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/cuda-keyring_1.1-1_all.deb
+        fi
+        if [ "arm64" = "$TARGETARCH" ]; then
+            if [ "${CUDA_MAJOR_VERSION}" = "13" ]; then
+                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/sbsa/cuda-keyring_1.1-1_all.deb
+            else
+                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/arm64/cuda-keyring_1.1-1_all.deb
+            fi
+        fi
+        dpkg -i cuda-keyring_1.1-1_all.deb && \
+        rm -f cuda-keyring_1.1-1_all.deb && \
+        apt-get update && \
+        apt-get install -y --no-install-recommends \
+            cuda-nvcc-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
+            libcufft-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
+            libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
+            libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
+            libcusparse-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
+            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
+        if [ "${CUDA_MAJOR_VERSION}" = "13" ] && [ "arm64" = "$TARGETARCH" ]; then
+            apt-get install -y --no-install-recommends \
+            libcufile-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libcudnn9-cuda-${CUDA_MAJOR_VERSION} cuda-cupti-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libnvjitlink-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
+        fi
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+    fi
+EOT
+
+
+# https://github.com/NVIDIA/Isaac-GR00T/issues/343
+RUN <<EOT bash
+    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "arm64" ]; then
+        wget https://developer.download.nvidia.com/compute/cudss/0.6.0/local_installers/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
+        dpkg -i cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
+        cp /var/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0/cudss-*-keyring.gpg /usr/share/keyrings/ && \
+        apt-get update && apt-get -y install cudss cudss-cuda-${CUDA_MAJOR_VERSION} && \
+        wget https://developer.download.nvidia.com/compute/nvpl/25.5/local_installers/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
+        dpkg -i nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
+        cp /var/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5/nvpl-*-keyring.gpg /usr/share/keyrings/ && \
+        apt-get update && apt-get install -y nvpl
+    fi
+EOT
+
+# If we are building with clblas support, we need the libraries for the builds
+RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
+        apt-get update && \
+        apt-get install -y --no-install-recommends \
+            libclblast-dev && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/* \
+    ; fi
+
+RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
+        apt-get update && \
+        apt-get install -y --no-install-recommends \
+            hipblas-dev \
+            rocblas-dev && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/* && \
+        # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able
+        # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency
+        ldconfig \
+    ; fi
+
+RUN echo "TARGETARCH: $TARGETARCH"
+
+# We need protoc installed, and the version in 22.04 is too old.  We will create one as part installing the GRPC build below
+# but that will also being in a newer version of absl which stablediffusion cannot compile with.  This version of protoc is only
+# here so that we can generate the grpc code for the stablediffusion build
+RUN <<EOT bash
+    if [ "amd64" = "$TARGETARCH" ]; then
+        curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v27.1/protoc-27.1-linux-x86_64.zip -o protoc.zip && \
+        unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
+        rm protoc.zip
+    fi
+    if [ "arm64" = "$TARGETARCH" ]; then
+        curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v27.1/protoc-27.1-linux-aarch_64.zip -o protoc.zip && \
+        unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
+        rm protoc.zip
+    fi
+EOT
+
+# Install CMake (the version in 22.04 is too old)
+RUN <<EOT bash
+    if [ "${CMAKE_FROM_SOURCE}" = "true" ]; then
+        curl -L -s https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -o cmake.tar.gz && tar xvf cmake.tar.gz && cd cmake-${CMAKE_VERSION} && ./configure && make && make install
+    else
+        apt-get update && \
+        apt-get install -y \
+            cmake && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+    fi
+EOT
+
+COPY --from=grpc /opt/grpc /usr/local
+
+
+COPY . /LocalAI
+
+RUN <<'EOT' bash
+set -euxo pipefail
+
+if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then
+  CUDA_ARCH_ESC="${CUDA_DOCKER_ARCH//;/\\;}"
+  export CMAKE_ARGS="${CMAKE_ARGS:-} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH_ESC}"
+  echo "CMAKE_ARGS(env) = ${CMAKE_ARGS}"
+  rm -rf /LocalAI/backend/cpp/ik-llama-cpp-*-build
+fi
+
+cd /LocalAI/backend/cpp/ik-llama-cpp
+
+if [ "${TARGETARCH}" = "arm64" ] || [ "${BUILD_TYPE}" = "hipblas" ]; then
+  # ARM64 / ROCm: build without x86 SIMD
+  make ik-llama-cpp-fallback
+else
+  # ik_llama.cpp's IQK kernels require at least AVX2
+  make ik-llama-cpp-avx2
+fi
+EOT
+
+
+# Copy libraries using a script to handle architecture differences
+RUN make -BC /LocalAI/backend/cpp/ik-llama-cpp package
+
+
+FROM scratch
+
+
+# Copy all available binaries (the build process only creates the appropriate ones for the target architecture)
+COPY --from=builder /LocalAI/backend/cpp/ik-llama-cpp/package/. ./

backend/Dockerfile.llama-cpp

@@ -209,7 +209,11 @@ RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         rm -rf /var/lib/apt/lists/* && \
         # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able
         # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency
-        ldconfig \
+        ldconfig && \
+        # Log which GPU architectures have rocBLAS kernel support
+        echo "rocBLAS library data architectures:" && \
+        (ls /opt/rocm*/lib/rocblas/library/Kernels* 2>/dev/null || ls /opt/rocm*/lib64/rocblas/library/Kernels* 2>/dev/null) | grep -oP 'gfx[0-9a-z+-]+' | sort -u || \
+        echo "WARNING: No rocBLAS kernel data found" \
     ; fi
 
 RUN echo "TARGETARCH: $TARGETARCH"

backend/Dockerfile.python

@@ -29,6 +29,7 @@ RUN apt-get update && \
         curl python3-pip \
         python-is-python3 \
         python3-dev llvm \
+        libnuma1 libgomp1 \
         python3-venv make cmake && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
@@ -195,6 +196,12 @@ COPY backend/backend.proto /${BACKEND}/backend.proto
 COPY backend/python/common/ /${BACKEND}/common
 COPY scripts/build/package-gpu-libs.sh /package-gpu-libs.sh
 
+# Optional per-backend source build toggle (e.g. vllm on CPU can set
+# FROM_SOURCE=true to compile against the build host SIMD instead of
+# pulling a prebuilt wheel). Default empty — most backends ignore it.
+ARG FROM_SOURCE=""
+ENV FROM_SOURCE=${FROM_SOURCE}
+
 RUN cd /${BACKEND} && PORTABLE_PYTHON=true make
 
 # Package GPU libraries into the backend's lib directory

backend/cpp/ik-llama-cpp/CMakeLists.txt

@@ -0,0 +1,78 @@
+## Clip/LLaVA library for multimodal support — built locally from copied sources
+set(TARGET myclip)
+add_library(${TARGET} clip.cpp clip.h llava.cpp llava.h)
+install(TARGETS ${TARGET} LIBRARY)
+target_include_directories(myclip PUBLIC .)
+target_include_directories(myclip PUBLIC ../..)
+target_include_directories(myclip PUBLIC ../../common)
+target_link_libraries(${TARGET} PRIVATE common ggml llama ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(${TARGET} PRIVATE cxx_std_11)
+if (NOT MSVC)
+    target_compile_options(${TARGET} PRIVATE -Wno-cast-qual)
+endif()
+
+set(TARGET grpc-server)
+set(CMAKE_CXX_STANDARD 17)
+cmake_minimum_required(VERSION 3.15)
+set(TARGET grpc-server)
+set(_PROTOBUF_LIBPROTOBUF libprotobuf)
+set(_REFLECTION grpc++_reflection)
+
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64")
+        set(HOMEBREW_DEFAULT_PREFIX "/opt/homebrew")
+    else()
+        set(HOMEBREW_DEFAULT_PREFIX "/usr/local")
+    endif()
+    link_directories("${HOMEBREW_DEFAULT_PREFIX}/lib")
+    include_directories("${HOMEBREW_DEFAULT_PREFIX}/include")
+endif()
+
+find_package(absl CONFIG REQUIRED)
+find_package(Protobuf CONFIG REQUIRED)
+find_package(gRPC CONFIG REQUIRED)
+
+find_program(_PROTOBUF_PROTOC protoc)
+set(_GRPC_GRPCPP grpc++)
+find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+include_directories(${Protobuf_INCLUDE_DIRS})
+
+message(STATUS "Using protobuf version ${Protobuf_VERSION} | Protobuf_INCLUDE_DIRS: ${Protobuf_INCLUDE_DIRS} | CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}")
+
+# Proto file
+get_filename_component(hw_proto "../../../../../../backend/backend.proto" ABSOLUTE)
+get_filename_component(hw_proto_path "${hw_proto}" PATH)
+
+set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.cc")
+set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.h")
+set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.cc")
+set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.h")
+
+add_custom_command(
+      OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}"
+      COMMAND ${_PROTOBUF_PROTOC}
+      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+        -I "${hw_proto_path}"
+        --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+        "${hw_proto}"
+      DEPENDS "${hw_proto}")
+
+add_library(hw_grpc_proto
+  ${hw_grpc_srcs}
+  ${hw_grpc_hdrs}
+  ${hw_proto_srcs}
+  ${hw_proto_hdrs} )
+
+add_executable(${TARGET} grpc-server.cpp json.hpp)
+target_link_libraries(${TARGET} PRIVATE common llama myclip ${CMAKE_THREAD_LIBS_INIT} absl::flags hw_grpc_proto
+  absl::flags_parse
+  gRPC::${_REFLECTION}
+  gRPC::${_GRPC_GRPCPP}
+  protobuf::${_PROTOBUF_LIBPROTOBUF})
+target_compile_features(${TARGET} PRIVATE cxx_std_11)
+if(TARGET BUILD_INFO)
+  add_dependencies(${TARGET} BUILD_INFO)
+endif()

backend/cpp/ik-llama-cpp/Makefile

@@ -0,0 +1,167 @@
+
+IK_LLAMA_VERSION?=08ae48c667e3dcd3025821a8585190b4a46c2f7c
+LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
+
+CMAKE_ARGS?=
+BUILD_TYPE?=
+NATIVE?=false
+ONEAPI_VARS?=/opt/intel/oneapi/setvars.sh
+TARGET?=--target grpc-server
+JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
+ARCH?=$(shell uname -m)
+
+# Disable Shared libs as we are linking on static gRPC and we can't mix shared and static
+CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF -DLLAMA_CURL=OFF
+
+CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+ifeq ($(NATIVE),false)
+	CMAKE_ARGS+=-DGGML_NATIVE=OFF -DLLAMA_OPENSSL=OFF
+endif
+# If build type is cublas, then we set -DGGML_CUDA=ON to CMAKE_ARGS automatically
+ifeq ($(BUILD_TYPE),cublas)
+	CMAKE_ARGS+=-DGGML_CUDA=ON
+# If build type is openblas then we set -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
+# to CMAKE_ARGS automatically
+else ifeq ($(BUILD_TYPE),openblas)
+	CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
+# If build type is clblas (openCL) we set -DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
+else ifeq ($(BUILD_TYPE),clblas)
+	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
+# If it's hipblas we do have also to set CC=/opt/rocm/llvm/bin/clang CXX=/opt/rocm/llvm/bin/clang++
+else ifeq ($(BUILD_TYPE),hipblas)
+	ROCM_HOME ?= /opt/rocm
+	ROCM_PATH ?= /opt/rocm
+	export CXX=$(ROCM_HOME)/llvm/bin/clang++
+	export CC=$(ROCM_HOME)/llvm/bin/clang
+	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
+else ifeq ($(BUILD_TYPE),vulkan)
+	CMAKE_ARGS+=-DGGML_VULKAN=1
+else ifeq ($(OS),Darwin)
+	ifeq ($(BUILD_TYPE),)
+		BUILD_TYPE=metal
+	endif
+	ifneq ($(BUILD_TYPE),metal)
+		CMAKE_ARGS+=-DGGML_METAL=OFF
+	else
+		CMAKE_ARGS+=-DGGML_METAL=ON
+		CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON
+		CMAKE_ARGS+=-DGGML_METAL_USE_BF16=ON
+		CMAKE_ARGS+=-DGGML_OPENMP=OFF
+	endif
+	TARGET+=--target ggml-metal
+endif
+
+ifeq ($(BUILD_TYPE),sycl_f16)
+	CMAKE_ARGS+=-DGGML_SYCL=ON \
+		-DCMAKE_C_COMPILER=icx \
+		-DCMAKE_CXX_COMPILER=icpx \
+		-DCMAKE_CXX_FLAGS="-fsycl" \
+		-DGGML_SYCL_F16=ON
+endif
+
+ifeq ($(BUILD_TYPE),sycl_f32)
+	CMAKE_ARGS+=-DGGML_SYCL=ON \
+		-DCMAKE_C_COMPILER=icx \
+		-DCMAKE_CXX_COMPILER=icpx \
+		-DCMAKE_CXX_FLAGS="-fsycl"
+endif
+
+INSTALLED_PACKAGES=$(CURDIR)/../grpc/installed_packages
+INSTALLED_LIB_CMAKE=$(INSTALLED_PACKAGES)/lib/cmake
+ADDED_CMAKE_ARGS=-Dabsl_DIR=${INSTALLED_LIB_CMAKE}/absl \
+				 -DProtobuf_DIR=${INSTALLED_LIB_CMAKE}/protobuf \
+				 -Dutf8_range_DIR=${INSTALLED_LIB_CMAKE}/utf8_range \
+				 -DgRPC_DIR=${INSTALLED_LIB_CMAKE}/grpc \
+				 -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${INSTALLED_PACKAGES}/include
+build-ik-llama-cpp-grpc-server:
+# Conditionally build grpc for the backend to use if needed
+ifdef BUILD_GRPC_FOR_BACKEND_LLAMA
+	$(MAKE) -C ../../grpc build
+	_PROTOBUF_PROTOC=${INSTALLED_PACKAGES}/bin/proto \
+	_GRPC_CPP_PLUGIN_EXECUTABLE=${INSTALLED_PACKAGES}/bin/grpc_cpp_plugin \
+	PATH="${INSTALLED_PACKAGES}/bin:${PATH}" \
+	CMAKE_ARGS="${CMAKE_ARGS} ${ADDED_CMAKE_ARGS}" \
+	IK_LLAMA_VERSION=$(IK_LLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../$(VARIANT) grpc-server
+else
+	echo "BUILD_GRPC_FOR_BACKEND_LLAMA is not defined."
+	IK_LLAMA_VERSION=$(IK_LLAMA_VERSION) $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../$(VARIANT) grpc-server
+endif
+
+ik-llama-cpp-avx2: llama.cpp
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx2-build
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx2-build purge
+	$(info ${GREEN}I ik-llama-cpp build info:avx2${RESET})
+	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) VARIANT="ik-llama-cpp-avx2-build" build-ik-llama-cpp-grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx2-build/grpc-server ik-llama-cpp-avx2
+
+ik-llama-cpp-avx512: llama.cpp
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx512-build
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx512-build purge
+	$(info ${GREEN}I ik-llama-cpp build info:avx512${RESET})
+	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) VARIANT="ik-llama-cpp-avx512-build" build-ik-llama-cpp-grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx512-build/grpc-server ik-llama-cpp-avx512
+
+ik-llama-cpp-avx: llama.cpp
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx-build
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx-build purge
+	$(info ${GREEN}I ik-llama-cpp build info:avx${RESET})
+	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) VARIANT="ik-llama-cpp-avx-build" build-ik-llama-cpp-grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx-build/grpc-server ik-llama-cpp-avx
+
+ik-llama-cpp-fallback: llama.cpp
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-fallback-build
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-fallback-build purge
+	$(info ${GREEN}I ik-llama-cpp build info:fallback${RESET})
+	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) VARIANT="ik-llama-cpp-fallback-build" build-ik-llama-cpp-grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-fallback-build/grpc-server ik-llama-cpp-fallback
+
+ik-llama-cpp-grpc: llama.cpp
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build purge
+	$(info ${GREEN}I ik-llama-cpp build info:grpc${RESET})
+	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" TARGET="--target grpc-server --target rpc-server" $(MAKE) VARIANT="ik-llama-cpp-grpc-build" build-ik-llama-cpp-grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build/grpc-server ik-llama-cpp-grpc
+
+ik-llama-cpp-rpc-server: ik-llama-cpp-grpc
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build/llama.cpp/build/bin/rpc-server ik-llama-cpp-rpc-server
+
+llama.cpp:
+	mkdir -p llama.cpp
+	cd llama.cpp && \
+	git init && \
+	git remote add origin $(LLAMA_REPO)  && \
+	git fetch origin && \
+	git checkout -b build $(IK_LLAMA_VERSION) && \
+	git submodule update --init --recursive --depth 1 --single-branch
+
+llama.cpp/examples/grpc-server: llama.cpp
+	mkdir -p llama.cpp/examples/grpc-server
+	bash prepare.sh
+
+rebuild:
+	bash prepare.sh
+	rm -rf grpc-server
+	$(MAKE) grpc-server
+
+package:
+	bash package.sh
+
+purge:
+	rm -rf llama.cpp/build
+	rm -rf llama.cpp/examples/grpc-server
+	rm -rf grpc-server
+
+clean: purge
+	rm -rf llama.cpp
+
+grpc-server: llama.cpp llama.cpp/examples/grpc-server
+	@echo "Building grpc-server with $(BUILD_TYPE) build type and $(CMAKE_ARGS)"
+ifneq (,$(findstring sycl,$(BUILD_TYPE)))
+	+bash -c "source $(ONEAPI_VARS); \
+	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release -j $(JOBS) $(TARGET)"
+else
+	+cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release -j $(JOBS) $(TARGET)
+endif
+	cp llama.cpp/build/bin/grpc-server .

backend/cpp/ik-llama-cpp/package.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Script to copy the appropriate libraries based on architecture
+# This script is used in the final stage of the Dockerfile
+
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+REPO_ROOT="${CURDIR}/../../.."
+
+# Create lib directory
+mkdir -p $CURDIR/package/lib
+
+cp -avrf $CURDIR/ik-llama-cpp-* $CURDIR/package/
+cp -rfv $CURDIR/run.sh $CURDIR/package/
+
+# Detect architecture and copy appropriate libraries
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    # x86_64 architecture
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    # ARM64 architecture
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+# Package GPU libraries based on BUILD_TYPE
+# The GPU library packaging script will detect BUILD_TYPE and copy appropriate GPU libraries
+GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
+if [ -f "$GPU_LIB_SCRIPT" ]; then
+    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
+    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    package_gpu_libs
+fi
+
+echo "Packaging completed successfully"
+ls -liah $CURDIR/package/
+ls -liah $CURDIR/package/lib/

backend/cpp/ik-llama-cpp/patches/0001-fix-missing-cstdint-include.patch

@@ -0,0 +1,10 @@
+--- a/ggml/src/iqk/iqk_common.h
++++ b/ggml/src/iqk/iqk_common.h
+@@ -9,6 +9,7 @@
+ #pragma once
+
+ #include "iqk_config.h"
++#include <cstdint>
+
+ #if defined IQK_IMPLEMENT
+

backend/cpp/ik-llama-cpp/prepare.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+## Patches
+
+## Apply patches from the `patches` directory
+if [ -d "patches" ]; then
+    for patch in $(ls patches); do
+        echo "Applying patch $patch"
+        patch -d llama.cpp/ -p1 < patches/$patch
+    done
+fi
+
+set -e
+
+cp -r CMakeLists.txt llama.cpp/examples/grpc-server/
+cp -r grpc-server.cpp llama.cpp/examples/grpc-server/
+cp -r utils.hpp llama.cpp/examples/grpc-server/
+cp -rfv llama.cpp/vendor/nlohmann/json.hpp llama.cpp/examples/grpc-server/
+
+## Copy clip/llava files for multimodal support (built as myclip library)
+cp -rfv llama.cpp/examples/llava/clip.h llama.cpp/examples/grpc-server/clip.h
+cp -rfv llama.cpp/examples/llava/clip.cpp llama.cpp/examples/grpc-server/clip.cpp
+cp -rfv llama.cpp/examples/llava/llava.cpp llama.cpp/examples/grpc-server/llava.cpp
+# Prepend llama.h include to llava.h
+echo '#include "llama.h"' > llama.cpp/examples/grpc-server/llava.h
+cat llama.cpp/examples/llava/llava.h >> llama.cpp/examples/grpc-server/llava.h
+# Copy clip-impl.h if it exists
+if [ -f llama.cpp/examples/llava/clip-impl.h ]; then
+    cp -rfv llama.cpp/examples/llava/clip-impl.h llama.cpp/examples/grpc-server/clip-impl.h
+fi
+# Copy stb_image.h
+if [ -f llama.cpp/vendor/stb/stb_image.h ]; then
+    cp -rfv llama.cpp/vendor/stb/stb_image.h llama.cpp/examples/grpc-server/stb_image.h
+elif [ -f llama.cpp/common/stb_image.h ]; then
+    cp -rfv llama.cpp/common/stb_image.h llama.cpp/examples/grpc-server/stb_image.h
+fi
+
+## Fix API compatibility in llava.cpp (llama_n_embd -> llama_model_n_embd)
+if [ -f llama.cpp/examples/grpc-server/llava.cpp ]; then
+    sed -i 's/llama_n_embd(/llama_model_n_embd(/g' llama.cpp/examples/grpc-server/llava.cpp
+fi
+
+set +e
+if grep -q "grpc-server" llama.cpp/examples/CMakeLists.txt; then
+    echo "grpc-server already added"
+else
+    echo "add_subdirectory(grpc-server)" >> llama.cpp/examples/CMakeLists.txt
+fi
+set -e

backend/cpp/ik-llama-cpp/run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -ex
+
+# Get the absolute current dir where the script is located
+CURDIR=$(dirname "$(realpath $0)")
+
+cd /
+
+echo "CPU info:"
+grep -e "model\sname" /proc/cpuinfo | head -1
+grep -e "flags" /proc/cpuinfo | head -1
+
+# ik_llama.cpp requires AVX2 — default to avx2 binary
+BINARY=ik-llama-cpp-avx2
+
+if [ -e $CURDIR/ik-llama-cpp-fallback ] && ! grep -q -e "\savx2\s" /proc/cpuinfo ; then
+	echo "CPU:    AVX2   NOT found, using fallback"
+	BINARY=ik-llama-cpp-fallback
+fi
+
+# Extend ld library path with the dir where this script is located/lib
+if [ "$(uname)" == "Darwin" ]; then
+	export DYLD_LIBRARY_PATH=$CURDIR/lib:$DYLD_LIBRARY_PATH
+	#export DYLD_FALLBACK_LIBRARY_PATH=$CURDIR/lib:$DYLD_FALLBACK_LIBRARY_PATH
+else
+	export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
+fi
+
+# If there is a lib/ld.so, use it
+if [ -f $CURDIR/lib/ld.so ]; then
+	echo "Using lib/ld.so"
+	echo "Using binary: $BINARY"
+	exec $CURDIR/lib/ld.so $CURDIR/$BINARY "$@"
+fi
+
+echo "Using binary: $BINARY"
+exec $CURDIR/$BINARY "$@"
+
+# We should never reach this point, however just in case we do, run fallback
+exec $CURDIR/ik-llama-cpp-fallback "$@"

backend/cpp/ik-llama-cpp/utils.hpp

@@ -0,0 +1,483 @@
+// https://github.com/ggerganov/llama.cpp/blob/master/examples/server/utils.hpp
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <set>
+#include <mutex>
+#include <condition_variable>
+#include <unordered_map>
+
+#include "json.hpp"
+
+#include "clip.h"
+
+using json = nlohmann::json;
+
+extern bool server_verbose;
+
+#ifndef SERVER_VERBOSE
+#define SERVER_VERBOSE 1
+#endif
+
+#if SERVER_VERBOSE != 1
+#define LOG_VERBOSE(MSG, ...)
+#else
+#define LOG_VERBOSE(MSG, ...)                                            \
+    do                                                                   \
+    {                                                                    \
+        if (server_verbose)                                              \
+        {                                                                \
+            server_log("VERBOSE", __func__, __LINE__, MSG, __VA_ARGS__); \
+        }                                                                \
+    } while (0)
+#endif
+
+#define LOG_ERROR(  MSG, ...) server_log("ERROR",   __func__, __LINE__, MSG, __VA_ARGS__)
+#define LOG_WARNING(MSG, ...) server_log("WARNING", __func__, __LINE__, MSG, __VA_ARGS__)
+#define LOG_INFO(   MSG, ...) server_log("INFO",    __func__, __LINE__, MSG, __VA_ARGS__)
+
+//
+// parallel
+//
+
+enum server_state {
+    SERVER_STATE_LOADING_MODEL,  // Server is starting up, model not fully loaded yet
+    SERVER_STATE_READY,          // Server is ready and model is loaded
+    SERVER_STATE_ERROR           // An error occurred, load_model failed
+};
+
+enum task_type {
+    TASK_TYPE_COMPLETION,
+    TASK_TYPE_CANCEL,
+    TASK_TYPE_NEXT_RESPONSE
+};
+
+struct task_server {
+    int id = -1; // to be filled by llama_server_queue
+    int target_id;
+    task_type type;
+    json data;
+    bool infill_mode = false;
+    bool embedding_mode = false;
+    int multitask_id = -1;
+};
+
+struct task_result {
+    int id;
+    int multitask_id = -1;
+    bool stop;
+    bool error;
+    json result_json;
+};
+
+struct task_multi {
+    int id;
+    std::set<int> subtasks_remaining{};
+    std::vector<task_result> results{};
+};
+
+// TODO: can become bool if we can't find use of more states
+enum slot_state
+{
+    IDLE,
+    PROCESSING,
+};
+
+enum slot_command
+{
+    NONE,
+    LOAD_PROMPT,
+    RELEASE,
+};
+
+struct slot_params
+{
+    bool stream       = true;
+    bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt
+
+    uint32_t seed      = -1; // RNG seed
+    int32_t  n_keep    =  0; // number of tokens to keep from initial prompt
+    int32_t  n_predict = -1; // new tokens to predict
+
+    std::vector<std::string> antiprompt;
+
+    json input_prefix;
+    json input_suffix;
+};
+
+struct slot_image
+{
+    int32_t id;
+
+    bool request_encode_image = false;
+    float * image_embedding = nullptr;
+    int32_t image_tokens = 0;
+
+    clip_image_u8 * img_data;
+
+    std::string prefix_prompt; // before of this image
+};
+
+// completion token output with probabilities
+struct completion_token_output
+{
+    struct token_prob
+    {
+        llama_token tok;
+        float prob;
+    };
+
+    std::vector<token_prob> probs;
+    llama_token tok;
+    std::string text_to_send;
+};
+
+static inline void server_log(const char *level, const char *function, int line,
+                       const char *message, const nlohmann::ordered_json &extra)
+{
+    nlohmann::ordered_json log
+    {
+        {"timestamp", time(nullptr)},
+        {"level",     level},
+        {"function",  function},
+        {"line",      line},
+        {"message",   message},
+    };
+
+    if (!extra.empty())
+    {
+        log.merge_patch(extra);
+    }
+
+    const std::string str = log.dump(-1, ' ', false, json::error_handler_t::replace);
+    printf("%.*s\n", (int)str.size(), str.data());
+    fflush(stdout);
+}
+
+//
+// server utils
+//
+
+template <typename T>
+static T json_value(const json &body, const std::string &key, const T &default_value)
+{
+    // Fallback null to default value
+    return body.contains(key) && !body.at(key).is_null()
+        ? body.value(key, default_value)
+        : default_value;
+}
+
+inline std::string format_chatml(std::vector<json> messages)
+{
+    std::ostringstream chatml_msgs;
+
+    for (auto it = messages.begin(); it != messages.end(); ++it) {
+        chatml_msgs << "<|im_start|>"
+                    << json_value(*it, "role",    std::string("user")) << '\n';
+        chatml_msgs << json_value(*it, "content", std::string(""))
+                    << "<|im_end|>\n";
+    }
+
+    chatml_msgs << "<|im_start|>assistant" << '\n';
+
+    return chatml_msgs.str();
+}
+
+//
+// work queue utils
+//
+
+struct llama_server_queue {
+    int id = 0;
+    std::mutex mutex_tasks;
+    // queues
+    std::vector<task_server> queue_tasks;
+    std::vector<task_server> queue_tasks_deferred;
+    std::vector<task_multi> queue_multitasks;
+    std::condition_variable condition_tasks;
+    // callback functions
+    std::function<void(task_server&)> callback_new_task;
+    std::function<void(task_multi&)> callback_finish_multitask;
+    std::function<void(void)> callback_all_task_finished;
+
+    // Add a new task to the end of the queue
+    int post(task_server task) {
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        if (task.id == -1) {
+            task.id = id++;
+        }
+        queue_tasks.push_back(std::move(task));
+        condition_tasks.notify_one();
+        return task.id;
+    }
+
+    // Add a new task, but defer until one slot is available
+    void defer(task_server task) {
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        queue_tasks_deferred.push_back(std::move(task));
+    }
+
+    // Get the next id for creating anew task
+    int get_new_id() {
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        return id++;
+    }
+
+    // Register function to process a new task
+    void on_new_task(std::function<void(task_server&)> callback) {
+        callback_new_task = callback;
+    }
+
+    // Register function to process a multitask
+    void on_finish_multitask(std::function<void(task_multi&)> callback) {
+        callback_finish_multitask = callback;
+    }
+
+    // Register the function to be called when the batch of tasks is finished
+    void on_all_tasks_finished(std::function<void(void)> callback) {
+        callback_all_task_finished = callback;
+    }
+
+    // Call when the state of one slot is changed
+    void notify_slot_changed() {
+        // move deferred tasks back to main loop
+        std::unique_lock<std::mutex> lock(mutex_tasks);
+        for (auto & task : queue_tasks_deferred) {
+            queue_tasks.push_back(std::move(task));
+        }
+        queue_tasks_deferred.clear();
+    }
+
+    // Start the main loop. This call is blocking
+    [[noreturn]]
+    void start_loop() {
+        while (true) {
+            // new task arrived
+            LOG_VERBOSE("have new task", {});
+            {
+                while (true)
+                {
+                    std::unique_lock<std::mutex> lock(mutex_tasks);
+                    if (queue_tasks.empty()) {
+                        lock.unlock();
+                        break;
+                    }
+                    task_server task = queue_tasks.front();
+                    queue_tasks.erase(queue_tasks.begin());
+                    lock.unlock();
+                    LOG_VERBOSE("callback_new_task", {});
+                    callback_new_task(task);
+                }
+                LOG_VERBOSE("callback_all_task_finished", {});
+                // process and update all the multitasks
+                auto queue_iterator = queue_multitasks.begin();
+                while (queue_iterator != queue_multitasks.end())
+                {
+                    if (queue_iterator->subtasks_remaining.empty())
+                    {
+                        // all subtasks done == multitask is done
+                        task_multi current_multitask = *queue_iterator;
+                        callback_finish_multitask(current_multitask);
+                        // remove this multitask
+                        queue_iterator = queue_multitasks.erase(queue_iterator);
+                    }
+                    else
+                    {
+                        ++queue_iterator;
+                    }
+                }
+                // all tasks in the current loop is finished
+                callback_all_task_finished();
+            }
+            LOG_VERBOSE("wait for new task", {});
+            // wait for new task
+            {
+                std::unique_lock<std::mutex> lock(mutex_tasks);
+                if (queue_tasks.empty()) {
+                    condition_tasks.wait(lock, [&]{
+                        return !queue_tasks.empty();
+                    });
+                }
+            }
+        }
+    }
+
+    //
+    // functions to manage multitasks
+    //
+
+    // add a multitask by specifying the id of all subtask (subtask is a task_server)
+    void add_multitask(int multitask_id, std::vector<int>& sub_ids)
+    {
+        std::lock_guard<std::mutex> lock(mutex_tasks);
+        task_multi multi;
+        multi.id = multitask_id;
+        std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));
+        queue_multitasks.push_back(multi);
+    }
+
+    // updatethe remaining subtasks, while appending results to multitask
+    void update_multitask(int multitask_id, int subtask_id, task_result& result)
+    {
+        std::lock_guard<std::mutex> lock(mutex_tasks);
+        for (auto& multitask : queue_multitasks)
+        {
+            if (multitask.id == multitask_id)
+            {
+                multitask.subtasks_remaining.erase(subtask_id);
+                multitask.results.push_back(result);
+            }
+        }
+    }
+};
+
+struct llama_server_response {
+    typedef std::function<void(int, int, task_result&)> callback_multitask_t;
+    callback_multitask_t callback_update_multitask;
+    // for keeping track of all tasks waiting for the result
+    std::set<int> waiting_task_ids;
+    // the main result queue
+    std::vector<task_result> queue_results;
+    std::mutex mutex_results;
+    std::condition_variable condition_results;
+
+    void add_waiting_task_id(int task_id) {
+        std::unique_lock<std::mutex> lock(mutex_results);
+        waiting_task_ids.insert(task_id);
+    }
+
+    void remove_waiting_task_id(int task_id) {
+        std::unique_lock<std::mutex> lock(mutex_results);
+        waiting_task_ids.erase(task_id);
+    }
+
+    // This function blocks the thread until there is a response for this task_id
+    task_result recv(int task_id) {
+        while (true)
+        {
+            std::unique_lock<std::mutex> lock(mutex_results);
+            condition_results.wait(lock, [&]{
+                return !queue_results.empty();
+            });
+            LOG_VERBOSE("condition_results unblock", {});
+
+            for (int i = 0; i < (int) queue_results.size(); i++)
+            {
+                if (queue_results[i].id == task_id)
+                {
+                    assert(queue_results[i].multitask_id == -1);
+                    task_result res = queue_results[i];
+                    queue_results.erase(queue_results.begin() + i);
+                    return res;
+                }
+            }
+        }
+
+        // should never reach here
+    }
+
+    // Register the function to update multitask
+    void on_multitask_update(callback_multitask_t callback) {
+        callback_update_multitask = callback;
+    }
+
+    // Send a new result to a waiting task_id
+    void send(task_result result) {
+        std::unique_lock<std::mutex> lock(mutex_results);
+        LOG_VERBOSE("send new result", {});
+        for (auto& task_id : waiting_task_ids) {
+            // LOG_TEE("waiting task id %i \n", task_id);
+            // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result
+            if (result.multitask_id == task_id)
+            {
+                LOG_VERBOSE("callback_update_multitask", {});
+                callback_update_multitask(task_id, result.id, result);
+                continue;
+            }
+
+            if (result.id == task_id)
+            {
+                LOG_VERBOSE("queue_results.push_back", {});
+                queue_results.push_back(result);
+                condition_results.notify_one();
+                return;
+            }
+        }
+    }
+};
+
+//
+// base64 utils (TODO: move to common in the future)
+//
+
+static const std::string base64_chars =
+             "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+             "abcdefghijklmnopqrstuvwxyz"
+             "0123456789+/";
+
+static inline bool is_base64(uint8_t c)
+{
+    return (isalnum(c) || (c == '+') || (c == '/'));
+}
+
+static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string)
+{
+    int i = 0;
+    int j = 0;
+    int in_ = 0;
+
+    int in_len = encoded_string.size();
+
+    uint8_t char_array_4[4];
+    uint8_t char_array_3[3];
+
+    std::vector<uint8_t> ret;
+
+    while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))
+    {
+        char_array_4[i++] = encoded_string[in_]; in_++;
+        if (i == 4)
+        {
+            for (i = 0; i <4; i++)
+            {
+                char_array_4[i] = base64_chars.find(char_array_4[i]);
+            }
+
+            char_array_3[0] = ((char_array_4[0]      ) << 2) + ((char_array_4[1] & 0x30) >> 4);
+            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) +   char_array_4[3];
+
+            for (i = 0; (i < 3); i++)
+            {
+                ret.push_back(char_array_3[i]);
+            }
+            i = 0;
+        }
+    }
+
+    if (i)
+    {
+        for (j = i; j <4; j++)
+        {
+            char_array_4[j] = 0;
+        }
+
+        for (j = 0; j <4; j++)
+        {
+            char_array_4[j] = base64_chars.find(char_array_4[j]);
+        }
+
+        char_array_3[0] = ((char_array_4[0]      ) << 2) + ((char_array_4[1] & 0x30) >> 4);
+        char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
+        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) +   char_array_4[3];
+
+        for (j = 0; (j < i - 1); j++)
+        {
+            ret.push_back(char_array_3[j]);
+        }
+    }
+
+    return ret;
+}

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=e62fa13c2497b2cd1958cb496e9489e86bbd5182
+LLAMA_VERSION?=ff5ef8278615a2462b79b50abdf3cc95cfb31c6f
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=
@@ -33,7 +33,7 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	ROCM_PATH ?= /opt/rocm
 	export CXX=$(ROCM_HOME)/llvm/bin/clang++
 	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
 	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=1

backend/cpp/llama-cpp/run.sh

@@ -46,6 +46,10 @@ if [ "$(uname)" == "Darwin" ]; then
 	#export DYLD_FALLBACK_LIBRARY_PATH=$CURDIR/lib:$DYLD_FALLBACK_LIBRARY_PATH
 else
 	export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
+	# Tell rocBLAS where to find TensileLibrary data (GPU kernel tuning files)
+	if [ -d "$CURDIR/lib/rocblas/library" ]; then
+		export ROCBLAS_TENSILE_LIBPATH=$CURDIR/lib/rocblas/library
+	fi
 fi
 
 # If there is a lib/ld.so, use it

backend/go/qwen3-tts-cpp/CMakeLists.txt

@@ -0,0 +1,56 @@
+cmake_minimum_required(VERSION 3.14)
+project(goqwen3ttscpp LANGUAGES C CXX)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+set(QWEN3TTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/sources/qwen3-tts.cpp)
+
+# Override upstream's CMAKE_CUDA_ARCHITECTURES before add_subdirectory.
+if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+    set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real")
+endif()
+
+# Build ggml from the upstream's submodule FIRST, so that ggml/ggml-base/ggml-cpu
+# CMake targets exist when the upstream project references them by name.
+# The upstream CMakeLists.txt uses target_link_libraries(... ggml ggml-base ggml-cpu)
+# with target_link_directories pointing at a pre-built ggml/build/. By adding ggml
+# as a subdirectory here, CMake resolves those names as targets instead.
+add_subdirectory(${QWEN3TTS_DIR}/ggml ggml EXCLUDE_FROM_ALL)
+
+# Now add the upstream project
+add_subdirectory(${QWEN3TTS_DIR} qwen3tts EXCLUDE_FROM_ALL)
+
+add_library(goqwen3ttscpp MODULE cpp/goqwen3ttscpp.cpp)
+target_link_libraries(goqwen3ttscpp PRIVATE qwen3_tts)
+
+target_include_directories(goqwen3ttscpp PRIVATE ${QWEN3TTS_DIR}/src)
+target_include_directories(goqwen3ttscpp SYSTEM PRIVATE ${QWEN3TTS_DIR}/ggml/include)
+
+# Link GPU backends if available
+foreach(backend blas cuda metal vulkan)
+    if(TARGET ggml-${backend})
+        target_link_libraries(goqwen3ttscpp PRIVATE ggml-${backend})
+        string(TOUPPER ${backend} BACKEND_UPPER)
+        target_compile_definitions(goqwen3ttscpp PRIVATE QWEN3TTS_HAVE_${BACKEND_UPPER})
+        if(backend STREQUAL "cuda")
+            find_package(CUDAToolkit QUIET)
+            if(CUDAToolkit_FOUND)
+                target_link_libraries(goqwen3ttscpp PRIVATE CUDA::cudart)
+            endif()
+        endif()
+    endif()
+endforeach()
+
+if(MSVC)
+    target_compile_options(goqwen3ttscpp PRIVATE /W4 /wd4100 /wd4505)
+else()
+    target_compile_options(goqwen3ttscpp PRIVATE -Wall -Wextra -Wshadow -Wconversion
+                          -Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
+endif()
+
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
+    target_link_libraries(goqwen3ttscpp PRIVATE stdc++fs)
+endif()
+
+set_property(TARGET goqwen3ttscpp PROPERTY CXX_STANDARD 17)
+set_target_properties(goqwen3ttscpp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

backend/go/qwen3-tts-cpp/Makefile

@@ -0,0 +1,126 @@
+CMAKE_ARGS?=
+BUILD_TYPE?=
+NATIVE?=false
+
+GOCMD?=go
+GO_TAGS?=
+JOBS?=$(shell nproc --ignore=1)
+
+# qwen3-tts.cpp version
+QWEN3TTS_REPO?=https://github.com/predict-woo/qwen3-tts.cpp
+QWEN3TTS_CPP_VERSION?=7a762e2ad4bacc6fdda81d81bf10a09ffb546f29
+SO_TARGET?=libgoqwen3ttscpp.so
+
+CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
+
+ifeq ($(NATIVE),false)
+	CMAKE_ARGS+=-DGGML_NATIVE=OFF
+endif
+
+ifeq ($(BUILD_TYPE),cublas)
+	CMAKE_ARGS+=-DGGML_CUDA=ON
+else ifeq ($(BUILD_TYPE),openblas)
+	CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
+else ifeq ($(BUILD_TYPE),clblas)
+	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
+else ifeq ($(BUILD_TYPE),hipblas)
+	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
+else ifeq ($(BUILD_TYPE),vulkan)
+	CMAKE_ARGS+=-DGGML_VULKAN=ON
+else ifeq ($(OS),Darwin)
+	ifneq ($(BUILD_TYPE),metal)
+		CMAKE_ARGS+=-DGGML_METAL=OFF
+	else
+		CMAKE_ARGS+=-DGGML_METAL=ON
+		CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON
+	endif
+endif
+
+ifeq ($(BUILD_TYPE),sycl_f16)
+	CMAKE_ARGS+=-DGGML_SYCL=ON \
+		-DCMAKE_C_COMPILER=icx \
+		-DCMAKE_CXX_COMPILER=icpx \
+		-DGGML_SYCL_F16=ON
+endif
+
+ifeq ($(BUILD_TYPE),sycl_f32)
+	CMAKE_ARGS+=-DGGML_SYCL=ON \
+		-DCMAKE_C_COMPILER=icx \
+		-DCMAKE_CXX_COMPILER=icpx
+endif
+
+sources/qwen3-tts.cpp:
+	mkdir -p sources/qwen3-tts.cpp
+	cd sources/qwen3-tts.cpp && \
+	git init && \
+	git remote add origin $(QWEN3TTS_REPO) && \
+	git fetch origin && \
+	git checkout $(QWEN3TTS_CPP_VERSION) && \
+	git submodule update --init --recursive --depth 1 --single-branch
+
+# Detect OS
+UNAME_S := $(shell uname -s)
+
+# Only build CPU variants on Linux
+ifeq ($(UNAME_S),Linux)
+	VARIANT_TARGETS = libgoqwen3ttscpp-avx.so libgoqwen3ttscpp-avx2.so libgoqwen3ttscpp-avx512.so libgoqwen3ttscpp-fallback.so
+else
+	# On non-Linux (e.g., Darwin), build only fallback variant
+	VARIANT_TARGETS = libgoqwen3ttscpp-fallback.so
+endif
+
+qwen3-tts-cpp: main.go goqwen3ttscpp.go $(VARIANT_TARGETS)
+	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o qwen3-tts-cpp ./
+
+package: qwen3-tts-cpp
+	bash package.sh
+
+build: package
+
+clean: purge
+	rm -rf libgoqwen3ttscpp*.so package sources/qwen3-tts.cpp qwen3-tts-cpp
+
+purge:
+	rm -rf build*
+
+# Variants must build sequentially
+.NOTPARALLEL:
+
+# Build all variants (Linux only)
+ifeq ($(UNAME_S),Linux)
+libgoqwen3ttscpp-avx.so: sources/qwen3-tts.cpp
+	$(info ${GREEN}I qwen3-tts-cpp build info:avx${RESET})
+	SO_TARGET=libgoqwen3ttscpp-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgoqwen3ttscpp-custom
+	rm -rf build-libgoqwen3ttscpp-avx.so
+
+libgoqwen3ttscpp-avx2.so: sources/qwen3-tts.cpp
+	$(info ${GREEN}I qwen3-tts-cpp build info:avx2${RESET})
+	SO_TARGET=libgoqwen3ttscpp-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgoqwen3ttscpp-custom
+	rm -rf build-libgoqwen3ttscpp-avx2.so
+
+libgoqwen3ttscpp-avx512.so: sources/qwen3-tts.cpp
+	$(info ${GREEN}I qwen3-tts-cpp build info:avx512${RESET})
+	SO_TARGET=libgoqwen3ttscpp-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgoqwen3ttscpp-custom
+	rm -rf build-libgoqwen3ttscpp-avx512.so
+endif
+
+# Build fallback variant (all platforms)
+libgoqwen3ttscpp-fallback.so: sources/qwen3-tts.cpp
+	$(info ${GREEN}I qwen3-tts-cpp build info:fallback${RESET})
+	SO_TARGET=libgoqwen3ttscpp-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgoqwen3ttscpp-custom
+	rm -rf build-libgoqwen3ttscpp-fallback.so
+
+libgoqwen3ttscpp-custom: CMakeLists.txt cpp/goqwen3ttscpp.cpp cpp/goqwen3ttscpp.h
+	mkdir -p build-$(SO_TARGET) && \
+	cd build-$(SO_TARGET) && \
+	cmake .. $(CMAKE_ARGS) && \
+	cmake --build . --config Release -j$(JOBS) --target goqwen3ttscpp && \
+	cd .. && \
+	mv build-$(SO_TARGET)/libgoqwen3ttscpp.so ./$(SO_TARGET)
+
+test: qwen3-tts-cpp
+	@echo "Running qwen3-tts-cpp tests..."
+	bash test.sh
+	@echo "qwen3-tts-cpp tests completed."
+
+all: qwen3-tts-cpp package

backend/go/qwen3-tts-cpp/cpp/goqwen3ttscpp.cpp

@@ -0,0 +1,161 @@
+#include "goqwen3ttscpp.h"
+#include "ggml-backend.h"
+#include "qwen3_tts.h"
+
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+
+using namespace qwen3_tts;
+
+// Global engine (loaded once, reused across requests)
+static Qwen3TTS *g_engine = nullptr;
+static bool g_loaded = false;
+static int g_threads = 4;
+
+static void ggml_log_cb(enum ggml_log_level level, const char *log, void *data) {
+    const char *level_str;
+    if (!log)
+        return;
+    switch (level) {
+    case GGML_LOG_LEVEL_DEBUG:
+        level_str = "DEBUG";
+        break;
+    case GGML_LOG_LEVEL_INFO:
+        level_str = "INFO";
+        break;
+    case GGML_LOG_LEVEL_WARN:
+        level_str = "WARN";
+        break;
+    case GGML_LOG_LEVEL_ERROR:
+        level_str = "ERROR";
+        break;
+    default:
+        level_str = "?????";
+        break;
+    }
+    fprintf(stderr, "[%-5s] ", level_str);
+    fputs(log, stderr);
+    fflush(stderr);
+}
+
+// Map language string to language_id token used by the model
+static int language_to_id(const char *lang) {
+    if (!lang || lang[0] == '\0')
+        return 2050; // default: English
+    std::string l(lang);
+    if (l == "en")
+        return 2050;
+    if (l == "ru")
+        return 2069;
+    if (l == "zh")
+        return 2055;
+    if (l == "ja")
+        return 2058;
+    if (l == "ko")
+        return 2064;
+    if (l == "de")
+        return 2053;
+    if (l == "fr")
+        return 2061;
+    if (l == "es")
+        return 2054;
+    if (l == "it")
+        return 2056;
+    if (l == "pt")
+        return 2057;
+    fprintf(stderr, "[qwen3-tts-cpp] Unknown language '%s', defaulting to English\n",
+            lang);
+    return 2050;
+}
+
+int load_model(const char *model_dir, int n_threads) {
+    ggml_log_set(ggml_log_cb, nullptr);
+    ggml_backend_load_all();
+
+    if (n_threads <= 0)
+        n_threads = 4;
+    g_threads = n_threads;
+
+    fprintf(stderr, "[qwen3-tts-cpp] Loading models from %s (threads=%d)\n",
+            model_dir, n_threads);
+
+    g_engine = new Qwen3TTS();
+    if (!g_engine->load_models(model_dir)) {
+        fprintf(stderr, "[qwen3-tts-cpp] FATAL: failed to load models from %s\n",
+                model_dir);
+        delete g_engine;
+        g_engine = nullptr;
+        return 1;
+    }
+
+    g_loaded = true;
+    fprintf(stderr, "[qwen3-tts-cpp] Models loaded successfully\n");
+    return 0;
+}
+
+int synthesize(const char *text, const char *ref_audio_path, const char *dst,
+               const char *language, float temperature, float top_p,
+               int top_k, float repetition_penalty, int max_audio_tokens,
+               int n_threads) {
+    if (!g_loaded || !g_engine) {
+        fprintf(stderr, "[qwen3-tts-cpp] ERROR: models not loaded\n");
+        return 1;
+    }
+
+    if (!text || !dst) {
+        fprintf(stderr, "[qwen3-tts-cpp] ERROR: text and dst are required\n");
+        return 2;
+    }
+
+    tts_params params;
+    params.max_audio_tokens = max_audio_tokens > 0 ? max_audio_tokens : 4096;
+    params.temperature = temperature;
+    params.top_p = top_p;
+    params.top_k = top_k;
+    params.repetition_penalty = repetition_penalty;
+    params.n_threads = n_threads > 0 ? n_threads : g_threads;
+    params.language_id = language_to_id(language);
+
+    fprintf(stderr, "[qwen3-tts-cpp] Synthesizing: text='%.50s%s', lang_id=%d, "
+                    "temp=%.2f, threads=%d\n",
+            text, (strlen(text) > 50 ? "..." : ""), params.language_id,
+            temperature, params.n_threads);
+
+    tts_result result;
+    bool has_ref = ref_audio_path && ref_audio_path[0] != '\0';
+
+    if (has_ref) {
+        fprintf(stderr, "[qwen3-tts-cpp] Voice cloning with ref: %s\n",
+                ref_audio_path);
+        result = g_engine->synthesize_with_voice(text, ref_audio_path, params);
+    } else {
+        result = g_engine->synthesize(text, params);
+    }
+
+    if (!result.success) {
+        fprintf(stderr, "[qwen3-tts-cpp] ERROR: synthesis failed: %s\n",
+                result.error_msg.c_str());
+        return 3;
+    }
+
+    int n_samples = (int)result.audio.size();
+    if (n_samples == 0) {
+        fprintf(stderr, "[qwen3-tts-cpp] ERROR: synthesis produced no samples\n");
+        return 4;
+    }
+
+    fprintf(stderr,
+            "[qwen3-tts-cpp] Synthesis done: %d samples (%.2fs @ 24kHz)\n",
+            n_samples, (float)n_samples / 24000.0f);
+
+    if (!save_audio_file(dst, result.audio, result.sample_rate)) {
+        fprintf(stderr, "[qwen3-tts-cpp] ERROR: failed to write %s\n", dst);
+        return 5;
+    }
+
+    fprintf(stderr, "[qwen3-tts-cpp] Wrote %s\n", dst);
+    return 0;
+}

backend/go/qwen3-tts-cpp/cpp/goqwen3ttscpp.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+extern "C" {
+int load_model(const char *model_dir, int n_threads);
+int synthesize(const char *text, const char *ref_audio_path, const char *dst,
+               const char *language, float temperature, float top_p,
+               int top_k, float repetition_penalty, int max_audio_tokens,
+               int n_threads);
+}

backend/go/qwen3-tts-cpp/goqwen3ttscpp.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/pkg/grpc/base"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+)
+
+var (
+	CppLoadModel  func(modelDir string, nThreads int) int
+	CppSynthesize func(text, refAudioPath, dst, language string,
+		temperature, topP float32, topK int,
+		repetitionPenalty float32, maxAudioTokens, nThreads int) int
+)
+
+type Qwen3TtsCpp struct {
+	base.SingleThread
+	threads int
+}
+
+func (q *Qwen3TtsCpp) Load(opts *pb.ModelOptions) error {
+	// ModelFile is the model directory path (containing GGUF files)
+	modelDir := opts.ModelFile
+	if modelDir == "" {
+		modelDir = opts.ModelPath
+	}
+
+	// Resolve relative paths
+	if !filepath.IsAbs(modelDir) && opts.ModelPath != "" {
+		modelDir = filepath.Join(opts.ModelPath, modelDir)
+	}
+
+	threads := int(opts.Threads)
+	if threads <= 0 {
+		threads = 4
+	}
+	q.threads = threads
+
+	fmt.Fprintf(os.Stderr, "[qwen3-tts-cpp] Loading models from: %s (threads=%d)\n", modelDir, threads)
+
+	if ret := CppLoadModel(modelDir, threads); ret != 0 {
+		return fmt.Errorf("failed to load qwen3-tts model (error code: %d)", ret)
+	}
+
+	return nil
+}
+
+func (q *Qwen3TtsCpp) TTS(req *pb.TTSRequest) error {
+	text := req.Text
+	voice := req.Voice // reference audio path for voice cloning (empty = no cloning)
+	dst := req.Dst
+	language := ""
+	if req.Language != nil {
+		language = *req.Language
+	}
+
+	// Synthesis parameters with sensible defaults
+	temperature := float32(0.9)
+	topP := float32(0.8)
+	topK := 50
+	repetitionPenalty := float32(1.05)
+	maxAudioTokens := 4096
+
+	if ret := CppSynthesize(text, voice, dst, language,
+		temperature, topP, topK, repetitionPenalty,
+		maxAudioTokens, q.threads); ret != 0 {
+		return fmt.Errorf("failed to synthesize audio (error code: %d)", ret)
+	}
+
+	return nil
+}

backend/go/qwen3-tts-cpp/main.go

@@ -0,0 +1,47 @@
+package main
+
+// Note: this is started internally by LocalAI and a server is allocated for each model
+import (
+	"flag"
+	"os"
+
+	"github.com/ebitengine/purego"
+	grpc "github.com/mudler/LocalAI/pkg/grpc"
+)
+
+var (
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+)
+
+type LibFuncs struct {
+	FuncPtr any
+	Name    string
+}
+
+func main() {
+	// Get library name from environment variable, default to fallback
+	libName := os.Getenv("QWEN3TTS_LIBRARY")
+	if libName == "" {
+		libName = "./libgoqwen3ttscpp-fallback.so"
+	}
+
+	gosd, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+	if err != nil {
+		panic(err)
+	}
+
+	libFuncs := []LibFuncs{
+		{&CppLoadModel, "load_model"},
+		{&CppSynthesize, "synthesize"},
+	}
+
+	for _, lf := range libFuncs {
+		purego.RegisterLibFunc(lf.FuncPtr, gosd, lf.Name)
+	}
+
+	flag.Parse()
+
+	if err := grpc.StartServer(*addr, &Qwen3TtsCpp{}); err != nil {
+		panic(err)
+	}
+}

backend/go/qwen3-tts-cpp/package.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Script to copy the appropriate libraries based on architecture
+# This script is used in the final stage of the Dockerfile
+
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+REPO_ROOT="${CURDIR}/../../.."
+
+# Create lib directory
+mkdir -p $CURDIR/package/lib
+
+cp -avf $CURDIR/qwen3-tts-cpp $CURDIR/package/
+cp -fv $CURDIR/libgoqwen3ttscpp-*.so $CURDIR/package/
+cp -fv $CURDIR/run.sh $CURDIR/package/
+
+# Detect architecture and copy appropriate libraries
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    # x86_64 architecture
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    # ARM64 architecture
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ $(uname -s) = "Darwin" ]; then
+    echo "Detected Darwin"
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+# Package GPU libraries based on BUILD_TYPE
+GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
+if [ -f "$GPU_LIB_SCRIPT" ]; then
+    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
+    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    package_gpu_libs
+fi
+
+echo "Packaging completed successfully"
+ls -liah $CURDIR/package/
+ls -liah $CURDIR/package/lib/

backend/go/qwen3-tts-cpp/qwen3ttscpp_test.go

@@ -0,0 +1,173 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const (
+	testAddr    = "localhost:50051"
+	startupWait = 5 * time.Second
+)
+
+func skipIfNoModel(t *testing.T) string {
+	t.Helper()
+	modelDir := os.Getenv("QWEN3TTS_MODEL_DIR")
+	if modelDir == "" {
+		t.Skip("QWEN3TTS_MODEL_DIR not set, skipping test (set to directory with GGUF models)")
+	}
+	if _, err := os.Stat(filepath.Join(modelDir, "qwen3-tts-0.6b-f16.gguf")); os.IsNotExist(err) {
+		t.Skipf("TTS model file not found in %s, skipping", modelDir)
+	}
+	if _, err := os.Stat(filepath.Join(modelDir, "qwen3-tts-tokenizer-f16.gguf")); os.IsNotExist(err) {
+		t.Skipf("Tokenizer model file not found in %s, skipping", modelDir)
+	}
+	return modelDir
+}
+
+func startServer(t *testing.T) *exec.Cmd {
+	t.Helper()
+	binary := os.Getenv("QWEN3TTS_BINARY")
+	if binary == "" {
+		binary = "./qwen3-tts-cpp"
+	}
+	if _, err := os.Stat(binary); os.IsNotExist(err) {
+		t.Skipf("Backend binary not found at %s, skipping", binary)
+	}
+	cmd := exec.Command(binary, "--addr", testAddr)
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start server: %v", err)
+	}
+	time.Sleep(startupWait)
+	return cmd
+}
+
+func stopServer(cmd *exec.Cmd) {
+	if cmd != nil && cmd.Process != nil {
+		cmd.Process.Kill()
+		cmd.Wait()
+	}
+}
+
+func dialGRPC(t *testing.T) *grpc.ClientConn {
+	t.Helper()
+	conn, err := grpc.Dial(testAddr,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultCallOptions(
+			grpc.MaxCallRecvMsgSize(50*1024*1024),
+			grpc.MaxCallSendMsgSize(50*1024*1024),
+		),
+	)
+	if err != nil {
+		t.Fatalf("Failed to dial gRPC: %v", err)
+	}
+	return conn
+}
+
+func TestServerHealth(t *testing.T) {
+	cmd := startServer(t)
+	defer stopServer(cmd)
+
+	conn := dialGRPC(t)
+	defer conn.Close()
+
+	client := pb.NewBackendClient(conn)
+	resp, err := client.Health(context.Background(), &pb.HealthMessage{})
+	if err != nil {
+		t.Fatalf("Health check failed: %v", err)
+	}
+	if string(resp.Message) != "OK" {
+		t.Fatalf("Expected OK, got %s", string(resp.Message))
+	}
+}
+
+func TestLoadModel(t *testing.T) {
+	modelDir := skipIfNoModel(t)
+	cmd := startServer(t)
+	defer stopServer(cmd)
+
+	conn := dialGRPC(t)
+	defer conn.Close()
+
+	client := pb.NewBackendClient(conn)
+
+	resp, err := client.LoadModel(context.Background(), &pb.ModelOptions{
+		ModelFile: modelDir,
+		Threads:   4,
+	})
+	if err != nil {
+		t.Fatalf("LoadModel failed: %v", err)
+	}
+	if !resp.Success {
+		t.Fatalf("LoadModel returned failure: %s", resp.Message)
+	}
+}
+
+func TestTTS(t *testing.T) {
+	modelDir := skipIfNoModel(t)
+
+	tmpDir, err := os.MkdirTemp("", "qwen3tts-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { os.RemoveAll(tmpDir) })
+
+	outputFile := filepath.Join(tmpDir, "output.wav")
+
+	cmd := startServer(t)
+	defer stopServer(cmd)
+
+	conn := dialGRPC(t)
+	defer conn.Close()
+
+	client := pb.NewBackendClient(conn)
+
+	// Load models
+	loadResp, err := client.LoadModel(context.Background(), &pb.ModelOptions{
+		ModelFile: modelDir,
+		Threads:   4,
+	})
+	if err != nil {
+		t.Fatalf("LoadModel failed: %v", err)
+	}
+	if !loadResp.Success {
+		t.Fatalf("LoadModel returned failure: %s", loadResp.Message)
+	}
+
+	// Synthesize speech
+	language := "en"
+	_, err = client.TTS(context.Background(), &pb.TTSRequest{
+		Text:     "Hello, this is a test of the Qwen3 text to speech system.",
+		Dst:      outputFile,
+		Language: &language,
+	})
+	if err != nil {
+		t.Fatalf("TTS failed: %v", err)
+	}
+
+	// Verify output file exists and has content
+	info, err := os.Stat(outputFile)
+	if os.IsNotExist(err) {
+		t.Fatal("Output audio file was not created")
+	}
+	if err != nil {
+		t.Fatalf("Failed to stat output file: %v", err)
+	}
+
+	t.Logf("Output file size: %d bytes", info.Size())
+
+	// WAV header is 44 bytes minimum; any real audio should be much larger
+	if info.Size() < 1000 {
+		t.Errorf("Output file too small (%d bytes), expected real audio data", info.Size())
+	}
+}

backend/go/qwen3-tts-cpp/run.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -ex
+
+# Get the absolute current dir where the script is located
+CURDIR=$(dirname "$(realpath $0)")
+
+cd /
+
+echo "CPU info:"
+if [ "$(uname)" != "Darwin" ]; then
+	grep -e "model\sname" /proc/cpuinfo | head -1
+	grep -e "flags" /proc/cpuinfo | head -1
+fi
+
+LIBRARY="$CURDIR/libgoqwen3ttscpp-fallback.so"
+
+if [ "$(uname)" != "Darwin" ]; then
+	if grep -q -e "\savx\s" /proc/cpuinfo ; then
+		echo "CPU:    AVX    found OK"
+		if [ -e $CURDIR/libgoqwen3ttscpp-avx.so ]; then
+			LIBRARY="$CURDIR/libgoqwen3ttscpp-avx.so"
+		fi
+	fi
+
+	if grep -q -e "\savx2\s" /proc/cpuinfo ; then
+		echo "CPU:    AVX2   found OK"
+		if [ -e $CURDIR/libgoqwen3ttscpp-avx2.so ]; then
+			LIBRARY="$CURDIR/libgoqwen3ttscpp-avx2.so"
+		fi
+	fi
+
+	# Check avx 512
+	if grep -q -e "\savx512f\s" /proc/cpuinfo ; then
+		echo "CPU:    AVX512F found OK"
+		if [ -e $CURDIR/libgoqwen3ttscpp-avx512.so ]; then
+			LIBRARY="$CURDIR/libgoqwen3ttscpp-avx512.so"
+		fi
+	fi
+fi
+
+export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
+export QWEN3TTS_LIBRARY=$LIBRARY
+
+# If there is a lib/ld.so, use it
+if [ -f $CURDIR/lib/ld.so ]; then
+	echo "Using lib/ld.so"
+	echo "Using library: $LIBRARY"
+	exec $CURDIR/lib/ld.so $CURDIR/qwen3-tts-cpp "$@"
+fi
+
+echo "Using library: $LIBRARY"
+exec $CURDIR/qwen3-tts-cpp "$@"

backend/go/qwen3-tts-cpp/test.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+
+echo "Running qwen3-tts-cpp backend tests..."
+
+# The test requires:
+#   - QWEN3TTS_MODEL_DIR: path to directory containing GGUF model files
+#   - QWEN3TTS_BINARY: path to the qwen3-tts-cpp binary (defaults to ./qwen3-tts-cpp)
+#
+# Tests that require the model will be skipped if QWEN3TTS_MODEL_DIR is not set
+# or the directory does not contain the required model files.
+
+cd "$CURDIR"
+
+# Only auto-download models when QWEN3TTS_MODEL_DIR is not explicitly set
+if [ -z "$QWEN3TTS_MODEL_DIR" ]; then
+    export QWEN3TTS_MODEL_DIR="./qwen3-tts-models"
+
+    if [ ! -d "$QWEN3TTS_MODEL_DIR" ]; then
+        echo "Creating qwen3-tts-models directory for tests..."
+        mkdir -p "$QWEN3TTS_MODEL_DIR"
+        REPO_ID="endo5501/qwen3-tts.cpp"
+        echo "Repository: ${REPO_ID}"
+        echo ""
+
+        # Files to download (smallest model for testing)
+        FILES=(
+            "qwen3-tts-0.6b-f16.gguf"
+            "qwen3-tts-tokenizer-f16.gguf"
+        )
+
+        BASE_URL="https://huggingface.co/${REPO_ID}/resolve/main"
+
+        for file in "${FILES[@]}"; do
+            dest="${QWEN3TTS_MODEL_DIR}/${file}"
+            if [ -f "${dest}" ]; then
+                echo "  [skip] ${file} (already exists)"
+            else
+                echo "  [download] ${file}..."
+                curl -L -o "${dest}" "${BASE_URL}/${file}" --progress-bar
+                echo "  [done] ${file}"
+            fi
+        done
+    fi
+fi
+
+# Run Go tests
+go test -v -timeout 600s .
+
+echo "All qwen3-tts-cpp tests passed."

backend/go/sam3-cpp/Makefile

@@ -26,7 +26,7 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	ROCM_PATH ?= /opt/rocm
 	export CXX=$(ROCM_HOME)/llvm/bin/clang++
 	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
 	CMAKE_ARGS+=-DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=e8323cabb0e4511ba18a50b1cb34cf1f87fc71ef
+STABLEDIFFUSION_GGML_VERSION?=6b675a5ede9b0edf0a0f44191e8b79d7ef27615a
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 
@@ -32,7 +32,7 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	ROCM_PATH ?= /opt/rocm
 	export CXX=$(ROCM_HOME)/llvm/bin/clang++
 	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
 	CMAKE_ARGS+=-DSD_HIPBLAS=ON -DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DSD_VULKAN=ON -DGGML_VULKAN=ON

backend/index.yaml

@@ -29,6 +29,20 @@
     nvidia-cuda-12: "cuda12-llama-cpp"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-llama-cpp"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-llama-cpp"
+- &ikllamacpp
+  name: "ik-llama-cpp"
+  alias: "ik-llama-cpp"
+  license: mit
+  description: |
+    Fork of llama.cpp optimized for CPU performance by ikawrakow
+  urls:
+    - https://github.com/ikawrakow/ik_llama.cpp
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+  capabilities:
+    default: "cpu-ik-llama-cpp"
 - &whispercpp
   name: "whisper"
   alias: "whisper"
@@ -183,6 +197,7 @@
     amd: "rocm-vllm"
     intel: "intel-vllm"
     nvidia-cuda-12: "cuda12-vllm"
+    cpu: "cpu-vllm"
 - &vllm-omni
   name: "vllm-omni"
   license: apache-2.0
@@ -412,6 +427,30 @@
     nvidia-l4t: "nvidia-l4t-arm64-acestep-cpp"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-acestep-cpp"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-acestep-cpp"
+- &qwen3ttscpp
+  name: "qwen3-tts-cpp"
+  description: |
+    Qwen3-TTS C++ backend using GGML. Native C++ text-to-speech with voice cloning support.
+    Generates 24kHz mono audio from text with optional reference audio for voice cloning via ECAPA-TDNN speaker embeddings.
+  urls:
+    - https://github.com/predict-woo/qwen3-tts.cpp
+  tags:
+    - text-to-speech
+    - tts
+    - voice-cloning
+  alias: "qwen3-tts-cpp"
+  capabilities:
+    default: "cpu-qwen3-tts-cpp"
+    nvidia: "cuda12-qwen3-tts-cpp"
+    nvidia-cuda-13: "cuda13-qwen3-tts-cpp"
+    nvidia-cuda-12: "cuda12-qwen3-tts-cpp"
+    intel: "intel-sycl-f16-qwen3-tts-cpp"
+    metal: "metal-qwen3-tts-cpp"
+    amd: "rocm-qwen3-tts-cpp"
+    vulkan: "vulkan-qwen3-tts-cpp"
+    nvidia-l4t: "nvidia-l4t-arm64-qwen3-tts-cpp"
+    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-qwen3-tts-cpp"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-qwen3-tts-cpp"
 - &faster-whisper
   icon: https://avatars.githubusercontent.com/u/1520500?s=200&v=4
   description: |
@@ -873,6 +912,10 @@
     nvidia-cuda-12: "cuda12-llama-cpp-development"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-llama-cpp-development"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-llama-cpp-development"
+- !!merge <<: *ikllamacpp
+  name: "ik-llama-cpp-development"
+  capabilities:
+    default: "cpu-ik-llama-cpp-development"
 - !!merge <<: *neutts
   name: "cpu-neutts"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-neutts"
@@ -1303,6 +1346,17 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-llama-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-llama-cpp
+## ik-llama-cpp
+- !!merge <<: *ikllamacpp
+  name: "cpu-ik-llama-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-ik-llama-cpp"
+  mirrors:
+    - localai/localai-backends:latest-cpu-ik-llama-cpp
+- !!merge <<: *ikllamacpp
+  name: "cpu-ik-llama-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-ik-llama-cpp"
+  mirrors:
+    - localai/localai-backends:master-cpu-ik-llama-cpp
 ## whisper
 - !!merge <<: *whispercpp
   name: "nvidia-l4t-arm64-whisper"
@@ -1510,6 +1564,7 @@
     nvidia: "cuda12-vllm-development"
     amd: "rocm-vllm-development"
     intel: "intel-vllm-development"
+    cpu: "cpu-vllm-development"
 - !!merge <<: *vllm
   name: "cuda12-vllm"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-vllm"
@@ -1525,6 +1580,11 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-vllm"
   mirrors:
     - localai/localai-backends:latest-gpu-intel-vllm
+- !!merge <<: *vllm
+  name: "cpu-vllm"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-vllm"
+  mirrors:
+    - localai/localai-backends:latest-cpu-vllm
 - !!merge <<: *vllm
   name: "cuda12-vllm-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-vllm"
@@ -1540,6 +1600,11 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-vllm"
   mirrors:
     - localai/localai-backends:master-gpu-intel-vllm
+- !!merge <<: *vllm
+  name: "cpu-vllm-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-vllm"
+  mirrors:
+    - localai/localai-backends:master-cpu-vllm
 # vllm-omni
 - !!merge <<: *vllm-omni
   name: "vllm-omni-development"
@@ -2107,6 +2172,107 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-acestep-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-acestep-cpp
+## qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "nvidia-l4t-arm64-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-arm64-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "nvidia-l4t-arm64-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-arm64-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cuda13-nvidia-l4t-arm64-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cuda13-nvidia-l4t-arm64-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cpu-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-cpu-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "metal-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "metal-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cpu-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-cpu-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cuda12-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "rocm-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-rocm-hipblas-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "intel-sycl-f32-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-intel-sycl-f32-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "intel-sycl-f16-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-intel-sycl-f16-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "vulkan-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-vulkan-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "vulkan-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-vulkan-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cuda12-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "rocm-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-rocm-hipblas-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "intel-sycl-f32-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-intel-sycl-f32-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "intel-sycl-f16-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-intel-sycl-f16-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cuda13-qwen3-tts-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-qwen3-tts-cpp
+- !!merge <<: *qwen3ttscpp
+  name: "cuda13-qwen3-tts-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-qwen3-tts-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-qwen3-tts-cpp
 ## kokoro
 - !!merge <<: *kokoro
   name: "kokoro-development"

backend/python/ace-step/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
 torchaudio
 torchvision
 

backend/python/chatterbox/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.9.1+rocm6.4
-torchaudio==2.9.1+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchaudio==2.10.0+rocm7.0
 transformers
 numpy>=1.24.0,<1.26.0
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289

backend/python/common/template/requirements-hipblas.txt

@@ -1,2 +1,2 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
 torch

backend/python/common/vllm_utils.py

@@ -0,0 +1,84 @@
+"""Shared utilities for vLLM-based backends."""
+import json
+import sys
+
+
+def parse_options(options_list):
+    """Parse Options[] list of 'key:value' strings into a dict.
+
+    Supports type inference for common cases (bool, int, float).
+    Used by LoadModel to extract backend-specific options.
+    """
+    opts = {}
+    for opt in options_list:
+        if ":" not in opt:
+            continue
+        key, value = opt.split(":", 1)
+        key = key.strip()
+        value = value.strip()
+        # Try type conversion
+        if value.lower() in ("true", "false"):
+            opts[key] = value.lower() == "true"
+        else:
+            try:
+                opts[key] = int(value)
+            except ValueError:
+                try:
+                    opts[key] = float(value)
+                except ValueError:
+                    opts[key] = value
+    return opts
+
+
+def messages_to_dicts(proto_messages):
+    """Convert proto Message objects to list of dicts for apply_chat_template().
+
+    Handles: role, content, name, tool_call_id, reasoning_content, tool_calls (JSON string -> list).
+    """
+    result = []
+    for msg in proto_messages:
+        d = {"role": msg.role, "content": msg.content or ""}
+        if msg.name:
+            d["name"] = msg.name
+        if msg.tool_call_id:
+            d["tool_call_id"] = msg.tool_call_id
+        if msg.reasoning_content:
+            d["reasoning_content"] = msg.reasoning_content
+        if msg.tool_calls:
+            try:
+                d["tool_calls"] = json.loads(msg.tool_calls)
+            except json.JSONDecodeError:
+                pass
+        result.append(d)
+    return result
+
+
+def setup_parsers(opts):
+    """Return (tool_parser_cls, reasoning_parser_cls) tuple from opts dict.
+
+    Uses vLLM's native ToolParserManager and ReasoningParserManager.
+    Returns (None, None) if vLLM is not installed or parsers not available.
+    """
+    tool_parser_cls = None
+    reasoning_parser_cls = None
+
+    tool_parser_name = opts.get("tool_parser")
+    reasoning_parser_name = opts.get("reasoning_parser")
+
+    if tool_parser_name:
+        try:
+            from vllm.tool_parsers import ToolParserManager
+            tool_parser_cls = ToolParserManager.get_tool_parser(tool_parser_name)
+            print(f"[vllm_utils] Loaded tool_parser: {tool_parser_name}", file=sys.stderr)
+        except Exception as e:
+            print(f"[vllm_utils] Failed to load tool_parser {tool_parser_name}: {e}", file=sys.stderr)
+
+    if reasoning_parser_name:
+        try:
+            from vllm.reasoning import ReasoningParserManager
+            reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
+            print(f"[vllm_utils] Loaded reasoning_parser: {reasoning_parser_name}", file=sys.stderr)
+        except Exception as e:
+            print(f"[vllm_utils] Failed to load reasoning_parser {reasoning_parser_name}: {e}", file=sys.stderr)
+
+    return tool_parser_cls, reasoning_parser_cls

backend/python/coqui/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
-torchaudio==2.8.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchaudio==2.10.0+rocm7.0
 transformers==4.48.3
 accelerate
 coqui-tts

backend/python/diffusers/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
-torchvision==0.23.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchvision==0.25.0+rocm7.0
 git+https://github.com/huggingface/diffusers
 opencv-python
 transformers

backend/python/faster-whisper/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
 torch
 faster-whisper

backend/python/fish-speech/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.3
-torch==2.7.1+rocm6.3
-torchaudio==2.7.1+rocm6.3
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchaudio==2.10.0+rocm7.0

backend/python/kokoro/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
-torchaudio==2.8.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchaudio==2.10.0+rocm7.0
 transformers
 accelerate
 kokoro

backend/python/nemo/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.3
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
 torch
 nemo_toolkit[asr]

backend/python/neutts/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
 transformers==4.56.1
 accelerate
 librosa==0.11.0

backend/python/outetts/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
 accelerate
 llvmlite==0.43.0
 numba==0.60.0

backend/python/pocket-tts/requirements-hipblas.txt

@@ -1,4 +1,4 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.3
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
 pocket-tts
 scipy
-torch==2.7.1+rocm6.3
+torch==2.10.0+rocm7.0

backend/python/qwen-asr/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.3
-torch==2.7.1+rocm6.3
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
 qwen-asr

backend/python/qwen-tts/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.3
-torch==2.7.1+rocm6.3
-torchaudio==2.7.1+rocm6.3
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchaudio==2.10.0+rocm7.0
 qwen-tts
 sox

backend/python/rerankers/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
 transformers
 accelerate
-torch==2.8.0+rocm6.4
+torch==2.10.0+rocm7.0
 rerankers[transformers]

backend/python/rfdetr/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
-torchvision==0.23.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchvision==0.25.0+rocm7.0
 rfdetr
 opencv-python
 accelerate

backend/python/transformers/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0+rocm6.4
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
 accelerate
 transformers>=5.0.0
 llvmlite==0.43.0

backend/python/vibevoice/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.3
-torch==2.7.1+rocm6.3
-torchvision==0.22.1+rocm6.3
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
+torchvision==0.25.0+rocm7.0
 git+https://github.com/huggingface/diffusers
 opencv-python
 transformers>=4.51.3,<5.0.0

backend/python/vllm-omni/backend.py

@@ -17,6 +17,8 @@ import time
 import os
 import base64
 import io
+import json
+import gc
 
 from PIL import Image
 import torch
@@ -30,6 +32,7 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
+from vllm_utils import parse_options, messages_to_dicts, setup_parsers
 
 
 from vllm_omni.entrypoints.omni import Omni
@@ -148,23 +151,20 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
     def LoadModel(self, request, context):
         try:
+            # CPU detection: if no CUDA, default vLLM target device to CPU.
+            try:
+                if not torch.cuda.is_available():
+                    os.environ.setdefault("VLLM_TARGET_DEVICE", "cpu")
+                    os.environ.setdefault("VLLM_CPU_KVCACHE_SPACE", "4")
+            except Exception:
+                pass
+
             print(f"Loading model {request.Model}...", file=sys.stderr)
             print(f"Request {request}", file=sys.stderr)
 
-            # Parse options from request.Options (key:value pairs)
-            self.options = {}
-            for opt in request.Options:
-                if ":" not in opt:
-                    continue
-                key, value = opt.split(":", 1)
-                # Convert value to appropriate type
-                if is_float(value):
-                    value = float(value)
-                elif is_int(value):
-                    value = int(value)
-                elif value.lower() in ["true", "false"]:
-                    value = value.lower() == "true"
-                self.options[key] = value
+            # Parse options from request.Options using shared helper
+            self.options = parse_options(request.Options)
+            opts = self.options
 
             print(f"Options: {self.options}", file=sys.stderr)
 
@@ -244,6 +244,24 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     omni_kwargs["max_model_len"] = request.MaxModelLen
 
             self.omni = Omni(**omni_kwargs)
+
+            # Load tokenizer for LLM/TTS so chat templates work
+            if self.model_type in ("llm", "tts"):
+                try:
+                    from vllm.transformers_utils.tokenizer import get_tokenizer
+                    self.tokenizer = get_tokenizer(
+                        request.Model,
+                        trust_remote_code=opts.get("trust_remote_code", False),
+                    )
+                except Exception as e:
+                    print(f"Failed to load tokenizer: {e}", file=sys.stderr)
+                    self.tokenizer = None
+            else:
+                self.tokenizer = None
+
+            # Setup optional tool / reasoning parsers
+            self.tool_parser_cls, self.reasoning_parser_cls = setup_parsers(opts)
+
             print("Model loaded successfully", file=sys.stderr)
             return backend_pb2.Result(message="Model loaded successfully", success=True)
 
@@ -466,14 +484,32 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # Extract prompt
             if request.Prompt:
                 prompt = request.Prompt
-            elif request.Messages and request.UseTokenizerTemplate:
-                # Build prompt from messages (simplified - would need tokenizer for full template)
-                prompt = ""
-                for msg in request.Messages:
-                    role = msg.role
-                    content = msg.content
-                    prompt += f"<|im_start|>{role}\n{content}<|im_end|>\n"
-                prompt += "<|im_start|>assistant\n"
+            elif request.Messages:
+                if getattr(self, "tokenizer", None) is not None:
+                    messages_dicts = messages_to_dicts(request.Messages)
+                    template_kwargs = {"tokenize": False, "add_generation_prompt": True}
+                    if request.Tools:
+                        try:
+                            template_kwargs["tools"] = json.loads(request.Tools)
+                        except json.JSONDecodeError:
+                            pass
+                    try:
+                        if request.Metadata.get("enable_thinking", "").lower() == "true":
+                            template_kwargs["enable_thinking"] = True
+                    except Exception:
+                        pass
+                    try:
+                        prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
+                    except TypeError:
+                        prompt = self.tokenizer.apply_chat_template(
+                            messages_dicts, tokenize=False, add_generation_prompt=True
+                        )
+                else:
+                    # Fallback: basic template
+                    prompt = ""
+                    for msg in request.Messages:
+                        prompt += f"<|im_start|>{msg.role}\n{msg.content}<|im_end|>\n"
+                    prompt += "<|im_start|>assistant\n"
             else:
                 yield backend_pb2.Reply(message=bytes("", 'utf-8'))
                 return
@@ -539,20 +575,79 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # Call omni.generate() (returns generator for LLM mode)
             omni_generator = self.omni.generate([inputs], sampling_params_list)
 
-            # Extract text from outputs
+            # Extract text from outputs and track token usage
             generated_text = ""
+            prompt_tokens = 0
+            completion_tokens = 0
             for stage_outputs in omni_generator:
                 if stage_outputs.final_output_type == "text":
                     for output in stage_outputs.request_output:
-                        text_output = output.outputs[0].text
+                        completion = output.outputs[0]
+                        text_output = completion.text
+                        # Track tokens when available
+                        try:
+                            if getattr(output, "prompt_token_ids", None) is not None:
+                                prompt_tokens = len(output.prompt_token_ids)
+                            if getattr(completion, "token_ids", None) is not None:
+                                completion_tokens = len(completion.token_ids)
+                        except Exception:
+                            pass
                         if streaming:
                             # Remove already sent text (vllm concatenates)
                             delta_text = text_output.removeprefix(generated_text)
-                            yield backend_pb2.Reply(message=bytes(delta_text, encoding='utf-8'))
+                            yield backend_pb2.Reply(
+                                message=bytes(delta_text, encoding='utf-8'),
+                                tokens=completion_tokens,
+                                prompt_tokens=prompt_tokens,
+                            )
                         generated_text = text_output
 
             if not streaming:
-                yield backend_pb2.Reply(message=bytes(generated_text, encoding='utf-8'))
+                # Build optional ChatDelta with parsed reasoning / tool calls
+                chat_deltas = []
+                content_text = generated_text
+                reasoning_text = ""
+                tool_call_deltas = []
+
+                if self.reasoning_parser_cls is not None:
+                    try:
+                        parser = self.reasoning_parser_cls(self.tokenizer) if self.tokenizer else self.reasoning_parser_cls()
+                        reasoning_text, content_text = parser.extract_reasoning_content(content_text, request=None)
+                        reasoning_text = reasoning_text or ""
+                        content_text = content_text or ""
+                    except Exception as e:
+                        print(f"reasoning_parser failed: {e}", file=sys.stderr)
+
+                if self.tool_parser_cls is not None:
+                    try:
+                        parser = self.tool_parser_cls(self.tokenizer) if self.tokenizer else self.tool_parser_cls()
+                        tool_info = parser.extract_tool_calls(content_text, request=None)
+                        if getattr(tool_info, "tools_called", False):
+                            content_text = tool_info.content or ""
+                            for tc in tool_info.tool_calls or []:
+                                fn = getattr(tc, "function", None)
+                                tool_call_deltas.append(backend_pb2.ToolCallDelta(
+                                    index=getattr(tc, "index", 0) or 0,
+                                    id=getattr(tc, "id", "") or "",
+                                    name=getattr(fn, "name", "") if fn else "",
+                                    arguments=getattr(fn, "arguments", "") if fn else "",
+                                ))
+                    except Exception as e:
+                        print(f"tool_parser failed: {e}", file=sys.stderr)
+
+                if self.tool_parser_cls is not None or self.reasoning_parser_cls is not None:
+                    chat_deltas.append(backend_pb2.ChatDelta(
+                        content=content_text,
+                        reasoning_content=reasoning_text,
+                        tool_calls=tool_call_deltas,
+                    ))
+
+                yield backend_pb2.Reply(
+                    message=bytes(generated_text, encoding='utf-8'),
+                    tokens=completion_tokens,
+                    prompt_tokens=prompt_tokens,
+                    chat_deltas=chat_deltas,
+                )
 
         except Exception as err:
             print(f"Error in Predict: {err}", file=sys.stderr)
@@ -647,6 +742,37 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             traceback.print_exc()
             return backend_pb2.Result(success=False, message=f"Error generating TTS: {err}")
 
+    def TokenizeString(self, request, context):
+        if not hasattr(self, 'tokenizer') or self.tokenizer is None:
+            context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
+            context.set_details("Model/tokenizer not loaded")
+            return backend_pb2.TokenizationResponse()
+        try:
+            tokens = self.tokenizer.encode(request.Prompt)
+            return backend_pb2.TokenizationResponse(length=len(tokens), tokens=tokens)
+        except Exception as e:
+            context.set_code(grpc.StatusCode.INTERNAL)
+            context.set_details(str(e))
+            return backend_pb2.TokenizationResponse()
+
+    def Free(self, request, context):
+        try:
+            if hasattr(self, 'omni'):
+                del self.omni
+            if hasattr(self, 'tokenizer'):
+                del self.tokenizer
+            self.tool_parser_cls = None
+            self.reasoning_parser_cls = None
+            gc.collect()
+            try:
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+            except Exception:
+                pass
+            return backend_pb2.Result(success=True, message="Model freed")
+        except Exception as e:
+            return backend_pb2.Result(success=False, message=str(e))
+
 
 def serve(address):
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),

backend/python/vllm-omni/requirements-hipblas.txt

@@ -1,4 +1,4 @@
---extra-index-url https://download.pytorch.org/whl/nightly/rocm6.4
+--extra-index-url https://download.pytorch.org/whl/nightly/rocm7.0
 accelerate
 torch
 transformers

backend/python/vllm/backend.py

@@ -5,6 +5,9 @@ import argparse
 import signal
 import sys
 import os
+import json
+import time
+import gc
 from typing import List
 from PIL import Image
 
@@ -26,6 +29,25 @@ from vllm.assets.video import VideoAsset
 import base64
 import io
 
+# Version-compat imports — wrap in try/except for older vLLM versions
+try:
+    from vllm.tool_parsers import ToolParserManager
+    HAS_TOOL_PARSERS = True
+except ImportError:
+    HAS_TOOL_PARSERS = False
+
+try:
+    from vllm.reasoning import ReasoningParserManager
+    HAS_REASONING_PARSERS = True
+except ImportError:
+    HAS_REASONING_PARSERS = False
+
+try:
+    from vllm.sampling_params import GuidedDecodingParams
+    HAS_GUIDED_DECODING = True
+except ImportError:
+    HAS_GUIDED_DECODING = False
+
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 # If MAX_WORKERS are specified in the environment use it, otherwise default to 1
@@ -69,6 +91,35 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 break
         return decoded_text
 
+    def _parse_options(self, options_list):
+        """Parse Options[] key:value string list into a dict."""
+        opts = {}
+        for opt in options_list:
+            if ":" not in opt:
+                continue
+            key, value = opt.split(":", 1)
+            opts[key.strip()] = value.strip()
+        return opts
+
+    def _messages_to_dicts(self, messages):
+        """Convert proto Messages to list of dicts suitable for apply_chat_template()."""
+        result = []
+        for msg in messages:
+            d = {"role": msg.role, "content": msg.content or ""}
+            if msg.name:
+                d["name"] = msg.name
+            if msg.tool_call_id:
+                d["tool_call_id"] = msg.tool_call_id
+            if msg.reasoning_content:
+                d["reasoning_content"] = msg.reasoning_content
+            if msg.tool_calls:
+                try:
+                    d["tool_calls"] = json.loads(msg.tool_calls)
+                except json.JSONDecodeError:
+                    pass
+            result.append(d)
+        return result
+
     def Health(self, request, context):
         """
         Returns a health check message.
@@ -132,15 +183,49 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
 
         try:
-           engine_model_config = await self.llm.get_model_config()
-           self.tokenizer = get_tokenizer(
-               engine_model_config.tokenizer,
-               tokenizer_mode=engine_model_config.tokenizer_mode,
-               trust_remote_code=engine_model_config.trust_remote_code,
-               truncation_side="left",
-           )
+            # vLLM >= 0.14 removed get_model_config() on AsyncLLM; the tokenizer
+            # is either already loaded on the engine or can be built from the
+            # Model name directly.
+            tokenizer = None
+            if hasattr(self.llm, "get_tokenizer"):
+                try:
+                    tokenizer = await self.llm.get_tokenizer()
+                except TypeError:
+                    tokenizer = self.llm.get_tokenizer()
+                except Exception:
+                    tokenizer = None
+            if tokenizer is None and hasattr(self.llm, "tokenizer"):
+                tokenizer = self.llm.tokenizer
+            if tokenizer is None:
+                tokenizer = get_tokenizer(
+                    request.Model,
+                    trust_remote_code=bool(request.TrustRemoteCode),
+                    truncation_side="left",
+                )
+            self.tokenizer = tokenizer
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+
+        # Parse options for parser selection
+        opts = self._parse_options(request.Options)
+
+        # Instantiate tool/reasoning parser classes (they'll be instantiated per-request with tokenizer)
+        self.tool_parser_cls = None
+        self.reasoning_parser_cls = None
+        if HAS_TOOL_PARSERS and opts.get("tool_parser"):
+            try:
+                self.tool_parser_cls = ToolParserManager.get_tool_parser(opts["tool_parser"])
+                print(f"Loaded tool_parser: {opts['tool_parser']}", file=sys.stderr)
+            except Exception as e:
+                print(f"Failed to load tool_parser {opts.get('tool_parser')}: {e}", file=sys.stderr)
+
+        if HAS_REASONING_PARSERS and opts.get("reasoning_parser"):
+            try:
+                self.reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser(opts["reasoning_parser"])
+                print(f"Loaded reasoning_parser: {opts['reasoning_parser']}", file=sys.stderr)
+            except Exception as e:
+                print(f"Failed to load reasoning_parser {opts.get('reasoning_parser')}: {e}", file=sys.stderr)
+
         print("Model loaded successfully", file=sys.stderr)
         return backend_pb2.Result(message="Model loaded successfully", success=True)
 
@@ -197,6 +282,38 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         finally:
             await iterations.aclose()
 
+    async def TokenizeString(self, request, context):
+        if not hasattr(self, 'tokenizer') or self.tokenizer is None:
+            context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
+            context.set_details("Model/tokenizer not loaded")
+            return backend_pb2.TokenizationResponse()
+        try:
+            tokens = self.tokenizer.encode(request.Prompt)
+            return backend_pb2.TokenizationResponse(length=len(tokens), tokens=tokens)
+        except Exception as e:
+            context.set_code(grpc.StatusCode.INTERNAL)
+            context.set_details(str(e))
+            return backend_pb2.TokenizationResponse()
+
+    async def Free(self, request, context):
+        try:
+            if hasattr(self, 'llm'):
+                del self.llm
+            if hasattr(self, 'tokenizer'):
+                del self.tokenizer
+            self.tool_parser_cls = None
+            self.reasoning_parser_cls = None
+            gc.collect()
+            try:
+                import torch
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+            except ImportError:
+                pass
+            return backend_pb2.Result(success=True, message="Model freed")
+        except Exception as e:
+            return backend_pb2.Result(success=False, message=str(e))
+
     async def _predict(self, request, context, streaming=False):
         # Build the sampling parameters
         # NOTE: this must stay in sync with the vllm backend
@@ -222,7 +339,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             "SkipSpecialTokens": "skip_special_tokens",
             "SpacesBetweenSpecialTokens": "spaces_between_special_tokens",
             "TruncatePromptTokens": "truncate_prompt_tokens",
-            "GuidedDecoding": "guided_decoding",
         }
 
         sampling_params = SamplingParams(top_p=0.9, max_tokens=200)
@@ -233,6 +349,14 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 if value not in (None, 0, [], False):
                     setattr(sampling_params, param_field, value)
 
+        # Guided decoding: use Grammar field to pass JSON schema or BNF
+        if HAS_GUIDED_DECODING and request.Grammar:
+            try:
+                json.loads(request.Grammar)  # valid JSON = JSON schema
+                sampling_params.guided_decoding = GuidedDecodingParams(json=request.Grammar)
+            except json.JSONDecodeError:
+                sampling_params.guided_decoding = GuidedDecodingParams(grammar=request.Grammar)
+
         # Extract image paths and process images
         prompt = request.Prompt
 
@@ -244,7 +368,27 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # If tokenizer template is enabled and messages are provided instead of prompt, apply the tokenizer template
         if not request.Prompt and request.UseTokenizerTemplate and request.Messages:
-            prompt = self.tokenizer.apply_chat_template(request.Messages, tokenize=False, add_generation_prompt=True)
+            messages_dicts = self._messages_to_dicts(request.Messages)
+            template_kwargs = {"tokenize": False, "add_generation_prompt": True}
+
+            # Pass tools for tool calling
+            if request.Tools:
+                try:
+                    template_kwargs["tools"] = json.loads(request.Tools)
+                except json.JSONDecodeError:
+                    pass
+
+            # Enable thinking mode if requested
+            if request.Metadata.get("enable_thinking", "").lower() == "true":
+                template_kwargs["enable_thinking"] = True
+
+            try:
+                prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
+            except TypeError:
+                # Some tokenizers don't support tools/enable_thinking kwargs — retry without them
+                prompt = self.tokenizer.apply_chat_template(
+                    messages_dicts, tokenize=False, add_generation_prompt=True
+                )
 
         # Generate text using the LLM engine
         request_id = random_uuid()
@@ -265,25 +409,26 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # Stream the results
         generated_text = ""
+        last_output = None
         try:
             async for request_output in outputs:
                 iteration_text = request_output.outputs[0].text
+                last_output = request_output
 
                 if streaming:
                     # Remove text already sent as vllm concatenates the text from previous yields
                     delta_iteration_text = iteration_text.removeprefix(generated_text)
                     # Send the partial result
-                    yield backend_pb2.Reply(message=bytes(delta_iteration_text, encoding='utf-8'))
+                    yield backend_pb2.Reply(
+                        message=bytes(delta_iteration_text, encoding='utf-8'),
+                        chat_deltas=[backend_pb2.ChatDelta(content=delta_iteration_text)],
+                    )
 
                 # Keep track of text generated
                 generated_text = iteration_text
         finally:
             await outputs.aclose()
 
-        # If streaming, we already sent everything
-        if streaming:
-            return
-
         # Remove the image files from /tmp folder
         for img_path in image_paths:
             try:
@@ -291,8 +436,99 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             except Exception as e:
                 print(f"Error removing image file: {img_path}, {e}", file=sys.stderr)
 
-        # Sending the final generated text
-        yield backend_pb2.Reply(message=bytes(generated_text, encoding='utf-8'))
+        # Parse reasoning and tool calls from final text using vLLM's native parsers
+        content = generated_text
+        reasoning_content = ""
+        tool_calls_proto = []
+
+        if self.reasoning_parser_cls:
+            try:
+                rp = self.reasoning_parser_cls(self.tokenizer)
+                r, c = rp.extract_reasoning(generated_text, request=None)
+                reasoning_content = r or ""
+                content = c if c is not None else generated_text
+            except Exception as e:
+                print(f"Reasoning parser error: {e}", file=sys.stderr)
+
+        if self.tool_parser_cls and request.Tools:
+            try:
+                tools = json.loads(request.Tools)
+                # Some concrete parsers only accept the tokenizer; only the
+                # abstract base declares the tools kwarg. Try with tools first,
+                # fall back to tokenizer-only.
+                try:
+                    tp = self.tool_parser_cls(self.tokenizer, tools=tools)
+                except TypeError:
+                    tp = self.tool_parser_cls(self.tokenizer)
+                info = tp.extract_tool_calls(content, request=None)
+                if info.tools_called:
+                    content = info.content or ""
+                    for i, tc in enumerate(info.tool_calls):
+                        tool_calls_proto.append(backend_pb2.ToolCallDelta(
+                            index=i,
+                            id=tc.id,
+                            name=tc.function.name,
+                            arguments=tc.function.arguments,
+                        ))
+            except Exception as e:
+                print(f"Tool parser error: {e}", file=sys.stderr)
+
+        # Extract token counts
+        prompt_tokens = 0
+        completion_tokens = 0
+        if last_output is not None:
+            try:
+                prompt_tokens = len(last_output.prompt_token_ids or [])
+            except Exception:
+                pass
+            try:
+                completion_tokens = len(last_output.outputs[0].token_ids or [])
+            except Exception:
+                pass
+
+        # Extract logprobs if requested
+        logprobs_bytes = b""
+        if last_output is not None and request.Logprobs > 0:
+            try:
+                lp = last_output.outputs[0].logprobs
+                if lp:
+                    logprobs_data = {"content": []}
+                    for token_lp_dict in lp:
+                        if token_lp_dict:
+                            first_tok_id, first_lp = next(iter(token_lp_dict.items()))
+                            logprobs_data["content"].append({
+                                "token": getattr(first_lp, "decoded_token", str(first_tok_id)),
+                                "logprob": first_lp.logprob,
+                            })
+                    logprobs_bytes = json.dumps(logprobs_data).encode("utf-8")
+            except Exception as e:
+                print(f"Logprobs extraction error: {e}", file=sys.stderr)
+
+        chat_delta = backend_pb2.ChatDelta(
+            content=content,
+            reasoning_content=reasoning_content,
+            tool_calls=tool_calls_proto,
+        )
+
+        if streaming:
+            # Final chunk with structured data
+            yield backend_pb2.Reply(
+                message=b"",
+                prompt_tokens=prompt_tokens,
+                tokens=completion_tokens,
+                chat_deltas=[chat_delta],
+                logprobs=logprobs_bytes,
+            )
+            return
+
+        # Non-streaming: single Reply with everything
+        yield backend_pb2.Reply(
+            message=bytes(content, encoding='utf-8'),
+            prompt_tokens=prompt_tokens,
+            tokens=completion_tokens,
+            chat_deltas=[chat_delta],
+            logprobs=logprobs_bytes,
+        )
 
     def load_image(self, image_path: str):
         """

backend/python/vllm/install.sh

@@ -26,20 +26,43 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then
     EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
 fi
 
-# We don't embed this into the images as it is a large dependency and not always needed.
-# Besides, the speed inference are not actually usable in the current state for production use-cases.
-if [ "x${BUILD_TYPE}" == "x" ] && [ "x${FROM_SOURCE:-}" == "xtrue" ]; then
-        ensureVenv
-        # https://docs.vllm.ai/en/v0.6.1/getting_started/cpu-installation.html
-        if [ ! -d vllm ]; then
-            git clone https://github.com/vllm-project/vllm
-        fi
-        pushd vllm
-            uv pip install wheel packaging ninja "setuptools>=49.4.0" numpy typing-extensions pillow setuptools-scm grpcio==1.68.1 protobuf bitsandbytes
-            uv pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
-            VLLM_TARGET_DEVICE=cpu python setup.py install
-        popd
-        rm -rf vllm
-    else
-        installRequirements
+# CPU builds need unsafe-best-match to pull torch==2.10.0+cpu from the
+# pytorch test channel while still resolving transformers/vllm from pypi.
+if [ "x${BUILD_PROFILE}" == "xcpu" ]; then
+    EXTRA_PIP_INSTALL_FLAGS+=" --index-strategy=unsafe-best-match"
+fi
+
+# FROM_SOURCE=true on a CPU build skips the prebuilt vllm wheel in
+# requirements-cpu-after.txt and compiles vllm locally against the host's
+# actual CPU. Not used by default because it takes ~30-40 minutes, but
+# kept here for hosts where the prebuilt wheel SIGILLs (CPU without the
+# required SIMD baseline, e.g. AVX-512 VNNI/BF16). Default CI uses a
+# bigger-runner with compatible hardware instead.
+if [ "x${BUILD_TYPE}" == "x" ] && [ "x${FROM_SOURCE:-}" == "xtrue" ]; then
+    # Temporarily hide the prebuilt wheel so installRequirements doesn't
+    # pull it — the rest of the requirements files (base deps, torch,
+    # transformers) are still installed normally.
+    _cpu_after="${backend_dir}/requirements-cpu-after.txt"
+    _cpu_after_bak=""
+    if [ -f "${_cpu_after}" ]; then
+        _cpu_after_bak="${_cpu_after}.from-source.bak"
+        mv "${_cpu_after}" "${_cpu_after_bak}"
+    fi
+    installRequirements
+    if [ -n "${_cpu_after_bak}" ]; then
+        mv "${_cpu_after_bak}" "${_cpu_after}"
+    fi
+
+    # Build vllm from source against the installed torch.
+    # https://docs.vllm.ai/en/latest/getting_started/installation/cpu/
+    _vllm_src=$(mktemp -d)
+    trap 'rm -rf "${_vllm_src}"' EXIT
+    git clone --depth 1 https://github.com/vllm-project/vllm "${_vllm_src}/vllm"
+    pushd "${_vllm_src}/vllm"
+        uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} wheel packaging ninja "setuptools>=49.4.0" numpy typing-extensions pillow setuptools-scm
+        # Respect pre-installed torch version — skip vllm's own requirements-build.txt torch pin.
+        VLLM_TARGET_DEVICE=cpu uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps .
+    popd
+else
+    installRequirements
 fi

backend/python/vllm/package.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Script to package runtime shared libraries for the vllm backend.
+#
+# The final Dockerfile.python stage is FROM scratch, so system libraries
+# must be explicitly copied into ${BACKEND}/lib so the backend can run on
+# any host without installing them. libbackend.sh automatically adds that
+# directory to LD_LIBRARY_PATH at run time.
+#
+# vllm's CPU C++ extension (vllm._C) dlopens libnuma.so.1 at import time;
+# if it's missing, the _C_utils torch ops are never registered and the
+# engine crashes with AttributeError on init_cpu_threads_env. libgomp is
+# used by torch's CPU kernels; on some stripped-down hosts it's also
+# absent, so we bundle it too.
+
+set -e
+
+CURDIR=$(dirname "$(realpath "$0")")
+LIB_DIR="${CURDIR}/lib"
+mkdir -p "${LIB_DIR}"
+
+copy_with_symlinks() {
+    local soname="$1"
+    local hit=""
+    for dir in /usr/lib/x86_64-linux-gnu /usr/lib/aarch64-linux-gnu /lib/x86_64-linux-gnu /lib/aarch64-linux-gnu /usr/lib /lib; do
+        if [ -e "${dir}/${soname}" ]; then
+            hit="${dir}/${soname}"
+            break
+        fi
+    done
+    if [ -z "${hit}" ]; then
+        echo "warning: ${soname} not found in standard lib paths" >&2
+        return 0
+    fi
+    # Follow the symlink to the real file, copy it, then recreate the symlink.
+    local real
+    real=$(readlink -f "${hit}")
+    cp -v "${real}" "${LIB_DIR}/"
+    local real_base
+    real_base=$(basename "${real}")
+    if [ "${real_base}" != "${soname}" ]; then
+        ln -sf "${real_base}" "${LIB_DIR}/${soname}"
+    fi
+}
+
+copy_with_symlinks libnuma.so.1
+copy_with_symlinks libgomp.so.1
+
+echo "vllm packaging completed successfully"
+ls -liah "${LIB_DIR}/"

backend/python/vllm/requirements-after.txt

@@ -1 +1,2 @@
-vllm
+# vllm is installed per-acceleration in requirements-{profile}-after.txt
+# (cublas12, hipblas, intel, cpu)

backend/python/vllm/requirements-cpu-after.txt

@@ -0,0 +1,2 @@
+vllm @ https://github.com/vllm-project/vllm/releases/download/v0.14.1/vllm-0.14.1+cpu-cp38-abi3-manylinux_2_35_x86_64.whl ; platform_machine == "x86_64"
+vllm @ https://github.com/vllm-project/vllm/releases/download/v0.14.1/vllm-0.14.1+cpu-cp38-abi3-manylinux_2_35_aarch64.whl ; platform_machine == "aarch64"

backend/python/vllm/requirements-cpu.txt

@@ -1,3 +1,6 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
 accelerate
-torch==2.7.0
-transformers
+torch==2.9.1+cpu
+torchvision
+torchaudio
+transformers

backend/python/vllm/requirements-cublas12-after.txt

@@ -1 +1,2 @@
 https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.7cxx11abiTRUE-cp310-cp310-linux_x86_64.whl
+vllm

backend/python/vllm/requirements-hipblas-after.txt

@@ -0,0 +1 @@
+vllm

backend/python/vllm/requirements-hipblas.txt

@@ -1,4 +1,4 @@
---extra-index-url https://download.pytorch.org/whl/nightly/rocm6.4
+--extra-index-url https://download.pytorch.org/whl/nightly/rocm7.0
 accelerate
 torch
 transformers

backend/python/vllm/requirements-intel-after.txt

@@ -0,0 +1 @@
+vllm

backend/python/vllm/test.py

@@ -122,6 +122,89 @@ class TestBackendServicer(unittest.TestCase):
             self.tearDown()
 
 
+    def test_messages_to_dicts(self):
+        """
+        Tests _messages_to_dicts conversion of proto Messages to dicts.
+        """
+        import sys, os
+        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+        from backend import BackendServicer
+        servicer = BackendServicer()
+        msgs = [
+            backend_pb2.Message(role="user", content="hello"),
+            backend_pb2.Message(
+                role="assistant",
+                content="",
+                tool_calls='[{"id":"call_1","type":"function","function":{"name":"foo","arguments":"{}"}}]',
+                reasoning_content="thinking...",
+            ),
+            backend_pb2.Message(role="tool", content="result", name="foo", tool_call_id="call_1"),
+        ]
+        result = servicer._messages_to_dicts(msgs)
+        self.assertEqual(len(result), 3)
+        self.assertEqual(result[0], {"role": "user", "content": "hello"})
+        self.assertEqual(result[1]["reasoning_content"], "thinking...")
+        self.assertIsInstance(result[1]["tool_calls"], list)
+        self.assertEqual(result[1]["tool_calls"][0]["id"], "call_1")
+        self.assertEqual(result[2]["tool_call_id"], "call_1")
+        self.assertEqual(result[2]["name"], "foo")
+
+    def test_parse_options(self):
+        """
+        Tests _parse_options correctly parses key:value strings.
+        """
+        import sys, os
+        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+        from backend import BackendServicer
+        servicer = BackendServicer()
+        opts = servicer._parse_options([
+            "tool_parser:hermes",
+            "reasoning_parser:deepseek_r1",
+            "invalid_no_colon",
+            "key_with_colons:a:b:c",
+        ])
+        self.assertEqual(opts["tool_parser"], "hermes")
+        self.assertEqual(opts["reasoning_parser"], "deepseek_r1")
+        self.assertEqual(opts["key_with_colons"], "a:b:c")
+        self.assertNotIn("invalid_no_colon", opts)
+
+    def test_tokenize_string(self):
+        """
+        Tests the TokenizeString RPC returns valid tokens.
+        """
+        try:
+            self.setUp()
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
+                self.assertTrue(response.success)
+                resp = stub.TokenizeString(backend_pb2.PredictOptions(Prompt="Hello world"))
+                self.assertGreater(resp.length, 0)
+                self.assertEqual(len(resp.tokens), resp.length)
+        except Exception as err:
+            print(err)
+            self.fail("TokenizeString service failed")
+        finally:
+            self.tearDown()
+
+    def test_free(self):
+        """
+        Tests the Free RPC doesn't crash.
+        """
+        try:
+            self.setUp()
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
+                self.assertTrue(response.success)
+                free_resp = stub.Free(backend_pb2.HealthMessage())
+                self.assertTrue(free_resp.success)
+        except Exception as err:
+            print(err)
+            self.fail("Free service failed")
+        finally:
+            self.tearDown()
+
     def test_embedding(self):
         """
         This method tests if the embeddings are generated successfully

backend/python/voxcpm/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.3
-torch==2.7.1+rocm6.3
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
 soundfile
 numpy
 voxcpm

backend/python/whisperx/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm6.4
-torch==2.8.0
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch==2.10.0+rocm7.0
 whisperx @ git+https://github.com/m-bain/whisperX.git

core/application/application.go

@@ -37,6 +37,9 @@ type Application struct {
 
 	// Distributed mode services (nil when not in distributed mode)
 	distributed *DistributedServices
+
+	// Upgrade checker (background service for detecting backend upgrades)
+	upgradeChecker *UpgradeChecker
 }
 
 func newApplication(appConfig *config.ApplicationConfig) *Application {
@@ -79,6 +82,19 @@ func (a *Application) AgentJobService() *agentpool.AgentJobService {
 	return a.agentJobService
 }
 
+func (a *Application) UpgradeChecker() *UpgradeChecker {
+	return a.upgradeChecker
+}
+
+// distributedDB returns the PostgreSQL database for distributed coordination,
+// or nil in standalone mode.
+func (a *Application) distributedDB() *gorm.DB {
+	if a.distributed != nil {
+		return a.authDB
+	}
+	return nil
+}
+
 func (a *Application) AgentPoolService() *agentpool.AgentPoolService {
 	return a.agentPoolService.Load()
 }

core/application/config_file_watcher.go

@@ -335,6 +335,12 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
 			if settings.AutoloadBackendGalleries != nil && !envAutoloadBackendGalleries {
 				appConfig.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries
 			}
+			if settings.AutoUpgradeBackends != nil {
+				appConfig.AutoUpgradeBackends = *settings.AutoUpgradeBackends
+			}
+			if settings.PreferDevelopmentBackends != nil {
+				appConfig.PreferDevelopmentBackends = *settings.PreferDevelopmentBackends
+			}
 			if settings.ApiKeys != nil {
 				// API keys from env vars (startup) should be kept, runtime settings keys replace all runtime keys
 				// If runtime_settings.json specifies ApiKeys (even if empty), it replaces all runtime keys

core/application/startup.go

@@ -231,6 +231,15 @@ func New(opts ...config.AppOption) (*Application, error) {
 		xlog.Error("error registering external backends", "error", err)
 	}
 
+	// Start background upgrade checker for backends.
+	// In distributed mode, uses PostgreSQL advisory lock so only one frontend
+	// instance runs periodic checks (avoids duplicate upgrades across replicas).
+	if len(options.BackendGalleries) > 0 {
+		uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB())
+		application.upgradeChecker = uc
+		go uc.Run(options.Context)
+	}
+
 	if options.ConfigFile != "" {
 		if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil {
 			xlog.Error("error loading config file", "error", err)

core/application/upgrade_checker.go

@@ -0,0 +1,198 @@
+package application
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/services/advisorylock"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+	"github.com/mudler/xlog"
+	"gorm.io/gorm"
+)
+
+// UpgradeChecker periodically checks for backend upgrades and optionally
+// auto-upgrades them. It caches the last check results for API queries.
+//
+// In standalone mode it runs a simple ticker loop.
+// In distributed mode it uses a PostgreSQL advisory lock so that only one
+// frontend instance performs periodic checks and auto-upgrades at a time.
+type UpgradeChecker struct {
+	appConfig   *config.ApplicationConfig
+	modelLoader *model.ModelLoader
+	galleries   []config.Gallery
+	systemState *system.SystemState
+	db          *gorm.DB // non-nil in distributed mode
+
+	checkInterval time.Duration
+	stop          chan struct{}
+	done          chan struct{}
+	triggerCh     chan struct{}
+
+	mu            sync.RWMutex
+	lastUpgrades  map[string]gallery.UpgradeInfo
+	lastCheckTime time.Time
+}
+
+// NewUpgradeChecker creates a new UpgradeChecker service.
+// Pass db=nil for standalone mode, or a *gorm.DB for distributed mode
+// (uses advisory locks so only one instance runs periodic checks).
+func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader, db *gorm.DB) *UpgradeChecker {
+	return &UpgradeChecker{
+		appConfig:     appConfig,
+		modelLoader:   ml,
+		galleries:     appConfig.BackendGalleries,
+		systemState:   appConfig.SystemState,
+		db:            db,
+		checkInterval: 6 * time.Hour,
+		stop:          make(chan struct{}),
+		done:          make(chan struct{}),
+		triggerCh:     make(chan struct{}, 1),
+		lastUpgrades:  make(map[string]gallery.UpgradeInfo),
+	}
+}
+
+// Run starts the upgrade checker loop. It waits 30 seconds after startup,
+// performs an initial check, then re-checks every 6 hours.
+//
+// In distributed mode, periodic checks are guarded by a PostgreSQL advisory
+// lock so only one frontend instance runs them. On-demand triggers (TriggerCheck)
+// and the initial check always run locally for fast API response cache warming.
+func (uc *UpgradeChecker) Run(ctx context.Context) {
+	defer close(uc.done)
+
+	// Initial delay: don't slow down startup
+	select {
+	case <-ctx.Done():
+		return
+	case <-uc.stop:
+		return
+	case <-time.After(30 * time.Second):
+	}
+
+	// First check always runs locally (to warm the cache on this instance)
+	uc.runCheck(ctx)
+
+	if uc.db != nil {
+		// Distributed mode: use advisory lock for periodic checks.
+		// RunLeaderLoop ticks every checkInterval; only the lock holder executes.
+		go advisorylock.RunLeaderLoop(ctx, uc.db, advisorylock.KeyBackendUpgradeCheck, uc.checkInterval, func() {
+			uc.runCheck(ctx)
+		})
+
+		// Still listen for on-demand triggers (from API / settings change)
+		// and stop signal — these run on every instance.
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-uc.stop:
+				return
+			case <-uc.triggerCh:
+				uc.runCheck(ctx)
+			}
+		}
+	} else {
+		// Standalone mode: simple ticker loop
+		ticker := time.NewTicker(uc.checkInterval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-uc.stop:
+				return
+			case <-ticker.C:
+				uc.runCheck(ctx)
+			case <-uc.triggerCh:
+				uc.runCheck(ctx)
+			}
+		}
+	}
+}
+
+// Shutdown stops the upgrade checker loop.
+func (uc *UpgradeChecker) Shutdown() {
+	close(uc.stop)
+	<-uc.done
+}
+
+// TriggerCheck forces an immediate upgrade check on this instance.
+func (uc *UpgradeChecker) TriggerCheck() {
+	select {
+	case uc.triggerCh <- struct{}{}:
+	default:
+		// Already triggered, skip
+	}
+}
+
+// GetAvailableUpgrades returns the cached upgrade check results.
+func (uc *UpgradeChecker) GetAvailableUpgrades() map[string]gallery.UpgradeInfo {
+	uc.mu.RLock()
+	defer uc.mu.RUnlock()
+
+	// Return a copy to avoid races
+	result := make(map[string]gallery.UpgradeInfo, len(uc.lastUpgrades))
+	for k, v := range uc.lastUpgrades {
+		result[k] = v
+	}
+	return result
+}
+
+func (uc *UpgradeChecker) runCheck(ctx context.Context) {
+	upgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState)
+
+	uc.mu.Lock()
+	uc.lastCheckTime = time.Now()
+	if err != nil {
+		xlog.Debug("Backend upgrade check failed", "error", err)
+		uc.mu.Unlock()
+		return
+	}
+	uc.lastUpgrades = upgrades
+	uc.mu.Unlock()
+
+	if len(upgrades) == 0 {
+		xlog.Debug("All backends up to date")
+		return
+	}
+
+	// Log available upgrades
+	for name, info := range upgrades {
+		if info.AvailableVersion != "" {
+			xlog.Info("Backend upgrade available",
+				"backend", name,
+				"installed", info.InstalledVersion,
+				"available", info.AvailableVersion)
+		} else {
+			xlog.Info("Backend upgrade available (new build)",
+				"backend", name)
+		}
+	}
+
+	// Auto-upgrade if enabled
+	if uc.appConfig.AutoUpgradeBackends {
+		for name, info := range upgrades {
+			xlog.Info("Auto-upgrading backend", "backend", name,
+				"from", info.InstalledVersion, "to", info.AvailableVersion)
+			if err := gallery.UpgradeBackend(ctx, uc.systemState, uc.modelLoader,
+				uc.galleries, name, nil); err != nil {
+				xlog.Error("Failed to auto-upgrade backend",
+					"backend", name, "error", err)
+			} else {
+				xlog.Info("Backend upgraded successfully", "backend", name,
+					"version", info.AvailableVersion)
+			}
+		}
+		// Re-check to update cache after upgrades
+		if freshUpgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState); err == nil {
+			uc.mu.Lock()
+			uc.lastUpgrades = freshUpgrades
+			uc.mu.Unlock()
+		}
+	}
+}

core/cli/backends.go

@@ -40,10 +40,17 @@ type BackendsUninstall struct {
 	BackendsCMDFlags `embed:""`
 }
 
+type BackendsUpgrade struct {
+	BackendArgs []string `arg:"" optional:"" name:"backends" help:"Backend names to upgrade (empty = upgrade all)"`
+
+	BackendsCMDFlags `embed:""`
+}
+
 type BackendsCMD struct {
 	List      BackendsList      `cmd:"" help:"List the backends available in your galleries" default:"withargs"`
 	Install   BackendsInstall   `cmd:"" help:"Install a backend from the gallery"`
 	Uninstall BackendsUninstall `cmd:"" help:"Uninstall a backend"`
+	Upgrade   BackendsUpgrade   `cmd:"" help:"Upgrade backends to latest versions"`
 }
 
 func (bl *BackendsList) Run(ctx *cliContext.Context) error {
@@ -64,11 +71,27 @@ func (bl *BackendsList) Run(ctx *cliContext.Context) error {
 	if err != nil {
 		return err
 	}
+
+	// Check for upgrades
+	upgrades, _ := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState)
+
 	for _, backend := range backends {
+		versionStr := ""
+		if backend.Version != "" {
+			versionStr = " v" + backend.Version
+		}
 		if backend.Installed {
-			fmt.Printf(" * %s@%s (installed)\n", backend.Gallery.Name, backend.Name)
+			if info, ok := upgrades[backend.Name]; ok {
+				upgradeStr := info.AvailableVersion
+				if upgradeStr == "" {
+					upgradeStr = "new build"
+				}
+				fmt.Printf(" * %s@%s%s (installed, upgrade available: %s)\n", backend.Gallery.Name, backend.Name, versionStr, upgradeStr)
+			} else {
+				fmt.Printf(" * %s@%s%s (installed)\n", backend.Gallery.Name, backend.Name, versionStr)
+			}
 		} else {
-			fmt.Printf(" - %s@%s\n", backend.Gallery.Name, backend.Name)
+			fmt.Printf(" - %s@%s%s\n", backend.Gallery.Name, backend.Name, versionStr)
 		}
 	}
 	return nil
@@ -111,6 +134,79 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
 	return nil
 }
 
+func (bu *BackendsUpgrade) Run(ctx *cliContext.Context) error {
+	var galleries []config.Gallery
+	if err := json.Unmarshal([]byte(bu.BackendGalleries), &galleries); err != nil {
+		xlog.Error("unable to load galleries", "error", err)
+	}
+
+	systemState, err := system.GetSystemState(
+		system.WithBackendSystemPath(bu.BackendsSystemPath),
+		system.WithBackendPath(bu.BackendsPath),
+	)
+	if err != nil {
+		return err
+	}
+
+	upgrades, err := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState)
+	if err != nil {
+		return fmt.Errorf("failed to check for upgrades: %w", err)
+	}
+
+	if len(upgrades) == 0 {
+		fmt.Println("All backends are up to date.")
+		return nil
+	}
+
+	// Filter to specified backends if args given
+	toUpgrade := upgrades
+	if len(bu.BackendArgs) > 0 {
+		toUpgrade = make(map[string]gallery.UpgradeInfo)
+		for _, name := range bu.BackendArgs {
+			if info, ok := upgrades[name]; ok {
+				toUpgrade[name] = info
+			} else {
+				fmt.Printf("Backend %s: no upgrade available\n", name)
+			}
+		}
+	}
+
+	if len(toUpgrade) == 0 {
+		fmt.Println("No upgrades to apply.")
+		return nil
+	}
+
+	modelLoader := model.NewModelLoader(systemState)
+	for name, info := range toUpgrade {
+		versionStr := ""
+		if info.AvailableVersion != "" {
+			versionStr = " to v" + info.AvailableVersion
+		}
+		fmt.Printf("Upgrading %s%s...\n", name, versionStr)
+
+		progressBar := progressbar.NewOptions(
+			1000,
+			progressbar.OptionSetDescription(fmt.Sprintf("downloading %s", name)),
+			progressbar.OptionShowBytes(false),
+			progressbar.OptionClearOnFinish(),
+		)
+		progressCallback := func(fileName string, current string, total string, percentage float64) {
+			v := int(percentage * 10)
+			if err := progressBar.Set(v); err != nil {
+				xlog.Error("error updating progress bar", "error", err)
+			}
+		}
+
+		if err := gallery.UpgradeBackend(context.Background(), systemState, modelLoader, galleries, name, progressCallback); err != nil {
+			fmt.Printf("Failed to upgrade %s: %v\n", name, err)
+		} else {
+			fmt.Printf("Backend %s upgraded successfully\n", name)
+		}
+	}
+
+	return nil
+}
+
 func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error {
 	for _, backendName := range bu.BackendArgs {
 		xlog.Info("uninstalling backend", "backend", backendName)

core/cli/run.go

@@ -47,7 +47,9 @@ type RunCMD struct {
 	BackendImagesReleaseTag  string   `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"`
 	BackendImagesBranchTag   string   `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"`
 	BackendDevSuffix         string   `env:"LOCALAI_BACKEND_DEV_SUFFIX,BACKEND_DEV_SUFFIX" help:"Development suffix for backend images" group:"backends" default:"development"`
-	PreloadModels            string   `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"`
+	AutoUpgradeBackends       bool     `env:"LOCALAI_AUTO_UPGRADE_BACKENDS,AUTO_UPGRADE_BACKENDS" help:"Automatically upgrade backends when new versions are detected" group:"backends" default:"false"`
+	PreferDevelopmentBackends bool     `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" help:"Prefer development backend versions (shows development backends by default in UI)" group:"backends" default:"false"`
+	PreloadModels             string   `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"`
 	Models                   []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"`
 	PreloadModelsConfig      string   `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"`
 
@@ -490,6 +492,14 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		opts = append(opts, config.EnableBackendGalleriesAutoload)
 	}
 
+	if r.AutoUpgradeBackends {
+		opts = append(opts, config.WithAutoUpgradeBackends(r.AutoUpgradeBackends))
+	}
+
+	if r.PreferDevelopmentBackends {
+		opts = append(opts, config.WithPreferDevelopmentBackends(r.PreferDevelopmentBackends))
+	}
+
 	if r.PreloadBackendOnly {
 		_, err := application.New(opts...)
 		return err

core/config/application_config.go

@@ -57,6 +57,8 @@ type ApplicationConfig struct {
 	ExternalGRPCBackends map[string]string
 
 	AutoloadGalleries, AutoloadBackendGalleries bool
+	AutoUpgradeBackends                         bool
+	PreferDevelopmentBackends                   bool
 
 	SingleBackend           bool // Deprecated: use MaxActiveBackends = 1 instead
 	MaxActiveBackends       int  // Maximum number of active backends (0 = unlimited, 1 = single backend mode)
@@ -390,6 +392,14 @@ var EnableBackendGalleriesAutoload = func(o *ApplicationConfig) {
 	o.AutoloadBackendGalleries = true
 }
 
+func WithAutoUpgradeBackends(v bool) AppOption {
+	return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v }
+}
+
+func WithPreferDevelopmentBackends(v bool) AppOption {
+	return func(o *ApplicationConfig) { o.PreferDevelopmentBackends = v }
+}
+
 var EnableFederated = func(o *ApplicationConfig) {
 	o.Federated = true
 }
@@ -862,6 +872,8 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
 	backendGalleries := o.BackendGalleries
 	autoloadGalleries := o.AutoloadGalleries
 	autoloadBackendGalleries := o.AutoloadBackendGalleries
+	autoUpgradeBackends := o.AutoUpgradeBackends
+	preferDevelopmentBackends := o.PreferDevelopmentBackends
 	apiKeys := o.ApiKeys
 	agentJobRetentionDays := o.AgentJobRetentionDays
 
@@ -935,6 +947,8 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
 		BackendGalleries:          &backendGalleries,
 		AutoloadGalleries:         &autoloadGalleries,
 		AutoloadBackendGalleries:  &autoloadBackendGalleries,
+		AutoUpgradeBackends:       &autoUpgradeBackends,
+		PreferDevelopmentBackends: &preferDevelopmentBackends,
 		ApiKeys:                   &apiKeys,
 		AgentJobRetentionDays:     &agentJobRetentionDays,
 		OpenResponsesStoreTTL:     &openResponsesStoreTTL,
@@ -1083,6 +1097,12 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req
 	if settings.AutoloadBackendGalleries != nil {
 		o.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries
 	}
+	if settings.AutoUpgradeBackends != nil {
+		o.AutoUpgradeBackends = *settings.AutoUpgradeBackends
+	}
+	if settings.PreferDevelopmentBackends != nil {
+		o.PreferDevelopmentBackends = *settings.PreferDevelopmentBackends
+	}
 	if settings.AgentJobRetentionDays != nil {
 		o.AgentJobRetentionDays = *settings.AgentJobRetentionDays
 	}

core/config/application_config_test.go

@@ -119,6 +119,13 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() {
 			Expect(*rs.AgentJobRetentionDays).To(Equal(30))
 		})
 
+		It("should include auto_upgrade_backends", func() {
+			appConfig := &ApplicationConfig{AutoUpgradeBackends: true}
+			rs := appConfig.ToRuntimeSettings()
+			Expect(rs.AutoUpgradeBackends).ToNot(BeNil())
+			Expect(*rs.AutoUpgradeBackends).To(BeTrue())
+		})
+
 		It("should use default timeouts when not set", func() {
 			appConfig := &ApplicationConfig{}
 
@@ -426,6 +433,14 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() {
 			Expect(appConfig.AutoloadBackendGalleries).To(BeTrue())
 		})
 
+		It("should apply auto_upgrade_backends setting", func() {
+			appConfig := &ApplicationConfig{}
+			v := true
+			rs := &RuntimeSettings{AutoUpgradeBackends: &v}
+			appConfig.ApplyRuntimeSettings(rs)
+			Expect(appConfig.AutoUpgradeBackends).To(BeTrue())
+		})
+
 		It("should apply agent settings", func() {
 			appConfig := &ApplicationConfig{}
 
@@ -465,6 +480,7 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() {
 				Federated:                true,
 				AutoloadGalleries:        true,
 				AutoloadBackendGalleries: false,
+				AutoUpgradeBackends:      true,
 				AgentJobRetentionDays:    60,
 			}
 
@@ -496,6 +512,7 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() {
 			Expect(target.Federated).To(Equal(original.Federated))
 			Expect(target.AutoloadGalleries).To(Equal(original.AutoloadGalleries))
 			Expect(target.AutoloadBackendGalleries).To(Equal(original.AutoloadBackendGalleries))
+			Expect(target.AutoUpgradeBackends).To(Equal(original.AutoUpgradeBackends))
 			Expect(target.AgentJobRetentionDays).To(Equal(original.AgentJobRetentionDays))
 		})
 

core/config/backend_hooks.go

@@ -0,0 +1,30 @@
+package config
+
+// BackendDefaultsHook is called during Prepare() and can modify cfg.
+// Only fills in values that are not already set by the user.
+type BackendDefaultsHook func(cfg *ModelConfig, modelPath string)
+
+var backendHooks = map[string][]BackendDefaultsHook{}
+
+// RegisterBackendHook registers a hook for a backend name.
+// Special keys:
+//   - "*"  = global catch-all, runs for EVERY backend (before specific hooks)
+//   - ""   = runs only when cfg.Backend is empty (auto-detect case)
+//   - "vllm", "llama-cpp" etc. = runs only for that specific backend
+//
+// Multiple hooks per key are supported; they run in registration order.
+func RegisterBackendHook(backend string, hook BackendDefaultsHook) {
+	backendHooks[backend] = append(backendHooks[backend], hook)
+}
+
+// runBackendHooks executes hooks in order:
+//  1. "*" (global) hooks for every backend
+//  2. Backend-specific hooks for cfg.Backend (includes "" when backend is empty)
+func runBackendHooks(cfg *ModelConfig, modelPath string) {
+	for _, h := range backendHooks["*"] {
+		h(cfg, modelPath)
+	}
+	for _, h := range backendHooks[cfg.Backend] {
+		h(cfg, modelPath)
+	}
+}

core/config/guesser.go

@@ -1,46 +0,0 @@
-package config
-
-import (
-	"os"
-	"path/filepath"
-
-	gguf "github.com/gpustack/gguf-parser-go"
-	"github.com/mudler/xlog"
-)
-
-func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
-	if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
-		xlog.Debug("guessDefaultsFromFile: guessing disabled with LOCALAI_DISABLE_GUESSING")
-		return
-	}
-
-	if modelPath == "" {
-		xlog.Debug("guessDefaultsFromFile: modelPath is empty")
-		return
-	}
-
-	// We try to guess only if we don't have a template defined already
-	guessPath := filepath.Join(modelPath, cfg.ModelFileName())
-
-	defer func() {
-		if r := recover(); r != nil {
-			xlog.Error("guessDefaultsFromFile: panic while parsing gguf file")
-		}
-	}()
-
-	defer func() {
-		if cfg.ContextSize == nil {
-			if defaultCtx == 0 {
-				defaultCtx = defaultContextSize
-			}
-			cfg.ContextSize = &defaultCtx
-		}
-	}()
-
-	// try to parse the gguf file
-	f, err := gguf.ParseGGUFFile(guessPath)
-	if err == nil {
-		guessGGUFFromFile(cfg, f, defaultCtx)
-		return
-	}
-}

core/config/hooks_llamacpp.go

@@ -0,0 +1,46 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/xlog"
+)
+
+func init() {
+	// Register for both explicit llama-cpp and empty backend (auto-detect from GGUF file)
+	RegisterBackendHook("llama-cpp", llamaCppDefaults)
+	RegisterBackendHook("", llamaCppDefaults)
+}
+
+func llamaCppDefaults(cfg *ModelConfig, modelPath string) {
+	if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
+		xlog.Debug("llamaCppDefaults: guessing disabled")
+		return
+	}
+	if modelPath == "" {
+		return
+	}
+
+	guessPath := filepath.Join(modelPath, cfg.ModelFileName())
+
+	defer func() {
+		if r := recover(); r != nil {
+			xlog.Error("llamaCppDefaults: panic while parsing gguf file")
+		}
+	}()
+
+	// Default context size if not set, regardless of whether GGUF parsing succeeds
+	defer func() {
+		if cfg.ContextSize == nil {
+			ctx := defaultContextSize
+			cfg.ContextSize = &ctx
+		}
+	}()
+
+	f, err := gguf.ParseGGUFFile(guessPath)
+	if err == nil {
+		guessGGUFFromFile(cfg, f, 0)
+	}
+}

core/config/hooks_test.go

@@ -0,0 +1,114 @@
+package config_test
+
+import (
+	. "github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/schema"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Backend hooks and parser defaults", func() {
+	Context("MatchParserDefaults", func() {
+		It("matches Qwen3 family", func() {
+			parsers := MatchParserDefaults("Qwen/Qwen3-8B")
+			Expect(parsers).NotTo(BeNil())
+			Expect(parsers["tool_parser"]).To(Equal("hermes"))
+			Expect(parsers["reasoning_parser"]).To(Equal("qwen3"))
+		})
+
+		It("matches Qwen3.5 with longest-prefix-first", func() {
+			parsers := MatchParserDefaults("Qwen/Qwen3.5-9B")
+			Expect(parsers).NotTo(BeNil())
+			Expect(parsers["tool_parser"]).To(Equal("qwen3_xml"))
+		})
+
+		It("matches Llama-3.3 not Llama-3.2", func() {
+			parsers := MatchParserDefaults("meta/Llama-3.3-70B-Instruct")
+			Expect(parsers).NotTo(BeNil())
+			Expect(parsers["tool_parser"]).To(Equal("llama3_json"))
+		})
+
+		It("matches deepseek-r1", func() {
+			parsers := MatchParserDefaults("deepseek-ai/DeepSeek-R1")
+			Expect(parsers).NotTo(BeNil())
+			Expect(parsers["reasoning_parser"]).To(Equal("deepseek_r1"))
+			Expect(parsers["tool_parser"]).To(Equal("deepseek_v3"))
+		})
+
+		It("returns nil for unknown families", func() {
+			Expect(MatchParserDefaults("acme/unknown-model-xyz")).To(BeNil())
+		})
+	})
+
+	Context("Backend hook registration and execution", func() {
+		It("runs registered hook for a backend", func() {
+			called := false
+			RegisterBackendHook("test-backend-hook", func(cfg *ModelConfig, modelPath string) {
+				called = true
+				cfg.Description = "modified-by-hook"
+			})
+
+			cfg := &ModelConfig{
+				Backend: "test-backend-hook",
+			}
+			// Use the public Prepare path indirectly is heavy; instead exercise via vllmDefaults
+			// path, but here just call RegisterBackendHook + we know runBackendHooks is internal.
+			// Verify by leveraging Prepare on a fresh ModelConfig with no model path.
+			cfg.PredictionOptions = schema.PredictionOptions{}
+
+			// Trigger via Prepare with empty options; this calls runBackendHooks internally.
+			cfg.SetDefaults()
+			Expect(called).To(BeTrue())
+			Expect(cfg.Description).To(Equal("modified-by-hook"))
+		})
+	})
+
+	Context("vllmDefaults hook", func() {
+		It("auto-sets parsers for known model families on vllm backend", func() {
+			cfg := &ModelConfig{
+				Backend: "vllm",
+				PredictionOptions: schema.PredictionOptions{
+					BasicModelRequest: schema.BasicModelRequest{
+						Model: "Qwen/Qwen3-8B",
+					},
+				},
+			}
+			cfg.SetDefaults()
+
+			foundTool := false
+			foundReasoning := false
+			for _, opt := range cfg.Options {
+				if opt == "tool_parser:hermes" {
+					foundTool = true
+				}
+				if opt == "reasoning_parser:qwen3" {
+					foundReasoning = true
+				}
+			}
+			Expect(foundTool).To(BeTrue())
+			Expect(foundReasoning).To(BeTrue())
+		})
+
+		It("does not override user-set tool_parser", func() {
+			cfg := &ModelConfig{
+				Backend: "vllm",
+				Options: []string{"tool_parser:custom"},
+				PredictionOptions: schema.PredictionOptions{
+					BasicModelRequest: schema.BasicModelRequest{
+						Model: "Qwen/Qwen3-8B",
+					},
+				},
+			}
+			cfg.SetDefaults()
+
+			count := 0
+			for _, opt := range cfg.Options {
+				if len(opt) >= len("tool_parser:") && opt[:len("tool_parser:")] == "tool_parser:" {
+					count++
+				}
+			}
+			Expect(count).To(Equal(1))
+		})
+	})
+})

core/config/hooks_vllm.go

@@ -0,0 +1,85 @@
+package config
+
+import (
+	_ "embed"
+	"encoding/json"
+	"strings"
+
+	"github.com/mudler/xlog"
+)
+
+//go:embed parser_defaults.json
+var parserDefaultsJSON []byte
+
+type parserDefaultsData struct {
+	Families map[string]map[string]string `json:"families"`
+	Patterns []string                     `json:"patterns"`
+}
+
+var parsersData *parserDefaultsData
+
+func init() {
+	parsersData = &parserDefaultsData{}
+	if err := json.Unmarshal(parserDefaultsJSON, parsersData); err != nil {
+		xlog.Warn("failed to parse parser_defaults.json", "error", err)
+	}
+
+	RegisterBackendHook("vllm", vllmDefaults)
+	RegisterBackendHook("vllm-omni", vllmDefaults)
+}
+
+// MatchParserDefaults returns parser defaults for the best-matching model family.
+// Returns nil if no family matches. Used both at load time (via hook) and at import time.
+func MatchParserDefaults(modelID string) map[string]string {
+	if parsersData == nil || len(parsersData.Patterns) == 0 {
+		return nil
+	}
+	normalized := normalizeModelID(modelID)
+	for _, pattern := range parsersData.Patterns {
+		if strings.Contains(normalized, pattern) {
+			if family, ok := parsersData.Families[pattern]; ok {
+				return family
+			}
+		}
+	}
+	return nil
+}
+
+func vllmDefaults(cfg *ModelConfig, modelPath string) {
+	// Check if user already set tool_parser or reasoning_parser in Options
+	hasToolParser := false
+	hasReasoningParser := false
+	for _, opt := range cfg.Options {
+		if strings.HasPrefix(opt, "tool_parser:") {
+			hasToolParser = true
+		}
+		if strings.HasPrefix(opt, "reasoning_parser:") {
+			hasReasoningParser = true
+		}
+	}
+	if hasToolParser && hasReasoningParser {
+		return
+	}
+
+	// Try matching against Model field, then Name
+	parsers := MatchParserDefaults(cfg.Model)
+	if parsers == nil {
+		parsers = MatchParserDefaults(cfg.Name)
+	}
+	if parsers == nil {
+		return
+	}
+
+	if !hasToolParser {
+		if tp, ok := parsers["tool_parser"]; ok {
+			cfg.Options = append(cfg.Options, "tool_parser:"+tp)
+			xlog.Debug("[parser_defaults] auto-set tool_parser", "parser", tp, "model", cfg.Model)
+		}
+	}
+	if !hasReasoningParser {
+		if rp, ok := parsers["reasoning_parser"]; ok {
+			cfg.Options = append(cfg.Options, "reasoning_parser:"+rp)
+			xlog.Debug("[parser_defaults] auto-set reasoning_parser", "parser", rp, "model", cfg.Model)
+		}
+	}
+}

core/config/model_config.go

@@ -497,7 +497,12 @@ func (cfg *ModelConfig) SetDefaults(opts ...ConfigLoaderOption) {
 		cfg.Debug = &trueV
 	}
 
-	guessDefaultsFromFile(cfg, lo.modelPath, ctx)
+	// If a context size was provided via LoadOptions, apply it before hooks so they
+	// don't override it with their own defaults.
+	if ctx != 0 && cfg.ContextSize == nil {
+		cfg.ContextSize = &ctx
+	}
+	runBackendHooks(cfg, lo.modelPath)
 	cfg.syncKnownUsecasesFromString()
 }
 

core/config/parser_defaults.json

@@ -0,0 +1,33 @@
+{
+  "families": {
+    "qwen3.5":       {"tool_parser": "qwen3_xml", "reasoning_parser": "qwen3"},
+    "qwen3-coder":   {"tool_parser": "qwen3_xml", "reasoning_parser": "qwen3"},
+    "qwen3":         {"tool_parser": "hermes",     "reasoning_parser": "qwen3"},
+    "qwen2.5":       {"tool_parser": "hermes"},
+    "qwq":           {"reasoning_parser": "deepseek_r1"},
+    "llama-4":       {"tool_parser": "llama4_pythonic"},
+    "llama-3.3":     {"tool_parser": "llama3_json"},
+    "llama-3.2":     {"tool_parser": "llama3_json"},
+    "llama-3.1":     {"tool_parser": "llama3_json"},
+    "mistral-nemo":  {"tool_parser": "mistral", "reasoning_parser": "mistral"},
+    "mistral-small": {"tool_parser": "mistral", "reasoning_parser": "mistral"},
+    "mistral-large": {"tool_parser": "mistral", "reasoning_parser": "mistral"},
+    "magistral":     {"tool_parser": "mistral", "reasoning_parser": "mistral"},
+    "deepseek-r1":   {"tool_parser": "deepseek_v3", "reasoning_parser": "deepseek_r1"},
+    "deepseek-v3":   {"tool_parser": "deepseek_v3", "reasoning_parser": "deepseek_v3"},
+    "glm-5":         {"tool_parser": "glm47"},
+    "glm-4":         {"tool_parser": "glm45", "reasoning_parser": "glm45"},
+    "gemma-4":       {"tool_parser": "gemma4", "reasoning_parser": "gemma4"},
+    "granite-4":     {"tool_parser": "granite4", "reasoning_parser": "granite"},
+    "minimax-m2.5":  {"tool_parser": "minimax_m2", "reasoning_parser": "minimax_m2"},
+    "minimax":       {"tool_parser": "minimax_m2", "reasoning_parser": "minimax_m2"},
+    "kimi-k2":       {"tool_parser": "kimi_k2", "reasoning_parser": "kimi_k2"},
+    "nemotron":      {"reasoning_parser": "nemotron_v3"},
+    "olmo":          {"tool_parser": "olmo3", "reasoning_parser": "olmo3"},
+    "ernie":         {"tool_parser": "ernie45", "reasoning_parser": "ernie45"},
+    "phi-4":         {"tool_parser": "phi4_mini_json"},
+    "gpt-oss":       {"tool_parser": "openai", "reasoning_parser": "openai_gptoss"},
+    "hermes":        {"tool_parser": "hermes"}
+  },
+  "patterns": ["qwen3.5","qwen3-coder","qwen3","qwen2.5","qwq","llama-4","llama-3.3","llama-3.2","llama-3.1","mistral-nemo","mistral-small","mistral-large","magistral","deepseek-r1","deepseek-v3","glm-5","glm-4","gemma-4","granite-4","minimax-m2.5","minimax","kimi-k2","nemotron","olmo","ernie","phi-4","gpt-oss","hermes"]
+}

core/config/runtime_settings.go

@@ -20,6 +20,8 @@ type RuntimeSettings struct {
 	// Backend management
 	SingleBackend           *bool `json:"single_backend,omitempty"`      // Deprecated: use MaxActiveBackends = 1 instead
 	MaxActiveBackends       *int  `json:"max_active_backends,omitempty"` // Maximum number of active backends (0 = unlimited, 1 = single backend mode)
+	AutoUpgradeBackends       *bool `json:"auto_upgrade_backends,omitempty"`       // Automatically upgrade backends when new versions are detected
+	PreferDevelopmentBackends *bool `json:"prefer_development_backends,omitempty"` // Prefer development backend versions by default in UI
 	// Memory Reclaimer settings (works with GPU if available, otherwise RAM)
 	MemoryReclaimerEnabled   *bool    `json:"memory_reclaimer_enabled,omitempty"`   // Enable memory threshold monitoring
 	MemoryReclaimerThreshold *float64 `json:"memory_reclaimer_threshold,omitempty"` // Threshold 0.0-1.0 (e.g., 0.95 = 95%)

core/gallery/backend_types.go

@@ -2,6 +2,7 @@ package gallery
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/system"
@@ -20,12 +21,19 @@ type BackendMetadata struct {
 	GalleryURL string `json:"gallery_url,omitempty"`
 	// InstalledAt is the timestamp when the backend was installed
 	InstalledAt string `json:"installed_at,omitempty"`
+	// Version is the version of the backend at install time
+	Version string `json:"version,omitempty"`
+	// URI is the original URI used to install the backend
+	URI string `json:"uri,omitempty"`
+	// Digest is the OCI image digest at install time (for upgrade detection)
+	Digest string `json:"digest,omitempty"`
 }
 
 type GalleryBackend struct {
 	Metadata        `json:",inline" yaml:",inline"`
 	Alias           string            `json:"alias,omitempty" yaml:"alias,omitempty"`
 	URI             string            `json:"uri,omitempty" yaml:"uri,omitempty"`
+	Version         string            `json:"version,omitempty" yaml:"version,omitempty"`
 	Mirrors         []string          `json:"mirrors,omitempty" yaml:"mirrors,omitempty"`
 	CapabilitiesMap map[string]string `json:"capabilities,omitempty" yaml:"capabilities,omitempty"`
 }
@@ -63,6 +71,13 @@ func (m *GalleryBackend) IsMeta() bool {
 	return len(m.CapabilitiesMap) > 0 && m.URI == ""
 }
 
+func (m *GalleryBackend) IsDevelopment(devSuffix string) bool {
+	if devSuffix == "" {
+		devSuffix = defaultDevSuffix
+	}
+	return strings.HasSuffix(m.Name, "-"+devSuffix)
+}
+
 // IsCompatibleWith checks if the backend is compatible with the current system capability.
 // For meta backends, it checks if any of the capabilities in the map match the system capability.
 // For concrete backends, it delegates to SystemState.IsBackendCompatible.

core/gallery/backends.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/downloader"
 	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/oci"
 	"github.com/mudler/LocalAI/pkg/system"
 	"github.com/mudler/xlog"
 	cp "github.com/otiai10/copy"
@@ -158,6 +159,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery,
 			Name:           name,
 			GalleryURL:     backend.Gallery.URL,
 			InstalledAt:    time.Now().Format(time.RFC3339),
+			Version:        bestBackend.Version,
 		}
 
 		if err := writeBackendMetadata(metaBackendPath, metaMetadata); err != nil {
@@ -279,6 +281,18 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL
 		Name:        name,
 		GalleryURL:  config.Gallery.URL,
 		InstalledAt: time.Now().Format(time.RFC3339),
+		Version:     config.Version,
+		URI:         string(uri),
+	}
+
+	// Record the OCI digest for upgrade detection (non-fatal on failure)
+	if uri.LooksLikeOCI() {
+		digest, digestErr := oci.GetImageDigest(string(uri), "", nil, nil)
+		if digestErr != nil {
+			xlog.Warn("Failed to get OCI image digest for backend", "uri", string(uri), "error", digestErr)
+		} else {
+			metadata.Digest = digest
+		}
 	}
 
 	if config.Alias != "" {
@@ -373,11 +387,13 @@ func DeleteBackendFromSystem(systemState *system.SystemState, name string) error
 }
 
 type SystemBackend struct {
-	Name     string
-	RunFile  string
-	IsMeta   bool
-	IsSystem bool
-	Metadata *BackendMetadata
+	Name             string
+	RunFile          string
+	IsMeta           bool
+	IsSystem         bool
+	Metadata         *BackendMetadata
+	UpgradeAvailable bool   `json:"upgrade_available,omitempty"`
+	AvailableVersion string `json:"available_version,omitempty"`
 }
 
 type SystemBackends map[string]SystemBackend

core/gallery/backends_version_test.go

@@ -0,0 +1,118 @@
+package gallery_test
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Backend versioning", func() {
+	var tempDir string
+	var systemState *system.SystemState
+	var modelLoader *model.ModelLoader
+
+	BeforeEach(func() {
+		var err error
+		tempDir, err = os.MkdirTemp("", "gallery-version-*")
+		Expect(err).NotTo(HaveOccurred())
+
+		systemState, err = system.GetSystemState(
+			system.WithBackendPath(tempDir),
+		)
+		Expect(err).NotTo(HaveOccurred())
+		modelLoader = model.NewModelLoader(systemState)
+	})
+
+	AfterEach(func() {
+		os.RemoveAll(tempDir)
+	})
+
+	It("records version in metadata when installing a backend with a version", func() {
+		// Create a fake backend source directory with a run.sh
+		srcDir, err := os.MkdirTemp("", "gallery-version-src-*")
+		Expect(err).NotTo(HaveOccurred())
+		defer os.RemoveAll(srcDir)
+		err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755)
+		Expect(err).NotTo(HaveOccurred())
+
+		backend := &gallery.GalleryBackend{}
+		backend.Name = "test-backend"
+		backend.URI = srcDir
+		backend.Version = "1.2.3"
+
+		err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		// Read the metadata file and check version
+		metadataPath := filepath.Join(tempDir, "test-backend", "metadata.json")
+		data, err := os.ReadFile(metadataPath)
+		Expect(err).NotTo(HaveOccurred())
+
+		var metadata map[string]any
+		err = json.Unmarshal(data, &metadata)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(metadata["version"]).To(Equal("1.2.3"))
+	})
+
+	It("records URI in metadata", func() {
+		srcDir, err := os.MkdirTemp("", "gallery-version-src-*")
+		Expect(err).NotTo(HaveOccurred())
+		defer os.RemoveAll(srcDir)
+		err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755)
+		Expect(err).NotTo(HaveOccurred())
+
+		backend := &gallery.GalleryBackend{}
+		backend.Name = "test-backend-uri"
+		backend.URI = srcDir
+		backend.Version = "2.0.0"
+
+		err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		metadataPath := filepath.Join(tempDir, "test-backend-uri", "metadata.json")
+		data, err := os.ReadFile(metadataPath)
+		Expect(err).NotTo(HaveOccurred())
+
+		var metadata map[string]any
+		err = json.Unmarshal(data, &metadata)
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(metadata["uri"]).To(Equal(srcDir))
+	})
+
+	It("omits version key when version is empty", func() {
+		srcDir, err := os.MkdirTemp("", "gallery-version-src-*")
+		Expect(err).NotTo(HaveOccurred())
+		defer os.RemoveAll(srcDir)
+		err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755)
+		Expect(err).NotTo(HaveOccurred())
+
+		backend := &gallery.GalleryBackend{}
+		backend.Name = "test-backend-noversion"
+		backend.URI = srcDir
+		// Version intentionally left empty
+
+		err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil)
+		Expect(err).NotTo(HaveOccurred())
+
+		metadataPath := filepath.Join(tempDir, "test-backend-noversion", "metadata.json")
+		data, err := os.ReadFile(metadataPath)
+		Expect(err).NotTo(HaveOccurred())
+
+		var metadata map[string]any
+		err = json.Unmarshal(data, &metadata)
+		Expect(err).NotTo(HaveOccurred())
+
+		// omitempty should exclude the version key entirely
+		_, hasVersion := metadata["version"]
+		Expect(hasVersion).To(BeFalse())
+	})
+})

core/gallery/importers/vllm.go

@@ -88,6 +88,18 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 	// Apply per-model-family inference parameter defaults
 	config.ApplyInferenceDefaults(&modelConfig, details.URI)
 
+	// Auto-detect tool_parser and reasoning_parser for known model families.
+	// Surfacing them in the generated YAML lets users see and edit the choices.
+	parsers := config.MatchParserDefaults(details.URI)
+	if parsers != nil {
+		if tp, ok := parsers["tool_parser"]; ok {
+			modelConfig.Options = append(modelConfig.Options, "tool_parser:"+tp)
+		}
+		if rp, ok := parsers["reasoning_parser"]; ok {
+			modelConfig.Options = append(modelConfig.Options, "reasoning_parser:"+rp)
+		}
+	}
+
 	data, err := yaml.Marshal(modelConfig)
 	if err != nil {
 		return gallery.ModelConfig{}, err

core/gallery/upgrade.go

@@ -0,0 +1,237 @@
+package gallery
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/downloader"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/oci"
+	"github.com/mudler/LocalAI/pkg/system"
+	"github.com/mudler/xlog"
+	cp "github.com/otiai10/copy"
+)
+
+// UpgradeInfo holds details about an available backend upgrade.
+type UpgradeInfo struct {
+	BackendName      string `json:"backend_name"`
+	InstalledVersion string `json:"installed_version"`
+	AvailableVersion string `json:"available_version"`
+	InstalledDigest  string `json:"installed_digest,omitempty"`
+	AvailableDigest  string `json:"available_digest,omitempty"`
+}
+
+// CheckBackendUpgrades compares installed backends against gallery entries
+// and returns a map of backend names to UpgradeInfo for those that have
+// newer versions or different OCI digests available.
+func CheckBackendUpgrades(ctx context.Context, galleries []config.Gallery, systemState *system.SystemState) (map[string]UpgradeInfo, error) {
+	galleryBackends, err := AvailableBackends(galleries, systemState)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list available backends: %w", err)
+	}
+
+	installedBackends, err := ListSystemBackends(systemState)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list installed backends: %w", err)
+	}
+
+	result := make(map[string]UpgradeInfo)
+
+	for _, installed := range installedBackends {
+		// Skip system backends — they are managed outside the gallery
+		if installed.IsSystem {
+			continue
+		}
+		if installed.Metadata == nil {
+			continue
+		}
+
+		// Find matching gallery entry by metadata name
+		galleryEntry := FindGalleryElement(galleryBackends, installed.Metadata.Name)
+		if galleryEntry == nil {
+			continue
+		}
+
+		installedVersion := installed.Metadata.Version
+		galleryVersion := galleryEntry.Version
+
+		// If both sides have versions, compare them
+		if galleryVersion != "" && installedVersion != "" {
+			if galleryVersion != installedVersion {
+				result[installed.Metadata.Name] = UpgradeInfo{
+					BackendName:      installed.Metadata.Name,
+					InstalledVersion: installedVersion,
+					AvailableVersion: galleryVersion,
+				}
+			}
+			// Versions match — no upgrade needed
+			continue
+		}
+
+		// Gallery has a version but installed doesn't — this happens for backends
+		// installed before version tracking was added. Flag as upgradeable so
+		// users can re-install to pick up version metadata.
+		if galleryVersion != "" && installedVersion == "" {
+			result[installed.Metadata.Name] = UpgradeInfo{
+				BackendName:      installed.Metadata.Name,
+				InstalledVersion: "",
+				AvailableVersion: galleryVersion,
+			}
+			continue
+		}
+
+		// Fall back to OCI digest comparison when versions are unavailable
+		if downloader.URI(galleryEntry.URI).LooksLikeOCI() {
+			remoteDigest, err := oci.GetImageDigest(galleryEntry.URI, "", nil, nil)
+			if err != nil {
+				xlog.Warn("Failed to get remote OCI digest for upgrade check", "backend", installed.Metadata.Name, "error", err)
+				continue
+			}
+			// If we have a stored digest, compare; otherwise any remote digest
+			// means we can't confirm we're up to date — flag as upgradeable
+			if installed.Metadata.Digest == "" || remoteDigest != installed.Metadata.Digest {
+				result[installed.Metadata.Name] = UpgradeInfo{
+					BackendName:     installed.Metadata.Name,
+					InstalledDigest: installed.Metadata.Digest,
+					AvailableDigest: remoteDigest,
+				}
+			}
+		}
+		// No version info and non-OCI URI — cannot determine, skip
+	}
+
+	return result, nil
+}
+
+// UpgradeBackend upgrades a single backend to the latest gallery version using
+// an atomic swap with backup-based rollback on failure.
+func UpgradeBackend(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, galleries []config.Gallery, backendName string, downloadStatus func(string, string, string, float64)) error {
+	// Look up the installed backend
+	installedBackends, err := ListSystemBackends(systemState)
+	if err != nil {
+		return fmt.Errorf("failed to list installed backends: %w", err)
+	}
+
+	installed, ok := installedBackends.Get(backendName)
+	if !ok {
+		return fmt.Errorf("backend %q: %w", backendName, ErrBackendNotFound)
+	}
+
+	if installed.IsSystem {
+		return fmt.Errorf("system backend %q cannot be upgraded via gallery", backendName)
+	}
+
+	// If this is a meta backend, recursively upgrade the concrete backend it points to
+	if installed.Metadata != nil && installed.Metadata.MetaBackendFor != "" {
+		xlog.Info("Meta backend detected, upgrading concrete backend", "meta", backendName, "concrete", installed.Metadata.MetaBackendFor)
+		return UpgradeBackend(ctx, systemState, modelLoader, galleries, installed.Metadata.MetaBackendFor, downloadStatus)
+	}
+
+	// Find the gallery entry
+	galleryBackends, err := AvailableBackends(galleries, systemState)
+	if err != nil {
+		return fmt.Errorf("failed to list available backends: %w", err)
+	}
+
+	galleryEntry := FindGalleryElement(galleryBackends, backendName)
+	if galleryEntry == nil {
+		return fmt.Errorf("no gallery entry found for backend %q", backendName)
+	}
+
+	backendPath := filepath.Join(systemState.Backend.BackendsPath, backendName)
+	tmpPath := backendPath + ".upgrade-tmp"
+	backupPath := backendPath + ".backup"
+
+	// Clean up any stale tmp/backup dirs from prior attempts
+	os.RemoveAll(tmpPath)
+	os.RemoveAll(backupPath)
+
+	// Step 1: Download the new backend into the tmp directory
+	if err := os.MkdirAll(tmpPath, 0750); err != nil {
+		return fmt.Errorf("failed to create upgrade tmp dir: %w", err)
+	}
+
+	uri := downloader.URI(galleryEntry.URI)
+	if uri.LooksLikeDir() {
+		if err := cp.Copy(string(uri), tmpPath); err != nil {
+			os.RemoveAll(tmpPath)
+			return fmt.Errorf("failed to copy backend from directory: %w", err)
+		}
+	} else {
+		if err := uri.DownloadFileWithContext(ctx, tmpPath, "", 1, 1, downloadStatus); err != nil {
+			os.RemoveAll(tmpPath)
+			return fmt.Errorf("failed to download backend: %w", err)
+		}
+	}
+
+	// Step 2: Validate — check that run.sh exists in the new content
+	newRunFile := filepath.Join(tmpPath, runFile)
+	if _, err := os.Stat(newRunFile); os.IsNotExist(err) {
+		os.RemoveAll(tmpPath)
+		return fmt.Errorf("upgrade validation failed: run.sh not found in new backend")
+	}
+
+	// Step 3: Atomic swap — rename current to backup, then tmp to current
+	if err := os.Rename(backendPath, backupPath); err != nil {
+		os.RemoveAll(tmpPath)
+		return fmt.Errorf("failed to move current backend to backup: %w", err)
+	}
+
+	if err := os.Rename(tmpPath, backendPath); err != nil {
+		// Restore backup on failure
+		xlog.Error("Failed to move new backend into place, restoring backup", "error", err)
+		if restoreErr := os.Rename(backupPath, backendPath); restoreErr != nil {
+			xlog.Error("Failed to restore backup", "error", restoreErr)
+		}
+		os.RemoveAll(tmpPath)
+		return fmt.Errorf("failed to move new backend into place: %w", err)
+	}
+
+	// Step 4: Write updated metadata, preserving alias from old metadata
+	var oldAlias string
+	if installed.Metadata != nil {
+		oldAlias = installed.Metadata.Alias
+	}
+
+	newMetadata := &BackendMetadata{
+		Name:        backendName,
+		Version:     galleryEntry.Version,
+		URI:         galleryEntry.URI,
+		InstalledAt: time.Now().Format(time.RFC3339),
+		Alias:       oldAlias,
+	}
+
+	if galleryEntry.Gallery.URL != "" {
+		newMetadata.GalleryURL = galleryEntry.Gallery.URL
+	}
+
+	// Record OCI digest if applicable (non-fatal on failure)
+	if uri.LooksLikeOCI() {
+		digest, digestErr := oci.GetImageDigest(galleryEntry.URI, "", nil, nil)
+		if digestErr != nil {
+			xlog.Warn("Failed to get OCI image digest after upgrade", "uri", galleryEntry.URI, "error", digestErr)
+		} else {
+			newMetadata.Digest = digest
+		}
+	}
+
+	if err := writeBackendMetadata(backendPath, newMetadata); err != nil {
+		// Metadata write failure is not worth rolling back the entire upgrade
+		xlog.Error("Failed to write metadata after upgrade", "error", err)
+	}
+
+	// Step 5: Re-register backends so the model loader picks up any changes
+	if err := RegisterBackends(systemState, modelLoader); err != nil {
+		xlog.Warn("Failed to re-register backends after upgrade", "error", err)
+	}
+
+	// Step 6: Remove backup
+	os.RemoveAll(backupPath)
+
+	xlog.Info("Backend upgraded successfully", "backend", backendName, "version", galleryEntry.Version)
+	return nil
+}

core/gallery/upgrade_test.go

@@ -0,0 +1,219 @@
+package gallery_test
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/mudler/LocalAI/core/config"
+	. "github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+)
+
+var _ = Describe("Upgrade Detection and Execution", func() {
+	var (
+		tempDir      string
+		backendsPath string
+		galleryPath  string
+		systemState  *system.SystemState
+		galleries    []config.Gallery
+	)
+
+	// installBackendWithVersion creates a fake installed backend directory with
+	// the given name, version, and optional run.sh content.
+	installBackendWithVersion := func(name, version string, runContent ...string) {
+		dir := filepath.Join(backendsPath, name)
+		Expect(os.MkdirAll(dir, 0750)).To(Succeed())
+
+		content := "#!/bin/sh\necho ok"
+		if len(runContent) > 0 {
+			content = runContent[0]
+		}
+		Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(content), 0755)).To(Succeed())
+
+		metadata := BackendMetadata{
+			Name:        name,
+			Version:     version,
+			InstalledAt: time.Now().Format(time.RFC3339),
+		}
+		data, err := json.MarshalIndent(metadata, "", "  ")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed())
+	}
+
+	// writeGalleryYAML writes a gallery YAML file with the given backends.
+	writeGalleryYAML := func(backends []GalleryBackend) {
+		data, err := yaml.Marshal(backends)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(galleryPath, data, 0644)).To(Succeed())
+	}
+
+	BeforeEach(func() {
+		var err error
+		tempDir, err = os.MkdirTemp("", "upgrade-test-*")
+		Expect(err).NotTo(HaveOccurred())
+
+		backendsPath = tempDir
+
+		galleryPath = filepath.Join(tempDir, "gallery.yaml")
+
+		// Write a default empty gallery
+		writeGalleryYAML([]GalleryBackend{})
+
+		galleries = []config.Gallery{
+			{
+				Name: "test-gallery",
+				URL:  "file://" + galleryPath,
+			},
+		}
+
+		systemState, err = system.GetSystemState(
+			system.WithBackendPath(backendsPath),
+		)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		os.RemoveAll(tempDir)
+	})
+
+	Describe("CheckBackendUpgrades", func() {
+		It("should detect upgrade when gallery version differs from installed version", func() {
+			// Install a backend at v1.0.0
+			installBackendWithVersion("my-backend", "1.0.0")
+
+			// Gallery advertises v2.0.0
+			writeGalleryYAML([]GalleryBackend{
+				{
+					Metadata: Metadata{
+						Name: "my-backend",
+					},
+					URI:     filepath.Join(tempDir, "some-source"),
+					Version: "2.0.0",
+				},
+			})
+
+			upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(HaveKey("my-backend"))
+			Expect(upgrades["my-backend"].InstalledVersion).To(Equal("1.0.0"))
+			Expect(upgrades["my-backend"].AvailableVersion).To(Equal("2.0.0"))
+		})
+
+		It("should NOT flag upgrade when versions match", func() {
+			installBackendWithVersion("my-backend", "2.0.0")
+
+			writeGalleryYAML([]GalleryBackend{
+				{
+					Metadata: Metadata{
+						Name: "my-backend",
+					},
+					URI:     filepath.Join(tempDir, "some-source"),
+					Version: "2.0.0",
+				},
+			})
+
+			upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(BeEmpty())
+		})
+
+		It("should skip backends without version info and without OCI digest", func() {
+			// Install without version
+			installBackendWithVersion("my-backend", "")
+
+			// Gallery also without version
+			writeGalleryYAML([]GalleryBackend{
+				{
+					Metadata: Metadata{
+						Name: "my-backend",
+					},
+					URI: filepath.Join(tempDir, "some-source"),
+				},
+			})
+
+			upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(upgrades).To(BeEmpty())
+		})
+	})
+
+	Describe("UpgradeBackend", func() {
+		It("should replace backend directory and update metadata", func() {
+			// Install v1
+			installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1")
+
+			// Create a source directory with v2 content
+			srcDir := filepath.Join(tempDir, "v2-source")
+			Expect(os.MkdirAll(srcDir, 0750)).To(Succeed())
+			Expect(os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho v2"), 0755)).To(Succeed())
+
+			// Gallery points to the v2 source dir
+			writeGalleryYAML([]GalleryBackend{
+				{
+					Metadata: Metadata{
+						Name: "my-backend",
+					},
+					URI:     srcDir,
+					Version: "2.0.0",
+				},
+			})
+
+			ml := model.NewModelLoader(systemState)
+			err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil)
+			Expect(err).NotTo(HaveOccurred())
+
+			// Verify run.sh was updated
+			content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh"))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(string(content)).To(Equal("#!/bin/sh\necho v2"))
+
+			// Verify metadata was updated
+			metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json"))
+			Expect(err).NotTo(HaveOccurred())
+			var meta BackendMetadata
+			Expect(json.Unmarshal(metaData, &meta)).To(Succeed())
+			Expect(meta.Version).To(Equal("2.0.0"))
+			Expect(meta.Name).To(Equal("my-backend"))
+		})
+
+		It("should restore backup on failure", func() {
+			// Install v1
+			installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1")
+
+			// Gallery points to a nonexistent path (no run.sh will be found)
+			nonExistentDir := filepath.Join(tempDir, "does-not-exist")
+			writeGalleryYAML([]GalleryBackend{
+				{
+					Metadata: Metadata{
+						Name: "my-backend",
+					},
+					URI:     nonExistentDir,
+					Version: "2.0.0",
+				},
+			})
+
+			ml := model.NewModelLoader(systemState)
+			err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil)
+			Expect(err).To(HaveOccurred())
+
+			// Verify v1 is still intact
+			content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh"))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(string(content)).To(Equal("#!/bin/sh\necho v1"))
+
+			// Verify metadata still says v1
+			metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json"))
+			Expect(err).NotTo(HaveOccurred())
+			var meta BackendMetadata
+			Expect(json.Unmarshal(metaData, &meta)).To(Succeed())
+			Expect(meta.Version).To(Equal("1.0.0"))
+		})
+	})
+})

core/http/endpoints/localai/backend.go

@@ -15,23 +15,31 @@ import (
 	"github.com/mudler/xlog"
 )
 
+// UpgradeInfoProvider is an interface for querying cached backend upgrade information.
+type UpgradeInfoProvider interface {
+	GetAvailableUpgrades() map[string]gallery.UpgradeInfo
+	TriggerCheck()
+}
+
 type BackendEndpointService struct {
 	galleries         []config.Gallery
 	backendPath       string
 	backendSystemPath string
 	backendApplier    *galleryop.GalleryService
+	upgradeChecker    UpgradeInfoProvider
 }
 
 type GalleryBackend struct {
 	ID string `json:"id"`
 }
 
-func CreateBackendEndpointService(galleries []config.Gallery, systemState *system.SystemState, backendApplier *galleryop.GalleryService) BackendEndpointService {
+func CreateBackendEndpointService(galleries []config.Gallery, systemState *system.SystemState, backendApplier *galleryop.GalleryService, upgradeChecker UpgradeInfoProvider) BackendEndpointService {
 	return BackendEndpointService{
 		galleries:         galleries,
 		backendPath:       systemState.Backend.BackendsPath,
 		backendSystemPath: systemState.Backend.BackendsSystemPath,
 		backendApplier:    backendApplier,
+		upgradeChecker:    upgradeChecker,
 	}
 }
 
@@ -146,6 +154,62 @@ func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFu
 	}
 }
 
+// GetUpgradesEndpoint returns the cached backend upgrade information
+// @Summary Get available backend upgrades
+// @Tags backends
+// @Success 200 {object} map[string]gallery.UpgradeInfo "Response"
+// @Router /backends/upgrades [get]
+func (mgs *BackendEndpointService) GetUpgradesEndpoint() echo.HandlerFunc {
+	return func(c echo.Context) error {
+		if mgs.upgradeChecker == nil {
+			return c.JSON(200, map[string]gallery.UpgradeInfo{})
+		}
+		return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades())
+	}
+}
+
+// CheckUpgradesEndpoint forces an immediate upgrade check
+// @Summary Force backend upgrade check
+// @Tags backends
+// @Success 200 {object} map[string]gallery.UpgradeInfo "Response"
+// @Router /backends/upgrades/check [post]
+func (mgs *BackendEndpointService) CheckUpgradesEndpoint() echo.HandlerFunc {
+	return func(c echo.Context) error {
+		if mgs.upgradeChecker == nil {
+			return c.JSON(200, map[string]gallery.UpgradeInfo{})
+		}
+		mgs.upgradeChecker.TriggerCheck()
+		// Return current cached results (the triggered check runs async)
+		return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades())
+	}
+}
+
+// UpgradeBackendEndpoint triggers an upgrade for a specific backend
+// @Summary Upgrade a backend
+// @Tags backends
+// @Param name path string true "Backend name"
+// @Success 200 {object} schema.BackendResponse "Response"
+// @Router /backends/upgrade/{name} [post]
+func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc {
+	return func(c echo.Context) error {
+		backendName := c.Param("name")
+
+		uuid, err := uuid.NewUUID()
+		if err != nil {
+			return err
+		}
+
+		mgs.backendApplier.BackendGalleryChannel <- galleryop.ManagementOp[gallery.GalleryBackend, any]{
+			ID:                 uuid.String(),
+			GalleryElementName: backendName,
+			Galleries:          mgs.galleries,
+			Upgrade:            true,
+		}
+
+		return c.JSON(200, schema.BackendResponse{ID: uuid.String(), StatusURL: fmt.Sprintf("%sbackends/jobs/%s", middleware.BaseURL(c), uuid.String())})
+	}
+}
+
 // ListAvailableBackendsEndpoint list the available backends in the galleries configured in LocalAI
 // @Summary List all available Backends
 // @Tags backends

core/http/react-ui/package-lock.json

@@ -8,7 +8,16 @@
       "name": "localai-react-ui",
       "version": "0.0.1",
       "dependencies": {
+        "@codemirror/autocomplete": "^6.18.6",
+        "@codemirror/commands": "^6.8.1",
+        "@codemirror/lang-yaml": "^6.1.2",
+        "@codemirror/language": "^6.11.0",
+        "@codemirror/lint": "^6.8.5",
+        "@codemirror/search": "^6.5.10",
+        "@codemirror/state": "^6.5.2",
+        "@codemirror/view": "^6.36.8",
         "@fortawesome/fontawesome-free": "^6.7.2",
+        "@lezer/highlight": "^1.2.1",
         "@modelcontextprotocol/ext-apps": "^1.2.2",
         "@modelcontextprotocol/sdk": "^1.25.1",
         "dompurify": "^3.2.5",
@@ -16,7 +25,8 @@
         "marked": "^15.0.7",
         "react": "^19.1.0",
         "react-dom": "^19.1.0",
-        "react-router-dom": "^7.6.1"
+        "react-router-dom": "^7.6.1",
+        "yaml": "^2.8.3"
       },
       "devDependencies": {
         "@eslint/js": "^9.27.0",
@@ -292,6 +302,94 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@codemirror/autocomplete": {
+      "version": "6.20.1",
+      "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.1.tgz",
+      "integrity": "sha512-1cvg3Vz1dSSToCNlJfRA2WSI4ht3K+WplO0UMOgmUYPivCyy2oueZY6Lx7M9wThm7SDUBViRmuT+OG/i8+ON9A==",
+      "dependencies": {
+        "@codemirror/language": "^6.0.0",
+        "@codemirror/state": "^6.0.0",
+        "@codemirror/view": "^6.17.0",
+        "@lezer/common": "^1.0.0"
+      }
+    },
+    "node_modules/@codemirror/commands": {
+      "version": "6.10.3",
+      "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.10.3.tgz",
+      "integrity": "sha512-JFRiqhKu+bvSkDLI+rUhJwSxQxYb759W5GBezE8Uc8mHLqC9aV/9aTC7yJSqCtB3F00pylrLCwnyS91Ap5ej4Q==",
+      "dependencies": {
+        "@codemirror/language": "^6.0.0",
+        "@codemirror/state": "^6.6.0",
+        "@codemirror/view": "^6.27.0",
+        "@lezer/common": "^1.1.0"
+      }
+    },
+    "node_modules/@codemirror/lang-yaml": {
+      "version": "6.1.3",
+      "resolved": "https://registry.npmjs.org/@codemirror/lang-yaml/-/lang-yaml-6.1.3.tgz",
+      "integrity": "sha512-AZ8DJBuXGVHybpBQhmZtgew5//4hv3tdkXnr3vDmOUMJRuB6vn/uuwtmTOTlqEaQFg3hQSVeA90NmvIQyUV6FQ==",
+      "dependencies": {
+        "@codemirror/autocomplete": "^6.0.0",
+        "@codemirror/language": "^6.0.0",
+        "@codemirror/state": "^6.0.0",
+        "@lezer/common": "^1.2.0",
+        "@lezer/highlight": "^1.2.0",
+        "@lezer/lr": "^1.0.0",
+        "@lezer/yaml": "^1.0.0"
+      }
+    },
+    "node_modules/@codemirror/language": {
+      "version": "6.12.3",
+      "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.12.3.tgz",
+      "integrity": "sha512-QwCZW6Tt1siP37Jet9Tb02Zs81TQt6qQrZR2H+eGMcFsL1zMrk2/b9CLC7/9ieP1fjIUMgviLWMmgiHoJrj+ZA==",
+      "dependencies": {
+        "@codemirror/state": "^6.0.0",
+        "@codemirror/view": "^6.23.0",
+        "@lezer/common": "^1.5.0",
+        "@lezer/highlight": "^1.0.0",
+        "@lezer/lr": "^1.0.0",
+        "style-mod": "^4.0.0"
+      }
+    },
+    "node_modules/@codemirror/lint": {
+      "version": "6.9.5",
+      "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.9.5.tgz",
+      "integrity": "sha512-GElsbU9G7QT9xXhpUg1zWGmftA/7jamh+7+ydKRuT0ORpWS3wOSP0yT1FOlIZa7mIJjpVPipErsyvVqB9cfTFA==",
+      "dependencies": {
+        "@codemirror/state": "^6.0.0",
+        "@codemirror/view": "^6.35.0",
+        "crelt": "^1.0.5"
+      }
+    },
+    "node_modules/@codemirror/search": {
+      "version": "6.6.0",
+      "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.6.0.tgz",
+      "integrity": "sha512-koFuNXcDvyyotWcgOnZGmY7LZqEOXZaaxD/j6n18TCLx2/9HieZJ5H6hs1g8FiRxBD0DNfs0nXn17g872RmYdw==",
+      "dependencies": {
+        "@codemirror/state": "^6.0.0",
+        "@codemirror/view": "^6.37.0",
+        "crelt": "^1.0.5"
+      }
+    },
+    "node_modules/@codemirror/state": {
+      "version": "6.6.0",
+      "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.6.0.tgz",
+      "integrity": "sha512-4nbvra5R5EtiCzr9BTHiTLc+MLXK2QGiAVYMyi8PkQd3SR+6ixar/Q/01Fa21TBIDOZXgeWV4WppsQolSreAPQ==",
+      "dependencies": {
+        "@marijn/find-cluster-break": "^1.0.0"
+      }
+    },
+    "node_modules/@codemirror/view": {
+      "version": "6.41.0",
+      "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.41.0.tgz",
+      "integrity": "sha512-6H/qadXsVuDY219Yljhohglve8xf4B8xJkVOEWfA5uiYKiTFppjqsvsfR5iPA0RbvRBoOyTZpbLIxe9+0UR8xA==",
+      "dependencies": {
+        "@codemirror/state": "^6.6.0",
+        "crelt": "^1.0.6",
+        "style-mod": "^4.1.0",
+        "w3c-keyname": "^2.2.4"
+      }
+    },
     "node_modules/@esbuild/aix-ppc64": {
       "version": "0.25.12",
       "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
@@ -966,6 +1064,42 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@lezer/common": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.5.2.tgz",
+      "integrity": "sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ=="
+    },
+    "node_modules/@lezer/highlight": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.3.tgz",
+      "integrity": "sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g==",
+      "dependencies": {
+        "@lezer/common": "^1.3.0"
+      }
+    },
+    "node_modules/@lezer/lr": {
+      "version": "1.4.8",
+      "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.8.tgz",
+      "integrity": "sha512-bPWa0Pgx69ylNlMlPvBPryqeLYQjyJjqPx+Aupm5zydLIF3NE+6MMLT8Yi23Bd9cif9VS00aUebn+6fDIGBcDA==",
+      "dependencies": {
+        "@lezer/common": "^1.0.0"
+      }
+    },
+    "node_modules/@lezer/yaml": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@lezer/yaml/-/yaml-1.0.4.tgz",
+      "integrity": "sha512-2lrrHqxalACEbxIbsjhqGpSW8kWpUKuY6RHgnSAFZa6qK62wvnPxA8hGOwOoDbwHcOFs5M4o27mjGu+P7TvBmw==",
+      "dependencies": {
+        "@lezer/common": "^1.2.0",
+        "@lezer/highlight": "^1.0.0",
+        "@lezer/lr": "^1.4.0"
+      }
+    },
+    "node_modules/@marijn/find-cluster-break": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz",
+      "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g=="
+    },
     "node_modules/@modelcontextprotocol/ext-apps": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/@modelcontextprotocol/ext-apps/-/ext-apps-1.2.2.tgz",
@@ -1829,6 +1963,11 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/crelt": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz",
+      "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g=="
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.6",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -3537,6 +3676,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/style-mod": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz",
+      "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ=="
+    },
     "node_modules/supports-color": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -3727,6 +3871,11 @@
         }
       }
     },
+    "node_modules/w3c-keyname": {
+      "version": "2.2.8",
+      "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz",
+      "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ=="
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -3761,6 +3910,20 @@
       "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
       "dev": true
     },
+    "node_modules/yaml": {
+      "version": "2.8.3",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz",
+      "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==",
+      "bin": {
+        "yaml": "bin.mjs"
+      },
+      "engines": {
+        "node": ">= 14.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/eemeli"
+      }
+    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

core/http/react-ui/src/pages/Backends.jsx

@@ -7,6 +7,7 @@ import { useOperations } from '../hooks/useOperations'
 import LoadingSpinner from '../components/LoadingSpinner'
 import { renderMarkdown } from '../utils/markdown'
 import ConfirmDialog from '../components/ConfirmDialog'
+import Toggle from '../components/Toggle'
 
 export default function Backends() {
   const { addToast } = useOutletContext()
@@ -26,6 +27,11 @@ export default function Backends() {
   const [expandedRow, setExpandedRow] = useState(null)
   const [confirmDialog, setConfirmDialog] = useState(null)
   const [allBackends, setAllBackends] = useState([])
+  const [upgrades, setUpgrades] = useState({})
+  const [upgradingAll, setUpgradingAll] = useState(false)
+  const [showAllBackends, setShowAllBackends] = useState(false)
+  const [showDevelopment, setShowDevelopment] = useState(false)
+  const [preferDevLoaded, setPreferDevLoaded] = useState(false)
 
   const fetchBackends = useCallback(async () => {
     try {
@@ -36,6 +42,11 @@ export default function Backends() {
       const list = Array.isArray(data?.backends) ? data.backends : Array.isArray(data) ? data : []
       setAllBackends(list)
       setInstalledCount(list.filter(b => b.installed).length)
+      // On first load, use server preference for development toggle
+      if (!preferDevLoaded && data?.preferDevelopmentBackends) {
+        setShowDevelopment(true)
+        setPreferDevLoaded(true)
+      }
     } catch (err) {
       addToast(`Failed to load backends: ${err.message}`, 'error')
     } finally {
@@ -52,17 +63,40 @@ export default function Backends() {
     if (!loading) fetchBackends()
   }, [operations.length])
 
-  // Client-side filtering by tag
-  const filteredBackends = filter
-    ? allBackends.filter(b => {
+  // Fetch available upgrades
+  useEffect(() => {
+    backendsApi.checkUpgrades()
+      .then(data => setUpgrades(data || {}))
+      .catch(() => {})
+  }, [operations.length])
+
+  // Client-side filtering by meta/development toggles and tag
+  const filteredBackends = (() => {
+    let result = allBackends
+
+    // Show only meta backends unless "Show all" is toggled
+    if (!showAllBackends) {
+      result = result.filter(b => b.isMeta)
+    }
+
+    // Hide development backends unless toggled on
+    if (!showDevelopment) {
+      result = result.filter(b => !b.isDevelopment)
+    }
+
+    // Apply tag filter
+    if (filter) {
+      result = result.filter(b => {
         const tags = (b.tags || []).map(t => t.toLowerCase())
         const name = (b.name || '').toLowerCase()
         const desc = (b.description || '').toLowerCase()
         const f = filter.toLowerCase()
-        // Match against tags, or name/description containing the filter keyword
         return tags.some(t => t.includes(f)) || name.includes(f) || desc.includes(f)
       })
-    : allBackends
+    }
+
+    return result
+  })()
 
   // Client-side pagination
   const ITEMS_PER_PAGE = 21
@@ -114,6 +148,31 @@ export default function Backends() {
     })
   }
 
+  const handleUpgrade = async (id) => {
+    try {
+      await backendsApi.upgrade(id)
+      addToast(`Upgrading ${id}...`, 'info')
+    } catch (err) {
+      addToast(`Upgrade failed: ${err.message}`, 'error')
+    }
+  }
+
+  const handleUpgradeAll = async () => {
+    const names = Object.keys(upgrades)
+    if (names.length === 0) return
+    setUpgradingAll(true)
+    try {
+      for (const name of names) {
+        await backendsApi.upgrade(name)
+      }
+      addToast(`Upgrading ${names.length} backend${names.length > 1 ? 's' : ''}...`, 'info')
+    } catch (err) {
+      addToast(`Upgrade failed: ${err.message}`, 'error')
+    } finally {
+      setUpgradingAll(false)
+    }
+  }
+
   const handleManualInstall = async (e) => {
     e.preventDefault()
     if (!manualUri.trim()) { addToast('Please enter a URI', 'warning'); return }
@@ -137,6 +196,9 @@ export default function Backends() {
     return operations.find(op => op.name === backend.name || op.name === backend.id) || null
   }
 
+  const handleToggleAllBackends = () => { setShowAllBackends(v => !v); setPage(1) }
+  const handleToggleDev = () => { setShowDevelopment(v => !v); setPage(1) }
+
   const FILTERS = [
     { key: '', label: 'All', icon: 'fa-layer-group' },
     { key: 'llm', label: 'LLM', icon: 'fa-brain' },
@@ -179,6 +241,14 @@ export default function Backends() {
                 <div style={{ color: 'var(--color-text-muted)' }}>Installed</div>
               </a>
             </div>
+            {Object.keys(upgrades).length > 0 && (
+              <div style={{ textAlign: 'center' }}>
+                <div style={{ fontSize: '1.25rem', fontWeight: 700, color: 'var(--color-warning)' }}>
+                  {Object.keys(upgrades).length}
+                </div>
+                <div style={{ color: 'var(--color-text-muted)' }}>Updates</div>
+              </div>
+            )}
           </div>
           <a className="btn btn-secondary btn-sm" href="https://localai.io/docs/getting-started/manual/" target="_blank" rel="noopener noreferrer">
             <i className="fas fa-book" /> Docs
@@ -186,6 +256,33 @@ export default function Backends() {
         </div>
       </div>
 
+      {/* Upgrade Banner */}
+      {Object.keys(upgrades).length > 0 && (
+        <div className="card" style={{
+          marginBottom: 'var(--spacing-md)',
+          display: 'flex', alignItems: 'center', justifyContent: 'space-between',
+          padding: 'var(--spacing-sm) var(--spacing-md)',
+          background: 'var(--color-warning-bg, #fef3cd)',
+          border: '1px solid var(--color-warning, #ffc107)',
+          borderRadius: 'var(--radius-md)',
+        }}>
+          <div style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-sm)' }}>
+            <i className="fas fa-arrow-up" style={{ color: 'var(--color-warning, #856404)' }} />
+            <span style={{ color: 'var(--color-warning, #856404)', fontWeight: 500, fontSize: '0.875rem' }}>
+              {Object.keys(upgrades).length} backend{Object.keys(upgrades).length > 1 ? 's have' : ' has'} updates available
+            </span>
+          </div>
+          <button
+            className="btn btn-primary btn-sm"
+            onClick={handleUpgradeAll}
+            disabled={upgradingAll}
+          >
+            <i className={`fas ${upgradingAll ? 'fa-spinner fa-spin' : 'fa-arrow-up'}`} style={{ marginRight: 4 }} />
+            Upgrade All
+          </button>
+        </div>
+      )}
+
       {/* Manual Install */}
       <div style={{ marginBottom: 'var(--spacing-md)' }}>
         <button className="btn btn-secondary btn-sm" onClick={() => setShowManualInstall(!showManualInstall)}>
@@ -227,17 +324,32 @@ export default function Backends() {
         </div>
       </div>
 
-      <div className="filter-bar" style={{ marginBottom: 'var(--spacing-md)' }}>
-        {FILTERS.map(f => (
-          <button
-            key={f.key}
-            className={`filter-btn ${filter === f.key ? 'active' : ''}`}
-            onClick={() => { setFilter(f.key); setPage(1) }}
-          >
+      <div style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-md)', marginBottom: 'var(--spacing-md)', flexWrap: 'wrap' }}>
+        <div className="filter-bar" style={{ margin: 0, flex: 1 }}>
+          {FILTERS.map(f => (
+            <button
+              key={f.key}
+              className={`filter-btn ${filter === f.key ? 'active' : ''}`}
+              onClick={() => { setFilter(f.key); setPage(1) }}
+            >
             <i className={`fas ${f.icon}`} style={{ marginRight: 4 }} />
             {f.label}
           </button>
         ))}
+        </div>
+
+        <div style={{ display: 'flex', gap: 'var(--spacing-md)', alignItems: 'center', borderLeft: '1px solid var(--color-border-subtle)', paddingLeft: 'var(--spacing-md)' }}>
+          <label style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-xs)', fontSize: '0.75rem', color: 'var(--color-text-secondary)', cursor: 'pointer', userSelect: 'none', whiteSpace: 'nowrap' }}>
+            <Toggle checked={showAllBackends} onChange={handleToggleAllBackends} />
+            <i className="fas fa-cubes" style={{ fontSize: '0.625rem' }} />
+            Show all
+          </label>
+          <label style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-xs)', fontSize: '0.75rem', color: 'var(--color-text-secondary)', cursor: 'pointer', userSelect: 'none', whiteSpace: 'nowrap' }}>
+            <Toggle checked={showDevelopment} onChange={handleToggleDev} />
+            <i className="fas fa-flask" style={{ fontSize: '0.625rem' }} />
+            Development
+          </label>
+        </div>
       </div>
 
       {/* Table */}
@@ -300,6 +412,11 @@ export default function Backends() {
                     {/* Name */}
                     <td>
                       <span style={{ fontWeight: 500 }}>{b.name || b.id}</span>
+                      {b.version && (
+                        <span className="badge" style={{ fontSize: '0.625rem', marginLeft: 4, background: 'var(--color-bg-tertiary)', color: 'var(--color-text-secondary)' }}>
+                          v{b.version}
+                        </span>
+                      )}
                     </td>
 
                     {/* Description */}
@@ -346,9 +463,17 @@ export default function Backends() {
                           </span>
                         </div>
                       ) : b.installed ? (
-                        <span className="badge badge-success">
-                          <i className="fas fa-check" style={{ fontSize: '0.5rem', marginRight: 2 }} /> Installed
-                        </span>
+                        <div style={{ display: 'flex', gap: 4, alignItems: 'center', flexWrap: 'wrap' }}>
+                          <span className="badge badge-success">
+                            <i className="fas fa-check" style={{ fontSize: '0.5rem', marginRight: 2 }} /> Installed
+                          </span>
+                          {upgrades[b.name] && (
+                            <span className="badge" style={{ fontSize: '0.625rem', background: '#fef3cd', color: '#856404' }}>
+                              <i className="fas fa-arrow-up" style={{ fontSize: '0.5rem', marginRight: 2 }} />
+                              {upgrades[b.name].available_version ? `v${upgrades[b.name].available_version}` : 'Update'}
+                            </span>
+                          )}
+                        </div>
                       ) : (
                         <span className="badge" style={{ background: 'var(--color-bg-tertiary)', color: 'var(--color-text-muted)' }}>
                           <i className="fas fa-circle" style={{ fontSize: '0.5rem', marginRight: 2 }} /> Not Installed
@@ -361,9 +486,15 @@ export default function Backends() {
                       <div style={{ display: 'flex', gap: 'var(--spacing-xs)', justifyContent: 'flex-end' }} onClick={e => e.stopPropagation()}>
                         {b.installed ? (
                           <>
-                            <button className="btn btn-secondary btn-sm" onClick={() => handleInstall(b.name || b.id)} title="Reinstall" disabled={isProcessing}>
-                              <i className={`fas ${isProcessing ? 'fa-spinner fa-spin' : 'fa-rotate'}`} />
-                            </button>
+                            {upgrades[b.name] ? (
+                              <button className="btn btn-primary btn-sm" onClick={() => handleUpgrade(b.name || b.id)} title={`Upgrade to ${upgrades[b.name]?.available_version ? 'v' + upgrades[b.name].available_version : 'latest'}`} disabled={isProcessing}>
+                                <i className={`fas ${isProcessing ? 'fa-spinner fa-spin' : 'fa-arrow-up'}`} />
+                              </button>
+                            ) : (
+                              <button className="btn btn-secondary btn-sm" onClick={() => handleInstall(b.name || b.id)} title="Reinstall" disabled={isProcessing}>
+                                <i className={`fas ${isProcessing ? 'fa-spinner fa-spin' : 'fa-rotate'}`} />
+                              </button>
+                            )}
                             <button className="btn btn-danger btn-sm" onClick={() => handleDelete(b.name || b.id)} title="Delete" disabled={isProcessing}>
                               <i className="fas fa-trash" />
                             </button>

core/http/react-ui/src/pages/Manage.jsx

@@ -2,6 +2,7 @@ import { useState, useEffect, useCallback } from 'react'
 import { useNavigate, useOutletContext, useSearchParams } from 'react-router-dom'
 import ResourceMonitor from '../components/ResourceMonitor'
 import ConfirmDialog from '../components/ConfirmDialog'
+import Toggle from '../components/Toggle'
 import { useModels } from '../hooks/useModels'
 import { backendControlApi, modelsApi, backendsApi, systemApi, nodesApi } from '../utils/api'
 
@@ -22,6 +23,7 @@ export default function Manage() {
   const [backendsLoading, setBackendsLoading] = useState(true)
   const [reloading, setReloading] = useState(false)
   const [reinstallingBackends, setReinstallingBackends] = useState(new Set())
+  const [upgrades, setUpgrades] = useState({})
   const [confirmDialog, setConfirmDialog] = useState(null)
   const [distributedMode, setDistributedMode] = useState(false)
   const [togglingModels, setTogglingModels] = useState(new Set())
@@ -62,6 +64,15 @@ export default function Manage() {
     nodesApi.list().then(() => setDistributedMode(true)).catch(() => {})
   }, [fetchLoadedModels, fetchBackends])
 
+  // Fetch available backend upgrades
+  useEffect(() => {
+    if (activeTab === 'backends') {
+      backendsApi.checkUpgrades()
+        .then(data => setUpgrades(data || {}))
+        .catch(() => {})
+    }
+  }, [activeTab])
+
   const handleStopModel = (modelName) => {
     setConfirmDialog({
       title: 'Stop Model',
@@ -169,6 +180,22 @@ export default function Manage() {
     }
   }
 
+  const handleUpgradeBackend = async (name) => {
+    try {
+      setReinstallingBackends(prev => new Set(prev).add(name))
+      await backendsApi.upgrade(name)
+      addToast(`Upgrading ${name}...`, 'info')
+    } catch (err) {
+      addToast(`Failed to upgrade: ${err.message}`, 'error')
+    } finally {
+      setReinstallingBackends(prev => {
+        const next = new Set(prev)
+        next.delete(name)
+        return next
+      })
+    }
+  }
+
   const handleDeleteBackend = (name) => {
     setConfirmDialog({
       title: 'Delete Backend',
@@ -252,6 +279,7 @@ export default function Manage() {
             <table className="table">
               <thead>
                 <tr>
+                  <th style={{ width: 36 }}>Enabled</th>
                   <th>Name</th>
                   <th>Status</th>
                   <th>Backend</th>
@@ -262,34 +290,49 @@ export default function Manage() {
               <tbody>
                 {models.map(model => (
                   <tr key={model.id} style={{ opacity: model.disabled ? 0.55 : 1, transition: 'opacity 0.2s' }}>
+                    {/* Enable/Disable toggle */}
+                    <td>
+                      <Toggle
+                        checked={!model.disabled}
+                        onChange={() => handleToggleModel(model.id, model.disabled)}
+                        disabled={togglingModels.has(model.id)}
+                      />
+                    </td>
+                    {/* Name */}
                     <td>
                       <div style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-sm)' }}>
-                        <i className="fas fa-brain" style={{ color: model.disabled ? 'var(--color-text-muted)' : 'var(--color-accent)' }} />
-                        <span className={`badge ${model.disabled ? '' : 'badge-success'}`} style={{ width: 6, height: 6, padding: 0, borderRadius: '50%', minWidth: 'auto', background: model.disabled ? 'var(--color-text-muted)' : undefined }} />
                         <span style={{ fontWeight: 500 }}>{model.id}</span>
-                        <a
-                          href="#"
-                          onClick={(e) => { e.preventDefault(); navigate(`/app/model-editor/${encodeURIComponent(model.id)}`) }}
-                          style={{ fontSize: '0.75rem', color: 'var(--color-primary)' }}
-                          title="Edit config"
-                        >
-                          <i className="fas fa-pen-to-square" />
-                        </a>
-                        {!distributedMode && (
+                        {model.pinned && (
+                          <i className="fas fa-thumbtack" style={{ fontSize: '0.625rem', color: 'var(--color-warning)' }} title="Pinned — won't be idle-unloaded" />
+                        )}
+                        <div style={{ display: 'flex', gap: '2px', marginLeft: 'auto' }}>
                           <a
                             href="#"
-                            onClick={(e) => { e.preventDefault(); navigate(`/app/backend-logs/${encodeURIComponent(model.id)}`) }}
-                            style={{ fontSize: '0.75rem', color: 'var(--color-primary)' }}
-                            title="Backend logs"
+                            onClick={(e) => { e.preventDefault(); navigate(`/app/model-editor/${encodeURIComponent(model.id)}`) }}
+                            className="btn btn-secondary btn-sm"
+                            style={{ padding: '2px 5px', fontSize: '0.625rem' }}
+                            title="Edit config"
                           >
-                            <i className="fas fa-terminal" />
+                            <i className="fas fa-pen-to-square" />
                           </a>
-                        )}
+                          {!distributedMode && (
+                            <a
+                              href="#"
+                              onClick={(e) => { e.preventDefault(); navigate(`/app/backend-logs/${encodeURIComponent(model.id)}`) }}
+                              className="btn btn-secondary btn-sm"
+                              style={{ padding: '2px 5px', fontSize: '0.625rem' }}
+                              title="Backend logs"
+                            >
+                              <i className="fas fa-terminal" />
+                            </a>
+                          )}
+                        </div>
                       </div>
                     </td>
+                    {/* Status */}
                     <td>
                       {model.disabled ? (
-                        <span className="badge" style={{ background: 'var(--color-danger, #ef4444)', color: 'white' }}>
+                        <span className="badge" style={{ background: 'var(--color-bg-tertiary)', color: 'var(--color-text-muted)' }}>
                           <i className="fas fa-ban" style={{ fontSize: '6px' }} /> Disabled
                         </span>
                       ) : loadedModelIds.has(model.id) ? (
@@ -302,17 +345,19 @@ export default function Manage() {
                         </span>
                       )}
                     </td>
+                    {/* Backend */}
                     <td>
                       <span className="badge badge-info">{model.backend || 'Auto'}</span>
                     </td>
+                    {/* Use Cases */}
                     <td>
                       <div style={{ display: 'flex', gap: '4px', flexWrap: 'wrap' }}>
                         <a href="#" onClick={(e) => { e.preventDefault(); navigate(`/app/chat/${encodeURIComponent(model.id)}`) }} className="badge badge-info" style={{ textDecoration: 'none', cursor: 'pointer' }}>Chat</a>
                       </div>
                     </td>
+                    {/* Actions */}
                     <td>
-                      <div style={{ display: 'flex', gap: 'var(--spacing-sm)', justifyContent: 'flex-end', alignItems: 'center' }}>
-                        {/* Stop button - shown when model is loaded */}
+                      <div style={{ display: 'flex', gap: 'var(--spacing-xs)', justifyContent: 'flex-end', alignItems: 'center' }}>
                         {loadedModelIds.has(model.id) && (
                           <button
                             className="btn btn-secondary btn-sm"
@@ -322,62 +367,17 @@ export default function Manage() {
                             <i className="fas fa-stop" />
                           </button>
                         )}
-                        {/* Pin button - prevents model from being unloaded */}
                         <button
-                          className="btn btn-sm"
+                          className="btn btn-secondary btn-sm"
                           onClick={() => handleTogglePinned(model.id, model.pinned)}
                           disabled={pinningModels.has(model.id) || model.disabled}
                           title={model.pinned ? 'Unpin model (allow idle unloading)' : 'Pin model (prevent idle unloading)'}
                           style={{
-                            padding: '2px 6px',
-                            minWidth: 28,
-                            color: model.pinned ? 'var(--color-warning, #f59e0b)' : 'var(--color-text-muted)',
-                            opacity: model.disabled ? 0.3 : (pinningModels.has(model.id) ? 0.5 : 1),
-                            cursor: pinningModels.has(model.id) ? 'wait' : (model.disabled ? 'not-allowed' : 'pointer'),
+                            color: model.pinned ? 'var(--color-warning)' : undefined,
                           }}
                         >
                           <i className={`fas fa-thumbtack${pinningModels.has(model.id) ? ' fa-spin' : ''}`} />
                         </button>
-                        {/* Toggle switch for enabling/disabling model loading on demand */}
-                        <label
-                          title={model.disabled ? 'Model is disabled — click to enable loading on demand' : 'Model is enabled — click to disable loading on demand'}
-                          style={{
-                            position: 'relative',
-                            display: 'inline-block',
-                            width: 36,
-                            height: 20,
-                            cursor: togglingModels.has(model.id) ? 'wait' : 'pointer',
-                            opacity: togglingModels.has(model.id) ? 0.5 : 1,
-                            flexShrink: 0,
-                          }}
-                        >
-                          <input
-                            type="checkbox"
-                            checked={!model.disabled}
-                            onChange={() => handleToggleModel(model.id, model.disabled)}
-                            disabled={togglingModels.has(model.id)}
-                            style={{ opacity: 0, width: 0, height: 0 }}
-                          />
-                          <span style={{
-                            position: 'absolute',
-                            top: 0, left: 0, right: 0, bottom: 0,
-                            backgroundColor: model.disabled ? 'var(--color-bg-tertiary)' : 'var(--color-success, #22c55e)',
-                            borderRadius: 20,
-                            transition: 'background-color 0.2s',
-                          }}>
-                            <span style={{
-                              position: 'absolute',
-                              content: '""',
-                              height: 14,
-                              width: 14,
-                              left: model.disabled ? 3 : 19,
-                              bottom: 3,
-                              backgroundColor: 'white',
-                              borderRadius: '50%',
-                              transition: 'left 0.2s',
-                            }} />
-                          </span>
-                        </label>
                         <button
                           className="btn btn-danger btn-sm"
                           onClick={() => handleDeleteModel(model.id)}
@@ -471,6 +471,17 @@ export default function Manage() {
                             For: <span style={{ color: 'var(--color-accent)' }}>{backend.Metadata.meta_backend_for}</span>
                           </span>
                         )}
+                        {backend.Metadata?.version && (
+                          <span>
+                            <i className="fas fa-code-branch" style={{ fontSize: '0.5rem', marginRight: 4 }} />
+                            Version: <span style={{ color: 'var(--color-text-primary)' }}>v{backend.Metadata.version}</span>
+                            {upgrades[backend.Name] && (
+                              <span style={{ color: '#856404', marginLeft: 4 }}>
+                                → v{upgrades[backend.Name].available_version}
+                              </span>
+                            )}
+                          </span>
+                        )}
                         {backend.Metadata?.installed_at && (
                           <span>
                             <i className="fas fa-calendar" style={{ fontSize: '0.5rem', marginRight: 4 }} />
@@ -485,12 +496,12 @@ export default function Manage() {
                         {!backend.IsSystem ? (
                           <>
                             <button
-                              className="btn btn-secondary btn-sm"
-                              onClick={() => handleReinstallBackend(backend.Name)}
+                              className={`btn ${upgrades[backend.Name] ? 'btn-primary' : 'btn-secondary'} btn-sm`}
+                              onClick={() => upgrades[backend.Name] ? handleUpgradeBackend(backend.Name) : handleReinstallBackend(backend.Name)}
                               disabled={reinstallingBackends.has(backend.Name)}
-                              title="Reinstall"
+                              title={upgrades[backend.Name] ? `Upgrade to v${upgrades[backend.Name]?.available_version || 'latest'}` : 'Reinstall'}
                             >
-                              <i className={`fas ${reinstallingBackends.has(backend.Name) ? 'fa-spinner fa-spin' : 'fa-rotate'}`} />
+                              <i className={`fas ${reinstallingBackends.has(backend.Name) ? 'fa-spinner fa-spin' : upgrades[backend.Name] ? 'fa-arrow-up' : 'fa-rotate'}`} />
                             </button>
                             <button
                               className="btn btn-danger btn-sm"

core/http/react-ui/src/pages/Settings.jsx

@@ -266,6 +266,12 @@ export default function Settings() {
               <SettingRow label="Max Active Backends" description="Maximum models to keep loaded simultaneously (0 = unlimited)">
                 <input className="input" type="number" style={{ width: 120 }} value={settings.max_active_backends ?? ''} onChange={(e) => update('max_active_backends', parseInt(e.target.value) || 0)} placeholder="0" />
               </SettingRow>
+              <SettingRow label="Auto-upgrade Backends" description="Automatically upgrade backends when new versions are detected">
+                <Toggle checked={settings.auto_upgrade_backends} onChange={(v) => update('auto_upgrade_backends', v)} />
+              </SettingRow>
+              <SettingRow label="Prefer Development Backends" description="Default to showing development backend versions in the backends gallery">
+                <Toggle checked={settings.prefer_development_backends} onChange={(v) => update('prefer_development_backends', v)} />
+              </SettingRow>
             </div>
           </div>
 

core/http/react-ui/src/utils/api.js

@@ -120,6 +120,9 @@ export const backendsApi = {
   installExternal: (body) => postJSON(API_CONFIG.endpoints.installExternalBackend, body),
   getJob: (uid) => fetchJSON(API_CONFIG.endpoints.backendJob(uid)),
   deleteInstalled: (name) => postJSON(API_CONFIG.endpoints.deleteInstalledBackend(name), {}),
+  checkUpgrades: () => fetchJSON(API_CONFIG.endpoints.backendsUpgrades),
+  forceCheckUpgrades: () => postJSON(API_CONFIG.endpoints.backendsUpgradesCheck, {}),
+  upgrade: (name) => postJSON(API_CONFIG.endpoints.upgradeBackend(name), {}),
 }
 
 // Chat API (non-streaming)

core/http/react-ui/src/utils/config.js

@@ -23,6 +23,9 @@ export const API_CONFIG = {
     installExternalBackend: '/api/backends/install-external',
     backendJob: (uid) => `/api/backends/job/${uid}`,
     deleteInstalledBackend: (name) => `/api/backends/system/delete/${name}`,
+    backendsUpgrades: '/api/backends/upgrades',
+    backendsUpgradesCheck: '/api/backends/upgrades/check',
+    upgradeBackend: (name) => `/api/backends/upgrade/${name}`,
 
     // Resources
     resources: '/api/resources',

core/http/routes/localai.go

@@ -59,13 +59,17 @@ func RegisterLocalAIRoutes(router *echo.Echo,
 		backendGalleryEndpointService := localai.CreateBackendEndpointService(
 			appConfig.BackendGalleries,
 			appConfig.SystemState,
-			galleryService)
+			galleryService,
+			app.UpgradeChecker())
 		router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(), adminMiddleware)
 		router.POST("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint(), adminMiddleware)
 		router.GET("/backends", backendGalleryEndpointService.ListBackendsEndpoint(), adminMiddleware)
 		router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState), adminMiddleware)
 		router.GET("/backends/galleries", backendGalleryEndpointService.ListBackendGalleriesEndpoint(), adminMiddleware)
 		router.GET("/backends/jobs/:uuid", backendGalleryEndpointService.GetOpStatusEndpoint(), adminMiddleware)
+		router.GET("/backends/upgrades", backendGalleryEndpointService.GetUpgradesEndpoint(), adminMiddleware)
+		router.POST("/backends/upgrades/check", backendGalleryEndpointService.CheckUpgradesEndpoint(), adminMiddleware)
+		router.POST("/backends/upgrade/:name", backendGalleryEndpointService.UpgradeBackendEndpoint(), adminMiddleware)
 		// Custom model import endpoint
 		router.POST("/models/import", localai.ImportModelEndpoint(cl, appConfig), adminMiddleware)
 

core/http/routes/ui_api.go

@@ -816,7 +816,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			items = "9"
 		}
 
-		backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
+		backends, err := gallery.AvailableBackendsUnfiltered(appConfig.BackendGalleries, appConfig.SystemState)
 		if err != nil {
 			xlog.Error("could not list backends from galleries", "error", err)
 			return c.JSON(http.StatusInternalServerError, map[string]any{
@@ -894,6 +894,12 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			backends = backends.Paginate(pageNum, itemsNum)
 		}
 
+		// Get dev suffix from SystemState for development backend detection
+		devSuffix := ""
+		if appConfig.SystemState != nil {
+			devSuffix = appConfig.SystemState.BackendDevSuffix
+		}
+
 		// Convert backends to JSON-friendly format and deduplicate by ID
 		backendsJSON := make([]map[string]any, 0, len(backends))
 		seenBackendIDs := make(map[string]bool)
@@ -920,18 +926,21 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			}
 
 			backendsJSON = append(backendsJSON, map[string]any{
-				"id":          backendID,
-				"name":        b.Name,
-				"description": b.Description,
-				"icon":        b.Icon,
-				"license":     b.License,
-				"urls":        b.URLs,
-				"tags":        b.Tags,
-				"gallery":     b.Gallery.Name,
-				"installed":   b.Installed,
-				"processing":  currentlyProcessing,
-				"jobID":       jobID,
-				"isDeletion":  isDeletionOp,
+				"id":            backendID,
+				"name":          b.Name,
+				"description":   b.Description,
+				"icon":          b.Icon,
+				"license":       b.License,
+				"urls":          b.URLs,
+				"tags":          b.Tags,
+				"gallery":       b.Gallery.Name,
+				"installed":     b.Installed,
+				"version":       b.Version,
+				"processing":    currentlyProcessing,
+				"jobID":         jobID,
+				"isDeletion":    isDeletionOp,
+				"isMeta":        b.IsMeta(),
+				"isDevelopment": b.IsDevelopment(devSuffix),
 			})
 		}
 
@@ -973,7 +982,8 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			"totalPages":         totalPages,
 			"prevPage":           prevPage,
 			"nextPage":           nextPage,
-			"systemCapability":   detectedCapability,
+			"systemCapability":          detectedCapability,
+			"preferDevelopmentBackends": appConfig.PreferDevelopmentBackends,
 		})
 	}, adminMiddleware)
 
@@ -1194,6 +1204,49 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 		})
 	}, adminMiddleware)
 
+	// Backend upgrade APIs
+	app.GET("/api/backends/upgrades", func(c echo.Context) error {
+		if applicationInstance == nil || applicationInstance.UpgradeChecker() == nil {
+			return c.JSON(200, map[string]any{})
+		}
+		return c.JSON(200, applicationInstance.UpgradeChecker().GetAvailableUpgrades())
+	}, adminMiddleware)
+
+	app.POST("/api/backends/upgrades/check", func(c echo.Context) error {
+		if applicationInstance == nil || applicationInstance.UpgradeChecker() == nil {
+			return c.JSON(200, map[string]any{})
+		}
+		applicationInstance.UpgradeChecker().TriggerCheck()
+		return c.JSON(200, applicationInstance.UpgradeChecker().GetAvailableUpgrades())
+	}, adminMiddleware)
+
+	app.POST("/api/backends/upgrade/:name", func(c echo.Context) error {
+		backendName := c.Param("name")
+		backendName, err := url.QueryUnescape(backendName)
+		if err != nil {
+			return c.JSON(http.StatusBadRequest, map[string]any{
+				"error": "invalid backend name",
+			})
+		}
+
+		uid, err := uuid.NewUUID()
+		if err != nil {
+			return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()})
+		}
+
+		galleryService.BackendGalleryChannel <- galleryop.ManagementOp[gallery.GalleryBackend, any]{
+			ID:                 uid.String(),
+			GalleryElementName: backendName,
+			Galleries:          appConfig.BackendGalleries,
+			Upgrade:            true,
+		}
+
+		return c.JSON(200, map[string]any{
+			"uuid":      uid.String(),
+			"statusUrl": fmt.Sprintf("/api/backends/job/%s", uid.String()),
+		})
+	}, adminMiddleware)
+
 	// P2P APIs
 	app.GET("/api/p2p/workers", func(c echo.Context) error {
 		llamaNodes := p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.LlamaCPPWorkerID))