feat: upgrade banner with Upgrade All button, detect pre-existing backends

- Add upgrade banner on Backends page showing count and Upgrade All button
- Fix upgrade detection for backends installed before version tracking:
  flag as upgradeable when gallery has a version but installed has none
- Fix OCI digest check to flag backends with no stored digest as upgradeable

.agents/adding-backends.md

@@ -28,7 +28,7 @@ Add build matrix entries for each platform/GPU type you want to support. Look at
 - CUDA 13 builds: Add after other CUDA 13 builds (e.g., after `gpu-nvidia-cuda-13-chatterbox`)
 
 **Additional build types you may need:**
-- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:7.2.1"`
+- ROCm/HIP: Use `build-type: 'hipblas'` with `base-image: "rocm/dev-ubuntu-24.04:6.4.4"`
 - Intel/SYCL: Use `build-type: 'intel'` or `build-type: 'sycl_f16'`/`sycl_f32` with `base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"`
 - L4T (ARM): Use `build-type: 'l4t'` with `platforms: 'linux/arm64'` and `runs-on: 'ubuntu-24.04-arm'`
 

.agents/building-and-testing.md

@@ -10,7 +10,7 @@ Let's say the user wants to build a particular backend for a given platform. For
 - At a minimum we need to set the BUILD_TYPE, BASE_IMAGE build-args
   - Use .github/workflows/backend.yml as a reference it lists the needed args in the `include` job strategy matrix
   - l4t and cublas also requires the CUDA major and minor version
-- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:7.2.1 make docker-build-coqui`
+- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:6.4.4 make docker-build-coqui`
 - Unless the user specifies that they want you to run the command, then just print it because not all agent frontends handle long running jobs well and the output may overflow your context
 - The user may say they want to build AMD or ROCM instead of hipblas, or Intel instead of SYCL or NVIDIA insted of l4t or cublas. Ask for confirmation if there is ambiguity.
 - Sometimes the user may need extra parameters to be added to `docker build` (e.g. `--platform` for cross-platform builds or `--progress` to view the full logs), in which case you can generate the `docker build` command directly.

.github/workflows/backend.yml

@@ -53,19 +53,6 @@ jobs:
             dockerfile: "./backend/Dockerfile.python"
             context: "./"
             ubuntu-version: '2204'
-          - build-type: ''
-            cuda-major-version: ""
-            cuda-minor-version: ""
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-cpu-vllm'
-            runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:24.04"
-            skip-drivers: 'true'
-            backend: "vllm"
-            dockerfile: "./backend/Dockerfile.python"
-            context: "./"
-            ubuntu-version: '2404'
           - build-type: ''
             cuda-major-version: ""
             cuda-minor-version: ""
@@ -626,19 +613,6 @@ jobs:
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
             ubuntu-version: '2404'
-          - build-type: 'cublas'
-            cuda-major-version: "12"
-            cuda-minor-version: "8"
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-gpu-nvidia-cuda-12-qwen3-tts-cpp'
-            runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:24.04"
-            skip-drivers: 'false'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "8"
@@ -1251,19 +1225,6 @@ jobs:
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
             ubuntu-version: '2404'
-          - build-type: 'cublas'
-            cuda-major-version: "13"
-            cuda-minor-version: "0"
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-gpu-nvidia-cuda-13-qwen3-tts-cpp'
-            runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:24.04"
-            skip-drivers: 'false'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "13"
             cuda-minor-version: "0"
@@ -1277,19 +1238,6 @@ jobs:
             backend: "acestep-cpp"
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
-          - build-type: 'cublas'
-            cuda-major-version: "13"
-            cuda-minor-version: "0"
-            platforms: 'linux/arm64'
-            skip-drivers: 'false'
-            tag-latest: 'auto'
-            tag-suffix: '-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp'
-            base-image: "ubuntu:24.04"
-            ubuntu-version: '2404'
-            runs-on: 'ubuntu-24.04-arm'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
           - build-type: 'cublas'
             cuda-major-version: "13"
             cuda-minor-version: "0"
@@ -1311,7 +1259,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-rerankers'
             runs-on: 'ubuntu-latest'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "rerankers"
             dockerfile: "./backend/Dockerfile.python"
@@ -1324,7 +1272,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-llama-cpp'
             runs-on: 'ubuntu-latest'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "llama-cpp"
             dockerfile: "./backend/Dockerfile.llama-cpp"
@@ -1337,7 +1285,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-vllm'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "vllm"
             dockerfile: "./backend/Dockerfile.python"
@@ -1350,7 +1298,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-vllm-omni'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "vllm-omni"
             dockerfile: "./backend/Dockerfile.python"
@@ -1363,7 +1311,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-transformers'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "transformers"
             dockerfile: "./backend/Dockerfile.python"
@@ -1376,7 +1324,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-diffusers'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "diffusers"
             dockerfile: "./backend/Dockerfile.python"
@@ -1389,7 +1337,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-ace-step'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "ace-step"
             dockerfile: "./backend/Dockerfile.python"
@@ -1403,7 +1351,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-kokoro'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "kokoro"
             dockerfile: "./backend/Dockerfile.python"
@@ -1416,7 +1364,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-vibevoice'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "vibevoice"
             dockerfile: "./backend/Dockerfile.python"
@@ -1429,7 +1377,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-qwen-asr'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "qwen-asr"
             dockerfile: "./backend/Dockerfile.python"
@@ -1442,7 +1390,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-nemo'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "nemo"
             dockerfile: "./backend/Dockerfile.python"
@@ -1455,7 +1403,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-qwen-tts'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "qwen-tts"
             dockerfile: "./backend/Dockerfile.python"
@@ -1468,7 +1416,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-fish-speech'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "fish-speech"
             dockerfile: "./backend/Dockerfile.python"
@@ -1481,7 +1429,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-voxcpm'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "voxcpm"
             dockerfile: "./backend/Dockerfile.python"
@@ -1494,7 +1442,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-pocket-tts'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "pocket-tts"
             dockerfile: "./backend/Dockerfile.python"
@@ -1507,7 +1455,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-faster-whisper'
             runs-on: 'bigger-runner'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "faster-whisper"
             dockerfile: "./backend/Dockerfile.python"
@@ -1520,7 +1468,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-whisperx'
             runs-on: 'bigger-runner'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "whisperx"
             dockerfile: "./backend/Dockerfile.python"
@@ -1533,7 +1481,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-coqui'
             runs-on: 'bigger-runner'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "coqui"
             dockerfile: "./backend/Dockerfile.python"
@@ -1958,19 +1906,6 @@ jobs:
             dockerfile: "./backend/Dockerfile.llama-cpp"
             context: "./"
             ubuntu-version: '2404'
-          - build-type: ''
-            cuda-major-version: ""
-            cuda-minor-version: ""
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-cpu-ik-llama-cpp'
-            runs-on: 'bigger-runner'
-            base-image: "ubuntu:24.04"
-            skip-drivers: 'false'
-            backend: "ik-llama-cpp"
-            dockerfile: "./backend/Dockerfile.ik-llama-cpp"
-            context: "./"
-            ubuntu-version: '2404'
           - build-type: 'cublas'
             cuda-major-version: "12"
             cuda-minor-version: "0"
@@ -2201,7 +2136,7 @@ jobs:
             platforms: 'linux/amd64'
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-whisper'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             runs-on: 'ubuntu-latest'
             skip-drivers: 'false'
             backend: "whisper"
@@ -2280,89 +2215,10 @@ jobs:
             platforms: 'linux/amd64'
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-acestep-cpp'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
-            runs-on: 'ubuntu-latest'
-            skip-drivers: 'false'
-            backend: "acestep-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2404'
-          # qwen3-tts-cpp
-          - build-type: ''
-            cuda-major-version: ""
-            cuda-minor-version: ""
-            platforms: 'linux/amd64,linux/arm64'
-            tag-latest: 'auto'
-            tag-suffix: '-cpu-qwen3-tts-cpp'
-            runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:24.04"
-            skip-drivers: 'false'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2404'
-          - build-type: 'sycl_f32'
-            cuda-major-version: ""
-            cuda-minor-version: ""
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-gpu-intel-sycl-f32-qwen3-tts-cpp'
-            runs-on: 'ubuntu-latest'
-            base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
-            skip-drivers: 'false'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2404'
-          - build-type: 'sycl_f16'
-            cuda-major-version: ""
-            cuda-minor-version: ""
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-gpu-intel-sycl-f16-qwen3-tts-cpp'
-            runs-on: 'ubuntu-latest'
-            base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
-            skip-drivers: 'false'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2404'
-          - build-type: 'vulkan'
-            cuda-major-version: ""
-            cuda-minor-version: ""
-            platforms: 'linux/amd64,linux/arm64'
-            tag-latest: 'auto'
-            tag-suffix: '-gpu-vulkan-qwen3-tts-cpp'
-            runs-on: 'ubuntu-latest'
-            base-image: "ubuntu:24.04"
-            skip-drivers: 'false'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2404'
-          - build-type: 'cublas'
-            cuda-major-version: "12"
-            cuda-minor-version: "0"
-            platforms: 'linux/arm64'
-            skip-drivers: 'false'
-            tag-latest: 'auto'
-            tag-suffix: '-nvidia-l4t-arm64-qwen3-tts-cpp'
-            base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
-            runs-on: 'ubuntu-24.04-arm'
-            backend: "qwen3-tts-cpp"
-            dockerfile: "./backend/Dockerfile.golang"
-            context: "./"
-            ubuntu-version: '2204'
-          - build-type: 'hipblas'
-            cuda-major-version: ""
-            cuda-minor-version: ""
-            platforms: 'linux/amd64'
-            tag-latest: 'auto'
-            tag-suffix: '-gpu-rocm-hipblas-qwen3-tts-cpp'
             base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             runs-on: 'ubuntu-latest'
             skip-drivers: 'false'
-            backend: "qwen3-tts-cpp"
+            backend: "acestep-cpp"
             dockerfile: "./backend/Dockerfile.golang"
             context: "./"
             ubuntu-version: '2404'
@@ -2482,7 +2338,7 @@ jobs:
           #   platforms: 'linux/amd64'
           #   tag-latest: 'auto'
           #   tag-suffix: '-gpu-hipblas-rfdetr'
-          #   base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+          #   base-image: "rocm/dev-ubuntu-24.04:6.4.4"
           #   runs-on: 'ubuntu-latest'
           #   skip-drivers: 'false'
           #   backend: "rfdetr"
@@ -2523,7 +2379,7 @@ jobs:
             tag-latest: 'auto'
             tag-suffix: '-gpu-rocm-hipblas-neutts'
             runs-on: 'arc-runner-set'
-            base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+            base-image: "rocm/dev-ubuntu-24.04:6.4.4"
             skip-drivers: 'false'
             backend: "neutts"
             dockerfile: "./backend/Dockerfile.python"
@@ -2671,10 +2527,6 @@ jobs:
             tag-suffix: "-metal-darwin-arm64-acestep-cpp"
             build-type: "metal"
             lang: "go"
-          - backend: "qwen3-tts-cpp"
-            tag-suffix: "-metal-darwin-arm64-qwen3-tts-cpp"
-            build-type: "metal"
-            lang: "go"
           - backend: "voxtral"
             tag-suffix: "-metal-darwin-arm64-voxtral"
             build-type: "metal"

.github/workflows/bump_deps.yaml

@@ -14,10 +14,6 @@ jobs:
             variable: "LLAMA_VERSION"
             branch: "master"
             file: "backend/cpp/llama-cpp/Makefile"
-          - repository: "ikawrakow/ik_llama.cpp"
-            variable: "IK_LLAMA_VERSION"
-            branch: "main"
-            file: "backend/cpp/ik-llama-cpp/Makefile"
           - repository: "ggml-org/whisper.cpp"
             variable: "WHISPER_CPP_VERSION"
             branch: "master"
@@ -42,10 +38,6 @@ jobs:
             variable: "SAM3_VERSION"
             branch: "main"
             file: "backend/go/sam3-cpp/Makefile"
-          - repository: "predict-woo/qwen3-tts.cpp"
-            variable: "QWEN3TTS_CPP_VERSION"
-            branch: "main"
-            file: "backend/go/qwen3-tts-cpp/Makefile"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6

.github/workflows/image-pr.yml

@@ -59,7 +59,7 @@
               platforms: 'linux/amd64'
               tag-latest: 'false'
               tag-suffix: '-hipblas'
-              base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+              base-image: "rocm/dev-ubuntu-24.04:6.4.4"
               grpc-base-image: "ubuntu:24.04"
               runs-on: 'ubuntu-latest'
               makeflags: "--jobs=3 --output-sync=target"

.github/workflows/image.yml

@@ -41,7 +41,7 @@
               platforms: 'linux/amd64'
               tag-latest: 'auto'
               tag-suffix: '-gpu-hipblas'
-              base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+              base-image: "rocm/dev-ubuntu-24.04:6.4.4"
               grpc-base-image: "ubuntu:24.04"
               runs-on: 'ubuntu-latest'
               makeflags: "--jobs=3 --output-sync=target"

.github/workflows/test-extra.yml

@@ -29,11 +29,7 @@ jobs:
       nemo: ${{ steps.detect.outputs.nemo }}
       voxcpm: ${{ steps.detect.outputs.voxcpm }}
       llama-cpp-quantization: ${{ steps.detect.outputs.llama-cpp-quantization }}
-      llama-cpp: ${{ steps.detect.outputs.llama-cpp }}
-      ik-llama-cpp: ${{ steps.detect.outputs.ik-llama-cpp }}
-      vllm: ${{ steps.detect.outputs.vllm }}
       acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
-      qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }}
       voxtral: ${{ steps.detect.outputs.voxtral }}
       kokoros: ${{ steps.detect.outputs.kokoros }}
     steps:
@@ -468,86 +464,6 @@ jobs:
       - name: Test llama-cpp-quantization
         run: |
           make --jobs=5 --output-sync=target -C backend/python/llama-cpp-quantization test
-  tests-llama-cpp-grpc:
-    needs: detect-changes
-    if: needs.detect-changes.outputs.llama-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-    steps:
-      - name: Clone
-        uses: actions/checkout@v6
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.25.4'
-      - name: Build llama-cpp backend image and run gRPC e2e tests
-        run: |
-          make test-extra-backend-llama-cpp
-  tests-ik-llama-cpp-grpc:
-    needs: detect-changes
-    if: needs.detect-changes.outputs.ik-llama-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-    steps:
-      - name: Clone
-        uses: actions/checkout@v6
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.25.4'
-      - name: Build ik-llama-cpp backend image and run gRPC e2e tests
-        run: |
-          make test-extra-backend-ik-llama-cpp
-  # tests-vllm-grpc is currently disabled in CI.
-  #
-  # The prebuilt vllm CPU wheel is compiled with AVX-512 VNNI/BF16
-  # instructions, and neither ubuntu-latest nor the bigger-runner pool
-  # offers a stable CPU baseline that supports them — runners come
-  # back with different hardware between runs and SIGILL on import of
-  # vllm.model_executor.models.registry. Compiling vllm from source
-  # via FROM_SOURCE=true works on any CPU but takes 30-50 minutes per
-  # run, which is too slow for a smoke test.
-  #
-  # The test itself (tests/e2e-backends + make test-extra-backend-vllm)
-  # is fully working and validated locally on a host with the right
-  # SIMD baseline. Run it manually with:
-  #
-  #   make test-extra-backend-vllm
-  #
-  # Re-enable this job once we have a self-hosted runner label with
-  # guaranteed AVX-512 VNNI/BF16 support, or once the vllm project
-  # publishes a CPU wheel with a wider baseline.
-  #
-  # tests-vllm-grpc:
-  #   needs: detect-changes
-  #   if: needs.detect-changes.outputs.vllm == 'true' || needs.detect-changes.outputs.run-all == 'true'
-  #   runs-on: bigger-runner
-  #   timeout-minutes: 90
-  #   steps:
-  #     - name: Clone
-  #       uses: actions/checkout@v6
-  #       with:
-  #         submodules: true
-  #     - name: Dependencies
-  #       run: |
-  #         sudo apt-get update
-  #         sudo apt-get install -y --no-install-recommends \
-  #             make build-essential curl unzip ca-certificates git tar
-  #     - name: Setup Go
-  #       uses: actions/setup-go@v5
-  #       with:
-  #         go-version: '1.25.4'
-  #     - name: Free disk space
-  #       run: |
-  #         sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL || true
-  #         df -h
-  #     - name: Build vllm (cpu) backend image and run gRPC e2e tests
-  #       run: |
-  #         make test-extra-backend-vllm
   tests-acestep-cpp:
     needs: detect-changes
     if: needs.detect-changes.outputs.acestep-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
@@ -580,38 +496,6 @@ jobs:
       - name: Test acestep-cpp
         run: |
           make --jobs=5 --output-sync=target -C backend/go/acestep-cpp test
-  tests-qwen3-tts-cpp:
-    needs: detect-changes
-    if: needs.detect-changes.outputs.qwen3-tts-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Clone
-        uses: actions/checkout@v6
-        with:
-          submodules: true
-      - name: Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential cmake curl libopenblas-dev ffmpeg
-      - name: Setup Go
-        uses: actions/setup-go@v5
-      - name: Display Go version
-        run: go version
-      - name: Proto Dependencies
-        run: |
-          # Install protoc
-          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
-          unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
-          rm protoc.zip
-          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
-          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
-          PATH="$PATH:$HOME/go/bin" make protogen-go
-      - name: Build qwen3-tts-cpp
-        run: |
-          make --jobs=5 --output-sync=target -C backend/go/qwen3-tts-cpp
-      - name: Test qwen3-tts-cpp
-        run: |
-          make --jobs=5 --output-sync=target -C backend/go/qwen3-tts-cpp test
   tests-voxtral:
     needs: detect-changes
     if: needs.detect-changes.outputs.voxtral == 'true' || needs.detect-changes.outputs.run-all == 'true'

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -456,69 +456,6 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/trl test
 	$(MAKE) -C backend/rust/kokoros test
 
-##
-## End-to-end gRPC tests that exercise a built backend container image.
-##
-## The test suite in tests/e2e-backends is backend-agnostic. You drive it via env
-## vars (see tests/e2e-backends/backend_test.go for the full list) and the
-## capability-driven harness picks which gRPC RPCs to exercise:
-##
-##   BACKEND_IMAGE            Required. Docker image to test, e.g. local-ai-backend:llama-cpp.
-##   BACKEND_TEST_MODEL_URL   URL of a model file to download and load.
-##   BACKEND_TEST_MODEL_FILE  Path to an already-downloaded model (skips download).
-##   BACKEND_TEST_MODEL_NAME  HuggingFace repo id (e.g. Qwen/Qwen2.5-0.5B-Instruct).
-##                            Use this instead of MODEL_URL for backends that
-##                            resolve HF model ids natively (vllm, vllm-omni).
-##   BACKEND_TEST_CAPS        Comma-separated capabilities, default "health,load,predict,stream".
-##                            Adds "tools" to exercise ChatDelta tool call extraction.
-##   BACKEND_TEST_PROMPT      Override the prompt used in predict/stream specs.
-##   BACKEND_TEST_OPTIONS     Comma-separated Options[] entries forwarded to LoadModel,
-##                            e.g. "tool_parser:hermes,reasoning_parser:qwen3".
-##
-## Direct usage (image already built, no docker-build-* dependency):
-##
-##   make test-extra-backend BACKEND_IMAGE=local-ai-backend:llama-cpp \
-##       BACKEND_TEST_MODEL_URL=https://.../model.gguf
-##
-## Convenience wrappers below build a specific backend image first, then run the
-## suite against it.
-##
-BACKEND_TEST_MODEL_URL?=https://huggingface.co/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf
-
-## Generic target — runs the suite against whatever BACKEND_IMAGE points at.
-## Depends on protogen-go so pkg/grpc/proto is generated before `go test`.
-test-extra-backend: protogen-go
-	@test -n "$$BACKEND_IMAGE" || { echo "BACKEND_IMAGE must be set" >&2; exit 1; }
-	BACKEND_IMAGE="$$BACKEND_IMAGE" \
-	BACKEND_TEST_MODEL_URL="$${BACKEND_TEST_MODEL_URL:-$(BACKEND_TEST_MODEL_URL)}" \
-	BACKEND_TEST_MODEL_FILE="$$BACKEND_TEST_MODEL_FILE" \
-	BACKEND_TEST_MODEL_NAME="$$BACKEND_TEST_MODEL_NAME" \
-	BACKEND_TEST_CAPS="$$BACKEND_TEST_CAPS" \
-	BACKEND_TEST_PROMPT="$$BACKEND_TEST_PROMPT" \
-	BACKEND_TEST_OPTIONS="$$BACKEND_TEST_OPTIONS" \
-	BACKEND_TEST_TOOL_PROMPT="$$BACKEND_TEST_TOOL_PROMPT" \
-	BACKEND_TEST_TOOL_NAME="$$BACKEND_TEST_TOOL_NAME" \
-	go test -v -timeout 30m ./tests/e2e-backends/...
-
-## Convenience wrappers: build the image, then exercise it.
-test-extra-backend-llama-cpp: docker-build-llama-cpp
-	BACKEND_IMAGE=local-ai-backend:llama-cpp $(MAKE) test-extra-backend
-
-test-extra-backend-ik-llama-cpp: docker-build-ik-llama-cpp
-	BACKEND_IMAGE=local-ai-backend:ik-llama-cpp $(MAKE) test-extra-backend
-
-## vllm is resolved from a HuggingFace model id (no file download) and
-## exercises Predict + streaming + tool-call extraction via the hermes parser.
-## Requires a host CPU with the SIMD instructions the prebuilt vllm CPU
-## wheel was compiled against (AVX-512 VNNI/BF16); older CPUs will SIGILL
-## on import — on CI this means using the bigger-runner label.
-test-extra-backend-vllm: docker-build-vllm
-	BACKEND_IMAGE=local-ai-backend:vllm \
-	BACKEND_TEST_MODEL_NAME=Qwen/Qwen2.5-0.5B-Instruct \
-	BACKEND_TEST_CAPS=health,load,predict,stream,tools \
-	BACKEND_TEST_OPTIONS=tool_parser:hermes \
-	$(MAKE) test-extra-backend
-
 DOCKER_IMAGE?=local-ai
 IMAGE_TYPE?=core
 BASE_IMAGE?=ubuntu:24.04
@@ -612,8 +549,6 @@ backend-images:
 # Backend metadata: BACKEND_NAME | DOCKERFILE_TYPE | BUILD_CONTEXT | PROGRESS_FLAG | NEEDS_BACKEND_ARG
 # llama-cpp is special - uses llama-cpp Dockerfile and doesn't need BACKEND arg
 BACKEND_LLAMA_CPP = llama-cpp|llama-cpp|.|false|false
-# ik-llama-cpp is a fork of llama.cpp with superior CPU performance
-BACKEND_IK_LLAMA_CPP = ik-llama-cpp|ik-llama-cpp|.|false|false
 
 # Golang backends
 BACKEND_PIPER = piper|golang|.|false|true
@@ -624,7 +559,6 @@ BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|tr
 BACKEND_WHISPER = whisper|golang|.|false|true
 BACKEND_VOXTRAL = voxtral|golang|.|false|true
 BACKEND_ACESTEP_CPP = acestep-cpp|golang|.|false|true
-BACKEND_QWEN3_TTS_CPP = qwen3-tts-cpp|golang|.|false|true
 BACKEND_OPUS = opus|golang|.|false|true
 
 # Python backends with root context
@@ -672,7 +606,6 @@ define docker-build-backend
 		--build-arg CUDA_MINOR_VERSION=$(CUDA_MINOR_VERSION) \
 		--build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) \
 		--build-arg UBUNTU_CODENAME=$(UBUNTU_CODENAME) \
-		$(if $(FROM_SOURCE),--build-arg FROM_SOURCE=$(FROM_SOURCE)) \
 		$(if $(filter true,$(5)),--build-arg BACKEND=$(1)) \
 		-t local-ai-backend:$(1) -f backend/Dockerfile.$(2) $(3)
 endef
@@ -685,7 +618,6 @@ endef
 
 # Generate all docker-build targets
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
-$(eval $(call generate-docker-build-target,$(BACKEND_IK_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PIPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE)))
@@ -719,7 +651,6 @@ $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
 $(eval $(call generate-docker-build-target,$(BACKEND_ACE_STEP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_ACESTEP_CPP)))
-$(eval $(call generate-docker-build-target,$(BACKEND_QWEN3_TTS_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_MLX_DISTRIBUTED)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRL)))
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP_QUANTIZATION)))
@@ -730,7 +661,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SAM3_CPP)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp
+docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp
 
 ########################################################
 ### Mock Backend for E2E Tests

README.md

@@ -32,7 +32,7 @@
 **LocalAI** is the open-source AI engine. Run any model - LLMs, vision, voice, image, video - on any hardware. No GPU required.
 
 - **Drop-in API compatibility** — OpenAI, Anthropic, ElevenLabs APIs
-- **36+ backends** — llama.cpp, vLLM, transformers, whisper, diffusers, MLX...
+- **35+ backends** — llama.cpp, vLLM, transformers, whisper, diffusers, MLX...
 - **Any hardware** — NVIDIA, AMD, Intel, Apple Silicon, Vulkan, or CPU-only
 - **Multi-user ready** — API key auth, user quotas, role-based access
 - **Built-in AI agents** — autonomous agents with tool use, RAG, MCP, and skills
@@ -185,7 +185,7 @@ For older news and full release notes, see [GitHub Releases](https://github.com/
 
 ## Supported Backends & Acceleration
 
-LocalAI supports **36+ backends** including llama.cpp, vLLM, transformers, whisper.cpp, diffusers, MLX, MLX-VLM, and many more. Hardware acceleration is available for **NVIDIA** (CUDA 12/13), **AMD** (ROCm), **Intel** (oneAPI/SYCL), **Apple Silicon** (Metal), **Vulkan**, and **NVIDIA Jetson** (L4T). All backends can be installed on-the-fly from the [Backend Gallery](https://localai.io/backends/).
+LocalAI supports **35+ backends** including llama.cpp, vLLM, transformers, whisper.cpp, diffusers, MLX, MLX-VLM, and many more. Hardware acceleration is available for **NVIDIA** (CUDA 12/13), **AMD** (ROCm), **Intel** (oneAPI/SYCL), **Apple Silicon** (Metal), **Vulkan**, and **NVIDIA Jetson** (L4T). All backends can be installed on-the-fly from the [Backend Gallery](https://localai.io/backends/).
 
 See the full [Backend & Model Compatibility Table](https://localai.io/model-compatibility/) and [GPU Acceleration guide](https://localai.io/features/gpu-acceleration/).
 

backend/Dockerfile.ik-llama-cpp

@@ -1,281 +0,0 @@
-ARG BASE_IMAGE=ubuntu:24.04
-ARG GRPC_BASE_IMAGE=${BASE_IMAGE}
-
-
-# The grpc target does one thing, it builds and installs GRPC.  This is in it's own layer so that it can be effectively cached by CI.
-# You probably don't need to change anything here, and if you do, make sure that CI is adjusted so that the cache continues to work.
-FROM ${GRPC_BASE_IMAGE} AS grpc
-
-# This is a bit of a hack, but it's required in order to be able to effectively cache this layer in CI
-ARG GRPC_MAKEFLAGS="-j4 -Otarget"
-ARG GRPC_VERSION=v1.65.0
-ARG CMAKE_FROM_SOURCE=false
-# CUDA Toolkit 13.x compatibility: CMake 3.31.9+ fixes toolchain detection/arch table issues
-ARG CMAKE_VERSION=3.31.10
-
-ENV MAKEFLAGS=${GRPC_MAKEFLAGS}
-
-WORKDIR /build
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        ca-certificates \
-        build-essential curl libssl-dev \
-        git wget && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install CMake (the version in 22.04 is too old)
-RUN <<EOT bash
-    if [ "${CMAKE_FROM_SOURCE}" = "true" ]; then
-        curl -L -s https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -o cmake.tar.gz && tar xvf cmake.tar.gz && cd cmake-${CMAKE_VERSION} && ./configure && make && make install
-    else
-        apt-get update && \
-        apt-get install -y \
-            cmake && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
-    fi
-EOT
-
-# We install GRPC to a different prefix here so that we can copy in only the build artifacts later
-# saves several hundred MB on the final docker image size vs copying in the entire GRPC source tree
-# and running make install in the target container
-RUN git clone --recurse-submodules --jobs 4 -b ${GRPC_VERSION} --depth 1 --shallow-submodules https://github.com/grpc/grpc && \
-    mkdir -p /build/grpc/cmake/build && \
-    cd /build/grpc/cmake/build && \
-    sed -i "216i\  TESTONLY" "../../third_party/abseil-cpp/absl/container/CMakeLists.txt" && \
-    cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX:PATH=/opt/grpc ../.. && \
-    make && \
-    make install && \
-    rm -rf /build
-
-FROM ${BASE_IMAGE} AS builder
-ARG CMAKE_FROM_SOURCE=false
-ARG CMAKE_VERSION=3.31.10
-# We can target specific CUDA ARCHITECTURES like --build-arg CUDA_DOCKER_ARCH='75;86;89;120'
-ARG CUDA_DOCKER_ARCH
-ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
-ARG CMAKE_ARGS
-ENV CMAKE_ARGS=${CMAKE_ARGS}
-ARG BACKEND=rerankers
-ARG BUILD_TYPE
-ENV BUILD_TYPE=${BUILD_TYPE}
-ARG CUDA_MAJOR_VERSION
-ARG CUDA_MINOR_VERSION
-ARG SKIP_DRIVERS=false
-ENV CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION}
-ENV CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION}
-ENV DEBIAN_FRONTEND=noninteractive
-ARG TARGETARCH
-ARG TARGETVARIANT
-ARG GO_VERSION=1.25.4
-ARG UBUNTU_VERSION=2404
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        build-essential \
-        ccache git \
-        ca-certificates \
-        make \
-        pkg-config libcurl4-openssl-dev \
-        curl unzip \
-        libssl-dev wget && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Cuda
-ENV PATH=/usr/local/cuda/bin:${PATH}
-
-# HipBLAS requirements
-ENV PATH=/opt/rocm/bin:${PATH}
-
-
-# Vulkan requirements
-RUN <<EOT bash
-    if [ "${BUILD_TYPE}" = "vulkan" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
-        apt-get update && \
-        apt-get install -y  --no-install-recommends \
-            software-properties-common pciutils wget gpg-agent && \
-        apt-get install -y libglm-dev cmake libxcb-dri3-0 libxcb-present0 libpciaccess0 \
-            libpng-dev libxcb-keysyms1-dev libxcb-dri3-dev libx11-dev g++ gcc \
-            libwayland-dev libxrandr-dev libxcb-randr0-dev libxcb-ewmh-dev \
-            git python-is-python3 bison libx11-xcb-dev liblz4-dev libzstd-dev \
-            ocaml-core ninja-build pkg-config libxml2-dev wayland-protocols python3-jsonschema \
-            clang-format qtbase5-dev qt6-base-dev libxcb-glx0-dev sudo xz-utils
-        if [ "amd64" = "$TARGETARCH" ]; then
-            wget "https://sdk.lunarg.com/sdk/download/1.4.335.0/linux/vulkansdk-linux-x86_64-1.4.335.0.tar.xz" && \
-            tar -xf vulkansdk-linux-x86_64-1.4.335.0.tar.xz && \
-            rm vulkansdk-linux-x86_64-1.4.335.0.tar.xz && \
-            mkdir -p /opt/vulkan-sdk && \
-            mv 1.4.335.0 /opt/vulkan-sdk/ && \
-            cd /opt/vulkan-sdk/1.4.335.0 && \
-            ./vulkansdk --no-deps --maxjobs \
-                vulkan-loader \
-                vulkan-validationlayers \
-                vulkan-extensionlayer \
-                vulkan-tools \
-                shaderc && \
-            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/bin/* /usr/bin/ && \
-            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/lib/* /usr/lib/x86_64-linux-gnu/ && \
-            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/include/* /usr/include/ && \
-            cp -rfv /opt/vulkan-sdk/1.4.335.0/x86_64/share/* /usr/share/ && \
-            rm -rf /opt/vulkan-sdk
-        fi
-        if [ "arm64" = "$TARGETARCH" ]; then
-            mkdir vulkan && cd vulkan && \
-            curl -L -o vulkan-sdk.tar.xz https://github.com/mudler/vulkan-sdk-arm/releases/download/1.4.335.0/vulkansdk-ubuntu-24.04-arm-1.4.335.0.tar.xz && \
-            tar -xvf vulkan-sdk.tar.xz && \
-            rm vulkan-sdk.tar.xz && \
-            cd 1.4.335.0 && \
-            cp -rfv aarch64/bin/* /usr/bin/ && \
-            cp -rfv aarch64/lib/* /usr/lib/aarch64-linux-gnu/ && \
-            cp -rfv aarch64/include/* /usr/include/ && \
-            cp -rfv aarch64/share/* /usr/share/ && \
-            cd ../.. && \
-            rm -rf vulkan
-        fi
-        ldconfig && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
-    fi
-EOT
-
-# CuBLAS requirements
-RUN <<EOT bash
-    if ( [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "l4t" ] ) && [ "${SKIP_DRIVERS}" = "false" ]; then
-        apt-get update && \
-        apt-get install -y  --no-install-recommends \
-            software-properties-common pciutils
-        if [ "amd64" = "$TARGETARCH" ]; then
-            curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/x86_64/cuda-keyring_1.1-1_all.deb
-        fi
-        if [ "arm64" = "$TARGETARCH" ]; then
-            if [ "${CUDA_MAJOR_VERSION}" = "13" ]; then
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/sbsa/cuda-keyring_1.1-1_all.deb
-            else
-                curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UBUNTU_VERSION}/arm64/cuda-keyring_1.1-1_all.deb
-            fi
-        fi
-        dpkg -i cuda-keyring_1.1-1_all.deb && \
-        rm -f cuda-keyring_1.1-1_all.deb && \
-        apt-get update && \
-        apt-get install -y --no-install-recommends \
-            cuda-nvcc-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcufft-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcurand-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcublas-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcusparse-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \
-            libcusolver-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        if [ "${CUDA_MAJOR_VERSION}" = "13" ] && [ "arm64" = "$TARGETARCH" ]; then
-            apt-get install -y --no-install-recommends \
-            libcufile-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libcudnn9-cuda-${CUDA_MAJOR_VERSION} cuda-cupti-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} libnvjitlink-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION}
-        fi
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
-    fi
-EOT
-
-
-# https://github.com/NVIDIA/Isaac-GR00T/issues/343
-RUN <<EOT bash
-    if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "arm64" ]; then
-        wget https://developer.download.nvidia.com/compute/cudss/0.6.0/local_installers/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        dpkg -i cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0_0.6.0-1_arm64.deb && \
-        cp /var/cudss-local-tegra-repo-ubuntu${UBUNTU_VERSION}-0.6.0/cudss-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get -y install cudss cudss-cuda-${CUDA_MAJOR_VERSION} && \
-        wget https://developer.download.nvidia.com/compute/nvpl/25.5/local_installers/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        dpkg -i nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5_1.0-1_arm64.deb && \
-        cp /var/nvpl-local-repo-ubuntu${UBUNTU_VERSION}-25.5/nvpl-*-keyring.gpg /usr/share/keyrings/ && \
-        apt-get update && apt-get install -y nvpl
-    fi
-EOT
-
-# If we are building with clblas support, we need the libraries for the builds
-RUN if [ "${BUILD_TYPE}" = "clblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
-        apt-get update && \
-        apt-get install -y --no-install-recommends \
-            libclblast-dev && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/* \
-    ; fi
-
-RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then \
-        apt-get update && \
-        apt-get install -y --no-install-recommends \
-            hipblas-dev \
-            rocblas-dev && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/* && \
-        # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able
-        # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency
-        ldconfig \
-    ; fi
-
-RUN echo "TARGETARCH: $TARGETARCH"
-
-# We need protoc installed, and the version in 22.04 is too old.  We will create one as part installing the GRPC build below
-# but that will also being in a newer version of absl which stablediffusion cannot compile with.  This version of protoc is only
-# here so that we can generate the grpc code for the stablediffusion build
-RUN <<EOT bash
-    if [ "amd64" = "$TARGETARCH" ]; then
-        curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v27.1/protoc-27.1-linux-x86_64.zip -o protoc.zip && \
-        unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
-        rm protoc.zip
-    fi
-    if [ "arm64" = "$TARGETARCH" ]; then
-        curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v27.1/protoc-27.1-linux-aarch_64.zip -o protoc.zip && \
-        unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
-        rm protoc.zip
-    fi
-EOT
-
-# Install CMake (the version in 22.04 is too old)
-RUN <<EOT bash
-    if [ "${CMAKE_FROM_SOURCE}" = "true" ]; then
-        curl -L -s https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.tar.gz -o cmake.tar.gz && tar xvf cmake.tar.gz && cd cmake-${CMAKE_VERSION} && ./configure && make && make install
-    else
-        apt-get update && \
-        apt-get install -y \
-            cmake && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/*
-    fi
-EOT
-
-COPY --from=grpc /opt/grpc /usr/local
-
-
-COPY . /LocalAI
-
-RUN <<'EOT' bash
-set -euxo pipefail
-
-if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then
-  CUDA_ARCH_ESC="${CUDA_DOCKER_ARCH//;/\\;}"
-  export CMAKE_ARGS="${CMAKE_ARGS:-} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH_ESC}"
-  echo "CMAKE_ARGS(env) = ${CMAKE_ARGS}"
-  rm -rf /LocalAI/backend/cpp/ik-llama-cpp-*-build
-fi
-
-cd /LocalAI/backend/cpp/ik-llama-cpp
-
-if [ "${TARGETARCH}" = "arm64" ] || [ "${BUILD_TYPE}" = "hipblas" ]; then
-  # ARM64 / ROCm: build without x86 SIMD
-  make ik-llama-cpp-fallback
-else
-  # ik_llama.cpp's IQK kernels require at least AVX2
-  make ik-llama-cpp-avx2
-fi
-EOT
-
-
-# Copy libraries using a script to handle architecture differences
-RUN make -BC /LocalAI/backend/cpp/ik-llama-cpp package
-
-
-FROM scratch
-
-
-# Copy all available binaries (the build process only creates the appropriate ones for the target architecture)
-COPY --from=builder /LocalAI/backend/cpp/ik-llama-cpp/package/. ./

backend/Dockerfile.llama-cpp

@@ -209,11 +209,7 @@ RUN if [ "${BUILD_TYPE}" = "hipblas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
         rm -rf /var/lib/apt/lists/* && \
         # I have no idea why, but the ROCM lib packages don't trigger ldconfig after they install, which results in local-ai and others not being able
         # to locate the libraries. We run ldconfig ourselves to work around this packaging deficiency
-        ldconfig && \
-        # Log which GPU architectures have rocBLAS kernel support
-        echo "rocBLAS library data architectures:" && \
-        (ls /opt/rocm*/lib/rocblas/library/Kernels* 2>/dev/null || ls /opt/rocm*/lib64/rocblas/library/Kernels* 2>/dev/null) | grep -oP 'gfx[0-9a-z+-]+' | sort -u || \
-        echo "WARNING: No rocBLAS kernel data found" \
+        ldconfig \
     ; fi
 
 RUN echo "TARGETARCH: $TARGETARCH"

backend/Dockerfile.python

@@ -29,7 +29,6 @@ RUN apt-get update && \
         curl python3-pip \
         python-is-python3 \
         python3-dev llvm \
-        libnuma1 libgomp1 \
         python3-venv make cmake && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
@@ -196,12 +195,6 @@ COPY backend/backend.proto /${BACKEND}/backend.proto
 COPY backend/python/common/ /${BACKEND}/common
 COPY scripts/build/package-gpu-libs.sh /package-gpu-libs.sh
 
-# Optional per-backend source build toggle (e.g. vllm on CPU can set
-# FROM_SOURCE=true to compile against the build host SIMD instead of
-# pulling a prebuilt wheel). Default empty — most backends ignore it.
-ARG FROM_SOURCE=""
-ENV FROM_SOURCE=${FROM_SOURCE}
-
 RUN cd /${BACKEND} && PORTABLE_PYTHON=true make
 
 # Package GPU libraries into the backend's lib directory

backend/cpp/ik-llama-cpp/CMakeLists.txt

@@ -1,78 +0,0 @@
-## Clip/LLaVA library for multimodal support — built locally from copied sources
-set(TARGET myclip)
-add_library(${TARGET} clip.cpp clip.h llava.cpp llava.h)
-install(TARGETS ${TARGET} LIBRARY)
-target_include_directories(myclip PUBLIC .)
-target_include_directories(myclip PUBLIC ../..)
-target_include_directories(myclip PUBLIC ../../common)
-target_link_libraries(${TARGET} PRIVATE common ggml llama ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_11)
-if (NOT MSVC)
-    target_compile_options(${TARGET} PRIVATE -Wno-cast-qual)
-endif()
-
-set(TARGET grpc-server)
-set(CMAKE_CXX_STANDARD 17)
-cmake_minimum_required(VERSION 3.15)
-set(TARGET grpc-server)
-set(_PROTOBUF_LIBPROTOBUF libprotobuf)
-set(_REFLECTION grpc++_reflection)
-
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64")
-        set(HOMEBREW_DEFAULT_PREFIX "/opt/homebrew")
-    else()
-        set(HOMEBREW_DEFAULT_PREFIX "/usr/local")
-    endif()
-    link_directories("${HOMEBREW_DEFAULT_PREFIX}/lib")
-    include_directories("${HOMEBREW_DEFAULT_PREFIX}/include")
-endif()
-
-find_package(absl CONFIG REQUIRED)
-find_package(Protobuf CONFIG REQUIRED)
-find_package(gRPC CONFIG REQUIRED)
-
-find_program(_PROTOBUF_PROTOC protoc)
-set(_GRPC_GRPCPP grpc++)
-find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
-
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
-include_directories(${Protobuf_INCLUDE_DIRS})
-
-message(STATUS "Using protobuf version ${Protobuf_VERSION} | Protobuf_INCLUDE_DIRS: ${Protobuf_INCLUDE_DIRS} | CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}")
-
-# Proto file
-get_filename_component(hw_proto "../../../../../../backend/backend.proto" ABSOLUTE)
-get_filename_component(hw_proto_path "${hw_proto}" PATH)
-
-set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.cc")
-set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.h")
-set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.cc")
-set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.h")
-
-add_custom_command(
-      OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}"
-      COMMAND ${_PROTOBUF_PROTOC}
-      ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
-        --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
-        -I "${hw_proto_path}"
-        --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
-        "${hw_proto}"
-      DEPENDS "${hw_proto}")
-
-add_library(hw_grpc_proto
-  ${hw_grpc_srcs}
-  ${hw_grpc_hdrs}
-  ${hw_proto_srcs}
-  ${hw_proto_hdrs} )
-
-add_executable(${TARGET} grpc-server.cpp json.hpp)
-target_link_libraries(${TARGET} PRIVATE common llama myclip ${CMAKE_THREAD_LIBS_INIT} absl::flags hw_grpc_proto
-  absl::flags_parse
-  gRPC::${_REFLECTION}
-  gRPC::${_GRPC_GRPCPP}
-  protobuf::${_PROTOBUF_LIBPROTOBUF})
-target_compile_features(${TARGET} PRIVATE cxx_std_11)
-if(TARGET BUILD_INFO)
-  add_dependencies(${TARGET} BUILD_INFO)
-endif()

backend/cpp/ik-llama-cpp/Makefile

@@ -1,167 +0,0 @@
-
-IK_LLAMA_VERSION?=08ae48c667e3dcd3025821a8585190b4a46c2f7c
-LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
-
-CMAKE_ARGS?=
-BUILD_TYPE?=
-NATIVE?=false
-ONEAPI_VARS?=/opt/intel/oneapi/setvars.sh
-TARGET?=--target grpc-server
-JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
-ARCH?=$(shell uname -m)
-
-# Disable Shared libs as we are linking on static gRPC and we can't mix shared and static
-CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF -DLLAMA_CURL=OFF
-
-CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-ifeq ($(NATIVE),false)
-	CMAKE_ARGS+=-DGGML_NATIVE=OFF -DLLAMA_OPENSSL=OFF
-endif
-# If build type is cublas, then we set -DGGML_CUDA=ON to CMAKE_ARGS automatically
-ifeq ($(BUILD_TYPE),cublas)
-	CMAKE_ARGS+=-DGGML_CUDA=ON
-# If build type is openblas then we set -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
-# to CMAKE_ARGS automatically
-else ifeq ($(BUILD_TYPE),openblas)
-	CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
-# If build type is clblas (openCL) we set -DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
-else ifeq ($(BUILD_TYPE),clblas)
-	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
-# If it's hipblas we do have also to set CC=/opt/rocm/llvm/bin/clang CXX=/opt/rocm/llvm/bin/clang++
-else ifeq ($(BUILD_TYPE),hipblas)
-	ROCM_HOME ?= /opt/rocm
-	ROCM_PATH ?= /opt/rocm
-	export CXX=$(ROCM_HOME)/llvm/bin/clang++
-	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
-	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
-else ifeq ($(BUILD_TYPE),vulkan)
-	CMAKE_ARGS+=-DGGML_VULKAN=1
-else ifeq ($(OS),Darwin)
-	ifeq ($(BUILD_TYPE),)
-		BUILD_TYPE=metal
-	endif
-	ifneq ($(BUILD_TYPE),metal)
-		CMAKE_ARGS+=-DGGML_METAL=OFF
-	else
-		CMAKE_ARGS+=-DGGML_METAL=ON
-		CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON
-		CMAKE_ARGS+=-DGGML_METAL_USE_BF16=ON
-		CMAKE_ARGS+=-DGGML_OPENMP=OFF
-	endif
-	TARGET+=--target ggml-metal
-endif
-
-ifeq ($(BUILD_TYPE),sycl_f16)
-	CMAKE_ARGS+=-DGGML_SYCL=ON \
-		-DCMAKE_C_COMPILER=icx \
-		-DCMAKE_CXX_COMPILER=icpx \
-		-DCMAKE_CXX_FLAGS="-fsycl" \
-		-DGGML_SYCL_F16=ON
-endif
-
-ifeq ($(BUILD_TYPE),sycl_f32)
-	CMAKE_ARGS+=-DGGML_SYCL=ON \
-		-DCMAKE_C_COMPILER=icx \
-		-DCMAKE_CXX_COMPILER=icpx \
-		-DCMAKE_CXX_FLAGS="-fsycl"
-endif
-
-INSTALLED_PACKAGES=$(CURDIR)/../grpc/installed_packages
-INSTALLED_LIB_CMAKE=$(INSTALLED_PACKAGES)/lib/cmake
-ADDED_CMAKE_ARGS=-Dabsl_DIR=${INSTALLED_LIB_CMAKE}/absl \
-				 -DProtobuf_DIR=${INSTALLED_LIB_CMAKE}/protobuf \
-				 -Dutf8_range_DIR=${INSTALLED_LIB_CMAKE}/utf8_range \
-				 -DgRPC_DIR=${INSTALLED_LIB_CMAKE}/grpc \
-				 -DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES=${INSTALLED_PACKAGES}/include
-build-ik-llama-cpp-grpc-server:
-# Conditionally build grpc for the backend to use if needed
-ifdef BUILD_GRPC_FOR_BACKEND_LLAMA
-	$(MAKE) -C ../../grpc build
-	_PROTOBUF_PROTOC=${INSTALLED_PACKAGES}/bin/proto \
-	_GRPC_CPP_PLUGIN_EXECUTABLE=${INSTALLED_PACKAGES}/bin/grpc_cpp_plugin \
-	PATH="${INSTALLED_PACKAGES}/bin:${PATH}" \
-	CMAKE_ARGS="${CMAKE_ARGS} ${ADDED_CMAKE_ARGS}" \
-	IK_LLAMA_VERSION=$(IK_LLAMA_VERSION) \
-	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../$(VARIANT) grpc-server
-else
-	echo "BUILD_GRPC_FOR_BACKEND_LLAMA is not defined."
-	IK_LLAMA_VERSION=$(IK_LLAMA_VERSION) $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../$(VARIANT) grpc-server
-endif
-
-ik-llama-cpp-avx2: llama.cpp
-	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx2-build
-	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx2-build purge
-	$(info ${GREEN}I ik-llama-cpp build info:avx2${RESET})
-	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) VARIANT="ik-llama-cpp-avx2-build" build-ik-llama-cpp-grpc-server
-	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx2-build/grpc-server ik-llama-cpp-avx2
-
-ik-llama-cpp-avx512: llama.cpp
-	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx512-build
-	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx512-build purge
-	$(info ${GREEN}I ik-llama-cpp build info:avx512${RESET})
-	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on" $(MAKE) VARIANT="ik-llama-cpp-avx512-build" build-ik-llama-cpp-grpc-server
-	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx512-build/grpc-server ik-llama-cpp-avx512
-
-ik-llama-cpp-avx: llama.cpp
-	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx-build
-	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx-build purge
-	$(info ${GREEN}I ik-llama-cpp build info:avx${RESET})
-	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) VARIANT="ik-llama-cpp-avx-build" build-ik-llama-cpp-grpc-server
-	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-avx-build/grpc-server ik-llama-cpp-avx
-
-ik-llama-cpp-fallback: llama.cpp
-	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-fallback-build
-	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-fallback-build purge
-	$(info ${GREEN}I ik-llama-cpp build info:fallback${RESET})
-	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) VARIANT="ik-llama-cpp-fallback-build" build-ik-llama-cpp-grpc-server
-	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-fallback-build/grpc-server ik-llama-cpp-fallback
-
-ik-llama-cpp-grpc: llama.cpp
-	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build
-	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build purge
-	$(info ${GREEN}I ik-llama-cpp build info:grpc${RESET})
-	CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" TARGET="--target grpc-server --target rpc-server" $(MAKE) VARIANT="ik-llama-cpp-grpc-build" build-ik-llama-cpp-grpc-server
-	cp -rfv $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build/grpc-server ik-llama-cpp-grpc
-
-ik-llama-cpp-rpc-server: ik-llama-cpp-grpc
-	cp -rf $(CURRENT_MAKEFILE_DIR)/../ik-llama-cpp-grpc-build/llama.cpp/build/bin/rpc-server ik-llama-cpp-rpc-server
-
-llama.cpp:
-	mkdir -p llama.cpp
-	cd llama.cpp && \
-	git init && \
-	git remote add origin $(LLAMA_REPO)  && \
-	git fetch origin && \
-	git checkout -b build $(IK_LLAMA_VERSION) && \
-	git submodule update --init --recursive --depth 1 --single-branch
-
-llama.cpp/examples/grpc-server: llama.cpp
-	mkdir -p llama.cpp/examples/grpc-server
-	bash prepare.sh
-
-rebuild:
-	bash prepare.sh
-	rm -rf grpc-server
-	$(MAKE) grpc-server
-
-package:
-	bash package.sh
-
-purge:
-	rm -rf llama.cpp/build
-	rm -rf llama.cpp/examples/grpc-server
-	rm -rf grpc-server
-
-clean: purge
-	rm -rf llama.cpp
-
-grpc-server: llama.cpp llama.cpp/examples/grpc-server
-	@echo "Building grpc-server with $(BUILD_TYPE) build type and $(CMAKE_ARGS)"
-ifneq (,$(findstring sycl,$(BUILD_TYPE)))
-	+bash -c "source $(ONEAPI_VARS); \
-	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release -j $(JOBS) $(TARGET)"
-else
-	+cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release -j $(JOBS) $(TARGET)
-endif
-	cp llama.cpp/build/bin/grpc-server .

backend/cpp/ik-llama-cpp/package.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-
-# Script to copy the appropriate libraries based on architecture
-# This script is used in the final stage of the Dockerfile
-
-set -e
-
-CURDIR=$(dirname "$(realpath $0)")
-REPO_ROOT="${CURDIR}/../../.."
-
-# Create lib directory
-mkdir -p $CURDIR/package/lib
-
-cp -avrf $CURDIR/ik-llama-cpp-* $CURDIR/package/
-cp -rfv $CURDIR/run.sh $CURDIR/package/
-
-# Detect architecture and copy appropriate libraries
-if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
-    # x86_64 architecture
-    echo "Detected x86_64 architecture, copying x86_64 libraries..."
-    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
-    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
-    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
-elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
-    # ARM64 architecture
-    echo "Detected ARM64 architecture, copying ARM64 libraries..."
-    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
-    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
-    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
-else
-    echo "Error: Could not detect architecture"
-    exit 1
-fi
-
-# Package GPU libraries based on BUILD_TYPE
-# The GPU library packaging script will detect BUILD_TYPE and copy appropriate GPU libraries
-GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
-if [ -f "$GPU_LIB_SCRIPT" ]; then
-    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
-    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
-    package_gpu_libs
-fi
-
-echo "Packaging completed successfully"
-ls -liah $CURDIR/package/
-ls -liah $CURDIR/package/lib/

backend/cpp/ik-llama-cpp/patches/0001-fix-missing-cstdint-include.patch

@@ -1,10 +0,0 @@
---- a/ggml/src/iqk/iqk_common.h
-+++ b/ggml/src/iqk/iqk_common.h
-@@ -9,6 +9,7 @@
- #pragma once
-
- #include "iqk_config.h"
-+#include <cstdint>
-
- #if defined IQK_IMPLEMENT
-

backend/cpp/ik-llama-cpp/prepare.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-## Patches
-
-## Apply patches from the `patches` directory
-if [ -d "patches" ]; then
-    for patch in $(ls patches); do
-        echo "Applying patch $patch"
-        patch -d llama.cpp/ -p1 < patches/$patch
-    done
-fi
-
-set -e
-
-cp -r CMakeLists.txt llama.cpp/examples/grpc-server/
-cp -r grpc-server.cpp llama.cpp/examples/grpc-server/
-cp -r utils.hpp llama.cpp/examples/grpc-server/
-cp -rfv llama.cpp/vendor/nlohmann/json.hpp llama.cpp/examples/grpc-server/
-
-## Copy clip/llava files for multimodal support (built as myclip library)
-cp -rfv llama.cpp/examples/llava/clip.h llama.cpp/examples/grpc-server/clip.h
-cp -rfv llama.cpp/examples/llava/clip.cpp llama.cpp/examples/grpc-server/clip.cpp
-cp -rfv llama.cpp/examples/llava/llava.cpp llama.cpp/examples/grpc-server/llava.cpp
-# Prepend llama.h include to llava.h
-echo '#include "llama.h"' > llama.cpp/examples/grpc-server/llava.h
-cat llama.cpp/examples/llava/llava.h >> llama.cpp/examples/grpc-server/llava.h
-# Copy clip-impl.h if it exists
-if [ -f llama.cpp/examples/llava/clip-impl.h ]; then
-    cp -rfv llama.cpp/examples/llava/clip-impl.h llama.cpp/examples/grpc-server/clip-impl.h
-fi
-# Copy stb_image.h
-if [ -f llama.cpp/vendor/stb/stb_image.h ]; then
-    cp -rfv llama.cpp/vendor/stb/stb_image.h llama.cpp/examples/grpc-server/stb_image.h
-elif [ -f llama.cpp/common/stb_image.h ]; then
-    cp -rfv llama.cpp/common/stb_image.h llama.cpp/examples/grpc-server/stb_image.h
-fi
-
-## Fix API compatibility in llava.cpp (llama_n_embd -> llama_model_n_embd)
-if [ -f llama.cpp/examples/grpc-server/llava.cpp ]; then
-    sed -i 's/llama_n_embd(/llama_model_n_embd(/g' llama.cpp/examples/grpc-server/llava.cpp
-fi
-
-set +e
-if grep -q "grpc-server" llama.cpp/examples/CMakeLists.txt; then
-    echo "grpc-server already added"
-else
-    echo "add_subdirectory(grpc-server)" >> llama.cpp/examples/CMakeLists.txt
-fi
-set -e

backend/cpp/ik-llama-cpp/run.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-set -ex
-
-# Get the absolute current dir where the script is located
-CURDIR=$(dirname "$(realpath $0)")
-
-cd /
-
-echo "CPU info:"
-grep -e "model\sname" /proc/cpuinfo | head -1
-grep -e "flags" /proc/cpuinfo | head -1
-
-# ik_llama.cpp requires AVX2 — default to avx2 binary
-BINARY=ik-llama-cpp-avx2
-
-if [ -e $CURDIR/ik-llama-cpp-fallback ] && ! grep -q -e "\savx2\s" /proc/cpuinfo ; then
-	echo "CPU:    AVX2   NOT found, using fallback"
-	BINARY=ik-llama-cpp-fallback
-fi
-
-# Extend ld library path with the dir where this script is located/lib
-if [ "$(uname)" == "Darwin" ]; then
-	export DYLD_LIBRARY_PATH=$CURDIR/lib:$DYLD_LIBRARY_PATH
-	#export DYLD_FALLBACK_LIBRARY_PATH=$CURDIR/lib:$DYLD_FALLBACK_LIBRARY_PATH
-else
-	export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
-fi
-
-# If there is a lib/ld.so, use it
-if [ -f $CURDIR/lib/ld.so ]; then
-	echo "Using lib/ld.so"
-	echo "Using binary: $BINARY"
-	exec $CURDIR/lib/ld.so $CURDIR/$BINARY "$@"
-fi
-
-echo "Using binary: $BINARY"
-exec $CURDIR/$BINARY "$@"
-
-# We should never reach this point, however just in case we do, run fallback
-exec $CURDIR/ik-llama-cpp-fallback "$@"

backend/cpp/ik-llama-cpp/utils.hpp

@@ -1,483 +0,0 @@
-// https://github.com/ggerganov/llama.cpp/blob/master/examples/server/utils.hpp
-
-#pragma once
-
-#include <string>
-#include <vector>
-#include <set>
-#include <mutex>
-#include <condition_variable>
-#include <unordered_map>
-
-#include "json.hpp"
-
-#include "clip.h"
-
-using json = nlohmann::json;
-
-extern bool server_verbose;
-
-#ifndef SERVER_VERBOSE
-#define SERVER_VERBOSE 1
-#endif
-
-#if SERVER_VERBOSE != 1
-#define LOG_VERBOSE(MSG, ...)
-#else
-#define LOG_VERBOSE(MSG, ...)                                            \
-    do                                                                   \
-    {                                                                    \
-        if (server_verbose)                                              \
-        {                                                                \
-            server_log("VERBOSE", __func__, __LINE__, MSG, __VA_ARGS__); \
-        }                                                                \
-    } while (0)
-#endif
-
-#define LOG_ERROR(  MSG, ...) server_log("ERROR",   __func__, __LINE__, MSG, __VA_ARGS__)
-#define LOG_WARNING(MSG, ...) server_log("WARNING", __func__, __LINE__, MSG, __VA_ARGS__)
-#define LOG_INFO(   MSG, ...) server_log("INFO",    __func__, __LINE__, MSG, __VA_ARGS__)
-
-//
-// parallel
-//
-
-enum server_state {
-    SERVER_STATE_LOADING_MODEL,  // Server is starting up, model not fully loaded yet
-    SERVER_STATE_READY,          // Server is ready and model is loaded
-    SERVER_STATE_ERROR           // An error occurred, load_model failed
-};
-
-enum task_type {
-    TASK_TYPE_COMPLETION,
-    TASK_TYPE_CANCEL,
-    TASK_TYPE_NEXT_RESPONSE
-};
-
-struct task_server {
-    int id = -1; // to be filled by llama_server_queue
-    int target_id;
-    task_type type;
-    json data;
-    bool infill_mode = false;
-    bool embedding_mode = false;
-    int multitask_id = -1;
-};
-
-struct task_result {
-    int id;
-    int multitask_id = -1;
-    bool stop;
-    bool error;
-    json result_json;
-};
-
-struct task_multi {
-    int id;
-    std::set<int> subtasks_remaining{};
-    std::vector<task_result> results{};
-};
-
-// TODO: can become bool if we can't find use of more states
-enum slot_state
-{
-    IDLE,
-    PROCESSING,
-};
-
-enum slot_command
-{
-    NONE,
-    LOAD_PROMPT,
-    RELEASE,
-};
-
-struct slot_params
-{
-    bool stream       = true;
-    bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt
-
-    uint32_t seed      = -1; // RNG seed
-    int32_t  n_keep    =  0; // number of tokens to keep from initial prompt
-    int32_t  n_predict = -1; // new tokens to predict
-
-    std::vector<std::string> antiprompt;
-
-    json input_prefix;
-    json input_suffix;
-};
-
-struct slot_image
-{
-    int32_t id;
-
-    bool request_encode_image = false;
-    float * image_embedding = nullptr;
-    int32_t image_tokens = 0;
-
-    clip_image_u8 * img_data;
-
-    std::string prefix_prompt; // before of this image
-};
-
-// completion token output with probabilities
-struct completion_token_output
-{
-    struct token_prob
-    {
-        llama_token tok;
-        float prob;
-    };
-
-    std::vector<token_prob> probs;
-    llama_token tok;
-    std::string text_to_send;
-};
-
-static inline void server_log(const char *level, const char *function, int line,
-                       const char *message, const nlohmann::ordered_json &extra)
-{
-    nlohmann::ordered_json log
-    {
-        {"timestamp", time(nullptr)},
-        {"level",     level},
-        {"function",  function},
-        {"line",      line},
-        {"message",   message},
-    };
-
-    if (!extra.empty())
-    {
-        log.merge_patch(extra);
-    }
-
-    const std::string str = log.dump(-1, ' ', false, json::error_handler_t::replace);
-    printf("%.*s\n", (int)str.size(), str.data());
-    fflush(stdout);
-}
-
-//
-// server utils
-//
-
-template <typename T>
-static T json_value(const json &body, const std::string &key, const T &default_value)
-{
-    // Fallback null to default value
-    return body.contains(key) && !body.at(key).is_null()
-        ? body.value(key, default_value)
-        : default_value;
-}
-
-inline std::string format_chatml(std::vector<json> messages)
-{
-    std::ostringstream chatml_msgs;
-
-    for (auto it = messages.begin(); it != messages.end(); ++it) {
-        chatml_msgs << "<|im_start|>"
-                    << json_value(*it, "role",    std::string("user")) << '\n';
-        chatml_msgs << json_value(*it, "content", std::string(""))
-                    << "<|im_end|>\n";
-    }
-
-    chatml_msgs << "<|im_start|>assistant" << '\n';
-
-    return chatml_msgs.str();
-}
-
-//
-// work queue utils
-//
-
-struct llama_server_queue {
-    int id = 0;
-    std::mutex mutex_tasks;
-    // queues
-    std::vector<task_server> queue_tasks;
-    std::vector<task_server> queue_tasks_deferred;
-    std::vector<task_multi> queue_multitasks;
-    std::condition_variable condition_tasks;
-    // callback functions
-    std::function<void(task_server&)> callback_new_task;
-    std::function<void(task_multi&)> callback_finish_multitask;
-    std::function<void(void)> callback_all_task_finished;
-
-    // Add a new task to the end of the queue
-    int post(task_server task) {
-        std::unique_lock<std::mutex> lock(mutex_tasks);
-        if (task.id == -1) {
-            task.id = id++;
-        }
-        queue_tasks.push_back(std::move(task));
-        condition_tasks.notify_one();
-        return task.id;
-    }
-
-    // Add a new task, but defer until one slot is available
-    void defer(task_server task) {
-        std::unique_lock<std::mutex> lock(mutex_tasks);
-        queue_tasks_deferred.push_back(std::move(task));
-    }
-
-    // Get the next id for creating anew task
-    int get_new_id() {
-        std::unique_lock<std::mutex> lock(mutex_tasks);
-        return id++;
-    }
-
-    // Register function to process a new task
-    void on_new_task(std::function<void(task_server&)> callback) {
-        callback_new_task = callback;
-    }
-
-    // Register function to process a multitask
-    void on_finish_multitask(std::function<void(task_multi&)> callback) {
-        callback_finish_multitask = callback;
-    }
-
-    // Register the function to be called when the batch of tasks is finished
-    void on_all_tasks_finished(std::function<void(void)> callback) {
-        callback_all_task_finished = callback;
-    }
-
-    // Call when the state of one slot is changed
-    void notify_slot_changed() {
-        // move deferred tasks back to main loop
-        std::unique_lock<std::mutex> lock(mutex_tasks);
-        for (auto & task : queue_tasks_deferred) {
-            queue_tasks.push_back(std::move(task));
-        }
-        queue_tasks_deferred.clear();
-    }
-
-    // Start the main loop. This call is blocking
-    [[noreturn]]
-    void start_loop() {
-        while (true) {
-            // new task arrived
-            LOG_VERBOSE("have new task", {});
-            {
-                while (true)
-                {
-                    std::unique_lock<std::mutex> lock(mutex_tasks);
-                    if (queue_tasks.empty()) {
-                        lock.unlock();
-                        break;
-                    }
-                    task_server task = queue_tasks.front();
-                    queue_tasks.erase(queue_tasks.begin());
-                    lock.unlock();
-                    LOG_VERBOSE("callback_new_task", {});
-                    callback_new_task(task);
-                }
-                LOG_VERBOSE("callback_all_task_finished", {});
-                // process and update all the multitasks
-                auto queue_iterator = queue_multitasks.begin();
-                while (queue_iterator != queue_multitasks.end())
-                {
-                    if (queue_iterator->subtasks_remaining.empty())
-                    {
-                        // all subtasks done == multitask is done
-                        task_multi current_multitask = *queue_iterator;
-                        callback_finish_multitask(current_multitask);
-                        // remove this multitask
-                        queue_iterator = queue_multitasks.erase(queue_iterator);
-                    }
-                    else
-                    {
-                        ++queue_iterator;
-                    }
-                }
-                // all tasks in the current loop is finished
-                callback_all_task_finished();
-            }
-            LOG_VERBOSE("wait for new task", {});
-            // wait for new task
-            {
-                std::unique_lock<std::mutex> lock(mutex_tasks);
-                if (queue_tasks.empty()) {
-                    condition_tasks.wait(lock, [&]{
-                        return !queue_tasks.empty();
-                    });
-                }
-            }
-        }
-    }
-
-    //
-    // functions to manage multitasks
-    //
-
-    // add a multitask by specifying the id of all subtask (subtask is a task_server)
-    void add_multitask(int multitask_id, std::vector<int>& sub_ids)
-    {
-        std::lock_guard<std::mutex> lock(mutex_tasks);
-        task_multi multi;
-        multi.id = multitask_id;
-        std::copy(sub_ids.begin(), sub_ids.end(), std::inserter(multi.subtasks_remaining, multi.subtasks_remaining.end()));
-        queue_multitasks.push_back(multi);
-    }
-
-    // updatethe remaining subtasks, while appending results to multitask
-    void update_multitask(int multitask_id, int subtask_id, task_result& result)
-    {
-        std::lock_guard<std::mutex> lock(mutex_tasks);
-        for (auto& multitask : queue_multitasks)
-        {
-            if (multitask.id == multitask_id)
-            {
-                multitask.subtasks_remaining.erase(subtask_id);
-                multitask.results.push_back(result);
-            }
-        }
-    }
-};
-
-struct llama_server_response {
-    typedef std::function<void(int, int, task_result&)> callback_multitask_t;
-    callback_multitask_t callback_update_multitask;
-    // for keeping track of all tasks waiting for the result
-    std::set<int> waiting_task_ids;
-    // the main result queue
-    std::vector<task_result> queue_results;
-    std::mutex mutex_results;
-    std::condition_variable condition_results;
-
-    void add_waiting_task_id(int task_id) {
-        std::unique_lock<std::mutex> lock(mutex_results);
-        waiting_task_ids.insert(task_id);
-    }
-
-    void remove_waiting_task_id(int task_id) {
-        std::unique_lock<std::mutex> lock(mutex_results);
-        waiting_task_ids.erase(task_id);
-    }
-
-    // This function blocks the thread until there is a response for this task_id
-    task_result recv(int task_id) {
-        while (true)
-        {
-            std::unique_lock<std::mutex> lock(mutex_results);
-            condition_results.wait(lock, [&]{
-                return !queue_results.empty();
-            });
-            LOG_VERBOSE("condition_results unblock", {});
-
-            for (int i = 0; i < (int) queue_results.size(); i++)
-            {
-                if (queue_results[i].id == task_id)
-                {
-                    assert(queue_results[i].multitask_id == -1);
-                    task_result res = queue_results[i];
-                    queue_results.erase(queue_results.begin() + i);
-                    return res;
-                }
-            }
-        }
-
-        // should never reach here
-    }
-
-    // Register the function to update multitask
-    void on_multitask_update(callback_multitask_t callback) {
-        callback_update_multitask = callback;
-    }
-
-    // Send a new result to a waiting task_id
-    void send(task_result result) {
-        std::unique_lock<std::mutex> lock(mutex_results);
-        LOG_VERBOSE("send new result", {});
-        for (auto& task_id : waiting_task_ids) {
-            // LOG_TEE("waiting task id %i \n", task_id);
-            // for now, tasks that have associated parent multitasks just get erased once multitask picks up the result
-            if (result.multitask_id == task_id)
-            {
-                LOG_VERBOSE("callback_update_multitask", {});
-                callback_update_multitask(task_id, result.id, result);
-                continue;
-            }
-
-            if (result.id == task_id)
-            {
-                LOG_VERBOSE("queue_results.push_back", {});
-                queue_results.push_back(result);
-                condition_results.notify_one();
-                return;
-            }
-        }
-    }
-};
-
-//
-// base64 utils (TODO: move to common in the future)
-//
-
-static const std::string base64_chars =
-             "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-             "abcdefghijklmnopqrstuvwxyz"
-             "0123456789+/";
-
-static inline bool is_base64(uint8_t c)
-{
-    return (isalnum(c) || (c == '+') || (c == '/'));
-}
-
-static inline std::vector<uint8_t> base64_decode(const std::string & encoded_string)
-{
-    int i = 0;
-    int j = 0;
-    int in_ = 0;
-
-    int in_len = encoded_string.size();
-
-    uint8_t char_array_4[4];
-    uint8_t char_array_3[3];
-
-    std::vector<uint8_t> ret;
-
-    while (in_len-- && (encoded_string[in_] != '=') && is_base64(encoded_string[in_]))
-    {
-        char_array_4[i++] = encoded_string[in_]; in_++;
-        if (i == 4)
-        {
-            for (i = 0; i <4; i++)
-            {
-                char_array_4[i] = base64_chars.find(char_array_4[i]);
-            }
-
-            char_array_3[0] = ((char_array_4[0]      ) << 2) + ((char_array_4[1] & 0x30) >> 4);
-            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
-            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) +   char_array_4[3];
-
-            for (i = 0; (i < 3); i++)
-            {
-                ret.push_back(char_array_3[i]);
-            }
-            i = 0;
-        }
-    }
-
-    if (i)
-    {
-        for (j = i; j <4; j++)
-        {
-            char_array_4[j] = 0;
-        }
-
-        for (j = 0; j <4; j++)
-        {
-            char_array_4[j] = base64_chars.find(char_array_4[j]);
-        }
-
-        char_array_3[0] = ((char_array_4[0]      ) << 2) + ((char_array_4[1] & 0x30) >> 4);
-        char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);
-        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) +   char_array_4[3];
-
-        for (j = 0; (j < i - 1); j++)
-        {
-            ret.push_back(char_array_3[j]);
-        }
-    }
-
-    return ret;
-}

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=ff5ef8278615a2462b79b50abdf3cc95cfb31c6f
+LLAMA_VERSION?=e62fa13c2497b2cd1958cb496e9489e86bbd5182
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=
@@ -33,7 +33,7 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	ROCM_PATH ?= /opt/rocm
 	export CXX=$(ROCM_HOME)/llvm/bin/clang++
 	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
 	CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=1

backend/cpp/llama-cpp/run.sh

@@ -46,10 +46,6 @@ if [ "$(uname)" == "Darwin" ]; then
 	#export DYLD_FALLBACK_LIBRARY_PATH=$CURDIR/lib:$DYLD_FALLBACK_LIBRARY_PATH
 else
 	export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
-	# Tell rocBLAS where to find TensileLibrary data (GPU kernel tuning files)
-	if [ -d "$CURDIR/lib/rocblas/library" ]; then
-		export ROCBLAS_TENSILE_LIBPATH=$CURDIR/lib/rocblas/library
-	fi
 fi
 
 # If there is a lib/ld.so, use it

backend/go/qwen3-tts-cpp/CMakeLists.txt

@@ -1,56 +0,0 @@
-cmake_minimum_required(VERSION 3.14)
-project(goqwen3ttscpp LANGUAGES C CXX)
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
-set(QWEN3TTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/sources/qwen3-tts.cpp)
-
-# Override upstream's CMAKE_CUDA_ARCHITECTURES before add_subdirectory.
-if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
-    set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real")
-endif()
-
-# Build ggml from the upstream's submodule FIRST, so that ggml/ggml-base/ggml-cpu
-# CMake targets exist when the upstream project references them by name.
-# The upstream CMakeLists.txt uses target_link_libraries(... ggml ggml-base ggml-cpu)
-# with target_link_directories pointing at a pre-built ggml/build/. By adding ggml
-# as a subdirectory here, CMake resolves those names as targets instead.
-add_subdirectory(${QWEN3TTS_DIR}/ggml ggml EXCLUDE_FROM_ALL)
-
-# Now add the upstream project
-add_subdirectory(${QWEN3TTS_DIR} qwen3tts EXCLUDE_FROM_ALL)
-
-add_library(goqwen3ttscpp MODULE cpp/goqwen3ttscpp.cpp)
-target_link_libraries(goqwen3ttscpp PRIVATE qwen3_tts)
-
-target_include_directories(goqwen3ttscpp PRIVATE ${QWEN3TTS_DIR}/src)
-target_include_directories(goqwen3ttscpp SYSTEM PRIVATE ${QWEN3TTS_DIR}/ggml/include)
-
-# Link GPU backends if available
-foreach(backend blas cuda metal vulkan)
-    if(TARGET ggml-${backend})
-        target_link_libraries(goqwen3ttscpp PRIVATE ggml-${backend})
-        string(TOUPPER ${backend} BACKEND_UPPER)
-        target_compile_definitions(goqwen3ttscpp PRIVATE QWEN3TTS_HAVE_${BACKEND_UPPER})
-        if(backend STREQUAL "cuda")
-            find_package(CUDAToolkit QUIET)
-            if(CUDAToolkit_FOUND)
-                target_link_libraries(goqwen3ttscpp PRIVATE CUDA::cudart)
-            endif()
-        endif()
-    endif()
-endforeach()
-
-if(MSVC)
-    target_compile_options(goqwen3ttscpp PRIVATE /W4 /wd4100 /wd4505)
-else()
-    target_compile_options(goqwen3ttscpp PRIVATE -Wall -Wextra -Wshadow -Wconversion
-                          -Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
-endif()
-
-if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
-    target_link_libraries(goqwen3ttscpp PRIVATE stdc++fs)
-endif()
-
-set_property(TARGET goqwen3ttscpp PROPERTY CXX_STANDARD 17)
-set_target_properties(goqwen3ttscpp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

backend/go/qwen3-tts-cpp/Makefile

@@ -1,126 +0,0 @@
-CMAKE_ARGS?=
-BUILD_TYPE?=
-NATIVE?=false
-
-GOCMD?=go
-GO_TAGS?=
-JOBS?=$(shell nproc --ignore=1)
-
-# qwen3-tts.cpp version
-QWEN3TTS_REPO?=https://github.com/predict-woo/qwen3-tts.cpp
-QWEN3TTS_CPP_VERSION?=7a762e2ad4bacc6fdda81d81bf10a09ffb546f29
-SO_TARGET?=libgoqwen3ttscpp.so
-
-CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
-
-ifeq ($(NATIVE),false)
-	CMAKE_ARGS+=-DGGML_NATIVE=OFF
-endif
-
-ifeq ($(BUILD_TYPE),cublas)
-	CMAKE_ARGS+=-DGGML_CUDA=ON
-else ifeq ($(BUILD_TYPE),openblas)
-	CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
-else ifeq ($(BUILD_TYPE),clblas)
-	CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path
-else ifeq ($(BUILD_TYPE),hipblas)
-	CMAKE_ARGS+=-DGGML_HIPBLAS=ON
-else ifeq ($(BUILD_TYPE),vulkan)
-	CMAKE_ARGS+=-DGGML_VULKAN=ON
-else ifeq ($(OS),Darwin)
-	ifneq ($(BUILD_TYPE),metal)
-		CMAKE_ARGS+=-DGGML_METAL=OFF
-	else
-		CMAKE_ARGS+=-DGGML_METAL=ON
-		CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON
-	endif
-endif
-
-ifeq ($(BUILD_TYPE),sycl_f16)
-	CMAKE_ARGS+=-DGGML_SYCL=ON \
-		-DCMAKE_C_COMPILER=icx \
-		-DCMAKE_CXX_COMPILER=icpx \
-		-DGGML_SYCL_F16=ON
-endif
-
-ifeq ($(BUILD_TYPE),sycl_f32)
-	CMAKE_ARGS+=-DGGML_SYCL=ON \
-		-DCMAKE_C_COMPILER=icx \
-		-DCMAKE_CXX_COMPILER=icpx
-endif
-
-sources/qwen3-tts.cpp:
-	mkdir -p sources/qwen3-tts.cpp
-	cd sources/qwen3-tts.cpp && \
-	git init && \
-	git remote add origin $(QWEN3TTS_REPO) && \
-	git fetch origin && \
-	git checkout $(QWEN3TTS_CPP_VERSION) && \
-	git submodule update --init --recursive --depth 1 --single-branch
-
-# Detect OS
-UNAME_S := $(shell uname -s)
-
-# Only build CPU variants on Linux
-ifeq ($(UNAME_S),Linux)
-	VARIANT_TARGETS = libgoqwen3ttscpp-avx.so libgoqwen3ttscpp-avx2.so libgoqwen3ttscpp-avx512.so libgoqwen3ttscpp-fallback.so
-else
-	# On non-Linux (e.g., Darwin), build only fallback variant
-	VARIANT_TARGETS = libgoqwen3ttscpp-fallback.so
-endif
-
-qwen3-tts-cpp: main.go goqwen3ttscpp.go $(VARIANT_TARGETS)
-	CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o qwen3-tts-cpp ./
-
-package: qwen3-tts-cpp
-	bash package.sh
-
-build: package
-
-clean: purge
-	rm -rf libgoqwen3ttscpp*.so package sources/qwen3-tts.cpp qwen3-tts-cpp
-
-purge:
-	rm -rf build*
-
-# Variants must build sequentially
-.NOTPARALLEL:
-
-# Build all variants (Linux only)
-ifeq ($(UNAME_S),Linux)
-libgoqwen3ttscpp-avx.so: sources/qwen3-tts.cpp
-	$(info ${GREEN}I qwen3-tts-cpp build info:avx${RESET})
-	SO_TARGET=libgoqwen3ttscpp-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgoqwen3ttscpp-custom
-	rm -rf build-libgoqwen3ttscpp-avx.so
-
-libgoqwen3ttscpp-avx2.so: sources/qwen3-tts.cpp
-	$(info ${GREEN}I qwen3-tts-cpp build info:avx2${RESET})
-	SO_TARGET=libgoqwen3ttscpp-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgoqwen3ttscpp-custom
-	rm -rf build-libgoqwen3ttscpp-avx2.so
-
-libgoqwen3ttscpp-avx512.so: sources/qwen3-tts.cpp
-	$(info ${GREEN}I qwen3-tts-cpp build info:avx512${RESET})
-	SO_TARGET=libgoqwen3ttscpp-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgoqwen3ttscpp-custom
-	rm -rf build-libgoqwen3ttscpp-avx512.so
-endif
-
-# Build fallback variant (all platforms)
-libgoqwen3ttscpp-fallback.so: sources/qwen3-tts.cpp
-	$(info ${GREEN}I qwen3-tts-cpp build info:fallback${RESET})
-	SO_TARGET=libgoqwen3ttscpp-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgoqwen3ttscpp-custom
-	rm -rf build-libgoqwen3ttscpp-fallback.so
-
-libgoqwen3ttscpp-custom: CMakeLists.txt cpp/goqwen3ttscpp.cpp cpp/goqwen3ttscpp.h
-	mkdir -p build-$(SO_TARGET) && \
-	cd build-$(SO_TARGET) && \
-	cmake .. $(CMAKE_ARGS) && \
-	cmake --build . --config Release -j$(JOBS) --target goqwen3ttscpp && \
-	cd .. && \
-	mv build-$(SO_TARGET)/libgoqwen3ttscpp.so ./$(SO_TARGET)
-
-test: qwen3-tts-cpp
-	@echo "Running qwen3-tts-cpp tests..."
-	bash test.sh
-	@echo "qwen3-tts-cpp tests completed."
-
-all: qwen3-tts-cpp package

backend/go/qwen3-tts-cpp/cpp/goqwen3ttscpp.cpp

@@ -1,161 +0,0 @@
-#include "goqwen3ttscpp.h"
-#include "ggml-backend.h"
-#include "qwen3_tts.h"
-
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <string>
-
-using namespace qwen3_tts;
-
-// Global engine (loaded once, reused across requests)
-static Qwen3TTS *g_engine = nullptr;
-static bool g_loaded = false;
-static int g_threads = 4;
-
-static void ggml_log_cb(enum ggml_log_level level, const char *log, void *data) {
-    const char *level_str;
-    if (!log)
-        return;
-    switch (level) {
-    case GGML_LOG_LEVEL_DEBUG:
-        level_str = "DEBUG";
-        break;
-    case GGML_LOG_LEVEL_INFO:
-        level_str = "INFO";
-        break;
-    case GGML_LOG_LEVEL_WARN:
-        level_str = "WARN";
-        break;
-    case GGML_LOG_LEVEL_ERROR:
-        level_str = "ERROR";
-        break;
-    default:
-        level_str = "?????";
-        break;
-    }
-    fprintf(stderr, "[%-5s] ", level_str);
-    fputs(log, stderr);
-    fflush(stderr);
-}
-
-// Map language string to language_id token used by the model
-static int language_to_id(const char *lang) {
-    if (!lang || lang[0] == '\0')
-        return 2050; // default: English
-    std::string l(lang);
-    if (l == "en")
-        return 2050;
-    if (l == "ru")
-        return 2069;
-    if (l == "zh")
-        return 2055;
-    if (l == "ja")
-        return 2058;
-    if (l == "ko")
-        return 2064;
-    if (l == "de")
-        return 2053;
-    if (l == "fr")
-        return 2061;
-    if (l == "es")
-        return 2054;
-    if (l == "it")
-        return 2056;
-    if (l == "pt")
-        return 2057;
-    fprintf(stderr, "[qwen3-tts-cpp] Unknown language '%s', defaulting to English\n",
-            lang);
-    return 2050;
-}
-
-int load_model(const char *model_dir, int n_threads) {
-    ggml_log_set(ggml_log_cb, nullptr);
-    ggml_backend_load_all();
-
-    if (n_threads <= 0)
-        n_threads = 4;
-    g_threads = n_threads;
-
-    fprintf(stderr, "[qwen3-tts-cpp] Loading models from %s (threads=%d)\n",
-            model_dir, n_threads);
-
-    g_engine = new Qwen3TTS();
-    if (!g_engine->load_models(model_dir)) {
-        fprintf(stderr, "[qwen3-tts-cpp] FATAL: failed to load models from %s\n",
-                model_dir);
-        delete g_engine;
-        g_engine = nullptr;
-        return 1;
-    }
-
-    g_loaded = true;
-    fprintf(stderr, "[qwen3-tts-cpp] Models loaded successfully\n");
-    return 0;
-}
-
-int synthesize(const char *text, const char *ref_audio_path, const char *dst,
-               const char *language, float temperature, float top_p,
-               int top_k, float repetition_penalty, int max_audio_tokens,
-               int n_threads) {
-    if (!g_loaded || !g_engine) {
-        fprintf(stderr, "[qwen3-tts-cpp] ERROR: models not loaded\n");
-        return 1;
-    }
-
-    if (!text || !dst) {
-        fprintf(stderr, "[qwen3-tts-cpp] ERROR: text and dst are required\n");
-        return 2;
-    }
-
-    tts_params params;
-    params.max_audio_tokens = max_audio_tokens > 0 ? max_audio_tokens : 4096;
-    params.temperature = temperature;
-    params.top_p = top_p;
-    params.top_k = top_k;
-    params.repetition_penalty = repetition_penalty;
-    params.n_threads = n_threads > 0 ? n_threads : g_threads;
-    params.language_id = language_to_id(language);
-
-    fprintf(stderr, "[qwen3-tts-cpp] Synthesizing: text='%.50s%s', lang_id=%d, "
-                    "temp=%.2f, threads=%d\n",
-            text, (strlen(text) > 50 ? "..." : ""), params.language_id,
-            temperature, params.n_threads);
-
-    tts_result result;
-    bool has_ref = ref_audio_path && ref_audio_path[0] != '\0';
-
-    if (has_ref) {
-        fprintf(stderr, "[qwen3-tts-cpp] Voice cloning with ref: %s\n",
-                ref_audio_path);
-        result = g_engine->synthesize_with_voice(text, ref_audio_path, params);
-    } else {
-        result = g_engine->synthesize(text, params);
-    }
-
-    if (!result.success) {
-        fprintf(stderr, "[qwen3-tts-cpp] ERROR: synthesis failed: %s\n",
-                result.error_msg.c_str());
-        return 3;
-    }
-
-    int n_samples = (int)result.audio.size();
-    if (n_samples == 0) {
-        fprintf(stderr, "[qwen3-tts-cpp] ERROR: synthesis produced no samples\n");
-        return 4;
-    }
-
-    fprintf(stderr,
-            "[qwen3-tts-cpp] Synthesis done: %d samples (%.2fs @ 24kHz)\n",
-            n_samples, (float)n_samples / 24000.0f);
-
-    if (!save_audio_file(dst, result.audio, result.sample_rate)) {
-        fprintf(stderr, "[qwen3-tts-cpp] ERROR: failed to write %s\n", dst);
-        return 5;
-    }
-
-    fprintf(stderr, "[qwen3-tts-cpp] Wrote %s\n", dst);
-    return 0;
-}

backend/go/qwen3-tts-cpp/cpp/goqwen3ttscpp.h

@@ -1,12 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <cstdint>
-
-extern "C" {
-int load_model(const char *model_dir, int n_threads);
-int synthesize(const char *text, const char *ref_audio_path, const char *dst,
-               const char *language, float temperature, float top_p,
-               int top_k, float repetition_penalty, int max_audio_tokens,
-               int n_threads);
-}

backend/go/qwen3-tts-cpp/goqwen3ttscpp.go

@@ -1,74 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/mudler/LocalAI/pkg/grpc/base"
-	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
-)
-
-var (
-	CppLoadModel  func(modelDir string, nThreads int) int
-	CppSynthesize func(text, refAudioPath, dst, language string,
-		temperature, topP float32, topK int,
-		repetitionPenalty float32, maxAudioTokens, nThreads int) int
-)
-
-type Qwen3TtsCpp struct {
-	base.SingleThread
-	threads int
-}
-
-func (q *Qwen3TtsCpp) Load(opts *pb.ModelOptions) error {
-	// ModelFile is the model directory path (containing GGUF files)
-	modelDir := opts.ModelFile
-	if modelDir == "" {
-		modelDir = opts.ModelPath
-	}
-
-	// Resolve relative paths
-	if !filepath.IsAbs(modelDir) && opts.ModelPath != "" {
-		modelDir = filepath.Join(opts.ModelPath, modelDir)
-	}
-
-	threads := int(opts.Threads)
-	if threads <= 0 {
-		threads = 4
-	}
-	q.threads = threads
-
-	fmt.Fprintf(os.Stderr, "[qwen3-tts-cpp] Loading models from: %s (threads=%d)\n", modelDir, threads)
-
-	if ret := CppLoadModel(modelDir, threads); ret != 0 {
-		return fmt.Errorf("failed to load qwen3-tts model (error code: %d)", ret)
-	}
-
-	return nil
-}
-
-func (q *Qwen3TtsCpp) TTS(req *pb.TTSRequest) error {
-	text := req.Text
-	voice := req.Voice // reference audio path for voice cloning (empty = no cloning)
-	dst := req.Dst
-	language := ""
-	if req.Language != nil {
-		language = *req.Language
-	}
-
-	// Synthesis parameters with sensible defaults
-	temperature := float32(0.9)
-	topP := float32(0.8)
-	topK := 50
-	repetitionPenalty := float32(1.05)
-	maxAudioTokens := 4096
-
-	if ret := CppSynthesize(text, voice, dst, language,
-		temperature, topP, topK, repetitionPenalty,
-		maxAudioTokens, q.threads); ret != 0 {
-		return fmt.Errorf("failed to synthesize audio (error code: %d)", ret)
-	}
-
-	return nil
-}

backend/go/qwen3-tts-cpp/main.go

@@ -1,47 +0,0 @@
-package main
-
-// Note: this is started internally by LocalAI and a server is allocated for each model
-import (
-	"flag"
-	"os"
-
-	"github.com/ebitengine/purego"
-	grpc "github.com/mudler/LocalAI/pkg/grpc"
-)
-
-var (
-	addr = flag.String("addr", "localhost:50051", "the address to connect to")
-)
-
-type LibFuncs struct {
-	FuncPtr any
-	Name    string
-}
-
-func main() {
-	// Get library name from environment variable, default to fallback
-	libName := os.Getenv("QWEN3TTS_LIBRARY")
-	if libName == "" {
-		libName = "./libgoqwen3ttscpp-fallback.so"
-	}
-
-	gosd, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
-	if err != nil {
-		panic(err)
-	}
-
-	libFuncs := []LibFuncs{
-		{&CppLoadModel, "load_model"},
-		{&CppSynthesize, "synthesize"},
-	}
-
-	for _, lf := range libFuncs {
-		purego.RegisterLibFunc(lf.FuncPtr, gosd, lf.Name)
-	}
-
-	flag.Parse()
-
-	if err := grpc.StartServer(*addr, &Qwen3TtsCpp{}); err != nil {
-		panic(err)
-	}
-}

backend/go/qwen3-tts-cpp/package.sh

@@ -1,64 +0,0 @@
-#!/bin/bash
-
-# Script to copy the appropriate libraries based on architecture
-# This script is used in the final stage of the Dockerfile
-
-set -e
-
-CURDIR=$(dirname "$(realpath $0)")
-REPO_ROOT="${CURDIR}/../../.."
-
-# Create lib directory
-mkdir -p $CURDIR/package/lib
-
-cp -avf $CURDIR/qwen3-tts-cpp $CURDIR/package/
-cp -fv $CURDIR/libgoqwen3ttscpp-*.so $CURDIR/package/
-cp -fv $CURDIR/run.sh $CURDIR/package/
-
-# Detect architecture and copy appropriate libraries
-if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
-    # x86_64 architecture
-    echo "Detected x86_64 architecture, copying x86_64 libraries..."
-    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
-    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
-    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
-elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
-    # ARM64 architecture
-    echo "Detected ARM64 architecture, copying ARM64 libraries..."
-    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
-    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
-    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
-elif [ $(uname -s) = "Darwin" ]; then
-    echo "Detected Darwin"
-else
-    echo "Error: Could not detect architecture"
-    exit 1
-fi
-
-# Package GPU libraries based on BUILD_TYPE
-GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
-if [ -f "$GPU_LIB_SCRIPT" ]; then
-    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
-    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
-    package_gpu_libs
-fi
-
-echo "Packaging completed successfully"
-ls -liah $CURDIR/package/
-ls -liah $CURDIR/package/lib/

backend/go/qwen3-tts-cpp/qwen3ttscpp_test.go

@@ -1,173 +0,0 @@
-package main
-
-import (
-	"context"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"testing"
-	"time"
-
-	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
-)
-
-const (
-	testAddr    = "localhost:50051"
-	startupWait = 5 * time.Second
-)
-
-func skipIfNoModel(t *testing.T) string {
-	t.Helper()
-	modelDir := os.Getenv("QWEN3TTS_MODEL_DIR")
-	if modelDir == "" {
-		t.Skip("QWEN3TTS_MODEL_DIR not set, skipping test (set to directory with GGUF models)")
-	}
-	if _, err := os.Stat(filepath.Join(modelDir, "qwen3-tts-0.6b-f16.gguf")); os.IsNotExist(err) {
-		t.Skipf("TTS model file not found in %s, skipping", modelDir)
-	}
-	if _, err := os.Stat(filepath.Join(modelDir, "qwen3-tts-tokenizer-f16.gguf")); os.IsNotExist(err) {
-		t.Skipf("Tokenizer model file not found in %s, skipping", modelDir)
-	}
-	return modelDir
-}
-
-func startServer(t *testing.T) *exec.Cmd {
-	t.Helper()
-	binary := os.Getenv("QWEN3TTS_BINARY")
-	if binary == "" {
-		binary = "./qwen3-tts-cpp"
-	}
-	if _, err := os.Stat(binary); os.IsNotExist(err) {
-		t.Skipf("Backend binary not found at %s, skipping", binary)
-	}
-	cmd := exec.Command(binary, "--addr", testAddr)
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	if err := cmd.Start(); err != nil {
-		t.Fatalf("Failed to start server: %v", err)
-	}
-	time.Sleep(startupWait)
-	return cmd
-}
-
-func stopServer(cmd *exec.Cmd) {
-	if cmd != nil && cmd.Process != nil {
-		cmd.Process.Kill()
-		cmd.Wait()
-	}
-}
-
-func dialGRPC(t *testing.T) *grpc.ClientConn {
-	t.Helper()
-	conn, err := grpc.Dial(testAddr,
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-		grpc.WithDefaultCallOptions(
-			grpc.MaxCallRecvMsgSize(50*1024*1024),
-			grpc.MaxCallSendMsgSize(50*1024*1024),
-		),
-	)
-	if err != nil {
-		t.Fatalf("Failed to dial gRPC: %v", err)
-	}
-	return conn
-}
-
-func TestServerHealth(t *testing.T) {
-	cmd := startServer(t)
-	defer stopServer(cmd)
-
-	conn := dialGRPC(t)
-	defer conn.Close()
-
-	client := pb.NewBackendClient(conn)
-	resp, err := client.Health(context.Background(), &pb.HealthMessage{})
-	if err != nil {
-		t.Fatalf("Health check failed: %v", err)
-	}
-	if string(resp.Message) != "OK" {
-		t.Fatalf("Expected OK, got %s", string(resp.Message))
-	}
-}
-
-func TestLoadModel(t *testing.T) {
-	modelDir := skipIfNoModel(t)
-	cmd := startServer(t)
-	defer stopServer(cmd)
-
-	conn := dialGRPC(t)
-	defer conn.Close()
-
-	client := pb.NewBackendClient(conn)
-
-	resp, err := client.LoadModel(context.Background(), &pb.ModelOptions{
-		ModelFile: modelDir,
-		Threads:   4,
-	})
-	if err != nil {
-		t.Fatalf("LoadModel failed: %v", err)
-	}
-	if !resp.Success {
-		t.Fatalf("LoadModel returned failure: %s", resp.Message)
-	}
-}
-
-func TestTTS(t *testing.T) {
-	modelDir := skipIfNoModel(t)
-
-	tmpDir, err := os.MkdirTemp("", "qwen3tts-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Cleanup(func() { os.RemoveAll(tmpDir) })
-
-	outputFile := filepath.Join(tmpDir, "output.wav")
-
-	cmd := startServer(t)
-	defer stopServer(cmd)
-
-	conn := dialGRPC(t)
-	defer conn.Close()
-
-	client := pb.NewBackendClient(conn)
-
-	// Load models
-	loadResp, err := client.LoadModel(context.Background(), &pb.ModelOptions{
-		ModelFile: modelDir,
-		Threads:   4,
-	})
-	if err != nil {
-		t.Fatalf("LoadModel failed: %v", err)
-	}
-	if !loadResp.Success {
-		t.Fatalf("LoadModel returned failure: %s", loadResp.Message)
-	}
-
-	// Synthesize speech
-	language := "en"
-	_, err = client.TTS(context.Background(), &pb.TTSRequest{
-		Text:     "Hello, this is a test of the Qwen3 text to speech system.",
-		Dst:      outputFile,
-		Language: &language,
-	})
-	if err != nil {
-		t.Fatalf("TTS failed: %v", err)
-	}
-
-	// Verify output file exists and has content
-	info, err := os.Stat(outputFile)
-	if os.IsNotExist(err) {
-		t.Fatal("Output audio file was not created")
-	}
-	if err != nil {
-		t.Fatalf("Failed to stat output file: %v", err)
-	}
-
-	t.Logf("Output file size: %d bytes", info.Size())
-
-	// WAV header is 44 bytes minimum; any real audio should be much larger
-	if info.Size() < 1000 {
-		t.Errorf("Output file too small (%d bytes), expected real audio data", info.Size())
-	}
-}

backend/go/qwen3-tts-cpp/run.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-set -ex
-
-# Get the absolute current dir where the script is located
-CURDIR=$(dirname "$(realpath $0)")
-
-cd /
-
-echo "CPU info:"
-if [ "$(uname)" != "Darwin" ]; then
-	grep -e "model\sname" /proc/cpuinfo | head -1
-	grep -e "flags" /proc/cpuinfo | head -1
-fi
-
-LIBRARY="$CURDIR/libgoqwen3ttscpp-fallback.so"
-
-if [ "$(uname)" != "Darwin" ]; then
-	if grep -q -e "\savx\s" /proc/cpuinfo ; then
-		echo "CPU:    AVX    found OK"
-		if [ -e $CURDIR/libgoqwen3ttscpp-avx.so ]; then
-			LIBRARY="$CURDIR/libgoqwen3ttscpp-avx.so"
-		fi
-	fi
-
-	if grep -q -e "\savx2\s" /proc/cpuinfo ; then
-		echo "CPU:    AVX2   found OK"
-		if [ -e $CURDIR/libgoqwen3ttscpp-avx2.so ]; then
-			LIBRARY="$CURDIR/libgoqwen3ttscpp-avx2.so"
-		fi
-	fi
-
-	# Check avx 512
-	if grep -q -e "\savx512f\s" /proc/cpuinfo ; then
-		echo "CPU:    AVX512F found OK"
-		if [ -e $CURDIR/libgoqwen3ttscpp-avx512.so ]; then
-			LIBRARY="$CURDIR/libgoqwen3ttscpp-avx512.so"
-		fi
-	fi
-fi
-
-export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
-export QWEN3TTS_LIBRARY=$LIBRARY
-
-# If there is a lib/ld.so, use it
-if [ -f $CURDIR/lib/ld.so ]; then
-	echo "Using lib/ld.so"
-	echo "Using library: $LIBRARY"
-	exec $CURDIR/lib/ld.so $CURDIR/qwen3-tts-cpp "$@"
-fi
-
-echo "Using library: $LIBRARY"
-exec $CURDIR/qwen3-tts-cpp "$@"

backend/go/qwen3-tts-cpp/test.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-set -e
-
-CURDIR=$(dirname "$(realpath $0)")
-
-echo "Running qwen3-tts-cpp backend tests..."
-
-# The test requires:
-#   - QWEN3TTS_MODEL_DIR: path to directory containing GGUF model files
-#   - QWEN3TTS_BINARY: path to the qwen3-tts-cpp binary (defaults to ./qwen3-tts-cpp)
-#
-# Tests that require the model will be skipped if QWEN3TTS_MODEL_DIR is not set
-# or the directory does not contain the required model files.
-
-cd "$CURDIR"
-
-# Only auto-download models when QWEN3TTS_MODEL_DIR is not explicitly set
-if [ -z "$QWEN3TTS_MODEL_DIR" ]; then
-    export QWEN3TTS_MODEL_DIR="./qwen3-tts-models"
-
-    if [ ! -d "$QWEN3TTS_MODEL_DIR" ]; then
-        echo "Creating qwen3-tts-models directory for tests..."
-        mkdir -p "$QWEN3TTS_MODEL_DIR"
-        REPO_ID="endo5501/qwen3-tts.cpp"
-        echo "Repository: ${REPO_ID}"
-        echo ""
-
-        # Files to download (smallest model for testing)
-        FILES=(
-            "qwen3-tts-0.6b-f16.gguf"
-            "qwen3-tts-tokenizer-f16.gguf"
-        )
-
-        BASE_URL="https://huggingface.co/${REPO_ID}/resolve/main"
-
-        for file in "${FILES[@]}"; do
-            dest="${QWEN3TTS_MODEL_DIR}/${file}"
-            if [ -f "${dest}" ]; then
-                echo "  [skip] ${file} (already exists)"
-            else
-                echo "  [download] ${file}..."
-                curl -L -o "${dest}" "${BASE_URL}/${file}" --progress-bar
-                echo "  [done] ${file}"
-            fi
-        done
-    fi
-fi
-
-# Run Go tests
-go test -v -timeout 600s .
-
-echo "All qwen3-tts-cpp tests passed."

backend/go/sam3-cpp/Makefile

@@ -26,7 +26,7 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	ROCM_PATH ?= /opt/rocm
 	export CXX=$(ROCM_HOME)/llvm/bin/clang++
 	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
 	CMAKE_ARGS+=-DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DGGML_VULKAN=ON

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=6b675a5ede9b0edf0a0f44191e8b79d7ef27615a
+STABLEDIFFUSION_GGML_VERSION?=e8323cabb0e4511ba18a50b1cb34cf1f87fc71ef
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 
@@ -32,7 +32,7 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	ROCM_PATH ?= /opt/rocm
 	export CXX=$(ROCM_HOME)/llvm/bin/clang++
 	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
+	AMDGPU_TARGETS?=gfx803,gfx900,gfx906,gfx908,gfx90a,gfx942,gfx1010,gfx1030,gfx1032,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
 	CMAKE_ARGS+=-DSD_HIPBLAS=ON -DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DSD_VULKAN=ON -DGGML_VULKAN=ON

backend/index.yaml

@@ -29,20 +29,6 @@
     nvidia-cuda-12: "cuda12-llama-cpp"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-llama-cpp"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-llama-cpp"
-- &ikllamacpp
-  name: "ik-llama-cpp"
-  alias: "ik-llama-cpp"
-  license: mit
-  description: |
-    Fork of llama.cpp optimized for CPU performance by ikawrakow
-  urls:
-    - https://github.com/ikawrakow/ik_llama.cpp
-  tags:
-    - text-to-text
-    - LLM
-    - CPU
-  capabilities:
-    default: "cpu-ik-llama-cpp"
 - &whispercpp
   name: "whisper"
   alias: "whisper"
@@ -197,7 +183,6 @@
     amd: "rocm-vllm"
     intel: "intel-vllm"
     nvidia-cuda-12: "cuda12-vllm"
-    cpu: "cpu-vllm"
 - &vllm-omni
   name: "vllm-omni"
   license: apache-2.0
@@ -427,30 +412,6 @@
     nvidia-l4t: "nvidia-l4t-arm64-acestep-cpp"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-acestep-cpp"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-acestep-cpp"
-- &qwen3ttscpp
-  name: "qwen3-tts-cpp"
-  description: |
-    Qwen3-TTS C++ backend using GGML. Native C++ text-to-speech with voice cloning support.
-    Generates 24kHz mono audio from text with optional reference audio for voice cloning via ECAPA-TDNN speaker embeddings.
-  urls:
-    - https://github.com/predict-woo/qwen3-tts.cpp
-  tags:
-    - text-to-speech
-    - tts
-    - voice-cloning
-  alias: "qwen3-tts-cpp"
-  capabilities:
-    default: "cpu-qwen3-tts-cpp"
-    nvidia: "cuda12-qwen3-tts-cpp"
-    nvidia-cuda-13: "cuda13-qwen3-tts-cpp"
-    nvidia-cuda-12: "cuda12-qwen3-tts-cpp"
-    intel: "intel-sycl-f16-qwen3-tts-cpp"
-    metal: "metal-qwen3-tts-cpp"
-    amd: "rocm-qwen3-tts-cpp"
-    vulkan: "vulkan-qwen3-tts-cpp"
-    nvidia-l4t: "nvidia-l4t-arm64-qwen3-tts-cpp"
-    nvidia-l4t-cuda-12: "nvidia-l4t-arm64-qwen3-tts-cpp"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-qwen3-tts-cpp"
 - &faster-whisper
   icon: https://avatars.githubusercontent.com/u/1520500?s=200&v=4
   description: |
@@ -912,10 +873,6 @@
     nvidia-cuda-12: "cuda12-llama-cpp-development"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-llama-cpp-development"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-llama-cpp-development"
-- !!merge <<: *ikllamacpp
-  name: "ik-llama-cpp-development"
-  capabilities:
-    default: "cpu-ik-llama-cpp-development"
 - !!merge <<: *neutts
   name: "cpu-neutts"
   uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-neutts"
@@ -1346,17 +1303,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-llama-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-llama-cpp
-## ik-llama-cpp
-- !!merge <<: *ikllamacpp
-  name: "cpu-ik-llama-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-ik-llama-cpp"
-  mirrors:
-    - localai/localai-backends:latest-cpu-ik-llama-cpp
-- !!merge <<: *ikllamacpp
-  name: "cpu-ik-llama-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-ik-llama-cpp"
-  mirrors:
-    - localai/localai-backends:master-cpu-ik-llama-cpp
 ## whisper
 - !!merge <<: *whispercpp
   name: "nvidia-l4t-arm64-whisper"
@@ -1564,7 +1510,6 @@
     nvidia: "cuda12-vllm-development"
     amd: "rocm-vllm-development"
     intel: "intel-vllm-development"
-    cpu: "cpu-vllm-development"
 - !!merge <<: *vllm
   name: "cuda12-vllm"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-vllm"
@@ -1580,11 +1525,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-vllm"
   mirrors:
     - localai/localai-backends:latest-gpu-intel-vllm
-- !!merge <<: *vllm
-  name: "cpu-vllm"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-vllm"
-  mirrors:
-    - localai/localai-backends:latest-cpu-vllm
 - !!merge <<: *vllm
   name: "cuda12-vllm-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-vllm"
@@ -1600,11 +1540,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-vllm"
   mirrors:
     - localai/localai-backends:master-gpu-intel-vllm
-- !!merge <<: *vllm
-  name: "cpu-vllm-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-vllm"
-  mirrors:
-    - localai/localai-backends:master-cpu-vllm
 # vllm-omni
 - !!merge <<: *vllm-omni
   name: "vllm-omni-development"
@@ -2172,107 +2107,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-acestep-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-acestep-cpp
-## qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "nvidia-l4t-arm64-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-arm64-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "nvidia-l4t-arm64-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-arm64-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cuda13-nvidia-l4t-arm64-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cuda13-nvidia-l4t-arm64-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cpu-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-cpu-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "metal-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-metal-darwin-arm64-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "metal-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-metal-darwin-arm64-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cpu-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-cpu-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cuda12-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-12-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "rocm-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-rocm-hipblas-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "intel-sycl-f32-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-intel-sycl-f32-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "intel-sycl-f16-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-intel-sycl-f16-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "vulkan-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-vulkan-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "vulkan-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-vulkan-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cuda12-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-12-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "rocm-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-rocm-hipblas-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "intel-sycl-f32-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-intel-sycl-f32-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "intel-sycl-f16-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-intel-sycl-f16-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cuda13-qwen3-tts-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-qwen3-tts-cpp
-- !!merge <<: *qwen3ttscpp
-  name: "cuda13-qwen3-tts-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-qwen3-tts-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-qwen3-tts-cpp
 ## kokoro
 - !!merge <<: *kokoro
   name: "kokoro-development"

backend/python/ace-step/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
 torchaudio
 torchvision
 

backend/python/chatterbox/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchaudio==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.9.1+rocm6.4
+torchaudio==2.9.1+rocm6.4
 transformers
 numpy>=1.24.0,<1.26.0
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289

backend/python/common/template/requirements-hipblas.txt

@@ -1,2 +1,2 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
 torch

backend/python/common/vllm_utils.py

@@ -1,84 +0,0 @@
-"""Shared utilities for vLLM-based backends."""
-import json
-import sys
-
-
-def parse_options(options_list):
-    """Parse Options[] list of 'key:value' strings into a dict.
-
-    Supports type inference for common cases (bool, int, float).
-    Used by LoadModel to extract backend-specific options.
-    """
-    opts = {}
-    for opt in options_list:
-        if ":" not in opt:
-            continue
-        key, value = opt.split(":", 1)
-        key = key.strip()
-        value = value.strip()
-        # Try type conversion
-        if value.lower() in ("true", "false"):
-            opts[key] = value.lower() == "true"
-        else:
-            try:
-                opts[key] = int(value)
-            except ValueError:
-                try:
-                    opts[key] = float(value)
-                except ValueError:
-                    opts[key] = value
-    return opts
-
-
-def messages_to_dicts(proto_messages):
-    """Convert proto Message objects to list of dicts for apply_chat_template().
-
-    Handles: role, content, name, tool_call_id, reasoning_content, tool_calls (JSON string -> list).
-    """
-    result = []
-    for msg in proto_messages:
-        d = {"role": msg.role, "content": msg.content or ""}
-        if msg.name:
-            d["name"] = msg.name
-        if msg.tool_call_id:
-            d["tool_call_id"] = msg.tool_call_id
-        if msg.reasoning_content:
-            d["reasoning_content"] = msg.reasoning_content
-        if msg.tool_calls:
-            try:
-                d["tool_calls"] = json.loads(msg.tool_calls)
-            except json.JSONDecodeError:
-                pass
-        result.append(d)
-    return result
-
-
-def setup_parsers(opts):
-    """Return (tool_parser_cls, reasoning_parser_cls) tuple from opts dict.
-
-    Uses vLLM's native ToolParserManager and ReasoningParserManager.
-    Returns (None, None) if vLLM is not installed or parsers not available.
-    """
-    tool_parser_cls = None
-    reasoning_parser_cls = None
-
-    tool_parser_name = opts.get("tool_parser")
-    reasoning_parser_name = opts.get("reasoning_parser")
-
-    if tool_parser_name:
-        try:
-            from vllm.tool_parsers import ToolParserManager
-            tool_parser_cls = ToolParserManager.get_tool_parser(tool_parser_name)
-            print(f"[vllm_utils] Loaded tool_parser: {tool_parser_name}", file=sys.stderr)
-        except Exception as e:
-            print(f"[vllm_utils] Failed to load tool_parser {tool_parser_name}: {e}", file=sys.stderr)
-
-    if reasoning_parser_name:
-        try:
-            from vllm.reasoning import ReasoningParserManager
-            reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
-            print(f"[vllm_utils] Loaded reasoning_parser: {reasoning_parser_name}", file=sys.stderr)
-        except Exception as e:
-            print(f"[vllm_utils] Failed to load reasoning_parser {reasoning_parser_name}: {e}", file=sys.stderr)
-
-    return tool_parser_cls, reasoning_parser_cls

backend/python/coqui/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchaudio==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
+torchaudio==2.8.0+rocm6.4
 transformers==4.48.3
 accelerate
 coqui-tts

backend/python/diffusers/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchvision==0.25.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
+torchvision==0.23.0+rocm6.4
 git+https://github.com/huggingface/diffusers
 opencv-python
 transformers

backend/python/faster-whisper/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
 torch
 faster-whisper

backend/python/fish-speech/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchaudio==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
+torch==2.7.1+rocm6.3
+torchaudio==2.7.1+rocm6.3

backend/python/kokoro/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchaudio==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
+torchaudio==2.8.0+rocm6.4
 transformers
 accelerate
 kokoro

backend/python/nemo/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
 torch
 nemo_toolkit[asr]

backend/python/neutts/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
 transformers==4.56.1
 accelerate
 librosa==0.11.0

backend/python/outetts/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
 accelerate
 llvmlite==0.43.0
 numba==0.60.0

backend/python/pocket-tts/requirements-hipblas.txt

@@ -1,4 +1,4 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
 pocket-tts
 scipy
-torch==2.10.0+rocm7.0
+torch==2.7.1+rocm6.3

backend/python/qwen-asr/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
+torch==2.7.1+rocm6.3
 qwen-asr

backend/python/qwen-tts/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchaudio==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
+torch==2.7.1+rocm6.3
+torchaudio==2.7.1+rocm6.3
 qwen-tts
 sox

backend/python/rerankers/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
 transformers
 accelerate
-torch==2.10.0+rocm7.0
+torch==2.8.0+rocm6.4
 rerankers[transformers]

backend/python/rfdetr/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchvision==0.25.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
+torchvision==0.23.0+rocm6.4
 rfdetr
 opencv-python
 accelerate

backend/python/transformers/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
 accelerate
 transformers>=5.0.0
 llvmlite==0.43.0

backend/python/vibevoice/requirements-hipblas.txt

@@ -1,6 +1,6 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
-torchvision==0.25.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
+torch==2.7.1+rocm6.3
+torchvision==0.22.1+rocm6.3
 git+https://github.com/huggingface/diffusers
 opencv-python
 transformers>=4.51.3,<5.0.0

backend/python/vllm-omni/backend.py

@@ -17,8 +17,6 @@ import time
 import os
 import base64
 import io
-import json
-import gc
 
 from PIL import Image
 import torch
@@ -32,7 +30,6 @@ import grpc
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
-from vllm_utils import parse_options, messages_to_dicts, setup_parsers
 
 
 from vllm_omni.entrypoints.omni import Omni
@@ -151,20 +148,23 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
     def LoadModel(self, request, context):
         try:
-            # CPU detection: if no CUDA, default vLLM target device to CPU.
-            try:
-                if not torch.cuda.is_available():
-                    os.environ.setdefault("VLLM_TARGET_DEVICE", "cpu")
-                    os.environ.setdefault("VLLM_CPU_KVCACHE_SPACE", "4")
-            except Exception:
-                pass
-
             print(f"Loading model {request.Model}...", file=sys.stderr)
             print(f"Request {request}", file=sys.stderr)
 
-            # Parse options from request.Options using shared helper
-            self.options = parse_options(request.Options)
-            opts = self.options
+            # Parse options from request.Options (key:value pairs)
+            self.options = {}
+            for opt in request.Options:
+                if ":" not in opt:
+                    continue
+                key, value = opt.split(":", 1)
+                # Convert value to appropriate type
+                if is_float(value):
+                    value = float(value)
+                elif is_int(value):
+                    value = int(value)
+                elif value.lower() in ["true", "false"]:
+                    value = value.lower() == "true"
+                self.options[key] = value
 
             print(f"Options: {self.options}", file=sys.stderr)
 
@@ -244,24 +244,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     omni_kwargs["max_model_len"] = request.MaxModelLen
 
             self.omni = Omni(**omni_kwargs)
-
-            # Load tokenizer for LLM/TTS so chat templates work
-            if self.model_type in ("llm", "tts"):
-                try:
-                    from vllm.transformers_utils.tokenizer import get_tokenizer
-                    self.tokenizer = get_tokenizer(
-                        request.Model,
-                        trust_remote_code=opts.get("trust_remote_code", False),
-                    )
-                except Exception as e:
-                    print(f"Failed to load tokenizer: {e}", file=sys.stderr)
-                    self.tokenizer = None
-            else:
-                self.tokenizer = None
-
-            # Setup optional tool / reasoning parsers
-            self.tool_parser_cls, self.reasoning_parser_cls = setup_parsers(opts)
-
             print("Model loaded successfully", file=sys.stderr)
             return backend_pb2.Result(message="Model loaded successfully", success=True)
 
@@ -484,32 +466,14 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # Extract prompt
             if request.Prompt:
                 prompt = request.Prompt
-            elif request.Messages:
-                if getattr(self, "tokenizer", None) is not None:
-                    messages_dicts = messages_to_dicts(request.Messages)
-                    template_kwargs = {"tokenize": False, "add_generation_prompt": True}
-                    if request.Tools:
-                        try:
-                            template_kwargs["tools"] = json.loads(request.Tools)
-                        except json.JSONDecodeError:
-                            pass
-                    try:
-                        if request.Metadata.get("enable_thinking", "").lower() == "true":
-                            template_kwargs["enable_thinking"] = True
-                    except Exception:
-                        pass
-                    try:
-                        prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
-                    except TypeError:
-                        prompt = self.tokenizer.apply_chat_template(
-                            messages_dicts, tokenize=False, add_generation_prompt=True
-                        )
-                else:
-                    # Fallback: basic template
-                    prompt = ""
-                    for msg in request.Messages:
-                        prompt += f"<|im_start|>{msg.role}\n{msg.content}<|im_end|>\n"
-                    prompt += "<|im_start|>assistant\n"
+            elif request.Messages and request.UseTokenizerTemplate:
+                # Build prompt from messages (simplified - would need tokenizer for full template)
+                prompt = ""
+                for msg in request.Messages:
+                    role = msg.role
+                    content = msg.content
+                    prompt += f"<|im_start|>{role}\n{content}<|im_end|>\n"
+                prompt += "<|im_start|>assistant\n"
             else:
                 yield backend_pb2.Reply(message=bytes("", 'utf-8'))
                 return
@@ -575,79 +539,20 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # Call omni.generate() (returns generator for LLM mode)
             omni_generator = self.omni.generate([inputs], sampling_params_list)
 
-            # Extract text from outputs and track token usage
+            # Extract text from outputs
             generated_text = ""
-            prompt_tokens = 0
-            completion_tokens = 0
             for stage_outputs in omni_generator:
                 if stage_outputs.final_output_type == "text":
                     for output in stage_outputs.request_output:
-                        completion = output.outputs[0]
-                        text_output = completion.text
-                        # Track tokens when available
-                        try:
-                            if getattr(output, "prompt_token_ids", None) is not None:
-                                prompt_tokens = len(output.prompt_token_ids)
-                            if getattr(completion, "token_ids", None) is not None:
-                                completion_tokens = len(completion.token_ids)
-                        except Exception:
-                            pass
+                        text_output = output.outputs[0].text
                         if streaming:
                             # Remove already sent text (vllm concatenates)
                             delta_text = text_output.removeprefix(generated_text)
-                            yield backend_pb2.Reply(
-                                message=bytes(delta_text, encoding='utf-8'),
-                                tokens=completion_tokens,
-                                prompt_tokens=prompt_tokens,
-                            )
+                            yield backend_pb2.Reply(message=bytes(delta_text, encoding='utf-8'))
                         generated_text = text_output
 
             if not streaming:
-                # Build optional ChatDelta with parsed reasoning / tool calls
-                chat_deltas = []
-                content_text = generated_text
-                reasoning_text = ""
-                tool_call_deltas = []
-
-                if self.reasoning_parser_cls is not None:
-                    try:
-                        parser = self.reasoning_parser_cls(self.tokenizer) if self.tokenizer else self.reasoning_parser_cls()
-                        reasoning_text, content_text = parser.extract_reasoning_content(content_text, request=None)
-                        reasoning_text = reasoning_text or ""
-                        content_text = content_text or ""
-                    except Exception as e:
-                        print(f"reasoning_parser failed: {e}", file=sys.stderr)
-
-                if self.tool_parser_cls is not None:
-                    try:
-                        parser = self.tool_parser_cls(self.tokenizer) if self.tokenizer else self.tool_parser_cls()
-                        tool_info = parser.extract_tool_calls(content_text, request=None)
-                        if getattr(tool_info, "tools_called", False):
-                            content_text = tool_info.content or ""
-                            for tc in tool_info.tool_calls or []:
-                                fn = getattr(tc, "function", None)
-                                tool_call_deltas.append(backend_pb2.ToolCallDelta(
-                                    index=getattr(tc, "index", 0) or 0,
-                                    id=getattr(tc, "id", "") or "",
-                                    name=getattr(fn, "name", "") if fn else "",
-                                    arguments=getattr(fn, "arguments", "") if fn else "",
-                                ))
-                    except Exception as e:
-                        print(f"tool_parser failed: {e}", file=sys.stderr)
-
-                if self.tool_parser_cls is not None or self.reasoning_parser_cls is not None:
-                    chat_deltas.append(backend_pb2.ChatDelta(
-                        content=content_text,
-                        reasoning_content=reasoning_text,
-                        tool_calls=tool_call_deltas,
-                    ))
-
-                yield backend_pb2.Reply(
-                    message=bytes(generated_text, encoding='utf-8'),
-                    tokens=completion_tokens,
-                    prompt_tokens=prompt_tokens,
-                    chat_deltas=chat_deltas,
-                )
+                yield backend_pb2.Reply(message=bytes(generated_text, encoding='utf-8'))
 
         except Exception as err:
             print(f"Error in Predict: {err}", file=sys.stderr)
@@ -742,37 +647,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             traceback.print_exc()
             return backend_pb2.Result(success=False, message=f"Error generating TTS: {err}")
 
-    def TokenizeString(self, request, context):
-        if not hasattr(self, 'tokenizer') or self.tokenizer is None:
-            context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
-            context.set_details("Model/tokenizer not loaded")
-            return backend_pb2.TokenizationResponse()
-        try:
-            tokens = self.tokenizer.encode(request.Prompt)
-            return backend_pb2.TokenizationResponse(length=len(tokens), tokens=tokens)
-        except Exception as e:
-            context.set_code(grpc.StatusCode.INTERNAL)
-            context.set_details(str(e))
-            return backend_pb2.TokenizationResponse()
-
-    def Free(self, request, context):
-        try:
-            if hasattr(self, 'omni'):
-                del self.omni
-            if hasattr(self, 'tokenizer'):
-                del self.tokenizer
-            self.tool_parser_cls = None
-            self.reasoning_parser_cls = None
-            gc.collect()
-            try:
-                if torch.cuda.is_available():
-                    torch.cuda.empty_cache()
-            except Exception:
-                pass
-            return backend_pb2.Result(success=True, message="Model freed")
-        except Exception as e:
-            return backend_pb2.Result(success=False, message=str(e))
-
 
 def serve(address):
     server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),

backend/python/vllm-omni/requirements-hipblas.txt

@@ -1,4 +1,4 @@
---extra-index-url https://download.pytorch.org/whl/nightly/rocm7.0
+--extra-index-url https://download.pytorch.org/whl/nightly/rocm6.4
 accelerate
 torch
 transformers

backend/python/vllm/backend.py

@@ -5,9 +5,6 @@ import argparse
 import signal
 import sys
 import os
-import json
-import time
-import gc
 from typing import List
 from PIL import Image
 
@@ -29,25 +26,6 @@ from vllm.assets.video import VideoAsset
 import base64
 import io
 
-# Version-compat imports — wrap in try/except for older vLLM versions
-try:
-    from vllm.tool_parsers import ToolParserManager
-    HAS_TOOL_PARSERS = True
-except ImportError:
-    HAS_TOOL_PARSERS = False
-
-try:
-    from vllm.reasoning import ReasoningParserManager
-    HAS_REASONING_PARSERS = True
-except ImportError:
-    HAS_REASONING_PARSERS = False
-
-try:
-    from vllm.sampling_params import GuidedDecodingParams
-    HAS_GUIDED_DECODING = True
-except ImportError:
-    HAS_GUIDED_DECODING = False
-
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 # If MAX_WORKERS are specified in the environment use it, otherwise default to 1
@@ -91,35 +69,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 break
         return decoded_text
 
-    def _parse_options(self, options_list):
-        """Parse Options[] key:value string list into a dict."""
-        opts = {}
-        for opt in options_list:
-            if ":" not in opt:
-                continue
-            key, value = opt.split(":", 1)
-            opts[key.strip()] = value.strip()
-        return opts
-
-    def _messages_to_dicts(self, messages):
-        """Convert proto Messages to list of dicts suitable for apply_chat_template()."""
-        result = []
-        for msg in messages:
-            d = {"role": msg.role, "content": msg.content or ""}
-            if msg.name:
-                d["name"] = msg.name
-            if msg.tool_call_id:
-                d["tool_call_id"] = msg.tool_call_id
-            if msg.reasoning_content:
-                d["reasoning_content"] = msg.reasoning_content
-            if msg.tool_calls:
-                try:
-                    d["tool_calls"] = json.loads(msg.tool_calls)
-                except json.JSONDecodeError:
-                    pass
-            result.append(d)
-        return result
-
     def Health(self, request, context):
         """
         Returns a health check message.
@@ -183,49 +132,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
 
         try:
-            # vLLM >= 0.14 removed get_model_config() on AsyncLLM; the tokenizer
-            # is either already loaded on the engine or can be built from the
-            # Model name directly.
-            tokenizer = None
-            if hasattr(self.llm, "get_tokenizer"):
-                try:
-                    tokenizer = await self.llm.get_tokenizer()
-                except TypeError:
-                    tokenizer = self.llm.get_tokenizer()
-                except Exception:
-                    tokenizer = None
-            if tokenizer is None and hasattr(self.llm, "tokenizer"):
-                tokenizer = self.llm.tokenizer
-            if tokenizer is None:
-                tokenizer = get_tokenizer(
-                    request.Model,
-                    trust_remote_code=bool(request.TrustRemoteCode),
-                    truncation_side="left",
-                )
-            self.tokenizer = tokenizer
+           engine_model_config = await self.llm.get_model_config()
+           self.tokenizer = get_tokenizer(
+               engine_model_config.tokenizer,
+               tokenizer_mode=engine_model_config.tokenizer_mode,
+               trust_remote_code=engine_model_config.trust_remote_code,
+               truncation_side="left",
+           )
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
-
-        # Parse options for parser selection
-        opts = self._parse_options(request.Options)
-
-        # Instantiate tool/reasoning parser classes (they'll be instantiated per-request with tokenizer)
-        self.tool_parser_cls = None
-        self.reasoning_parser_cls = None
-        if HAS_TOOL_PARSERS and opts.get("tool_parser"):
-            try:
-                self.tool_parser_cls = ToolParserManager.get_tool_parser(opts["tool_parser"])
-                print(f"Loaded tool_parser: {opts['tool_parser']}", file=sys.stderr)
-            except Exception as e:
-                print(f"Failed to load tool_parser {opts.get('tool_parser')}: {e}", file=sys.stderr)
-
-        if HAS_REASONING_PARSERS and opts.get("reasoning_parser"):
-            try:
-                self.reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser(opts["reasoning_parser"])
-                print(f"Loaded reasoning_parser: {opts['reasoning_parser']}", file=sys.stderr)
-            except Exception as e:
-                print(f"Failed to load reasoning_parser {opts.get('reasoning_parser')}: {e}", file=sys.stderr)
-
         print("Model loaded successfully", file=sys.stderr)
         return backend_pb2.Result(message="Model loaded successfully", success=True)
 
@@ -282,38 +197,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         finally:
             await iterations.aclose()
 
-    async def TokenizeString(self, request, context):
-        if not hasattr(self, 'tokenizer') or self.tokenizer is None:
-            context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
-            context.set_details("Model/tokenizer not loaded")
-            return backend_pb2.TokenizationResponse()
-        try:
-            tokens = self.tokenizer.encode(request.Prompt)
-            return backend_pb2.TokenizationResponse(length=len(tokens), tokens=tokens)
-        except Exception as e:
-            context.set_code(grpc.StatusCode.INTERNAL)
-            context.set_details(str(e))
-            return backend_pb2.TokenizationResponse()
-
-    async def Free(self, request, context):
-        try:
-            if hasattr(self, 'llm'):
-                del self.llm
-            if hasattr(self, 'tokenizer'):
-                del self.tokenizer
-            self.tool_parser_cls = None
-            self.reasoning_parser_cls = None
-            gc.collect()
-            try:
-                import torch
-                if torch.cuda.is_available():
-                    torch.cuda.empty_cache()
-            except ImportError:
-                pass
-            return backend_pb2.Result(success=True, message="Model freed")
-        except Exception as e:
-            return backend_pb2.Result(success=False, message=str(e))
-
     async def _predict(self, request, context, streaming=False):
         # Build the sampling parameters
         # NOTE: this must stay in sync with the vllm backend
@@ -339,6 +222,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             "SkipSpecialTokens": "skip_special_tokens",
             "SpacesBetweenSpecialTokens": "spaces_between_special_tokens",
             "TruncatePromptTokens": "truncate_prompt_tokens",
+            "GuidedDecoding": "guided_decoding",
         }
 
         sampling_params = SamplingParams(top_p=0.9, max_tokens=200)
@@ -349,14 +233,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 if value not in (None, 0, [], False):
                     setattr(sampling_params, param_field, value)
 
-        # Guided decoding: use Grammar field to pass JSON schema or BNF
-        if HAS_GUIDED_DECODING and request.Grammar:
-            try:
-                json.loads(request.Grammar)  # valid JSON = JSON schema
-                sampling_params.guided_decoding = GuidedDecodingParams(json=request.Grammar)
-            except json.JSONDecodeError:
-                sampling_params.guided_decoding = GuidedDecodingParams(grammar=request.Grammar)
-
         # Extract image paths and process images
         prompt = request.Prompt
 
@@ -368,27 +244,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # If tokenizer template is enabled and messages are provided instead of prompt, apply the tokenizer template
         if not request.Prompt and request.UseTokenizerTemplate and request.Messages:
-            messages_dicts = self._messages_to_dicts(request.Messages)
-            template_kwargs = {"tokenize": False, "add_generation_prompt": True}
-
-            # Pass tools for tool calling
-            if request.Tools:
-                try:
-                    template_kwargs["tools"] = json.loads(request.Tools)
-                except json.JSONDecodeError:
-                    pass
-
-            # Enable thinking mode if requested
-            if request.Metadata.get("enable_thinking", "").lower() == "true":
-                template_kwargs["enable_thinking"] = True
-
-            try:
-                prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
-            except TypeError:
-                # Some tokenizers don't support tools/enable_thinking kwargs — retry without them
-                prompt = self.tokenizer.apply_chat_template(
-                    messages_dicts, tokenize=False, add_generation_prompt=True
-                )
+            prompt = self.tokenizer.apply_chat_template(request.Messages, tokenize=False, add_generation_prompt=True)
 
         # Generate text using the LLM engine
         request_id = random_uuid()
@@ -409,26 +265,25 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # Stream the results
         generated_text = ""
-        last_output = None
         try:
             async for request_output in outputs:
                 iteration_text = request_output.outputs[0].text
-                last_output = request_output
 
                 if streaming:
                     # Remove text already sent as vllm concatenates the text from previous yields
                     delta_iteration_text = iteration_text.removeprefix(generated_text)
                     # Send the partial result
-                    yield backend_pb2.Reply(
-                        message=bytes(delta_iteration_text, encoding='utf-8'),
-                        chat_deltas=[backend_pb2.ChatDelta(content=delta_iteration_text)],
-                    )
+                    yield backend_pb2.Reply(message=bytes(delta_iteration_text, encoding='utf-8'))
 
                 # Keep track of text generated
                 generated_text = iteration_text
         finally:
             await outputs.aclose()
 
+        # If streaming, we already sent everything
+        if streaming:
+            return
+
         # Remove the image files from /tmp folder
         for img_path in image_paths:
             try:
@@ -436,99 +291,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             except Exception as e:
                 print(f"Error removing image file: {img_path}, {e}", file=sys.stderr)
 
-        # Parse reasoning and tool calls from final text using vLLM's native parsers
-        content = generated_text
-        reasoning_content = ""
-        tool_calls_proto = []
-
-        if self.reasoning_parser_cls:
-            try:
-                rp = self.reasoning_parser_cls(self.tokenizer)
-                r, c = rp.extract_reasoning(generated_text, request=None)
-                reasoning_content = r or ""
-                content = c if c is not None else generated_text
-            except Exception as e:
-                print(f"Reasoning parser error: {e}", file=sys.stderr)
-
-        if self.tool_parser_cls and request.Tools:
-            try:
-                tools = json.loads(request.Tools)
-                # Some concrete parsers only accept the tokenizer; only the
-                # abstract base declares the tools kwarg. Try with tools first,
-                # fall back to tokenizer-only.
-                try:
-                    tp = self.tool_parser_cls(self.tokenizer, tools=tools)
-                except TypeError:
-                    tp = self.tool_parser_cls(self.tokenizer)
-                info = tp.extract_tool_calls(content, request=None)
-                if info.tools_called:
-                    content = info.content or ""
-                    for i, tc in enumerate(info.tool_calls):
-                        tool_calls_proto.append(backend_pb2.ToolCallDelta(
-                            index=i,
-                            id=tc.id,
-                            name=tc.function.name,
-                            arguments=tc.function.arguments,
-                        ))
-            except Exception as e:
-                print(f"Tool parser error: {e}", file=sys.stderr)
-
-        # Extract token counts
-        prompt_tokens = 0
-        completion_tokens = 0
-        if last_output is not None:
-            try:
-                prompt_tokens = len(last_output.prompt_token_ids or [])
-            except Exception:
-                pass
-            try:
-                completion_tokens = len(last_output.outputs[0].token_ids or [])
-            except Exception:
-                pass
-
-        # Extract logprobs if requested
-        logprobs_bytes = b""
-        if last_output is not None and request.Logprobs > 0:
-            try:
-                lp = last_output.outputs[0].logprobs
-                if lp:
-                    logprobs_data = {"content": []}
-                    for token_lp_dict in lp:
-                        if token_lp_dict:
-                            first_tok_id, first_lp = next(iter(token_lp_dict.items()))
-                            logprobs_data["content"].append({
-                                "token": getattr(first_lp, "decoded_token", str(first_tok_id)),
-                                "logprob": first_lp.logprob,
-                            })
-                    logprobs_bytes = json.dumps(logprobs_data).encode("utf-8")
-            except Exception as e:
-                print(f"Logprobs extraction error: {e}", file=sys.stderr)
-
-        chat_delta = backend_pb2.ChatDelta(
-            content=content,
-            reasoning_content=reasoning_content,
-            tool_calls=tool_calls_proto,
-        )
-
-        if streaming:
-            # Final chunk with structured data
-            yield backend_pb2.Reply(
-                message=b"",
-                prompt_tokens=prompt_tokens,
-                tokens=completion_tokens,
-                chat_deltas=[chat_delta],
-                logprobs=logprobs_bytes,
-            )
-            return
-
-        # Non-streaming: single Reply with everything
-        yield backend_pb2.Reply(
-            message=bytes(content, encoding='utf-8'),
-            prompt_tokens=prompt_tokens,
-            tokens=completion_tokens,
-            chat_deltas=[chat_delta],
-            logprobs=logprobs_bytes,
-        )
+        # Sending the final generated text
+        yield backend_pb2.Reply(message=bytes(generated_text, encoding='utf-8'))
 
     def load_image(self, image_path: str):
         """

backend/python/vllm/install.sh

@@ -26,43 +26,20 @@ if [ "x${BUILD_PROFILE}" == "xintel" ]; then
     EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
 fi
 
-# CPU builds need unsafe-best-match to pull torch==2.10.0+cpu from the
-# pytorch test channel while still resolving transformers/vllm from pypi.
-if [ "x${BUILD_PROFILE}" == "xcpu" ]; then
-    EXTRA_PIP_INSTALL_FLAGS+=" --index-strategy=unsafe-best-match"
-fi
-
-# FROM_SOURCE=true on a CPU build skips the prebuilt vllm wheel in
-# requirements-cpu-after.txt and compiles vllm locally against the host's
-# actual CPU. Not used by default because it takes ~30-40 minutes, but
-# kept here for hosts where the prebuilt wheel SIGILLs (CPU without the
-# required SIMD baseline, e.g. AVX-512 VNNI/BF16). Default CI uses a
-# bigger-runner with compatible hardware instead.
+# We don't embed this into the images as it is a large dependency and not always needed.
+# Besides, the speed inference are not actually usable in the current state for production use-cases.
 if [ "x${BUILD_TYPE}" == "x" ] && [ "x${FROM_SOURCE:-}" == "xtrue" ]; then
-    # Temporarily hide the prebuilt wheel so installRequirements doesn't
-    # pull it — the rest of the requirements files (base deps, torch,
-    # transformers) are still installed normally.
-    _cpu_after="${backend_dir}/requirements-cpu-after.txt"
-    _cpu_after_bak=""
-    if [ -f "${_cpu_after}" ]; then
-        _cpu_after_bak="${_cpu_after}.from-source.bak"
-        mv "${_cpu_after}" "${_cpu_after_bak}"
-    fi
-    installRequirements
-    if [ -n "${_cpu_after_bak}" ]; then
-        mv "${_cpu_after_bak}" "${_cpu_after}"
-    fi
-
-    # Build vllm from source against the installed torch.
-    # https://docs.vllm.ai/en/latest/getting_started/installation/cpu/
-    _vllm_src=$(mktemp -d)
-    trap 'rm -rf "${_vllm_src}"' EXIT
-    git clone --depth 1 https://github.com/vllm-project/vllm "${_vllm_src}/vllm"
-    pushd "${_vllm_src}/vllm"
-        uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} wheel packaging ninja "setuptools>=49.4.0" numpy typing-extensions pillow setuptools-scm
-        # Respect pre-installed torch version — skip vllm's own requirements-build.txt torch pin.
-        VLLM_TARGET_DEVICE=cpu uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps .
-    popd
-else
-    installRequirements
+        ensureVenv
+        # https://docs.vllm.ai/en/v0.6.1/getting_started/cpu-installation.html
+        if [ ! -d vllm ]; then
+            git clone https://github.com/vllm-project/vllm
+        fi
+        pushd vllm
+            uv pip install wheel packaging ninja "setuptools>=49.4.0" numpy typing-extensions pillow setuptools-scm grpcio==1.68.1 protobuf bitsandbytes
+            uv pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
+            VLLM_TARGET_DEVICE=cpu python setup.py install
+        popd
+        rm -rf vllm
+    else
+        installRequirements
 fi

backend/python/vllm/package.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-# Script to package runtime shared libraries for the vllm backend.
-#
-# The final Dockerfile.python stage is FROM scratch, so system libraries
-# must be explicitly copied into ${BACKEND}/lib so the backend can run on
-# any host without installing them. libbackend.sh automatically adds that
-# directory to LD_LIBRARY_PATH at run time.
-#
-# vllm's CPU C++ extension (vllm._C) dlopens libnuma.so.1 at import time;
-# if it's missing, the _C_utils torch ops are never registered and the
-# engine crashes with AttributeError on init_cpu_threads_env. libgomp is
-# used by torch's CPU kernels; on some stripped-down hosts it's also
-# absent, so we bundle it too.
-
-set -e
-
-CURDIR=$(dirname "$(realpath "$0")")
-LIB_DIR="${CURDIR}/lib"
-mkdir -p "${LIB_DIR}"
-
-copy_with_symlinks() {
-    local soname="$1"
-    local hit=""
-    for dir in /usr/lib/x86_64-linux-gnu /usr/lib/aarch64-linux-gnu /lib/x86_64-linux-gnu /lib/aarch64-linux-gnu /usr/lib /lib; do
-        if [ -e "${dir}/${soname}" ]; then
-            hit="${dir}/${soname}"
-            break
-        fi
-    done
-    if [ -z "${hit}" ]; then
-        echo "warning: ${soname} not found in standard lib paths" >&2
-        return 0
-    fi
-    # Follow the symlink to the real file, copy it, then recreate the symlink.
-    local real
-    real=$(readlink -f "${hit}")
-    cp -v "${real}" "${LIB_DIR}/"
-    local real_base
-    real_base=$(basename "${real}")
-    if [ "${real_base}" != "${soname}" ]; then
-        ln -sf "${real_base}" "${LIB_DIR}/${soname}"
-    fi
-}
-
-copy_with_symlinks libnuma.so.1
-copy_with_symlinks libgomp.so.1
-
-echo "vllm packaging completed successfully"
-ls -liah "${LIB_DIR}/"

backend/python/vllm/requirements-after.txt

@@ -1,2 +1 @@
-# vllm is installed per-acceleration in requirements-{profile}-after.txt
-# (cublas12, hipblas, intel, cpu)
+vllm

backend/python/vllm/requirements-cpu-after.txt

@@ -1,2 +0,0 @@
-vllm @ https://github.com/vllm-project/vllm/releases/download/v0.14.1/vllm-0.14.1+cpu-cp38-abi3-manylinux_2_35_x86_64.whl ; platform_machine == "x86_64"
-vllm @ https://github.com/vllm-project/vllm/releases/download/v0.14.1/vllm-0.14.1+cpu-cp38-abi3-manylinux_2_35_aarch64.whl ; platform_machine == "aarch64"

backend/python/vllm/requirements-cpu.txt

@@ -1,6 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/cpu
 accelerate
-torch==2.9.1+cpu
-torchvision
-torchaudio
-transformers
+torch==2.7.0
+transformers

backend/python/vllm/requirements-cublas12-after.txt

@@ -1,2 +1 @@
 https://github.com/Dao-AILab/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu12torch2.7cxx11abiTRUE-cp310-cp310-linux_x86_64.whl
-vllm

backend/python/vllm/requirements-hipblas-after.txt

@@ -1 +0,0 @@
-vllm

backend/python/vllm/requirements-hipblas.txt

@@ -1,4 +1,4 @@
---extra-index-url https://download.pytorch.org/whl/nightly/rocm7.0
+--extra-index-url https://download.pytorch.org/whl/nightly/rocm6.4
 accelerate
 torch
 transformers

backend/python/vllm/requirements-intel-after.txt

@@ -1 +0,0 @@
-vllm

backend/python/vllm/test.py

@@ -122,89 +122,6 @@ class TestBackendServicer(unittest.TestCase):
             self.tearDown()
 
 
-    def test_messages_to_dicts(self):
-        """
-        Tests _messages_to_dicts conversion of proto Messages to dicts.
-        """
-        import sys, os
-        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-        from backend import BackendServicer
-        servicer = BackendServicer()
-        msgs = [
-            backend_pb2.Message(role="user", content="hello"),
-            backend_pb2.Message(
-                role="assistant",
-                content="",
-                tool_calls='[{"id":"call_1","type":"function","function":{"name":"foo","arguments":"{}"}}]',
-                reasoning_content="thinking...",
-            ),
-            backend_pb2.Message(role="tool", content="result", name="foo", tool_call_id="call_1"),
-        ]
-        result = servicer._messages_to_dicts(msgs)
-        self.assertEqual(len(result), 3)
-        self.assertEqual(result[0], {"role": "user", "content": "hello"})
-        self.assertEqual(result[1]["reasoning_content"], "thinking...")
-        self.assertIsInstance(result[1]["tool_calls"], list)
-        self.assertEqual(result[1]["tool_calls"][0]["id"], "call_1")
-        self.assertEqual(result[2]["tool_call_id"], "call_1")
-        self.assertEqual(result[2]["name"], "foo")
-
-    def test_parse_options(self):
-        """
-        Tests _parse_options correctly parses key:value strings.
-        """
-        import sys, os
-        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-        from backend import BackendServicer
-        servicer = BackendServicer()
-        opts = servicer._parse_options([
-            "tool_parser:hermes",
-            "reasoning_parser:deepseek_r1",
-            "invalid_no_colon",
-            "key_with_colons:a:b:c",
-        ])
-        self.assertEqual(opts["tool_parser"], "hermes")
-        self.assertEqual(opts["reasoning_parser"], "deepseek_r1")
-        self.assertEqual(opts["key_with_colons"], "a:b:c")
-        self.assertNotIn("invalid_no_colon", opts)
-
-    def test_tokenize_string(self):
-        """
-        Tests the TokenizeString RPC returns valid tokens.
-        """
-        try:
-            self.setUp()
-            with grpc.insecure_channel("localhost:50051") as channel:
-                stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
-                self.assertTrue(response.success)
-                resp = stub.TokenizeString(backend_pb2.PredictOptions(Prompt="Hello world"))
-                self.assertGreater(resp.length, 0)
-                self.assertEqual(len(resp.tokens), resp.length)
-        except Exception as err:
-            print(err)
-            self.fail("TokenizeString service failed")
-        finally:
-            self.tearDown()
-
-    def test_free(self):
-        """
-        Tests the Free RPC doesn't crash.
-        """
-        try:
-            self.setUp()
-            with grpc.insecure_channel("localhost:50051") as channel:
-                stub = backend_pb2_grpc.BackendStub(channel)
-                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
-                self.assertTrue(response.success)
-                free_resp = stub.Free(backend_pb2.HealthMessage())
-                self.assertTrue(free_resp.success)
-        except Exception as err:
-            print(err)
-            self.fail("Free service failed")
-        finally:
-            self.tearDown()
-
     def test_embedding(self):
         """
         This method tests if the embeddings are generated successfully

backend/python/voxcpm/requirements-hipblas.txt

@@ -1,5 +1,5 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
+torch==2.7.1+rocm6.3
 soundfile
 numpy
 voxcpm

backend/python/whisperx/requirements-hipblas.txt

@@ -1,3 +1,3 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch==2.10.0+rocm7.0
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0
 whisperx @ git+https://github.com/m-bain/whisperX.git

core/application/config_file_watcher.go

@@ -338,9 +338,6 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand
 			if settings.AutoUpgradeBackends != nil {
 				appConfig.AutoUpgradeBackends = *settings.AutoUpgradeBackends
 			}
-			if settings.PreferDevelopmentBackends != nil {
-				appConfig.PreferDevelopmentBackends = *settings.PreferDevelopmentBackends
-			}
 			if settings.ApiKeys != nil {
 				// API keys from env vars (startup) should be kept, runtime settings keys replace all runtime keys
 				// If runtime_settings.json specifies ApiKeys (even if empty), it replaces all runtime keys

core/cli/run.go

@@ -47,9 +47,8 @@ type RunCMD struct {
 	BackendImagesReleaseTag  string   `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"`
 	BackendImagesBranchTag   string   `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"`
 	BackendDevSuffix         string   `env:"LOCALAI_BACKEND_DEV_SUFFIX,BACKEND_DEV_SUFFIX" help:"Development suffix for backend images" group:"backends" default:"development"`
-	AutoUpgradeBackends       bool     `env:"LOCALAI_AUTO_UPGRADE_BACKENDS,AUTO_UPGRADE_BACKENDS" help:"Automatically upgrade backends when new versions are detected" group:"backends" default:"false"`
-	PreferDevelopmentBackends bool     `env:"LOCALAI_PREFER_DEV_BACKENDS,PREFER_DEV_BACKENDS" help:"Prefer development backend versions (shows development backends by default in UI)" group:"backends" default:"false"`
-	PreloadModels             string   `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"`
+	AutoUpgradeBackends      bool     `env:"LOCALAI_AUTO_UPGRADE_BACKENDS,AUTO_UPGRADE_BACKENDS" help:"Automatically upgrade backends when new versions are detected" group:"backends" default:"false"`
+	PreloadModels            string   `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"`
 	Models                   []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"`
 	PreloadModelsConfig      string   `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"`
 
@@ -496,10 +495,6 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		opts = append(opts, config.WithAutoUpgradeBackends(r.AutoUpgradeBackends))
 	}
 
-	if r.PreferDevelopmentBackends {
-		opts = append(opts, config.WithPreferDevelopmentBackends(r.PreferDevelopmentBackends))
-	}
-
 	if r.PreloadBackendOnly {
 		_, err := application.New(opts...)
 		return err

core/config/application_config.go

@@ -58,7 +58,6 @@ type ApplicationConfig struct {
 
 	AutoloadGalleries, AutoloadBackendGalleries bool
 	AutoUpgradeBackends                         bool
-	PreferDevelopmentBackends                   bool
 
 	SingleBackend           bool // Deprecated: use MaxActiveBackends = 1 instead
 	MaxActiveBackends       int  // Maximum number of active backends (0 = unlimited, 1 = single backend mode)
@@ -396,10 +395,6 @@ func WithAutoUpgradeBackends(v bool) AppOption {
 	return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v }
 }
 
-func WithPreferDevelopmentBackends(v bool) AppOption {
-	return func(o *ApplicationConfig) { o.PreferDevelopmentBackends = v }
-}
-
 var EnableFederated = func(o *ApplicationConfig) {
 	o.Federated = true
 }
@@ -873,7 +868,6 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
 	autoloadGalleries := o.AutoloadGalleries
 	autoloadBackendGalleries := o.AutoloadBackendGalleries
 	autoUpgradeBackends := o.AutoUpgradeBackends
-	preferDevelopmentBackends := o.PreferDevelopmentBackends
 	apiKeys := o.ApiKeys
 	agentJobRetentionDays := o.AgentJobRetentionDays
 
@@ -948,7 +942,6 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings {
 		AutoloadGalleries:         &autoloadGalleries,
 		AutoloadBackendGalleries:  &autoloadBackendGalleries,
 		AutoUpgradeBackends:       &autoUpgradeBackends,
-		PreferDevelopmentBackends: &preferDevelopmentBackends,
 		ApiKeys:                   &apiKeys,
 		AgentJobRetentionDays:     &agentJobRetentionDays,
 		OpenResponsesStoreTTL:     &openResponsesStoreTTL,
@@ -1100,9 +1093,6 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req
 	if settings.AutoUpgradeBackends != nil {
 		o.AutoUpgradeBackends = *settings.AutoUpgradeBackends
 	}
-	if settings.PreferDevelopmentBackends != nil {
-		o.PreferDevelopmentBackends = *settings.PreferDevelopmentBackends
-	}
 	if settings.AgentJobRetentionDays != nil {
 		o.AgentJobRetentionDays = *settings.AgentJobRetentionDays
 	}

core/config/backend_hooks.go

@@ -1,30 +0,0 @@
-package config
-
-// BackendDefaultsHook is called during Prepare() and can modify cfg.
-// Only fills in values that are not already set by the user.
-type BackendDefaultsHook func(cfg *ModelConfig, modelPath string)
-
-var backendHooks = map[string][]BackendDefaultsHook{}
-
-// RegisterBackendHook registers a hook for a backend name.
-// Special keys:
-//   - "*"  = global catch-all, runs for EVERY backend (before specific hooks)
-//   - ""   = runs only when cfg.Backend is empty (auto-detect case)
-//   - "vllm", "llama-cpp" etc. = runs only for that specific backend
-//
-// Multiple hooks per key are supported; they run in registration order.
-func RegisterBackendHook(backend string, hook BackendDefaultsHook) {
-	backendHooks[backend] = append(backendHooks[backend], hook)
-}
-
-// runBackendHooks executes hooks in order:
-//  1. "*" (global) hooks for every backend
-//  2. Backend-specific hooks for cfg.Backend (includes "" when backend is empty)
-func runBackendHooks(cfg *ModelConfig, modelPath string) {
-	for _, h := range backendHooks["*"] {
-		h(cfg, modelPath)
-	}
-	for _, h := range backendHooks[cfg.Backend] {
-		h(cfg, modelPath)
-	}
-}

core/config/guesser.go

@@ -0,0 +1,46 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	gguf "github.com/gpustack/gguf-parser-go"
+	"github.com/mudler/xlog"
+)
+
+func guessDefaultsFromFile(cfg *ModelConfig, modelPath string, defaultCtx int) {
+	if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
+		xlog.Debug("guessDefaultsFromFile: guessing disabled with LOCALAI_DISABLE_GUESSING")
+		return
+	}
+
+	if modelPath == "" {
+		xlog.Debug("guessDefaultsFromFile: modelPath is empty")
+		return
+	}
+
+	// We try to guess only if we don't have a template defined already
+	guessPath := filepath.Join(modelPath, cfg.ModelFileName())
+
+	defer func() {
+		if r := recover(); r != nil {
+			xlog.Error("guessDefaultsFromFile: panic while parsing gguf file")
+		}
+	}()
+
+	defer func() {
+		if cfg.ContextSize == nil {
+			if defaultCtx == 0 {
+				defaultCtx = defaultContextSize
+			}
+			cfg.ContextSize = &defaultCtx
+		}
+	}()
+
+	// try to parse the gguf file
+	f, err := gguf.ParseGGUFFile(guessPath)
+	if err == nil {
+		guessGGUFFromFile(cfg, f, defaultCtx)
+		return
+	}
+}

core/config/hooks_llamacpp.go

@@ -1,46 +0,0 @@
-package config
-
-import (
-	"os"
-	"path/filepath"
-
-	gguf "github.com/gpustack/gguf-parser-go"
-	"github.com/mudler/xlog"
-)
-
-func init() {
-	// Register for both explicit llama-cpp and empty backend (auto-detect from GGUF file)
-	RegisterBackendHook("llama-cpp", llamaCppDefaults)
-	RegisterBackendHook("", llamaCppDefaults)
-}
-
-func llamaCppDefaults(cfg *ModelConfig, modelPath string) {
-	if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
-		xlog.Debug("llamaCppDefaults: guessing disabled")
-		return
-	}
-	if modelPath == "" {
-		return
-	}
-
-	guessPath := filepath.Join(modelPath, cfg.ModelFileName())
-
-	defer func() {
-		if r := recover(); r != nil {
-			xlog.Error("llamaCppDefaults: panic while parsing gguf file")
-		}
-	}()
-
-	// Default context size if not set, regardless of whether GGUF parsing succeeds
-	defer func() {
-		if cfg.ContextSize == nil {
-			ctx := defaultContextSize
-			cfg.ContextSize = &ctx
-		}
-	}()
-
-	f, err := gguf.ParseGGUFFile(guessPath)
-	if err == nil {
-		guessGGUFFromFile(cfg, f, 0)
-	}
-}

core/config/hooks_test.go

@@ -1,114 +0,0 @@
-package config_test
-
-import (
-	. "github.com/mudler/LocalAI/core/config"
-	"github.com/mudler/LocalAI/core/schema"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Backend hooks and parser defaults", func() {
-	Context("MatchParserDefaults", func() {
-		It("matches Qwen3 family", func() {
-			parsers := MatchParserDefaults("Qwen/Qwen3-8B")
-			Expect(parsers).NotTo(BeNil())
-			Expect(parsers["tool_parser"]).To(Equal("hermes"))
-			Expect(parsers["reasoning_parser"]).To(Equal("qwen3"))
-		})
-
-		It("matches Qwen3.5 with longest-prefix-first", func() {
-			parsers := MatchParserDefaults("Qwen/Qwen3.5-9B")
-			Expect(parsers).NotTo(BeNil())
-			Expect(parsers["tool_parser"]).To(Equal("qwen3_xml"))
-		})
-
-		It("matches Llama-3.3 not Llama-3.2", func() {
-			parsers := MatchParserDefaults("meta/Llama-3.3-70B-Instruct")
-			Expect(parsers).NotTo(BeNil())
-			Expect(parsers["tool_parser"]).To(Equal("llama3_json"))
-		})
-
-		It("matches deepseek-r1", func() {
-			parsers := MatchParserDefaults("deepseek-ai/DeepSeek-R1")
-			Expect(parsers).NotTo(BeNil())
-			Expect(parsers["reasoning_parser"]).To(Equal("deepseek_r1"))
-			Expect(parsers["tool_parser"]).To(Equal("deepseek_v3"))
-		})
-
-		It("returns nil for unknown families", func() {
-			Expect(MatchParserDefaults("acme/unknown-model-xyz")).To(BeNil())
-		})
-	})
-
-	Context("Backend hook registration and execution", func() {
-		It("runs registered hook for a backend", func() {
-			called := false
-			RegisterBackendHook("test-backend-hook", func(cfg *ModelConfig, modelPath string) {
-				called = true
-				cfg.Description = "modified-by-hook"
-			})
-
-			cfg := &ModelConfig{
-				Backend: "test-backend-hook",
-			}
-			// Use the public Prepare path indirectly is heavy; instead exercise via vllmDefaults
-			// path, but here just call RegisterBackendHook + we know runBackendHooks is internal.
-			// Verify by leveraging Prepare on a fresh ModelConfig with no model path.
-			cfg.PredictionOptions = schema.PredictionOptions{}
-
-			// Trigger via Prepare with empty options; this calls runBackendHooks internally.
-			cfg.SetDefaults()
-			Expect(called).To(BeTrue())
-			Expect(cfg.Description).To(Equal("modified-by-hook"))
-		})
-	})
-
-	Context("vllmDefaults hook", func() {
-		It("auto-sets parsers for known model families on vllm backend", func() {
-			cfg := &ModelConfig{
-				Backend: "vllm",
-				PredictionOptions: schema.PredictionOptions{
-					BasicModelRequest: schema.BasicModelRequest{
-						Model: "Qwen/Qwen3-8B",
-					},
-				},
-			}
-			cfg.SetDefaults()
-
-			foundTool := false
-			foundReasoning := false
-			for _, opt := range cfg.Options {
-				if opt == "tool_parser:hermes" {
-					foundTool = true
-				}
-				if opt == "reasoning_parser:qwen3" {
-					foundReasoning = true
-				}
-			}
-			Expect(foundTool).To(BeTrue())
-			Expect(foundReasoning).To(BeTrue())
-		})
-
-		It("does not override user-set tool_parser", func() {
-			cfg := &ModelConfig{
-				Backend: "vllm",
-				Options: []string{"tool_parser:custom"},
-				PredictionOptions: schema.PredictionOptions{
-					BasicModelRequest: schema.BasicModelRequest{
-						Model: "Qwen/Qwen3-8B",
-					},
-				},
-			}
-			cfg.SetDefaults()
-
-			count := 0
-			for _, opt := range cfg.Options {
-				if len(opt) >= len("tool_parser:") && opt[:len("tool_parser:")] == "tool_parser:" {
-					count++
-				}
-			}
-			Expect(count).To(Equal(1))
-		})
-	})
-})

core/config/hooks_vllm.go

@@ -1,85 +0,0 @@
-package config
-
-import (
-	_ "embed"
-	"encoding/json"
-	"strings"
-
-	"github.com/mudler/xlog"
-)
-
-//go:embed parser_defaults.json
-var parserDefaultsJSON []byte
-
-type parserDefaultsData struct {
-	Families map[string]map[string]string `json:"families"`
-	Patterns []string                     `json:"patterns"`
-}
-
-var parsersData *parserDefaultsData
-
-func init() {
-	parsersData = &parserDefaultsData{}
-	if err := json.Unmarshal(parserDefaultsJSON, parsersData); err != nil {
-		xlog.Warn("failed to parse parser_defaults.json", "error", err)
-	}
-
-	RegisterBackendHook("vllm", vllmDefaults)
-	RegisterBackendHook("vllm-omni", vllmDefaults)
-}
-
-// MatchParserDefaults returns parser defaults for the best-matching model family.
-// Returns nil if no family matches. Used both at load time (via hook) and at import time.
-func MatchParserDefaults(modelID string) map[string]string {
-	if parsersData == nil || len(parsersData.Patterns) == 0 {
-		return nil
-	}
-	normalized := normalizeModelID(modelID)
-	for _, pattern := range parsersData.Patterns {
-		if strings.Contains(normalized, pattern) {
-			if family, ok := parsersData.Families[pattern]; ok {
-				return family
-			}
-		}
-	}
-	return nil
-}
-
-func vllmDefaults(cfg *ModelConfig, modelPath string) {
-	// Check if user already set tool_parser or reasoning_parser in Options
-	hasToolParser := false
-	hasReasoningParser := false
-	for _, opt := range cfg.Options {
-		if strings.HasPrefix(opt, "tool_parser:") {
-			hasToolParser = true
-		}
-		if strings.HasPrefix(opt, "reasoning_parser:") {
-			hasReasoningParser = true
-		}
-	}
-	if hasToolParser && hasReasoningParser {
-		return
-	}
-
-	// Try matching against Model field, then Name
-	parsers := MatchParserDefaults(cfg.Model)
-	if parsers == nil {
-		parsers = MatchParserDefaults(cfg.Name)
-	}
-	if parsers == nil {
-		return
-	}
-
-	if !hasToolParser {
-		if tp, ok := parsers["tool_parser"]; ok {
-			cfg.Options = append(cfg.Options, "tool_parser:"+tp)
-			xlog.Debug("[parser_defaults] auto-set tool_parser", "parser", tp, "model", cfg.Model)
-		}
-	}
-	if !hasReasoningParser {
-		if rp, ok := parsers["reasoning_parser"]; ok {
-			cfg.Options = append(cfg.Options, "reasoning_parser:"+rp)
-			xlog.Debug("[parser_defaults] auto-set reasoning_parser", "parser", rp, "model", cfg.Model)
-		}
-	}
-}

core/config/model_config.go

@@ -497,12 +497,7 @@ func (cfg *ModelConfig) SetDefaults(opts ...ConfigLoaderOption) {
 		cfg.Debug = &trueV
 	}
 
-	// If a context size was provided via LoadOptions, apply it before hooks so they
-	// don't override it with their own defaults.
-	if ctx != 0 && cfg.ContextSize == nil {
-		cfg.ContextSize = &ctx
-	}
-	runBackendHooks(cfg, lo.modelPath)
+	guessDefaultsFromFile(cfg, lo.modelPath, ctx)
 	cfg.syncKnownUsecasesFromString()
 }
 

core/config/parser_defaults.json

@@ -1,33 +0,0 @@
-{
-  "families": {
-    "qwen3.5":       {"tool_parser": "qwen3_xml", "reasoning_parser": "qwen3"},
-    "qwen3-coder":   {"tool_parser": "qwen3_xml", "reasoning_parser": "qwen3"},
-    "qwen3":         {"tool_parser": "hermes",     "reasoning_parser": "qwen3"},
-    "qwen2.5":       {"tool_parser": "hermes"},
-    "qwq":           {"reasoning_parser": "deepseek_r1"},
-    "llama-4":       {"tool_parser": "llama4_pythonic"},
-    "llama-3.3":     {"tool_parser": "llama3_json"},
-    "llama-3.2":     {"tool_parser": "llama3_json"},
-    "llama-3.1":     {"tool_parser": "llama3_json"},
-    "mistral-nemo":  {"tool_parser": "mistral", "reasoning_parser": "mistral"},
-    "mistral-small": {"tool_parser": "mistral", "reasoning_parser": "mistral"},
-    "mistral-large": {"tool_parser": "mistral", "reasoning_parser": "mistral"},
-    "magistral":     {"tool_parser": "mistral", "reasoning_parser": "mistral"},
-    "deepseek-r1":   {"tool_parser": "deepseek_v3", "reasoning_parser": "deepseek_r1"},
-    "deepseek-v3":   {"tool_parser": "deepseek_v3", "reasoning_parser": "deepseek_v3"},
-    "glm-5":         {"tool_parser": "glm47"},
-    "glm-4":         {"tool_parser": "glm45", "reasoning_parser": "glm45"},
-    "gemma-4":       {"tool_parser": "gemma4", "reasoning_parser": "gemma4"},
-    "granite-4":     {"tool_parser": "granite4", "reasoning_parser": "granite"},
-    "minimax-m2.5":  {"tool_parser": "minimax_m2", "reasoning_parser": "minimax_m2"},
-    "minimax":       {"tool_parser": "minimax_m2", "reasoning_parser": "minimax_m2"},
-    "kimi-k2":       {"tool_parser": "kimi_k2", "reasoning_parser": "kimi_k2"},
-    "nemotron":      {"reasoning_parser": "nemotron_v3"},
-    "olmo":          {"tool_parser": "olmo3", "reasoning_parser": "olmo3"},
-    "ernie":         {"tool_parser": "ernie45", "reasoning_parser": "ernie45"},
-    "phi-4":         {"tool_parser": "phi4_mini_json"},
-    "gpt-oss":       {"tool_parser": "openai", "reasoning_parser": "openai_gptoss"},
-    "hermes":        {"tool_parser": "hermes"}
-  },
-  "patterns": ["qwen3.5","qwen3-coder","qwen3","qwen2.5","qwq","llama-4","llama-3.3","llama-3.2","llama-3.1","mistral-nemo","mistral-small","mistral-large","magistral","deepseek-r1","deepseek-v3","glm-5","glm-4","gemma-4","granite-4","minimax-m2.5","minimax","kimi-k2","nemotron","olmo","ernie","phi-4","gpt-oss","hermes"]
-}

core/config/runtime_settings.go

@@ -20,8 +20,7 @@ type RuntimeSettings struct {
 	// Backend management
 	SingleBackend           *bool `json:"single_backend,omitempty"`      // Deprecated: use MaxActiveBackends = 1 instead
 	MaxActiveBackends       *int  `json:"max_active_backends,omitempty"` // Maximum number of active backends (0 = unlimited, 1 = single backend mode)
-	AutoUpgradeBackends       *bool `json:"auto_upgrade_backends,omitempty"`       // Automatically upgrade backends when new versions are detected
-	PreferDevelopmentBackends *bool `json:"prefer_development_backends,omitempty"` // Prefer development backend versions by default in UI
+	AutoUpgradeBackends     *bool `json:"auto_upgrade_backends,omitempty"` // Automatically upgrade backends when new versions are detected
 	// Memory Reclaimer settings (works with GPU if available, otherwise RAM)
 	MemoryReclaimerEnabled   *bool    `json:"memory_reclaimer_enabled,omitempty"`   // Enable memory threshold monitoring
 	MemoryReclaimerThreshold *float64 `json:"memory_reclaimer_threshold,omitempty"` // Threshold 0.0-1.0 (e.g., 0.95 = 95%)

core/gallery/backend_types.go

@@ -2,7 +2,6 @@ package gallery
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/system"
@@ -71,13 +70,6 @@ func (m *GalleryBackend) IsMeta() bool {
 	return len(m.CapabilitiesMap) > 0 && m.URI == ""
 }
 
-func (m *GalleryBackend) IsDevelopment(devSuffix string) bool {
-	if devSuffix == "" {
-		devSuffix = defaultDevSuffix
-	}
-	return strings.HasSuffix(m.Name, "-"+devSuffix)
-}
-
 // IsCompatibleWith checks if the backend is compatible with the current system capability.
 // For meta backends, it checks if any of the capabilities in the map match the system capability.
 // For concrete backends, it delegates to SystemState.IsBackendCompatible.

core/gallery/importers/vllm.go

@@ -88,18 +88,6 @@ func (i *VLLMImporter) Import(details Details) (gallery.ModelConfig, error) {
 	// Apply per-model-family inference parameter defaults
 	config.ApplyInferenceDefaults(&modelConfig, details.URI)
 
-	// Auto-detect tool_parser and reasoning_parser for known model families.
-	// Surfacing them in the generated YAML lets users see and edit the choices.
-	parsers := config.MatchParserDefaults(details.URI)
-	if parsers != nil {
-		if tp, ok := parsers["tool_parser"]; ok {
-			modelConfig.Options = append(modelConfig.Options, "tool_parser:"+tp)
-		}
-		if rp, ok := parsers["reasoning_parser"]; ok {
-			modelConfig.Options = append(modelConfig.Options, "reasoning_parser:"+rp)
-		}
-	}
-
 	data, err := yaml.Marshal(modelConfig)
 	if err != nil {
 		return gallery.ModelConfig{}, err

core/http/react-ui/package-lock.json

@@ -8,16 +8,7 @@
       "name": "localai-react-ui",
       "version": "0.0.1",
       "dependencies": {
-        "@codemirror/autocomplete": "^6.18.6",
-        "@codemirror/commands": "^6.8.1",
-        "@codemirror/lang-yaml": "^6.1.2",
-        "@codemirror/language": "^6.11.0",
-        "@codemirror/lint": "^6.8.5",
-        "@codemirror/search": "^6.5.10",
-        "@codemirror/state": "^6.5.2",
-        "@codemirror/view": "^6.36.8",
         "@fortawesome/fontawesome-free": "^6.7.2",
-        "@lezer/highlight": "^1.2.1",
         "@modelcontextprotocol/ext-apps": "^1.2.2",
         "@modelcontextprotocol/sdk": "^1.25.1",
         "dompurify": "^3.2.5",
@@ -25,8 +16,7 @@
         "marked": "^15.0.7",
         "react": "^19.1.0",
         "react-dom": "^19.1.0",
-        "react-router-dom": "^7.6.1",
-        "yaml": "^2.8.3"
+        "react-router-dom": "^7.6.1"
       },
       "devDependencies": {
         "@eslint/js": "^9.27.0",
@@ -302,94 +292,6 @@
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@codemirror/autocomplete": {
-      "version": "6.20.1",
-      "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.20.1.tgz",
-      "integrity": "sha512-1cvg3Vz1dSSToCNlJfRA2WSI4ht3K+WplO0UMOgmUYPivCyy2oueZY6Lx7M9wThm7SDUBViRmuT+OG/i8+ON9A==",
-      "dependencies": {
-        "@codemirror/language": "^6.0.0",
-        "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.17.0",
-        "@lezer/common": "^1.0.0"
-      }
-    },
-    "node_modules/@codemirror/commands": {
-      "version": "6.10.3",
-      "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.10.3.tgz",
-      "integrity": "sha512-JFRiqhKu+bvSkDLI+rUhJwSxQxYb759W5GBezE8Uc8mHLqC9aV/9aTC7yJSqCtB3F00pylrLCwnyS91Ap5ej4Q==",
-      "dependencies": {
-        "@codemirror/language": "^6.0.0",
-        "@codemirror/state": "^6.6.0",
-        "@codemirror/view": "^6.27.0",
-        "@lezer/common": "^1.1.0"
-      }
-    },
-    "node_modules/@codemirror/lang-yaml": {
-      "version": "6.1.3",
-      "resolved": "https://registry.npmjs.org/@codemirror/lang-yaml/-/lang-yaml-6.1.3.tgz",
-      "integrity": "sha512-AZ8DJBuXGVHybpBQhmZtgew5//4hv3tdkXnr3vDmOUMJRuB6vn/uuwtmTOTlqEaQFg3hQSVeA90NmvIQyUV6FQ==",
-      "dependencies": {
-        "@codemirror/autocomplete": "^6.0.0",
-        "@codemirror/language": "^6.0.0",
-        "@codemirror/state": "^6.0.0",
-        "@lezer/common": "^1.2.0",
-        "@lezer/highlight": "^1.2.0",
-        "@lezer/lr": "^1.0.0",
-        "@lezer/yaml": "^1.0.0"
-      }
-    },
-    "node_modules/@codemirror/language": {
-      "version": "6.12.3",
-      "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.12.3.tgz",
-      "integrity": "sha512-QwCZW6Tt1siP37Jet9Tb02Zs81TQt6qQrZR2H+eGMcFsL1zMrk2/b9CLC7/9ieP1fjIUMgviLWMmgiHoJrj+ZA==",
-      "dependencies": {
-        "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.23.0",
-        "@lezer/common": "^1.5.0",
-        "@lezer/highlight": "^1.0.0",
-        "@lezer/lr": "^1.0.0",
-        "style-mod": "^4.0.0"
-      }
-    },
-    "node_modules/@codemirror/lint": {
-      "version": "6.9.5",
-      "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.9.5.tgz",
-      "integrity": "sha512-GElsbU9G7QT9xXhpUg1zWGmftA/7jamh+7+ydKRuT0ORpWS3wOSP0yT1FOlIZa7mIJjpVPipErsyvVqB9cfTFA==",
-      "dependencies": {
-        "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.35.0",
-        "crelt": "^1.0.5"
-      }
-    },
-    "node_modules/@codemirror/search": {
-      "version": "6.6.0",
-      "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.6.0.tgz",
-      "integrity": "sha512-koFuNXcDvyyotWcgOnZGmY7LZqEOXZaaxD/j6n18TCLx2/9HieZJ5H6hs1g8FiRxBD0DNfs0nXn17g872RmYdw==",
-      "dependencies": {
-        "@codemirror/state": "^6.0.0",
-        "@codemirror/view": "^6.37.0",
-        "crelt": "^1.0.5"
-      }
-    },
-    "node_modules/@codemirror/state": {
-      "version": "6.6.0",
-      "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.6.0.tgz",
-      "integrity": "sha512-4nbvra5R5EtiCzr9BTHiTLc+MLXK2QGiAVYMyi8PkQd3SR+6ixar/Q/01Fa21TBIDOZXgeWV4WppsQolSreAPQ==",
-      "dependencies": {
-        "@marijn/find-cluster-break": "^1.0.0"
-      }
-    },
-    "node_modules/@codemirror/view": {
-      "version": "6.41.0",
-      "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.41.0.tgz",
-      "integrity": "sha512-6H/qadXsVuDY219Yljhohglve8xf4B8xJkVOEWfA5uiYKiTFppjqsvsfR5iPA0RbvRBoOyTZpbLIxe9+0UR8xA==",
-      "dependencies": {
-        "@codemirror/state": "^6.6.0",
-        "crelt": "^1.0.6",
-        "style-mod": "^4.1.0",
-        "w3c-keyname": "^2.2.4"
-      }
-    },
     "node_modules/@esbuild/aix-ppc64": {
       "version": "0.25.12",
       "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
@@ -1064,42 +966,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
-    "node_modules/@lezer/common": {
-      "version": "1.5.2",
-      "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.5.2.tgz",
-      "integrity": "sha512-sxQE460fPZyU3sdc8lafxiPwJHBzZRy/udNFynGQky1SePYBdhkBl1kOagA9uT3pxR8K09bOrmTUqA9wb/PjSQ=="
-    },
-    "node_modules/@lezer/highlight": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.3.tgz",
-      "integrity": "sha512-qXdH7UqTvGfdVBINrgKhDsVTJTxactNNxLk7+UMwZhU13lMHaOBlJe9Vqp907ya56Y3+ed2tlqzys7jDkTmW0g==",
-      "dependencies": {
-        "@lezer/common": "^1.3.0"
-      }
-    },
-    "node_modules/@lezer/lr": {
-      "version": "1.4.8",
-      "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.8.tgz",
-      "integrity": "sha512-bPWa0Pgx69ylNlMlPvBPryqeLYQjyJjqPx+Aupm5zydLIF3NE+6MMLT8Yi23Bd9cif9VS00aUebn+6fDIGBcDA==",
-      "dependencies": {
-        "@lezer/common": "^1.0.0"
-      }
-    },
-    "node_modules/@lezer/yaml": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@lezer/yaml/-/yaml-1.0.4.tgz",
-      "integrity": "sha512-2lrrHqxalACEbxIbsjhqGpSW8kWpUKuY6RHgnSAFZa6qK62wvnPxA8hGOwOoDbwHcOFs5M4o27mjGu+P7TvBmw==",
-      "dependencies": {
-        "@lezer/common": "^1.2.0",
-        "@lezer/highlight": "^1.0.0",
-        "@lezer/lr": "^1.4.0"
-      }
-    },
-    "node_modules/@marijn/find-cluster-break": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz",
-      "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g=="
-    },
     "node_modules/@modelcontextprotocol/ext-apps": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/@modelcontextprotocol/ext-apps/-/ext-apps-1.2.2.tgz",
@@ -1963,11 +1829,6 @@
         "url": "https://opencollective.com/express"
       }
     },
-    "node_modules/crelt": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz",
-      "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g=="
-    },
     "node_modules/cross-spawn": {
       "version": "7.0.6",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -3676,11 +3537,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/style-mod": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz",
-      "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ=="
-    },
     "node_modules/supports-color": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -3871,11 +3727,6 @@
         }
       }
     },
-    "node_modules/w3c-keyname": {
-      "version": "2.2.8",
-      "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz",
-      "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ=="
-    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -3910,20 +3761,6 @@
       "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
       "dev": true
     },
-    "node_modules/yaml": {
-      "version": "2.8.3",
-      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz",
-      "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==",
-      "bin": {
-        "yaml": "bin.mjs"
-      },
-      "engines": {
-        "node": ">= 14.6"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/eemeli"
-      }
-    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

core/http/react-ui/src/pages/Backends.jsx

@@ -341,12 +341,10 @@ export default function Backends() {
         <div style={{ display: 'flex', gap: 'var(--spacing-md)', alignItems: 'center', borderLeft: '1px solid var(--color-border-subtle)', paddingLeft: 'var(--spacing-md)' }}>
           <label style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-xs)', fontSize: '0.75rem', color: 'var(--color-text-secondary)', cursor: 'pointer', userSelect: 'none', whiteSpace: 'nowrap' }}>
             <Toggle checked={showAllBackends} onChange={handleToggleAllBackends} />
-            <i className="fas fa-cubes" style={{ fontSize: '0.625rem' }} />
             Show all
           </label>
           <label style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-xs)', fontSize: '0.75rem', color: 'var(--color-text-secondary)', cursor: 'pointer', userSelect: 'none', whiteSpace: 'nowrap' }}>
             <Toggle checked={showDevelopment} onChange={handleToggleDev} />
-            <i className="fas fa-flask" style={{ fontSize: '0.625rem' }} />
             Development
           </label>
         </div>

core/http/react-ui/src/pages/Manage.jsx

@@ -2,7 +2,6 @@ import { useState, useEffect, useCallback } from 'react'
 import { useNavigate, useOutletContext, useSearchParams } from 'react-router-dom'
 import ResourceMonitor from '../components/ResourceMonitor'
 import ConfirmDialog from '../components/ConfirmDialog'
-import Toggle from '../components/Toggle'
 import { useModels } from '../hooks/useModels'
 import { backendControlApi, modelsApi, backendsApi, systemApi, nodesApi } from '../utils/api'
 
@@ -279,7 +278,6 @@ export default function Manage() {
             <table className="table">
               <thead>
                 <tr>
-                  <th style={{ width: 36 }}>Enabled</th>
                   <th>Name</th>
                   <th>Status</th>
                   <th>Backend</th>
@@ -290,49 +288,34 @@ export default function Manage() {
               <tbody>
                 {models.map(model => (
                   <tr key={model.id} style={{ opacity: model.disabled ? 0.55 : 1, transition: 'opacity 0.2s' }}>
-                    {/* Enable/Disable toggle */}
-                    <td>
-                      <Toggle
-                        checked={!model.disabled}
-                        onChange={() => handleToggleModel(model.id, model.disabled)}
-                        disabled={togglingModels.has(model.id)}
-                      />
-                    </td>
-                    {/* Name */}
                     <td>
                       <div style={{ display: 'flex', alignItems: 'center', gap: 'var(--spacing-sm)' }}>
+                        <i className="fas fa-brain" style={{ color: model.disabled ? 'var(--color-text-muted)' : 'var(--color-accent)' }} />
+                        <span className={`badge ${model.disabled ? '' : 'badge-success'}`} style={{ width: 6, height: 6, padding: 0, borderRadius: '50%', minWidth: 'auto', background: model.disabled ? 'var(--color-text-muted)' : undefined }} />
                         <span style={{ fontWeight: 500 }}>{model.id}</span>
-                        {model.pinned && (
-                          <i className="fas fa-thumbtack" style={{ fontSize: '0.625rem', color: 'var(--color-warning)' }} title="Pinned — won't be idle-unloaded" />
-                        )}
-                        <div style={{ display: 'flex', gap: '2px', marginLeft: 'auto' }}>
+                        <a
+                          href="#"
+                          onClick={(e) => { e.preventDefault(); navigate(`/app/model-editor/${encodeURIComponent(model.id)}`) }}
+                          style={{ fontSize: '0.75rem', color: 'var(--color-primary)' }}
+                          title="Edit config"
+                        >
+                          <i className="fas fa-pen-to-square" />
+                        </a>
+                        {!distributedMode && (
                           <a
                             href="#"
-                            onClick={(e) => { e.preventDefault(); navigate(`/app/model-editor/${encodeURIComponent(model.id)}`) }}
-                            className="btn btn-secondary btn-sm"
-                            style={{ padding: '2px 5px', fontSize: '0.625rem' }}
-                            title="Edit config"
+                            onClick={(e) => { e.preventDefault(); navigate(`/app/backend-logs/${encodeURIComponent(model.id)}`) }}
+                            style={{ fontSize: '0.75rem', color: 'var(--color-primary)' }}
+                            title="Backend logs"
                           >
-                            <i className="fas fa-pen-to-square" />
+                            <i className="fas fa-terminal" />
                           </a>
-                          {!distributedMode && (
-                            <a
-                              href="#"
-                              onClick={(e) => { e.preventDefault(); navigate(`/app/backend-logs/${encodeURIComponent(model.id)}`) }}
-                              className="btn btn-secondary btn-sm"
-                              style={{ padding: '2px 5px', fontSize: '0.625rem' }}
-                              title="Backend logs"
-                            >
-                              <i className="fas fa-terminal" />
-                            </a>
-                          )}
-                        </div>
+                        )}
                       </div>
                     </td>
-                    {/* Status */}
                     <td>
                       {model.disabled ? (
-                        <span className="badge" style={{ background: 'var(--color-bg-tertiary)', color: 'var(--color-text-muted)' }}>
+                        <span className="badge" style={{ background: 'var(--color-danger, #ef4444)', color: 'white' }}>
                           <i className="fas fa-ban" style={{ fontSize: '6px' }} /> Disabled
                         </span>
                       ) : loadedModelIds.has(model.id) ? (
@@ -345,19 +328,17 @@ export default function Manage() {
                         </span>
                       )}
                     </td>
-                    {/* Backend */}
                     <td>
                       <span className="badge badge-info">{model.backend || 'Auto'}</span>
                     </td>
-                    {/* Use Cases */}
                     <td>
                       <div style={{ display: 'flex', gap: '4px', flexWrap: 'wrap' }}>
                         <a href="#" onClick={(e) => { e.preventDefault(); navigate(`/app/chat/${encodeURIComponent(model.id)}`) }} className="badge badge-info" style={{ textDecoration: 'none', cursor: 'pointer' }}>Chat</a>
                       </div>
                     </td>
-                    {/* Actions */}
                     <td>
-                      <div style={{ display: 'flex', gap: 'var(--spacing-xs)', justifyContent: 'flex-end', alignItems: 'center' }}>
+                      <div style={{ display: 'flex', gap: 'var(--spacing-sm)', justifyContent: 'flex-end', alignItems: 'center' }}>
+                        {/* Stop button - shown when model is loaded */}
                         {loadedModelIds.has(model.id) && (
                           <button
                             className="btn btn-secondary btn-sm"
@@ -367,17 +348,62 @@ export default function Manage() {
                             <i className="fas fa-stop" />
                           </button>
                         )}
+                        {/* Pin button - prevents model from being unloaded */}
                         <button
-                          className="btn btn-secondary btn-sm"
+                          className="btn btn-sm"
                           onClick={() => handleTogglePinned(model.id, model.pinned)}
                           disabled={pinningModels.has(model.id) || model.disabled}
                           title={model.pinned ? 'Unpin model (allow idle unloading)' : 'Pin model (prevent idle unloading)'}
                           style={{
-                            color: model.pinned ? 'var(--color-warning)' : undefined,
+                            padding: '2px 6px',
+                            minWidth: 28,
+                            color: model.pinned ? 'var(--color-warning, #f59e0b)' : 'var(--color-text-muted)',
+                            opacity: model.disabled ? 0.3 : (pinningModels.has(model.id) ? 0.5 : 1),
+                            cursor: pinningModels.has(model.id) ? 'wait' : (model.disabled ? 'not-allowed' : 'pointer'),
                           }}
                         >
                           <i className={`fas fa-thumbtack${pinningModels.has(model.id) ? ' fa-spin' : ''}`} />
                         </button>
+                        {/* Toggle switch for enabling/disabling model loading on demand */}
+                        <label
+                          title={model.disabled ? 'Model is disabled — click to enable loading on demand' : 'Model is enabled — click to disable loading on demand'}
+                          style={{
+                            position: 'relative',
+                            display: 'inline-block',
+                            width: 36,
+                            height: 20,
+                            cursor: togglingModels.has(model.id) ? 'wait' : 'pointer',
+                            opacity: togglingModels.has(model.id) ? 0.5 : 1,
+                            flexShrink: 0,
+                          }}
+                        >
+                          <input
+                            type="checkbox"
+                            checked={!model.disabled}
+                            onChange={() => handleToggleModel(model.id, model.disabled)}
+                            disabled={togglingModels.has(model.id)}
+                            style={{ opacity: 0, width: 0, height: 0 }}
+                          />
+                          <span style={{
+                            position: 'absolute',
+                            top: 0, left: 0, right: 0, bottom: 0,
+                            backgroundColor: model.disabled ? 'var(--color-bg-tertiary)' : 'var(--color-success, #22c55e)',
+                            borderRadius: 20,
+                            transition: 'background-color 0.2s',
+                          }}>
+                            <span style={{
+                              position: 'absolute',
+                              content: '""',
+                              height: 14,
+                              width: 14,
+                              left: model.disabled ? 3 : 19,
+                              bottom: 3,
+                              backgroundColor: 'white',
+                              borderRadius: '50%',
+                              transition: 'left 0.2s',
+                            }} />
+                          </span>
+                        </label>
                         <button
                           className="btn btn-danger btn-sm"
                           onClick={() => handleDeleteModel(model.id)}

core/http/react-ui/src/pages/Settings.jsx

@@ -269,9 +269,6 @@ export default function Settings() {
               <SettingRow label="Auto-upgrade Backends" description="Automatically upgrade backends when new versions are detected">
                 <Toggle checked={settings.auto_upgrade_backends} onChange={(v) => update('auto_upgrade_backends', v)} />
               </SettingRow>
-              <SettingRow label="Prefer Development Backends" description="Default to showing development backend versions in the backends gallery">
-                <Toggle checked={settings.prefer_development_backends} onChange={(v) => update('prefer_development_backends', v)} />
-              </SettingRow>
             </div>
           </div>
 

core/http/routes/ui_api.go

@@ -816,7 +816,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			items = "9"
 		}
 
-		backends, err := gallery.AvailableBackendsUnfiltered(appConfig.BackendGalleries, appConfig.SystemState)
+		backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
 		if err != nil {
 			xlog.Error("could not list backends from galleries", "error", err)
 			return c.JSON(http.StatusInternalServerError, map[string]any{
@@ -894,12 +894,6 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			backends = backends.Paginate(pageNum, itemsNum)
 		}
 
-		// Get dev suffix from SystemState for development backend detection
-		devSuffix := ""
-		if appConfig.SystemState != nil {
-			devSuffix = appConfig.SystemState.BackendDevSuffix
-		}
-
 		// Convert backends to JSON-friendly format and deduplicate by ID
 		backendsJSON := make([]map[string]any, 0, len(backends))
 		seenBackendIDs := make(map[string]bool)
@@ -926,21 +920,19 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			}
 
 			backendsJSON = append(backendsJSON, map[string]any{
-				"id":            backendID,
-				"name":          b.Name,
-				"description":   b.Description,
-				"icon":          b.Icon,
-				"license":       b.License,
-				"urls":          b.URLs,
-				"tags":          b.Tags,
-				"gallery":       b.Gallery.Name,
-				"installed":     b.Installed,
-				"version":       b.Version,
-				"processing":    currentlyProcessing,
-				"jobID":         jobID,
-				"isDeletion":    isDeletionOp,
-				"isMeta":        b.IsMeta(),
-				"isDevelopment": b.IsDevelopment(devSuffix),
+				"id":          backendID,
+				"name":        b.Name,
+				"description": b.Description,
+				"icon":        b.Icon,
+				"license":     b.License,
+				"urls":        b.URLs,
+				"tags":        b.Tags,
+				"gallery":     b.Gallery.Name,
+				"installed":   b.Installed,
+				"version":     b.Version,
+				"processing":  currentlyProcessing,
+				"jobID":       jobID,
+				"isDeletion":  isDeletionOp,
 			})
 		}
 
@@ -982,8 +974,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			"totalPages":         totalPages,
 			"prevPage":           prevPage,
 			"nextPage":           nextPage,
-			"systemCapability":          detectedCapability,
-			"preferDevelopmentBackends": appConfig.PreferDevelopmentBackends,
+			"systemCapability":   detectedCapability,
 		})
 	}, adminMiddleware)
 

core/schema/message.go

@@ -83,12 +83,8 @@ func (messages Messages) ToProto() []*proto.Message {
 			}
 		}
 
-		if message.ToolCallID != "" {
-			protoMessages[i].ToolCallId = message.ToolCallID
-		}
-		if message.Reasoning != nil {
-			protoMessages[i].ReasoningContent = *message.Reasoning
-		}
+		// Note: tool_call_id is not in schema.Message yet
+		// Reasoning field is now available in schema.Message but not yet in proto.Message
 	}
 	return protoMessages
 }

core/schema/message_test.go

@@ -237,24 +237,6 @@ var _ = Describe("LLM tests", func() {
 			Expect(protoMessages[0].Content).To(Equal(""))
 		})
 
-		It("should serialize ToolCallID and Reasoning fields", func() {
-			reasoning := "thinking..."
-			messages := Messages{
-				{
-					Role:       "tool",
-					Content:    "result",
-					ToolCallID: "call_123",
-					Reasoning:  &reasoning,
-				},
-			}
-
-			protoMessages := messages.ToProto()
-
-			Expect(protoMessages).To(HaveLen(1))
-			Expect(protoMessages[0].ToolCallId).To(Equal("call_123"))
-			Expect(protoMessages[0].ReasoningContent).To(Equal("thinking..."))
-		})
-
 		It("should handle message with array content containing non-text parts", func() {
 			messages := Messages{
 				{

docs/content/features/GPU-acceleration.md

@@ -151,15 +151,15 @@ llama_init_from_file: kv self size  =  512.00 MB
 
 ## ROCM(AMD) acceleration
 
-There are a limited number of tested configurations for ROCm systems however most newer dedicated GPU consumer grade devices seem to be supported under the current ROCm 7 implementation.
+There are a limited number of tested configurations for ROCm systems however most newer deditated GPU consumer grade devices seem to be supported under the current ROCm6 implementation.
 
 Due to the nature of ROCm it is best to run all implementations in containers as this limits the number of packages required for installation on host system, compatibility and package versions for dependencies across all variations of OS must be tested independently if desired, please refer to the [build]({{%relref "installation/build#Acceleration" %}}) documentation.
 
 ### Requirements
 
-- `ROCm 7.x.x` compatible GPU/accelerator
-- OS: `Ubuntu` (24.04, 22.04), `RHEL` (9.x), `SLES` (15.x)
-- Installed to host: `amdgpu-dkms` and `rocm` >=7.0.0 as per ROCm documentation.
+- `ROCm 6.x.x` compatible GPU/accelerator
+- OS: `Ubuntu` (22.04, 20.04), `RHEL` (9.3, 9.2, 8.9, 8.8), `SLES` (15.5, 15.4)
+- Installed to host: `amdgpu-dkms` and `rocm` >=6.0.0 as per ROCm documentation.
 
 ### Recommendations
 
@@ -171,23 +171,24 @@ Due to the nature of ROCm it is best to run all implementations in containers as
 Ongoing verification testing of ROCm compatibility with integrated backends.
 Please note the following list of verified backends and devices.
 
-LocalAI hipblas images are built against the following targets: gfx908, gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1200, gfx1201
+LocalAI hipblas images are built against the following targets: gfx900,gfx906,gfx908,gfx940,gfx941,gfx942,gfx90a,gfx1030,gfx1031,gfx1100,gfx1101
 
-**Note:** Starting with ROCm 6.4, AMD removed rocBLAS kernel support for older architectures (gfx803, gfx900, gfx906). Since llama.cpp and other backends depend on rocBLAS for matrix operations, these GPUs (e.g. Radeon VII) are no longer supported in pre-built images.
-
-If your device is not one of the above targets, you must specify the corresponding `GPU_TARGETS` and specify `REBUILD=true`. However, rebuilding will not help for architectures that lack rocBLAS kernel support in your ROCm version.
+If your device is not one of these you must specify the corresponding `GPU_TARGETS` and specify `REBUILD=true`. Otherwise you don't need to specify these in the commands below.
 
 ### Verified
 
-The devices in the following list have been tested with `hipblas` images.
+The devices in the following list have been tested with `hipblas` images running `ROCm 6.0.0`
 
 | Backend | Verified | Devices |
 | ---- | ---- | ---- |
-| llama.cpp | yes | MI100 (gfx908), MI210/250 (gfx90a) |
-| diffusers | yes | MI100 (gfx908), MI210/250 (gfx90a) |
+| llama.cpp | yes | Radeon VII (gfx906) |
+| diffusers | yes | Radeon VII (gfx906) |
+| piper | yes | Radeon VII (gfx906) |
 | whisper | no | none |
 | coqui | no | none |
 | transformers | no | none |
+| sentencetransformers | no | none |
+| transformers-musicgen | no | none |
 | vllm | no | none |
 
 **You can help by expanding this list.**
@@ -195,8 +196,8 @@ The devices in the following list have been tested with `hipblas` images.
 ### System Prep
 
 1. Check your GPU LLVM target is compatible with the version of ROCm. This can be found in the [LLVM Docs](https://llvm.org/docs/AMDGPUUsage.html).
-2. Check which ROCm version is compatible with your LLVM target and your chosen OS (pay special attention to supported kernel versions). See the [ROCm compatibility matrix](https://rocm.docs.amd.com/en/latest/compatibility/compatibility-matrix.html).
-3. Install your chosen version of the `dkms` and `rocm` (it is recommended that the native package manager be used for this process for any OS as version changes are executed more easily via this method if updates are required). Take care to restart after installing `amdgpu-dkms` and before installing `rocm`, for details regarding this see the [ROCm installation documentation](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/native-install/index.html).
+2. Check which ROCm version is compatible with your LLVM target and your chosen OS (pay special attention to supported kernel versions). See the following for compatibility for ([ROCm 6.0.0](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.0.0/reference/system-requirements.html)) or ([ROCm 6.0.2](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html))
+3. Install you chosen version of the `dkms` and `rocm` (it is recommended that the native package manager be used for this process for any OS as version changes are executed more easily via this method if updates are required). Take care to restart after installing `amdgpu-dkms` and before installing `rocm`, for details regarding this see the installation documentation for your chosen OS ([6.0.2](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/native-install/index.html) or [6.0.0](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.0.0/how-to/native-install/index.html))
 4. Deploy. Yes it's that easy.
 
 #### Setup Example (Docker/containerd)
@@ -211,7 +212,7 @@ The following are examples of the ROCm specific configuration elements required.
       # If your gpu is not already included in the current list of default targets the following build details are required.
       - REBUILD=true
       - BUILD_TYPE=hipblas
-      - GPU_TARGETS=gfx1100 # Example for RX 7900 XTX
+      - GPU_TARGETS=gfx906 # Example for Radeon VII
     devices:
       # AMD GPU only require the following devices be passed through to the container for offloading to occur.
       - /dev/dri
@@ -225,7 +226,7 @@ docker run \
  -e DEBUG=true \
  -e REBUILD=true \
  -e BUILD_TYPE=hipblas \
- -e GPU_TARGETS=gfx1100 \
+ -e GPU_TARGETS=gfx906 \
  --device /dev/dri \
  --device /dev/kfd \
  quay.io/go-skynet/local-ai:master-gpu-hipblas

docs/content/features/text-generation.md

@@ -539,47 +539,6 @@ options:
 - [llama](https://github.com/ggerganov/llama.cpp)
 
 
-### ik_llama.cpp
-
-[ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) is a hard fork of `llama.cpp` by Iwan Kawrakow that focuses on superior CPU and hybrid GPU/CPU performance. It ships additional quantization types (IQK quants), custom quantization mixes, Multi-head Latent Attention (MLA) for DeepSeek models, and fine-grained tensor offload controls — particularly useful for running very large models on commodity CPU hardware.
-
-{{% notice note %}}
-
-The `ik-llama-cpp` backend requires a CPU with **AVX2** support. The IQK kernels are not compatible with older CPUs.
-
-{{% /notice %}}
-
-#### Features
-
-The `ik-llama-cpp` backend supports the following features:
-- [📖 Text generation (GPT)]({{%relref "features/text-generation" %}})
-- [🧠 Embeddings]({{%relref "features/embeddings" %}})
-- IQK quantization types for better CPU inference performance
-- Multimodal models (via clip/llava)
-
-#### Setup
-
-The backend is distributed as a separate container image and can be installed from the LocalAI backend gallery, or specified directly in a model configuration. GGUF models loaded with this backend benefit from ik_llama.cpp's optimized CPU kernels — especially useful for MoE models and large quantized models that would otherwise be GPU-bound.
-
-#### YAML configuration
-
-To use the `ik-llama-cpp` backend, specify it as the backend in the YAML file:
-
-```yaml
-name: my-model
-backend: ik-llama-cpp
-parameters:
-  # Relative to the models path
-  model: file.gguf
-```
-
-The aliases `ik-llama` and `ik_llama` are also accepted.
-
-#### Reference
-
-- [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp)
-
-
 ### vLLM
 
 [vLLM](https://github.com/vllm-project/vllm) is a fast and easy-to-use library for LLM inference.

docs/content/getting-started/troubleshooting.md

@@ -163,7 +163,7 @@ rocminfo
 docker run --device=/dev/kfd --device=/dev/dri --group-add=video ...
 ```
 
-If your GPU is not in the default target list, open up an Issue. Supported targets include: gfx908, gfx90a, gfx942, gfx950, gfx1030, gfx1100, gfx1101, gfx1102, gfx1200, gfx1201.
+If your GPU is not in the default target list, open up an Issue. Supported targets include: gfx900, gfx906, gfx908, gfx90a, gfx940, gfx941, gfx942, gfx1030, gfx1031, gfx1100, gfx1101.
 
 **Intel (SYCL):**
 

docs/content/reference/compatibility-table.md

@@ -19,7 +19,6 @@ LocalAI will attempt to automatically load models which are not explicitly confi
 | Backend | Description | Capability | Embeddings | Streaming | Acceleration |
 |---------|-------------|------------|------------|-----------|-------------|
 | [llama.cpp](https://github.com/ggerganov/llama.cpp) | LLM inference in C/C++. Supports LLaMA, Mamba, RWKV, Falcon, Starcoder, GPT-2, [and many others](https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#description) | GPT, Functions | yes | yes | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T |
-| [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) | Hard fork of llama.cpp optimized for CPU/hybrid CPU+GPU with IQK quants, custom quant mixes, and MLA for DeepSeek | GPT | yes | yes | CPU (AVX2+) |
 | [vLLM](https://github.com/vllm-project/vllm) | Fast LLM serving with PagedAttention | GPT | no | no | CUDA 12, ROCm, Intel |
 | [vLLM Omni](https://github.com/vllm-project/vllm) | Unified multimodal generation (text, image, video, audio) | Multimodal GPT | no | no | CUDA 12, ROCm |
 | [transformers](https://github.com/huggingface/transformers) | HuggingFace Transformers framework | GPT, Embeddings, Multimodal | yes | yes* | CPU, CUDA 12/13, ROCm, Intel, Metal |

gallery/index.yaml

@@ -150,8 +150,8 @@
       use_tokenizer_template: true
   files:
     - filename: gemma-4-31B-it-Q4_K_M.gguf
+      sha256: 5783acab7217360984ea957abc36f89d35cdeba2f5dd30b0e9e33f9f294bad82
       uri: huggingface://unsloth/gemma-4-31B-it-GGUF/gemma-4-31B-it-Q4_K_M.gguf
-      sha256: 3bf13fff7c0d4deff809b7ab168428f54855f3ef0279339dda278a81ea853474
     - filename: mmproj-F16.gguf
       sha256: 1be2a32013e0d29c6c746513b5f5e7d38f47b694351e74a69c7172acdbbb11a6
       uri: https://huggingface.co/unsloth/gemma-4-31B-it-GGUF/resolve/main/mmproj-F16.gguf
@@ -994,67 +994,6 @@
     - filename: acestep-cpp/vae-BF16.gguf
       uri: huggingface://Serveurperso/ACE-Step-1.5-GGUF/vae-BF16.gguf
       sha256: 0599862ac5d15cd308e1d2e368373aea6c02e25ebd1737ad4a4562a0901b0ef8
-- name: "qwen3-tts-cpp"
-  license: apache-2.0
-  tags:
-    - tts
-    - text-to-speech
-    - qwen3-tts
-    - qwen3-tts-cpp
-    - gguf
-  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
-  urls:
-    - https://huggingface.co/endo5501/qwen3-tts.cpp
-    - https://github.com/predict-woo/qwen3-tts.cpp
-  description: |
-    Qwen3-TTS 0.6B (C++ / GGML) — native C++ text-to-speech from text input.
-    Generates 24kHz mono audio. Supports 10 languages (en, zh, ja, ko, de, fr, es, it, pt, ru).
-    Uses F16 GGUF models (~2 GB total).
-  overrides:
-    name: qwen3-tts-cpp
-    backend: qwen3-tts-cpp
-    parameters:
-      model: qwen3-tts-cpp
-    known_usecases:
-      - tts
-  files:
-    - filename: qwen3-tts-cpp/qwen3-tts-0.6b-f16.gguf
-      uri: huggingface://endo5501/qwen3-tts.cpp/qwen3-tts-0.6b-f16.gguf
-      sha256: 0b89770118463af8f2467d824a8de57d96df6a09f927a9769a3f7b7fffa7087d
-    - filename: qwen3-tts-cpp/qwen3-tts-tokenizer-f16.gguf
-      uri: huggingface://endo5501/qwen3-tts.cpp/qwen3-tts-tokenizer-f16.gguf
-      sha256: d1ad9660bd99343f4851d5a4b17e31f65648feb3559f6ea062ae6575e5cd9d90
-- name: "qwen3-tts-cpp-customvoice"
-  license: apache-2.0
-  tags:
-    - tts
-    - text-to-speech
-    - voice-cloning
-    - qwen3-tts
-    - qwen3-tts-cpp
-    - gguf
-  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
-  urls:
-    - https://huggingface.co/endo5501/qwen3-tts.cpp
-    - https://github.com/predict-woo/qwen3-tts.cpp
-  description: |
-    Qwen3-TTS 0.6B Custom Voice (C++ / GGML) — text-to-speech with voice cloning support.
-    Generates 24kHz mono audio with optional reference audio for voice cloning via ECAPA-TDNN speaker embeddings.
-    Supports 10 languages (en, zh, ja, ko, de, fr, es, it, pt, ru).
-  overrides:
-    name: qwen3-tts-cpp-customvoice
-    backend: qwen3-tts-cpp
-    parameters:
-      model: qwen3-tts-cpp-customvoice
-    known_usecases:
-      - tts
-  files:
-    - filename: qwen3-tts-cpp-customvoice/qwen3-tts-0.6b-customvoice-f16.gguf
-      uri: huggingface://endo5501/qwen3-tts.cpp/qwen3-tts-0.6b-customvoice-f16.gguf
-      sha256: 40b985b71be0970d41eb042488766db556cf17290aa1cff631cabfa0bd3b0431
-    - filename: qwen3-tts-cpp-customvoice/qwen3-tts-tokenizer-f16.gguf
-      uri: huggingface://endo5501/qwen3-tts.cpp/qwen3-tts-tokenizer-f16.gguf
-      sha256: d1ad9660bd99343f4851d5a4b17e31f65648feb3559f6ea062ae6575e5cd9d90
 - name: "qwen3-coder-next-mxfp4_moe"
   url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
   urls:

pkg/model/initializers.go

@@ -14,15 +14,12 @@ import (
 )
 
 const (
-	LLamaCPP   = "llama-cpp"
-	IKLLamaCPP = "ik-llama-cpp"
+	LLamaCPP = "llama-cpp"
 )
 
 var Aliases = map[string]string{
 	"go-llama":               LLamaCPP,
 	"llama":                  LLamaCPP,
-	"ik_llama":               IKLLamaCPP,
-	"ik-llama":               IKLLamaCPP,
 	"embedded-store":         LocalStoreBackend,
 	"huggingface-embeddings": TransformersBackend,
 	"transformers-musicgen":  TransformersBackend,

scripts/build/package-gpu-libs.sh

@@ -198,25 +198,17 @@ package_rocm_libs() {
         fi
     done
 
-    # Copy rocblas library data (tuning files, TensileLibrary, etc.)
+    # Copy rocblas library data (tuning files, etc.)
     local old_nullglob=$(shopt -p nullglob)
     shopt -s nullglob
     local rocm_dirs=(/opt/rocm /opt/rocm-*)
     eval "$old_nullglob"
-    local rocblas_found=false
     for rocm_base in "${rocm_dirs[@]}"; do
-        for lib_subdir in lib lib64; do
-            if [ -d "$rocm_base/$lib_subdir/rocblas" ]; then
-                echo "Found rocblas data at $rocm_base/$lib_subdir/rocblas"
-                mkdir -p "$TARGET_LIB_DIR/rocblas"
-                cp -arfL "$rocm_base/$lib_subdir/rocblas/"* "$TARGET_LIB_DIR/rocblas/" || echo "WARNING: Failed to copy rocblas data from $rocm_base/$lib_subdir/rocblas"
-                rocblas_found=true
-            fi
-        done
+        if [ -d "$rocm_base/lib/rocblas" ]; then
+            mkdir -p "$TARGET_LIB_DIR/rocblas"
+            cp -arfL "$rocm_base/lib/rocblas/"* "$TARGET_LIB_DIR/rocblas/" 2>/dev/null || true
+        fi
     done
-    if [ "$rocblas_found" = false ]; then
-        echo "WARNING: No rocblas library data found in /opt/rocm*/lib{,64}/rocblas"
-    fi
 
     # Copy libomp from LLVM (required for ROCm)
     shopt -s nullglob

scripts/changed-backends.js

@@ -24,9 +24,6 @@ function inferBackendPath(item) {
   if (item.dockerfile.endsWith("rust")) {
     return `backend/rust/${item.backend}/`;
   }
-  if (item.dockerfile.endsWith("ik-llama-cpp")) {
-    return `backend/cpp/ik-llama-cpp/`;
-  }
   if (item.dockerfile.endsWith("llama-cpp")) {
     return `backend/cpp/llama-cpp/`;
   }

swagger/docs.go

@@ -771,57 +771,6 @@ const docTemplate = `{
                 }
             }
         },
-        "/api/models/toggle-pinned/{name}/{action}": {
-            "put": {
-                "description": "Pin or unpin a model. Pinned models stay loaded and are excluded from automatic eviction.",
-                "tags": [
-                    "config"
-                ],
-                "summary": "Toggle model pinned status",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Model name",
-                        "name": "name",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "type": "string",
-                        "description": "Action: 'pin' or 'unpin'",
-                        "name": "action",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    }
-                }
-            }
-        },
         "/api/models/vram-estimate": {
             "post": {
                 "description": "Estimates VRAM based on model weight files, context size, and GPU layers",

swagger/swagger.json

@@ -768,57 +768,6 @@
                 }
             }
         },
-        "/api/models/toggle-pinned/{name}/{action}": {
-            "put": {
-                "description": "Pin or unpin a model. Pinned models stay loaded and are excluded from automatic eviction.",
-                "tags": [
-                    "config"
-                ],
-                "summary": "Toggle model pinned status",
-                "parameters": [
-                    {
-                        "type": "string",
-                        "description": "Model name",
-                        "name": "name",
-                        "in": "path",
-                        "required": true
-                    },
-                    {
-                        "type": "string",
-                        "description": "Action: 'pin' or 'unpin'",
-                        "name": "action",
-                        "in": "path",
-                        "required": true
-                    }
-                ],
-                "responses": {
-                    "200": {
-                        "description": "OK",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    },
-                    "400": {
-                        "description": "Bad Request",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    },
-                    "404": {
-                        "description": "Not Found",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    },
-                    "500": {
-                        "description": "Internal Server Error",
-                        "schema": {
-                            "$ref": "#/definitions/localai.ModelResponse"
-                        }
-                    }
-                }
-            }
-        },
         "/api/models/vram-estimate": {
             "post": {
                 "description": "Estimates VRAM based on model weight files, context size, and GPU layers",

swagger/swagger.yaml

@@ -2241,41 +2241,6 @@ paths:
       summary: Get dynamic autocomplete values for a config field
       tags:
       - config
-  /api/models/toggle-pinned/{name}/{action}:
-    put:
-      description: Pin or unpin a model. Pinned models stay loaded and are excluded
-        from automatic eviction.
-      parameters:
-      - description: Model name
-        in: path
-        name: name
-        required: true
-        type: string
-      - description: 'Action: ''pin'' or ''unpin'''
-        in: path
-        name: action
-        required: true
-        type: string
-      responses:
-        "200":
-          description: OK
-          schema:
-            $ref: '#/definitions/localai.ModelResponse'
-        "400":
-          description: Bad Request
-          schema:
-            $ref: '#/definitions/localai.ModelResponse'
-        "404":
-          description: Not Found
-          schema:
-            $ref: '#/definitions/localai.ModelResponse'
-        "500":
-          description: Internal Server Error
-          schema:
-            $ref: '#/definitions/localai.ModelResponse'
-      summary: Toggle model pinned status
-      tags:
-      - config
   /api/models/vram-estimate:
     post:
       consumes: