feat(gallery): add Ornith 9B Q5 and Q6 variants

Offer intermediate GGUF sizes with the existing vision projector and
llama.cpp settings. Pin downloads and document explicit Q6 selection.

Assisted-by: Codex:gpt-6

.github/backend-matrix.yml

@@ -166,19 +166,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: ''
-    cuda-major-version: ""
-    cuda-minor-version: ""
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-cpu-whisper-medusa'
-    runs-on: 'ubuntu-latest'
-    base-image: "ubuntu:24.04"
-    skip-drivers: 'true'
-    backend: "whisper-medusa"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -367,19 +354,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: 'cublas'
-    cuda-major-version: "12"
-    cuda-minor-version: "8"
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-gpu-nvidia-cuda-12-funasr'
-    runs-on: 'ubuntu-latest'
-    base-image: "ubuntu:24.04"
-    skip-drivers: 'false'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -652,19 +626,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: 'cublas'
-    cuda-major-version: "12"
-    cuda-minor-version: "1"
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-gpu-nvidia-cuda-12-whisper-medusa'
-    runs-on: 'ubuntu-latest'
-    base-image: "ubuntu:24.04"
-    skip-drivers: 'false'
-    backend: "whisper-medusa"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1108,19 +1069,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: 'cublas'
-    cuda-major-version: "13"
-    cuda-minor-version: "0"
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-gpu-nvidia-cuda-13-funasr'
-    runs-on: 'ubuntu-latest'
-    base-image: "ubuntu:24.04"
-    skip-drivers: 'false'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1456,19 +1404,6 @@ include:
     backend: "qwen-asr"
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
-  - build-type: 'l4t'
-    cuda-major-version: "13"
-    cuda-minor-version: "0"
-    platforms: 'linux/arm64'
-    tag-latest: 'auto'
-    tag-suffix: '-nvidia-l4t-cuda-13-arm64-funasr'
-    runs-on: 'ubuntu-24.04-arm'
-    base-image: "ubuntu:24.04"
-    skip-drivers: 'false'
-    ubuntu-version: '2404'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
   - build-type: 'l4t'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -2453,19 +2388,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: 'hipblas'
-    cuda-major-version: ""
-    cuda-minor-version: ""
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-gpu-rocm-hipblas-funasr'
-    runs-on: 'ubuntu-latest'
-    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
-    skip-drivers: 'false'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: 'hipblas'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -2746,19 +2668,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2204'
-  - build-type: 'l4t'
-    cuda-major-version: "12"
-    cuda-minor-version: "0"
-    platforms: 'linux/arm64'
-    tag-latest: 'auto'
-    tag-suffix: '-nvidia-l4t-funasr'
-    runs-on: 'ubuntu-24.04-arm'
-    base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
-    skip-drivers: 'true'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2204'
   - build-type: 'l4t'
     cuda-major-version: "12"
     cuda-minor-version: "0"
@@ -2968,19 +2877,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: 'intel'
-    cuda-major-version: ""
-    cuda-minor-version: ""
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-gpu-intel-funasr'
-    runs-on: 'ubuntu-latest'
-    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
-    skip-drivers: 'false'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: 'intel'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -3312,10 +3208,9 @@ include:
   # consumed. Same reason CUDA needs its toolkit in base-image rather than in a
   # builder image: this is the ds4 shape, not the llama-cpp one.
   #
-  # ROCm uses upstream's HIP backend and the project-wide ROCm 7.2.1 base. No
-  # CUDA arm64 or L4T entry: upstream documents and validates CUDA on x86 only.
-  # Darwin/Metal is in the includeDarwin matrix below, built by
-  # scripts/build/audio-cpp-darwin.sh.
+  # No ROCm entry: upstream has no HIP configuration. No CUDA arm64 or L4T
+  # entry: upstream documents and validates CUDA on x86 only. Darwin/Metal is in
+  # the includeDarwin matrix below, built by scripts/build/audio-cpp-darwin.sh.
   #
   # No vulkan entry either, though Dockerfile.audio-cpp and the backend Makefile
   # both handle BUILD_TYPE=vulkan for local builds. Every other vulkan backend
@@ -3383,19 +3278,6 @@ include:
     dockerfile: "./backend/Dockerfile.audio-cpp"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: 'hipblas'
-    cuda-major-version: ""
-    cuda-minor-version: ""
-    platforms: 'linux/amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-gpu-rocm-hipblas-audio-cpp'
-    runs-on: 'ubuntu-latest'
-    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
-    skip-drivers: 'false'
-    backend: "audio-cpp"
-    dockerfile: "./backend/Dockerfile.audio-cpp"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -6189,20 +6071,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: ''
-    cuda-major-version: ""
-    cuda-minor-version: ""
-    platforms: 'linux/amd64'
-    platform-tag: 'amd64'
-    tag-latest: 'auto'
-    tag-suffix: '-cpu-funasr'
-    runs-on: 'ubuntu-latest'
-    base-image: "ubuntu:24.04"
-    skip-drivers: 'false'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -6217,20 +6085,6 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
-  - build-type: ''
-    cuda-major-version: ""
-    cuda-minor-version: ""
-    platforms: 'linux/arm64'
-    platform-tag: 'arm64'
-    tag-latest: 'auto'
-    tag-suffix: '-cpu-funasr'
-    runs-on: 'ubuntu-24.04-arm'
-    base-image: "ubuntu:24.04"
-    skip-drivers: 'false'
-    backend: "funasr"
-    dockerfile: "./backend/Dockerfile.python"
-    context: "./"
-    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -6470,9 +6324,6 @@ includeDarwin:
   - backend: "mlx"
     tag-suffix: "-metal-darwin-arm64-mlx"
     build-type: "mps"
-  - backend: "mlx-video"
-    tag-suffix: "-metal-darwin-arm64-mlx-video"
-    build-type: "mps"
   - backend: "chatterbox"
     tag-suffix: "-metal-darwin-arm64-chatterbox"
     build-type: "mps"
@@ -6609,9 +6460,6 @@ includeDarwin:
   - backend: "qwen-asr"
     tag-suffix: "-metal-darwin-arm64-qwen-asr"
     build-type: "mps"
-  - backend: "funasr"
-    tag-suffix: "-metal-darwin-arm64-funasr"
-    build-type: "mps"
   - backend: "nemo"
     tag-suffix: "-metal-darwin-arm64-nemo"
     build-type: "mps"

.github/bump_ctranslate2_rocm_wheel.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-# Bump the CTranslate2 ROCm Python wheel release pin used by faster-whisper.
-set -xe
-
-REPO=$1   # OpenNMT/CTranslate2
-FILE=$2   # backend/python/faster-whisper/install.sh
-VAR=$3    # CTRANSLATE2_VERSION (used for output file names so the workflow can read them)
-
-if [ -z "$FILE" ] || [ -z "$REPO" ] || [ -z "$VAR" ]; then
-    echo "usage: $0 <repo> <install-script> <var-name>" >&2
-    exit 1
-fi
-
-LATEST_RELEASE=$(curl -sS -H "Accept: application/vnd.github+json" \
-    "https://api.github.com/repos/$REPO/releases/latest")
-LATEST_TAG=$(python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'])" <<< "$LATEST_RELEASE")
-
-set +e
-CURRENT_VERSION=$(grep -m1 "^${VAR}=" "$FILE" | cut -d= -f2 | sed -E 's/^\$\{[^:]+:-([^}]+)\}$/\1/')
-CTRANSLATE2_ROCM_WHEEL_OS=$(grep -m1 '^CTRANSLATE2_ROCM_WHEEL_OS=' "$FILE" | cut -d= -f2 | sed -E 's/^\$\{[^:]+:-([^}]+)\}$/\1/')
-set -e
-
-if [ -z "$CURRENT_VERSION" ]; then
-    echo "Could not find $VAR in $FILE."
-    exit 0
-fi
-
-if [ -z "$CTRANSLATE2_ROCM_WHEEL_OS" ]; then
-    echo "Could not find CTRANSLATE2_ROCM_WHEEL_OS in $FILE."
-    exit 0
-fi
-
-ASSET_NAME="rocm-python-wheels-${CTRANSLATE2_ROCM_WHEEL_OS}.zip"
-LATEST_RELEASE="$LATEST_RELEASE" python3 - "$ASSET_NAME" <<'PY'
-import json
-import os
-import sys
-
-asset_name = sys.argv[1]
-release = json.loads(os.environ["LATEST_RELEASE"])
-assets = {asset.get("name") for asset in release.get("assets", [])}
-if asset_name not in assets:
-    raise SystemExit(f"Could not find release asset {asset_name!r}")
-PY
-
-sed -i "$FILE" -e "s/^${VAR}=.*/${VAR}=\${${VAR}:-${LATEST_TAG}}/"
-
-echo "Changes: https://github.com/$REPO/compare/${CURRENT_VERSION}...${LATEST_TAG}" >> "${VAR}_message.txt"
-echo "${LATEST_TAG}" >> "${VAR}_commit.txt"

.github/workflows/bump_deps.yaml

@@ -170,39 +170,6 @@ jobs:
           body: ${{ steps.bump.outputs.message }}
           signoff: true
 
-  bump-ctranslate2-rocm-wheel:
-    # CTranslate2's ROCm wheels are published as release assets, so the
-    # faster-whisper hipblas install path pins the release tag used in the URL.
-    if: github.repository == 'mudler/LocalAI'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-      - name: Bump CTranslate2 ROCm wheel pin 🔧
-        id: bump
-        run: |
-          bash .github/bump_ctranslate2_rocm_wheel.sh OpenNMT/CTranslate2 backend/python/faster-whisper/install.sh CTRANSLATE2_VERSION
-          {
-            echo 'message<<EOF'
-            cat "CTRANSLATE2_VERSION_message.txt"
-            echo EOF
-          } >> "$GITHUB_OUTPUT"
-          {
-            echo 'commit<<EOF'
-            cat "CTRANSLATE2_VERSION_commit.txt"
-            echo EOF
-          } >> "$GITHUB_OUTPUT"
-          rm -rfv CTRANSLATE2_VERSION_message.txt CTRANSLATE2_VERSION_commit.txt
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v8
-        with:
-          token: ${{ secrets.UPDATE_BOT_TOKEN }}
-          push-to-fork: ci-forks/LocalAI
-          commit-message: ':arrow_up: Update OpenNMT/CTranslate2 ROCm wheel'
-          title: 'chore: :arrow_up: Update OpenNMT/CTranslate2 ROCm wheel to `${{ steps.bump.outputs.commit }}`'
-          branch: "update/CTRANSLATE2_VERSION"
-          body: ${{ steps.bump.outputs.message }}
-          signoff: true
-
   bump-vllm-wheel:
     # vLLM's cu130 wheel comes from a per-tag index URL (no /latest/ alias),
     # so the cublas13 requirements file pins both a URL segment and a version

.github/workflows/secscan.yaml

@@ -31,7 +31,7 @@ jobs:
         if: ${{ !github.repository.fork && github.actor != 'dependabot[bot]' }}
       - name: Run Gosec Security Scanner
         if: ${{ !github.repository.fork && github.actor != 'dependabot[bot]' }}
-        uses: securego/gosec@v2.29.0
+        uses: securego/gosec@v2.27.1
         with:
           # we let the report trigger content trigger a failure using the GitHub Security features.
           # backend/go/supertonic is excluded: it vendors upstream supertone-inc/supertonic

Makefile

@@ -1,7 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
-.NOTPARALLEL: backends/whisper-medusa
-.NOTPARALLEL: backends/funasr
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -614,7 +612,6 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/vllm
 	$(MAKE) -C backend/python/vllm-omni
 	$(MAKE) -C backend/python/longcat-video
-	$(MAKE) -C backend/python/mlx-video
 	$(MAKE) -C backend/python/sglang
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/liquid-audio
@@ -624,11 +621,9 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/fish-speech
 	$(MAKE) -C backend/python/faster-qwen3-tts
 	$(MAKE) -C backend/python/qwen-asr
-	$(MAKE) -C backend/python/funasr
 	$(MAKE) -C backend/python/nemo
 	$(MAKE) -C backend/python/voxcpm
 	$(MAKE) -C backend/python/faster-whisper
-	$(MAKE) -C backend/python/whisper-medusa
 	$(MAKE) -C backend/python/whisperx
 	$(MAKE) -C backend/python/ace-step
 	$(MAKE) -C backend/python/trl
@@ -649,7 +644,6 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/vllm test
 	$(MAKE) -C backend/python/vllm-omni test
 	$(MAKE) -C backend/python/longcat-video test
-	$(MAKE) -C backend/python/mlx-video test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/liquid-audio test
 	$(MAKE) -C backend/python/moonshine test
@@ -658,11 +652,9 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/fish-speech test
 	$(MAKE) -C backend/python/faster-qwen3-tts test
 	$(MAKE) -C backend/python/qwen-asr test
-	$(MAKE) -C backend/python/funasr test
 	$(MAKE) -C backend/python/nemo test
 	$(MAKE) -C backend/python/voxcpm test
 	$(MAKE) -C backend/python/faster-whisper test
-	$(MAKE) -C backend/python/whisper-medusa test
 	$(MAKE) -C backend/python/whisperx test
 	$(MAKE) -C backend/python/ace-step test
 	$(MAKE) -C backend/python/trl test
@@ -1266,10 +1258,6 @@ backends/mlx:
 	BACKEND=mlx $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx.tar)"
 
-backends/mlx-video:
-	BACKEND=mlx-video $(MAKE) build-darwin-python-backend
-	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx-video.tar)"
-
 backends/diffuser-darwin:
 	BACKEND=diffusers $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/diffusers.tar)"
@@ -1356,7 +1344,6 @@ BACKEND_RERANKERS = rerankers|python|.|false|true
 BACKEND_TRANSFORMERS = transformers|python|.|false|true
 BACKEND_OUTETTS = outetts|python|.|false|true
 BACKEND_FASTER_WHISPER = faster-whisper|python|.|false|true
-BACKEND_WHISPER_MEDUSA = whisper-medusa|python|.|false|true
 BACKEND_COQUI = coqui|python|.|false|true
 BACKEND_RFDETR = rfdetr|python|.|false|true
 BACKEND_INSIGHTFACE = insightface|python|.|false|true
@@ -1378,7 +1365,6 @@ BACKEND_QWEN_TTS = qwen-tts|python|.|false|true
 BACKEND_FISH_SPEECH = fish-speech|python|.|false|true
 BACKEND_FASTER_QWEN3_TTS = faster-qwen3-tts|python|.|false|true
 BACKEND_QWEN_ASR = qwen-asr|python|.|false|true
-BACKEND_FUNASR = funasr|python|.|false|true
 BACKEND_NEMO = nemo|python|.|false|true
 BACKEND_VOXCPM = voxcpm|python|.|false|true
 BACKEND_WHISPERX = whisperx|python|.|false|true
@@ -1449,7 +1435,6 @@ $(eval $(call generate-docker-build-target,$(BACKEND_RERANKERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRANSFORMERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_OUTETTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FASTER_WHISPER)))
-$(eval $(call generate-docker-build-target,$(BACKEND_WHISPER_MEDUSA)))
 $(eval $(call generate-docker-build-target,$(BACKEND_COQUI)))
 $(eval $(call generate-docker-build-target,$(BACKEND_RFDETR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_INSIGHTFACE)))
@@ -1471,7 +1456,6 @@ $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FISH_SPEECH)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FASTER_QWEN3_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_ASR)))
-$(eval $(call generate-docker-build-target,$(BACKEND_FUNASR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_NEMO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
@@ -1501,8 +1485,6 @@ docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
 docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-nemo-speech-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-audio-cpp
-docker-build-backends: docker-build-whisper-medusa
-docker-build-backends: docker-build-funasr
 
 ########################################################
 ### Mock Backend for E2E Tests

backend/Dockerfile.audio-cpp

@@ -6,14 +6,13 @@ ARG APT_PORTS_MIRROR=""
 # ASR, VAD, diarization, source separation and music generation, wrapped as a
 # LocalAI gRPC backend.
 #
-# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds,
-# nvidia/cuda:<ver>-devel-ubuntu24.04 for cublas builds, or
-# rocm/dev-ubuntu-24.04:<ver> for hipblas builds. All ship apt and Ubuntu Noble
-# packages; the GPU bases also provide their toolkits. BUILD_TYPE selects the
-# engine backend in the Makefile: "" = portable CPU with all ggml CPU variants,
-# "cublas" -> -DENGINE_ENABLE_CUDA=ON, "hipblas" -> -DENGINE_ENABLE_HIP=ON,
-# and "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin (Metal) builds bypass this
-# Dockerfile entirely.
+# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds, or
+# nvidia/cuda:<ver>-devel-ubuntu24.04 for cublas builds; both ship apt and
+# Ubuntu Noble packages, and the CUDA base additionally provides
+# /usr/local/cuda. BUILD_TYPE selects the engine backend in the Makefile:
+# "" = portable CPU with all ggml CPU variants, "cublas" ->
+# -DENGINE_ENABLE_CUDA=ON, "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin
+# (Metal) builds bypass this Dockerfile entirely.
 #
 # Upstream needs GCC 13 or newer, which ubuntu:24.04 and the CUDA 12/13
 # devel-ubuntu24.04 images all provide.
@@ -63,7 +62,7 @@ ENV BUILD_TYPE=${BUILD_TYPE} \
     APT_MIRROR=${APT_MIRROR} \
     APT_PORTS_MIRROR=${APT_PORTS_MIRROR} \
     DEBIAN_FRONTEND=noninteractive \
-    PATH=/opt/rocm/bin:/usr/local/cuda/bin:${PATH}
+    PATH=/usr/local/cuda/bin:${PATH}
 
 WORKDIR /build
 
@@ -74,7 +73,7 @@ WORKDIR /build
 # fallback of its own.
 #
 # BUILD_TYPE=vulkan additionally needs the loader headers and glslc; both are in
-# Noble. The CUDA and ROCm toolkits come from their matching BASE_IMAGE.
+# Noble. The CUDA toolkit for BUILD_TYPE=cublas comes from BASE_IMAGE.
 RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
     sh /usr/local/sbin/apt-mirror && \
     apt-get update && \
@@ -84,9 +83,6 @@ RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mi
     if [ "${BUILD_TYPE}" = "vulkan" ]; then \
         apt-get install -y --no-install-recommends libvulkan-dev glslc; \
     fi && \
-    if [ "${BUILD_TYPE}" = "hipblas" ]; then \
-        apt-get install -y --no-install-recommends hipblas-dev rocblas-dev; \
-    fi && \
     if [ "${TARGETARCH}" = "arm64" ]; then \
         apt-get install -y --no-install-recommends gcc-14 g++-14; \
     fi && \

backend/README.md

@@ -47,7 +47,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
 - **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation
-- **Audio**: coqui, faster-whisper, funasr, kitten-tts
+- **Audio**: coqui, faster-whisper, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro
 

backend/backend.proto

@@ -499,10 +499,6 @@ message ModelOptions {
   // applied verbatim to the backend's engine constructor (e.g. vLLM AsyncEngineArgs).
   // Unknown keys produce an error at LoadModel time.
   string EngineArgs = 73;
-  string OriginalConfigFile = 77;
-
-  // EnvVars carries environment variables to be passed to the backend process.
-  map<string, string> EnvVars = 76;
 
   // Proxy carries the cloud-proxy backend's per-model configuration.
   // Empty for non-proxy backends.

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=efb04233dab73aeee4b2912042a90e7b36329061
+AUDIO_CPP_VERSION?=05e508a70e3600b01454c647cdb122133ba8e64c
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -77,16 +77,6 @@ endif
 
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DENGINE_ENABLE_CUDA=ON "-DCMAKE_CUDA_ARCHITECTURES=$(CUDA_ARCHITECTURES)"
-else ifeq ($(BUILD_TYPE),hipblas)
-    ROCM_HOME ?= /opt/rocm
-    ROCM_PATH ?= /opt/rocm
-    export CXX=$(ROCM_HOME)/llvm/bin/clang++
-    export CC=$(ROCM_HOME)/llvm/bin/clang
-    AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
-    # audio.cpp forwards GPU_TARGETS to CMake's semicolon-delimited HIP list.
-    comma := ,
-    HIP_TARGETS := $(subst $(comma),;,$(AMDGPU_TARGETS))
-    CMAKE_ARGS += -DENGINE_ENABLE_HIP=ON "-DGPU_TARGETS=$(HIP_TARGETS)"
 else ifeq ($(BUILD_TYPE),vulkan)
     CMAKE_ARGS += -DENGINE_ENABLE_VULKAN=ON
 else ifeq ($(UNAME_S),Darwin)

backend/cpp/audio-cpp/generation_request.cpp

@@ -22,8 +22,7 @@ bool voice_is_reference_file(const std::string &voice) {
 
 RequestShape build_tts_shape(const backend::TTSRequest &request) {
     RequestShape shape;
-    shape.has_voice_reference = voice_is_reference_file(request.voice()) ||
-                                request.params().find("multi_reference_cond") != request.params().end();
+    shape.has_voice_reference = voice_is_reference_file(request.voice());
     // !empty() as well as has_instructions(), and it must match the guard in
     // build_tts_request: a request whose instructions are an empty string
     // carries no style condition, so telling routing to prefer VoiceDesign for
@@ -135,13 +134,6 @@ build_tts_request(const backend::TTSRequest &request,
         task.options["language"] = request.language();
     }
 
-    // Saved voice profiles send ref_text; Fish Audio reads reference_text.
-    // Derive the alias before copying params so an explicit canonical key wins.
-    const auto reference_text = request.params().find("ref_text");
-    if (reference_text != request.params().end()) {
-        task.options["reference_text"] = reference_text->second;
-    }
-
     // LAST, so an explicit params entry wins over anything derived above. That
     // matters for "caption": a caller who sets params[caption] has named the
     // exact string they want, and it must not be overwritten by `instructions`.

backend/cpp/audio-cpp/generation_request_ctest.cpp

@@ -128,14 +128,6 @@ static void test_tts_shape() {
         check(!shape.has_voice_reference,
               "shape: a preset name is not a voice reference");
     }
-    {
-        backend::TTSRequest request;
-        (*request.mutable_params())["multi_reference_cond"] =
-            R"([{"audio":"one.wav","text":"one"}])";
-        const auto shape = build_tts_shape(request);
-        check(shape.has_voice_reference,
-              "shape: multi-reference conditioning is a voice reference");
-    }
     {
         backend::TTSRequest request;
         request.set_voice(dir.string());
@@ -375,40 +367,6 @@ static void test_tts_language_and_params() {
           "tts params: an explicit param overrides the derived caption");
 }
 
-static void test_tts_reference_transcript() {
-    backend::TTSRequest request;
-    request.set_text("New speech to generate.");
-    request.set_voice("reference.wav");
-    (*request.mutable_params())["ref_text"] = "The saved voice transcript.";
-
-    const auto task = build_tts_request(request, clip(24000, 1));
-    check(option_or(task.options, "reference_text", "") ==
-              "The saved voice transcript.",
-          "tts reference: saved transcript reaches Fish Audio's option");
-    check(option_or(task.options, "ref_text", "") ==
-              "The saved voice transcript.",
-          "tts reference: original transcript parameter is preserved");
-    check(task.text_input->text == "New speech to generate.",
-          "tts reference: transcript does not replace synthesis text");
-
-    (*request.mutable_params())["reference_text"] = "Explicit transcript.";
-    const auto explicit_task = build_tts_request(request, clip(24000, 1));
-    check(option_or(explicit_task.options, "reference_text", "") ==
-              "Explicit transcript.",
-          "tts reference: explicit canonical parameter wins over alias");
-
-    (*request.mutable_params())["reference_text"] = "";
-    const auto empty_task = build_tts_request(request, clip(24000, 1));
-    check(has_key(empty_task.options, "reference_text") &&
-              empty_task.options.at("reference_text").empty(),
-          "tts reference: explicit empty canonical parameter is preserved");
-
-    request.mutable_params()->clear();
-    const auto missing_task = build_tts_request(request, clip(24000, 1));
-    check(!has_key(missing_task.options, "reference_text"),
-          "tts reference: no transcript is invented when none was supplied");
-}
-
 static void test_sound_generation_minimal() {
     backend::SoundGenerationRequest request;
     request.set_text("a distant thunderstorm");
@@ -596,7 +554,6 @@ int main() {
     test_tts_empty_language_is_not_a_language();
     test_tts_clip_and_instructions();
     test_tts_language_and_params();
-    test_tts_reference_transcript();
     test_sound_generation_minimal();
     test_sound_generation_full();
     test_transform_text_absent();

backend/cpp/audio-cpp/loaded_model.cpp

@@ -103,9 +103,6 @@ engine::core::BackendType parse_backend_type(const std::string &value) {
     if (value == "cuda") {
         return engine::core::BackendType::Cuda;
     }
-    if (value == "hip" || value == "rocm") {
-        return engine::core::BackendType::Hip;
-    }
     if (value == "vulkan") {
         return engine::core::BackendType::Vulkan;
     }
@@ -119,7 +116,7 @@ engine::core::BackendType parse_backend_type(const std::string &value) {
         return engine::core::BackendType::Cpu;
     }
     throw ConfigError("audio-cpp: unknown backend option '" + value +
-                      "'. Known backends: cpu, cuda, hip, rocm, vulkan, metal, best");
+                      "'. Known backends: cpu, cuda, vulkan, metal, best");
 }
 
 std::filesystem::path executable_directory() {

backend/cpp/audio-cpp/model_options.cpp

@@ -59,12 +59,6 @@ bool starts_with(const std::string &value, const std::string &prefix) {
            value.compare(0, prefix.size(), prefix) == 0;
 }
 
-bool is_known_backend(const std::string &value) {
-    return value == "cpu" || value == "cuda" || value == "hip" ||
-           value == "rocm" || value == "vulkan" || value == "metal" ||
-           value == "best";
-}
-
 } // namespace
 
 ParsedOptions parse_model_options(const std::vector<std::string> &entries) {
@@ -113,12 +107,6 @@ ParsedOptions parse_model_options(const std::vector<std::string> &entries) {
         } else if (key == "task") {
             parsed.options.task = value;
         } else if (key == "backend") {
-            if (!is_known_backend(value)) {
-                parsed.error = "audio-cpp: unknown backend option '" + value +
-                               "'. Known backends: cpu, cuda, hip, rocm, "
-                               "vulkan, metal, best";
-                return parsed;
-            }
             parsed.options.backend = value;
         } else if (key == "model_spec_override") {
             parsed.options.model_spec_override = value;

backend/cpp/audio-cpp/model_options.h

@@ -17,7 +17,7 @@ struct ModelOptions {
     std::string family;
     // Pins the audio.cpp task, overriding RPC-based routing. Empty means route.
     std::string task;
-    // ggml backend: cpu, cuda, hip (or rocm), vulkan, metal, best.
+    // ggml backend: cpu, cuda, vulkan, metal, best.
     std::string backend = "cpu";
     int device = 0;
     // True once a `device:` entry has been seen. 0 is both the default and a

backend/cpp/audio-cpp/model_options_test.cpp

@@ -75,11 +75,6 @@ static void test_scalar_options() {
     check(parse_model_options({"live_idle_timeout_ms:0"}).options.live_idle_timeout_ms == 0,
           "an explicit 0 turns the live idle limit off rather than reverting to "
           "the default");
-
-    check(parse_model_options({"backend:hip"}).error.empty(),
-          "HIP backend option is accepted");
-    check(parse_model_options({"backend:rocm"}).error.empty(),
-          "ROCm backend alias is accepted");
 }
 
 // Values containing colons must survive: split on the FIRST colon only.
@@ -129,8 +124,6 @@ static void test_errors() {
           "negative device is rejected");
     check(!parse_model_options({"threads:x"}).error.empty(),
           "non-numeric threads is rejected");
-    check(!parse_model_options({"backend:unknown"}).error.empty(),
-          "unknown compute backend is rejected before model loading");
 
     // Values too large for int must be rejected, not silently wrapped into a
     // negative device index that then reaches the ggml backend selector.

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=3bb386eb68ffee0a5dc7db21da0735d594929eeb
+IK_LLAMA_VERSION?=1a2a8604a6c6c6413c06bf9adfc2f64329af4366
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/CMakeLists.txt

@@ -125,15 +125,4 @@ if(LLAMA_GRPC_BUILD_TESTS)
     target_include_directories(thread_params_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_compile_features(thread_params_test PRIVATE cxx_std_17)
     add_test(NAME thread_params_test COMMAND thread_params_test)
-
-    add_executable(model_load_error_test model_load_error_test.cpp model_load_error.h)
-    target_include_directories(model_load_error_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-    target_compile_features(model_load_error_test PRIVATE cxx_std_17)
-    add_test(NAME model_load_error_test COMMAND model_load_error_test)
-
-    # Dead-stream tracker (standard library only).
-    add_executable(stream_peer_test stream_peer_test.cpp stream_peer.h)
-    target_include_directories(stream_peer_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-    target_compile_features(stream_peer_test PRIVATE cxx_std_17)
-    add_test(NAME stream_peer_test COMMAND stream_peer_test)
 endif()

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=df03399b885831b2a1603b3abb0d8c156808e363
+LLAMA_VERSION?=f3f1a8f2760f28325a5ec20c05b171e5b7c83a29
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -53,11 +53,9 @@
 #include "arg.h"
 #include "chat-auto-parser.h"
 #include "llama_compat.h" // fork-skew switches, generated by prepare.sh
-#include "model_load_error.h"
 #include "thread_params.h"
 #include "message_content.h"
 #include "passthrough_options.h"
-#include "stream_peer.h"
 #include "tts_request_options.h"
 #include <getopt.h>
 #include <grpcpp/ext/proto_server_reflection_plugin.h>
@@ -1630,8 +1628,7 @@ public:
             {
                 std::lock_guard<std::mutex> lock(error_capture_data.error_mutex);
                 if (!error_capture_data.captured_error.empty()) {
-                    error_msg += ". Error: " +
-                                 localai::model_load_error_with_hint(error_capture_data.captured_error);
+                    error_msg += ". Error: " + error_capture_data.captured_error;
                 } else {
                     error_msg += ". Model file may not exist or be invalid.";
                 }
@@ -2269,11 +2266,6 @@ public:
         // such concept, so there is nothing to emit — the real tokens arrive in
         // the loop below. Feeding this null into build_reply_from_json would
         // throw (uncaught) and surface as a generic RPC error.
-        // A write that returns false means the peer is gone for good. Track it
-        // so the loop below stops decoding instead of feeding a dead stream —
-        // see stream_peer.h for why that matters to everyone else's requests.
-        llama_grpc::StreamPeer peer;
-
         if (first_res_json.is_null()) {
             // skip the begin-of-stream marker
         } else if (first_res_json.is_array()) {
@@ -2286,21 +2278,17 @@ public:
                 if (!is_role_init) {
                     attach_chat_deltas(reply, first_result.get());
                 }
-                peer.observe_write(writer->Write(reply));
-                if (peer.gone()) {
-                    break;
-                }
+                writer->Write(reply);
             }
         } else {
             auto reply = build_reply_from_json(first_res_json, first_result.get());
             attach_chat_deltas(reply, first_result.get());
-            peer.observe_write(writer->Write(reply));
+            writer->Write(reply);
         }
 
         // Process subsequent results
         while (rd.has_next()) {
-            peer.observe_cancelled(context->IsCancelled());
-            if (peer.gone()) {
+            if (context->IsCancelled()) {
                 break;
             }
 
@@ -2321,22 +2309,17 @@ public:
                     if (!is_role_init) {
                         attach_chat_deltas(reply, result.get());
                     }
-                    peer.observe_write(writer->Write(reply));
-                    if (peer.gone()) {
-                        break;
-                    }
+                    writer->Write(reply);
                 }
             } else {
                 auto reply = build_reply_from_json(res_json, result.get());
                 attach_chat_deltas(reply, result.get());
-                peer.observe_write(writer->Write(reply));
+                writer->Write(reply);
             }
         }
 
-        // Returning here is what releases the slot: ~server_response_reader()
-        // posts SERVER_TASK_TYPE_CANCEL for whatever is still decoding.
-        peer.observe_cancelled(context->IsCancelled());
-        if (peer.gone()) {
+        // Check if context was cancelled during processing
+        if (context->IsCancelled()) {
             return grpc::Status(grpc::StatusCode::CANCELLED, "Request cancelled by client");
         }
 

backend/cpp/llama-cpp/model_load_error.h

@@ -1,24 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-#pragma once
-
-#include <string>
-
-namespace localai {
-
-inline std::string model_load_error_with_hint(const std::string& error) {
-  const std::string mismatch = "wrong number of tensors; expected ";
-  const std::string got = ", got ";
-  const std::string::size_type mismatch_pos = error.find(mismatch);
-  if (mismatch_pos == std::string::npos ||
-      error.find(got, mismatch_pos + mismatch.size()) == std::string::npos) {
-    return error;
-  }
-
-  return error +
-         " Hint: the model may be incompatible with this llama.cpp backend "
-         "or the GGUF file may be corrupt. Try a newer compatible backend "
-         "and verify or re-download the model file.";
-}
-
-}  // namespace localai

backend/cpp/llama-cpp/model_load_error_test.cpp

@@ -1,25 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-#include "model_load_error.h"
-
-#include <cassert>
-#include <string>
-
-int main() {
-  const std::string issue_error =
-      "llama_model_load: error loading model: done_getting_tensors: wrong number of tensors; expected 2131, got 720; "
-      "llama_model_load_from_file_impl: failed to load model";
-  const std::string issue_result = localai::model_load_error_with_hint(issue_error);
-  assert(issue_result.compare(0, issue_error.size(), issue_error) == 0);
-  assert(issue_result.find("incompatible") != std::string::npos);
-  assert(issue_result.find("corrupt") != std::string::npos);
-
-  const std::string generic_error =
-      "wrong number of tensors; expected 42, got 17";
-  const std::string generic_result = localai::model_load_error_with_hint(generic_error);
-  assert(generic_result.compare(0, generic_error.size(), generic_error) == 0);
-  assert(generic_result.size() > generic_error.size());
-
-  const std::string unrelated_error = "failed to open GGUF file";
-  assert(localai::model_load_error_with_hint(unrelated_error) == unrelated_error);
-}

backend/cpp/llama-cpp/prepare.sh

@@ -45,10 +45,6 @@ done
 
 cp -r CMakeLists.txt llama.cpp/tools/grpc-server/
 cp -r grpc-server.cpp llama.cpp/tools/grpc-server/
-# Model-load diagnostics (included by grpc-server.cpp) and their standalone
-# regression test.
-cp -r model_load_error.h llama.cpp/tools/grpc-server/
-cp -r model_load_error_test.cpp llama.cpp/tools/grpc-server/
 # Shared message-reconstruction helpers (included by grpc-server.cpp) and their
 # unit test (compiled only when -DLLAMA_GRPC_BUILD_TESTS=ON).
 cp -r message_content.h llama.cpp/tools/grpc-server/
@@ -67,10 +63,6 @@ cp -r thread_params_test.cpp llama.cpp/tools/grpc-server/
 # test (run via backend/cpp/run-unit-tests.sh; also buildable under ctest).
 cp -r parent_watch.h llama.cpp/tools/grpc-server/
 cp -r parent_watch_test.cpp llama.cpp/tools/grpc-server/
-# Dead-stream tracker (included by grpc-server.cpp) and its standalone unit
-# test (run via backend/cpp/run-unit-tests.sh; also buildable under ctest).
-cp -r stream_peer.h llama.cpp/tools/grpc-server/
-cp -r stream_peer_test.cpp llama.cpp/tools/grpc-server/
 cp -rfv llama.cpp/vendor/nlohmann/json.hpp llama.cpp/tools/grpc-server/
 cp -rfv llama.cpp/vendor/cpp-httplib/httplib.h llama.cpp/tools/grpc-server/
 

backend/cpp/llama-cpp/stream_peer.h

@@ -1,44 +0,0 @@
-// SPDX-License-Identifier: MIT
-#pragma once
-
-namespace llama_grpc {
-
-// Tracks whether a server-streaming RPC still has somewhere to send tokens.
-//
-// grpc::ServerWriter::Write() returns false once the peer is gone, and a
-// stream never recovers afterwards. Ignoring that result is not harmless: the
-// handler goes on draining decoded tokens into a dead stream, so the llama.cpp
-// slot stays busy for the rest of the request's token budget. A model config
-// with no max_tokens and a large context turns that into tens of minutes per
-// abandoned request, and the slots are exactly what every other request queues
-// behind.
-//
-// Returning as soon as the peer is gone is what frees the slot: the handler's
-// server_response_reader then goes out of scope and its destructor posts
-// SERVER_TASK_TYPE_CANCEL for whatever is still decoding.
-class StreamPeer {
-public:
-    // Records the outcome of a Write(). Once a write has failed the peer stays
-    // gone -- a later write cannot succeed on a broken stream.
-    void observe_write(bool ok) noexcept {
-        if (!ok) {
-            gone_ = true;
-        }
-    }
-
-    // Folds in the RPC's own cancellation flag, so callers have a single
-    // predicate to test rather than two that can disagree.
-    void observe_cancelled(bool cancelled) noexcept {
-        if (cancelled) {
-            gone_ = true;
-        }
-    }
-
-    bool gone() const noexcept { return gone_; }
-    bool alive() const noexcept { return !gone_; }
-
-private:
-    bool gone_ = false;
-};
-
-} // namespace llama_grpc

backend/cpp/llama-cpp/stream_peer_test.cpp

@@ -1,67 +0,0 @@
-#include "stream_peer.h"
-
-#include <cstdio>
-
-namespace {
-
-int failures = 0;
-
-void check(bool condition, const char *what) {
-    if (!condition) {
-        std::fprintf(stderr, "FAIL: %s\n", what);
-        ++failures;
-    }
-}
-
-} // namespace
-
-int main() {
-    {
-        llama_grpc::StreamPeer peer;
-        check(peer.alive(), "a fresh peer is alive");
-        check(!peer.gone(), "a fresh peer is not gone");
-    }
-
-    {
-        llama_grpc::StreamPeer peer;
-        peer.observe_write(true);
-        peer.observe_write(true);
-        check(peer.alive(), "successful writes keep the peer alive");
-    }
-
-    {
-        llama_grpc::StreamPeer peer;
-        peer.observe_write(false);
-        check(peer.gone(), "a failed write marks the peer gone");
-    }
-
-    {
-        // The whole point of the guard: a stream never comes back, so a later
-        // success must not resurrect a peer an earlier failure retired.
-        llama_grpc::StreamPeer peer;
-        peer.observe_write(false);
-        peer.observe_write(true);
-        check(peer.gone(), "a failed write is sticky across later writes");
-    }
-
-    {
-        llama_grpc::StreamPeer peer;
-        peer.observe_cancelled(false);
-        check(peer.alive(), "an uncancelled RPC keeps the peer alive");
-        peer.observe_cancelled(true);
-        check(peer.gone(), "cancellation marks the peer gone");
-    }
-
-    {
-        llama_grpc::StreamPeer peer;
-        peer.observe_cancelled(true);
-        peer.observe_cancelled(false);
-        check(peer.gone(), "cancellation is sticky across later checks");
-    }
-
-    if (failures != 0) {
-        std::fprintf(stderr, "%d check(s) failed\n", failures);
-        return 1;
-    }
-    return 0;
-}

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=b68d58624d227682eb4b95ef8bcf569cd1311eb5
+STABLEDIFFUSION_GGML_VERSION?=d04e8950c1ec8d30248cbe996682b3182fb1adf6
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/whisper/addr_test.go

@@ -1,35 +0,0 @@
-package main
-
-import (
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-)
-
-// Specs for resolveAddr run under the suite bootstrap in gowhisper_test.go
-// (TestWhisper); they need no native library, so they never skip.
-var _ = Describe("resolveAddr", func() {
-	It("prefers an explicitly set -addr over a positional argument", func() {
-		Expect(resolveAddr("127.0.0.1:12345", true, []string{"127.0.0.1:59999"})).To(Equal("127.0.0.1:12345"))
-	})
-
-	It("keeps an explicit -addr equal to the default over a positional argument", func() {
-		Expect(resolveAddr(defaultAddr, true, []string{"127.0.0.1:59999"})).To(Equal(defaultAddr))
-	})
-
-	It("falls back to the positional argument when -addr is unset", func() {
-		Expect(resolveAddr(defaultAddr, false, []string{"127.0.0.1:59999"})).To(Equal("127.0.0.1:59999"))
-	})
-
-	It("keeps the default when -addr is unset and no positional argument is given", func() {
-		Expect(resolveAddr(defaultAddr, false, nil)).To(Equal(defaultAddr))
-	})
-
-	It("uses the first positional argument among several", func() {
-		Expect(resolveAddr(defaultAddr, false, []string{"127.0.0.1:59999", "extra"})).To(Equal("127.0.0.1:59999"))
-	})
-
-	It("treats an explicitly empty -addr as unset", func() {
-		Expect(resolveAddr("", true, []string{"127.0.0.1:59999"})).To(Equal("127.0.0.1:59999"))
-		Expect(resolveAddr("", true, nil)).To(Equal(defaultAddr))
-	})
-})

backend/go/whisper/main.go

@@ -10,31 +10,10 @@ import (
 	grpc "github.com/mudler/LocalAI/pkg/grpc"
 )
 
-const (
-	defaultAddr = "localhost:50051"
-)
-
 var (
-	addr = flag.String("addr", defaultAddr, "the address to listen on")
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
 )
 
-// resolveAddr picks the address the gRPC server binds to. An explicitly set
-// -addr always wins. Launchers may hand us the listen address as a bare
-// positional argument, which Go's flag package silently drops — honour it
-// next so the server binds the port its caller actually allocated instead of
-// the default one (#11623). An explicitly empty -addr counts as unset:
-// binding the empty address would listen on an OS-chosen port on every
-// interface instead of the one the caller allocated.
-func resolveAddr(flagAddr string, addrSet bool, args []string) string {
-	if addrSet && flagAddr != "" {
-		return flagAddr
-	}
-	if len(args) > 0 {
-		return args[0]
-	}
-	return defaultAddr
-}
-
 type LibFuncs struct {
 	FuncPtr any
 	Name    string
@@ -82,16 +61,7 @@ func main() {
 
 	flag.Parse()
 
-	// flag.Visit reports only flags that were explicitly set, so an -addr
-	// equal to the default is still distinguished from an untouched one.
-	addrSet := false
-	flag.Visit(func(f *flag.Flag) {
-		if f.Name == "addr" {
-			addrSet = true
-		}
-	})
-
-	if err := grpc.StartServer(resolveAddr(*addr, addrSet, flag.Args()), &Whisper{}); err != nil {
+	if err := grpc.StartServer(*addr, &Whisper{}); err != nil {
 		panic(err)
 	}
 }

backend/index.yaml

@@ -150,7 +150,6 @@
     - audio-transcription
     - CPU
     - CUDA
-    - HIP
     - Metal
   # No vulkan key: the vulkan image would carry a Vulkan loader with no Mesa ICD
   # (see the audio-cpp block in .github/backend-matrix.yml). Pointing a
@@ -162,7 +161,6 @@
     nvidia: "cuda12-audio-cpp"
     nvidia-cuda-12: "cuda12-audio-cpp"
     nvidia-cuda-13: "cuda13-audio-cpp"
-    amd: "rocm-audio-cpp"
     metal: "metal-audio-cpp"
     metal-darwin-arm64: "metal-audio-cpp"
 - &whispercpp
@@ -1062,21 +1060,6 @@
     nvidia-l4t: "nvidia-l4t-mlx"
     nvidia-l4t-cuda-12: "nvidia-l4t-mlx"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-mlx"
-- &mlx-video
-  name: "mlx-video"
-  icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
-  urls:
-    - https://github.com/Blaizzy/mlx-video
-  license: MIT
-  description: |
-      Generate videos with LTX-2 and converted Wan2.1/Wan2.2 checkpoints using
-      MLX on Apple Silicon.
-  tags:
-    - text-to-video
-    - image-to-video
-    - MLX
-  capabilities:
-    metal: "metal-mlx-video"
 - &mlx-vlm
   name: "mlx-vlm"
   icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
@@ -1211,7 +1194,6 @@
   tags:
     - image-generation
     - video-generation
-    - sound-generation
     - diffusion-models
   license: apache-2.0
   alias: "diffusers"
@@ -1536,25 +1518,6 @@
     nvidia-cuda-12: "cuda12-faster-whisper"
     nvidia-l4t: "nvidia-l4t-arm64-faster-whisper"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-faster-whisper"
-- &whisper-medusa
-  description: |
-    Whisper-Medusa accelerates Whisper transcription by predicting multiple tokens per decoding step.
-    The upstream checkpoints use custom Transformers generation code and accept audio clips up to 30 seconds.
-  urls:
-    - https://github.com/aiola-lab/whisper-medusa
-    - https://huggingface.co/collections/aiola/whisper-medusa
-  tags:
-    - speech-to-text
-    - transcription
-    - Whisper
-    - Medusa
-  license: MIT
-  name: "whisper-medusa"
-  alias: "whisper-medusa"
-  capabilities:
-    default: "cpu-whisper-medusa"
-    nvidia: "cuda12-whisper-medusa"
-    nvidia-cuda-12: "cuda12-whisper-medusa"
 - &moonshine
   description: |
     Moonshine is a fast, accurate, and efficient speech-to-text transcription model using ONNX Runtime.
@@ -1822,32 +1785,6 @@
     nvidia-l4t-cuda-12: "nvidia-l4t-faster-qwen3-tts"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-faster-qwen3-tts"
   icon: https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png
-- &funasr
-  urls:
-    - https://github.com/modelscope/FunASR
-  description: |
-    FunASR is an industrial-grade speech recognition toolkit supporting 50+ languages.
-    Includes SenseVoice (170x realtime, emotion detection), Paraformer (highest Chinese
-    accuracy), and built-in VAD, punctuation restoration, and speaker diarization.
-  tags:
-    - speech-recognition
-    - ASR
-    - multilingual
-  license: mit
-  name: "funasr"
-  alias: "funasr"
-  capabilities:
-    nvidia: "cuda12-funasr"
-    intel: "intel-funasr"
-    amd: "rocm-funasr"
-    metal: "metal-funasr"
-    default: "cpu-funasr"
-    nvidia-cuda-13: "cuda13-funasr"
-    nvidia-cuda-12: "cuda12-funasr"
-    nvidia-l4t: "nvidia-l4t-funasr"
-    nvidia-l4t-cuda-12: "nvidia-l4t-funasr"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr"
-  icon: https://avatars.githubusercontent.com/u/109454077
 - &qwen-asr
   urls:
     - https://github.com/QwenLM/Qwen3-ASR
@@ -2164,7 +2101,6 @@
     nvidia: "cuda12-audio-cpp-development"
     nvidia-cuda-12: "cuda12-audio-cpp-development"
     nvidia-cuda-13: "cuda13-audio-cpp-development"
-    amd: "rocm-audio-cpp-development"
     metal: "metal-audio-cpp-development"
     metal-darwin-arm64: "metal-audio-cpp-development"
 - !!merge <<: *stablediffusionggml
@@ -2228,20 +2164,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx"
   mirrors:
     - localai/localai-backends:latest-metal-darwin-arm64-mlx
-- !!merge <<: *mlx-video
-  name: "metal-mlx-video"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-video"
-  mirrors:
-    - localai/localai-backends:latest-metal-darwin-arm64-mlx-video
-- !!merge <<: *mlx-video
-  name: "metal-mlx-video-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-video"
-  mirrors:
-    - localai/localai-backends:master-metal-darwin-arm64-mlx-video
-- !!merge <<: *mlx-video
-  name: "mlx-video-development"
-  capabilities:
-    metal: "metal-mlx-video-development"
 - !!merge <<: *mlx
   name: "metal-mlx-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx"
@@ -6446,34 +6368,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-faster-qwen3-tts"
   mirrors:
     - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-faster-qwen3-tts
-## whisper-medusa
-- !!merge <<: *whisper-medusa
-  name: "whisper-medusa-development"
-  capabilities:
-    default: "cpu-whisper-medusa-development"
-    nvidia: "cuda12-whisper-medusa-development"
-    nvidia-cuda-12: "cuda12-whisper-medusa-development"
-- !!merge <<: *whisper-medusa
-  name: "cpu-whisper-medusa"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-whisper-medusa"
-  mirrors:
-    - localai/localai-backends:latest-cpu-whisper-medusa
-- !!merge <<: *whisper-medusa
-  name: "cpu-whisper-medusa-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-whisper-medusa"
-  mirrors:
-    - localai/localai-backends:master-cpu-whisper-medusa
-- !!merge <<: *whisper-medusa
-  name: "cuda12-whisper-medusa"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-whisper-medusa"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-12-whisper-medusa
-- !!merge <<: *whisper-medusa
-  name: "cuda12-whisper-medusa-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-whisper-medusa"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-12-whisper-medusa
-
 ## qwen-asr
 - !!merge <<: *qwen-asr
   name: "qwen-asr-development"
@@ -7160,16 +7054,6 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-audio-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-audio-cpp
-- !!merge <<: *audiocpp
-  name: "rocm-audio-cpp"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-audio-cpp"
-  mirrors:
-    - localai/localai-backends:latest-gpu-rocm-hipblas-audio-cpp
-- !!merge <<: *audiocpp
-  name: "rocm-audio-cpp-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-audio-cpp"
-  mirrors:
-    - localai/localai-backends:master-gpu-rocm-hipblas-audio-cpp
 - !!merge <<: *audiocpp
   name: "metal-audio-cpp"
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-audio-cpp"
@@ -7180,97 +7064,3 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-audio-cpp"
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-audio-cpp
-## funasr
-- !!merge <<: *funasr
-  name: "funasr-development"
-  capabilities:
-    nvidia: "cuda12-funasr-development"
-    intel: "intel-funasr-development"
-    amd: "rocm-funasr-development"
-    nvidia-l4t: "nvidia-l4t-funasr-development"
-    metal: "metal-funasr-development"
-    default: "cpu-funasr-development"
-    nvidia-cuda-13: "cuda13-funasr-development"
-    nvidia-cuda-12: "cuda12-funasr-development"
-    nvidia-l4t-cuda-12: "nvidia-l4t-funasr-development"
-    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr-development"
-- !!merge <<: *funasr
-  name: "cpu-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-funasr"
-  mirrors:
-    - localai/localai-backends:latest-cpu-funasr
-- !!merge <<: *funasr
-  name: "cpu-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-funasr"
-  mirrors:
-    - localai/localai-backends:master-cpu-funasr
-- !!merge <<: *funasr
-  name: "cuda12-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-funasr"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-12-funasr
-- !!merge <<: *funasr
-  name: "cuda12-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-funasr"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-12-funasr
-- !!merge <<: *funasr
-  name: "cuda13-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-funasr"
-  mirrors:
-    - localai/localai-backends:latest-gpu-nvidia-cuda-13-funasr
-- !!merge <<: *funasr
-  name: "cuda13-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-funasr"
-  mirrors:
-    - localai/localai-backends:master-gpu-nvidia-cuda-13-funasr
-- !!merge <<: *funasr
-  name: "intel-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-funasr"
-  mirrors:
-    - localai/localai-backends:latest-gpu-intel-funasr
-- !!merge <<: *funasr
-  name: "intel-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-funasr"
-  mirrors:
-    - localai/localai-backends:master-gpu-intel-funasr
-- !!merge <<: *funasr
-  name: "rocm-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-funasr"
-  mirrors:
-    - localai/localai-backends:latest-gpu-rocm-hipblas-funasr
-- !!merge <<: *funasr
-  name: "rocm-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-funasr"
-  mirrors:
-    - localai/localai-backends:master-gpu-rocm-hipblas-funasr
-- !!merge <<: *funasr
-  name: "nvidia-l4t-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-funasr"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-funasr
-- !!merge <<: *funasr
-  name: "nvidia-l4t-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-funasr"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-funasr
-- !!merge <<: *funasr
-  name: "cuda13-nvidia-l4t-arm64-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr"
-  mirrors:
-    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr
-- !!merge <<: *funasr
-  name: "cuda13-nvidia-l4t-arm64-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-funasr"
-  mirrors:
-    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-funasr
-- !!merge <<: *funasr
-  name: "metal-funasr"
-  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-funasr"
-  mirrors:
-    - localai/localai-backends:latest-metal-darwin-arm64-funasr
-- !!merge <<: *funasr
-  name: "metal-funasr-development"
-  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr"
-  mirrors:
-    - localai/localai-backends:master-metal-darwin-arm64-funasr

backend/python/README.md

@@ -20,7 +20,6 @@ The Python backends use a unified build system based on `libbackend.sh` that pro
 ### Audio & Speech
 - **coqui** - Coqui TTS models
 - **faster-whisper** - Fast Whisper speech recognition
-- **funasr** - Local multilingual transcription with FunASR and SenseVoice
 - **kitten-tts** - Lightweight TTS
 - **mlx-audio** - Apple Silicon audio processing
 - **chatterbox** - TTS model

backend/python/common/libbackend.sh

@@ -563,18 +563,6 @@ function startBackend() {
         echo "Added ${EDIR}/lib to LD_LIBRARY_PATH for GPU libraries"
     fi
 
-    # XPU wheels carry a matching SYCL/Unified Runtime set. Prefer it over
-    # packaged oneAPI or host libraries, which may expose an older loader ABI.
-    if [ "$(uname -s)" = "Linux" ]; then
-        local sycl_runtime
-        for sycl_runtime in "${EDIR}/venv/lib"/libsycl.so*; do
-            if [ -f "${sycl_runtime}" ]; then
-                export LD_LIBRARY_PATH="${EDIR}/venv/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
-                break
-            fi
-        done
-    fi
-
     if [ ! -z "${BACKEND_FILE:-}" ]; then
         exec "${EDIR}/venv/bin/python" "${BACKEND_FILE}" "$@"
     elif [ -e "${MY_DIR}/server.py" ]; then

backend/python/diffusers/audio_utils.py

@@ -1,24 +0,0 @@
-import array
-import sys
-import wave
-
-
-def write_pcm_wav(destination, samples, sampling_rate):
-    """Write normalized floating-point audio samples as mono 16-bit PCM."""
-    pcm = array.array(
-        "h",
-        (
-            max(-32768, min(32767, round(float(sample) * 32768)))
-            for sample in samples
-        ),
-    )
-    if pcm.itemsize != 2:
-        raise RuntimeError("16-bit PCM requires two-byte signed integers")
-    if sys.byteorder != "little":
-        pcm.byteswap()
-
-    with wave.open(destination, "wb") as output:
-        output.setnchannels(1)
-        output.setsampwidth(2)
-        output.setframerate(sampling_rate)
-        output.writeframes(pcm.tobytes())

backend/python/diffusers/backend.py

@@ -26,7 +26,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
 from model_utils import resolve_model_reference
-from audio_utils import write_pcm_wav
 
 
 # Import dynamic loader for pipeline discovery
@@ -36,7 +35,6 @@ from diffusers_dynamic_loader import (
     get_available_pipelines,
     load_diffusers_pipeline,
 )
-from load_options import single_file_load_kwargs
 
 # Import specific items still needed for special cases and safety checker
 from diffusers import DiffusionPipeline, ControlNetModel
@@ -481,9 +479,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # Build kwargs for dynamic loading
         load_kwargs = {"torch_dtype": torchType}
-        load_kwargs.update(
-            single_file_load_kwargs(request.OriginalConfigFile, from_single_file)
-        )
 
         # Add variant if not loading from single file
         if not from_single_file and variant:
@@ -904,49 +899,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         return backend_pb2.Result(message="Media generated", success=True)
 
-    def SoundGeneration(self, request, context):
-        if not request.dst:
-            return backend_pb2.Result(success=False, message="request.dst is required")
-
-        prompt = request.text or request.caption
-        if not prompt:
-            return backend_pb2.Result(success=False, message="request.text is required")
-
-        try:
-            generation_options = dict(self.options)
-            if "num_inference_steps" in generation_options:
-                generation_options["num_inference_steps"] = int(
-                    generation_options["num_inference_steps"]
-                )
-            generation_options["prompt"] = prompt
-            if request.HasField("duration"):
-                generation_options["audio_length_in_s"] = request.duration
-            if request.HasField("temperature"):
-                generation_options["guidance_scale"] = request.temperature
-
-            generated = self.pipe(**generation_options)
-            if not hasattr(generated, "audios") or len(generated.audios) == 0:
-                return backend_pb2.Result(
-                    success=False,
-                    message="The diffusers pipeline returned no audio",
-                )
-
-            samples = generated.audios[0]
-            if hasattr(samples, "reshape"):
-                samples = samples.reshape(-1)
-            if hasattr(samples, "tolist"):
-                samples = samples.tolist()
-
-            sampling_rate = getattr(
-                getattr(getattr(self.pipe, "vae", None), "config", None),
-                "sampling_rate",
-                16000,
-            )
-            write_pcm_wav(request.dst, samples, sampling_rate)
-            return backend_pb2.Result(success=True, message="Sound generated successfully")
-        except Exception as err:
-            return backend_pb2.Result(success=False, message=f"SoundGeneration error: {err}")
-
     def UpscaleImage(self, request, context):
         try:
             if not request.src:

backend/python/diffusers/load_options.py

@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: MIT
-
-
-def single_file_load_kwargs(original_config_file: str, from_single_file: bool) -> dict:
-    if from_single_file and original_config_file:
-        return {"original_config_file": original_config_file}
-    return {}

backend/python/diffusers/test.py

@@ -4,9 +4,6 @@ A test script to test the gRPC service and dynamic loader
 import unittest
 import subprocess
 import time
-import os
-import tempfile
-import wave
 from unittest.mock import patch, MagicMock
 
 # Import dynamic loader for testing (these don't need gRPC)
@@ -448,64 +445,3 @@ class TestDeviceSelection(unittest.TestCase):
 
     def test_mps_overrides(self):
         self.assertEqual(backend.select_device(False, None, True, False, True), "mps")
-
-
-class TestWritePcmWav(unittest.TestCase):
-    def test_writes_clipped_float_samples_as_mono_pcm(self):
-        from audio_utils import write_pcm_wav
-
-        destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
-        destination.close()
-
-        try:
-            write_pcm_wav(destination.name, [0.0, 0.5, -0.5, 2.0], 16000)
-
-            with wave.open(destination.name, "rb") as generated:
-                self.assertEqual(generated.getframerate(), 16000)
-                self.assertEqual(generated.getnchannels(), 1)
-                self.assertEqual(generated.getsampwidth(), 2)
-                self.assertEqual(generated.getnframes(), 4)
-                self.assertEqual(
-                    generated.readframes(4),
-                    b"\x00\x00\x00@\x00\xc0\xff\x7f",
-                )
-        finally:
-            os.unlink(destination.name)
-
-
-@unittest.skipUnless(GRPC_AVAILABLE, "gRPC modules not available")
-class TestSoundGeneration(unittest.TestCase):
-    def test_maps_request_options_and_writes_pipeline_audio(self):
-        from backend import BackendServicer
-
-        service = BackendServicer.__new__(BackendServicer)
-        service.options = {"num_inference_steps": 200.0}
-        service.pipe = MagicMock()
-        service.pipe.return_value.audios = [[0.0, 0.5, -0.5]]
-        service.pipe.vae.config.sampling_rate = 16000
-
-        destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
-        destination.close()
-
-        try:
-            request = backend_pb2.SoundGenerationRequest(
-                text="ocean waves",
-                dst=destination.name,
-                duration=2.5,
-                temperature=0,
-            )
-
-            result = service.SoundGeneration(request, context=None)
-
-            self.assertTrue(result.success, result.message)
-            service.pipe.assert_called_once_with(
-                num_inference_steps=200,
-                prompt="ocean waves",
-                audio_length_in_s=2.5,
-                guidance_scale=0,
-            )
-            with wave.open(destination.name, "rb") as generated:
-                self.assertEqual(generated.getframerate(), 16000)
-                self.assertEqual(generated.getnframes(), 3)
-        finally:
-            os.unlink(destination.name)

backend/python/diffusers/test_load_options.py

@@ -1,26 +0,0 @@
-# SPDX-License-Identifier: MIT
-
-import unittest
-
-from backend.python.diffusers.load_options import single_file_load_kwargs
-
-
-class SingleFileLoadKwargsTest(unittest.TestCase):
-    def test_includes_original_config_for_single_file(self):
-        self.assertEqual(
-            single_file_load_kwargs("configs/v1-inference.yaml", True),
-            {"original_config_file": "configs/v1-inference.yaml"},
-        )
-
-    def test_omits_original_config_for_pretrained_model(self):
-        self.assertEqual(
-            single_file_load_kwargs("configs/v1-inference.yaml", False),
-            {},
-        )
-
-    def test_omits_empty_original_config(self):
-        self.assertEqual(single_file_load_kwargs("", True), {})
-
-
-if __name__ == "__main__":
-    unittest.main()

backend/python/faster-whisper/install.sh

@@ -26,17 +26,4 @@ if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then
     USE_PIP=true
 fi
 
-CTRANSLATE2_VERSION=${CTRANSLATE2_VERSION:-v4.7.1}
-CTRANSLATE2_ROCM_WHEEL_OS=${CTRANSLATE2_ROCM_WHEEL_OS:-Linux}
-CTRANSLATE2_ROCM_WHEEL_ARCHIVE="rocm-python-wheels-${CTRANSLATE2_ROCM_WHEEL_OS}.zip"
-
-if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then
-  ensureVenv
-  mkdir /tmp/ctranslate2-rocm
-  wget -O "/tmp/ctranslate2-rocm/${CTRANSLATE2_ROCM_WHEEL_ARCHIVE}" "https://github.com/OpenNMT/CTranslate2/releases/download/${CTRANSLATE2_VERSION}/${CTRANSLATE2_ROCM_WHEEL_ARCHIVE}"
-  unzip "/tmp/ctranslate2-rocm/${CTRANSLATE2_ROCM_WHEEL_ARCHIVE}" -d /tmp/ctranslate2-rocm/
-  python3 -m ensurepip
-  python3 -m pip install --no-dependencies --no-index --find-links=/tmp/ctranslate2-rocm/temp-linux/ ctranslate2
-fi
-
 installRequirements

backend/python/fish-speech/Makefile

@@ -1,26 +1,18 @@
-.PHONY: fish-speech test-source-preparation test-runtime-import
-fish-speech: test-source-preparation test-runtime-import
+.PHONY: fish-speech test-source-preparation
+fish-speech: test-source-preparation
 	bash install.sh
 
 test-source-preparation:
 	bash prepare-source_test.sh
 
-test-runtime-import:
-	bash run_test.sh
-
 .PHONY: run
 run: fish-speech
 	@echo "Running fish-speech..."
 	bash run.sh
 	@echo "fish-speech run."
 
-.PHONY: test-unit
-test-unit:
-	python3 -m unittest -v prepare_upstream_test.py
-	bash run_test.sh
-
 .PHONY: test
-test: fish-speech test-unit
+test: fish-speech
 	@echo "Testing fish-speech..."
 	bash test.sh
 	@echo "fish-speech tested."

backend/python/fish-speech/backend.py

@@ -326,29 +326,11 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             max_new_tokens = self.options.get("max_new_tokens", 1024)
             chunk_length = self.options.get("chunk_length", 200)
 
-            # Build references list for voice cloning. Saved LocalAI
-            # personalities use the same ordered JSON shape as audio.cpp.
+            # Build references list for voice cloning
             references = []
             voice_name = request.voice if request.voice else None
 
-            multi_reference_cond = request.params.get("multi_reference_cond", "") if hasattr(request, "params") else ""
-            if multi_reference_cond:
-                reference_entries = json.loads(multi_reference_cond)
-                if not isinstance(reference_entries, list) or not reference_entries:
-                    raise ValueError("multi_reference_cond must be a non-empty JSON array")
-                for entry in reference_entries:
-                    if not isinstance(entry, dict) or not entry.get("audio") or not entry.get("text"):
-                        raise ValueError("multi_reference_cond entries require audio and text")
-                    ref_audio_path = self._get_ref_audio_path(entry["audio"])
-                    with open(ref_audio_path, "rb") as f:
-                        audio_bytes = f.read()
-                    references.append(ServeReferenceAudio(audio=audio_bytes, text=entry["text"]))
-                print(
-                    f"[INFO] Using {len(references)} per-request reference audios",
-                    file=sys.stderr,
-                )
-
-            elif voice_name and os.path.isfile(voice_name):
+            if voice_name and os.path.isfile(voice_name):
                 ref_audio_path = self._get_ref_audio_path(voice_name)
                 with open(ref_audio_path, "rb") as f:
                     audio_bytes = f.read()

backend/python/fish-speech/install.sh

@@ -44,13 +44,6 @@ fi
 # editable install. pyaudio is only used by the upstream playback client.
 bash "${backend_dir}/prepare-source.sh" "${BUILD_TYPE:-}" "${FISH_SPEECH_DIR}/pyproject.toml"
 
-# CUDA 13 has no torch 2.8 wheels, so fish-speech's exact upstream pin would
-# make pip select the CPU-only aarch64 wheel from PyPI. Prepare the cloned tree
-# before resolving it, and use soundfile for reference audio because torchcodec
-# does not publish Linux aarch64 wheels.
-python3 "${backend_dir}/prepare_upstream.py" "${FISH_SPEECH_DIR}" \
-    --cuda-major "${CUDA_MAJOR_VERSION:-}"
-
 # Install fish-speech deps from source (without the package itself since we use PYTHONPATH)
 ensureVenv
 if [ "x${USE_PIP}" == "xtrue" ]; then

backend/python/fish-speech/prepare_upstream.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-
-import argparse
-from pathlib import Path
-
-
-TORCH_28 = '"torch==2.8.0"'
-TORCH_29 = '"torch==2.9.1"'
-TORCHAUDIO_28 = '"torchaudio==2.8.0"'
-TORCHAUDIO_29 = '"torchaudio==2.9.1"'
-TORCHAUDIO_LOAD = (
-    "        waveform, original_sr = "
-    "torchaudio.load(reference_audio, backend=self.backend)"
-)
-SOUNDFILE_LOAD = "\n".join(
-    (
-        "        import soundfile as _sf",
-        "        import torch as _torch",
-        "",
-        "        data, original_sr = _sf.read(",
-        '            reference_audio, dtype="float32", always_2d=True',
-        "        )",
-        "        waveform = _torch.from_numpy(data.T.copy())",
-    )
-)
-
-
-def patch_cuda13_dependencies(pyproject: Path) -> None:
-    content = pyproject.read_text()
-    if (
-        TORCH_28 not in content
-        and TORCHAUDIO_28 not in content
-        and TORCH_29 in content
-        and TORCHAUDIO_29 in content
-    ):
-        return
-    if TORCH_28 not in content or TORCHAUDIO_28 not in content:
-        raise RuntimeError("fish-speech's torch 2.8 dependency pins have changed")
-
-    content = content.replace(TORCH_28, TORCH_29)
-    content = content.replace(TORCHAUDIO_28, TORCHAUDIO_29)
-    pyproject.write_text(content)
-
-
-def patch_reference_loader(loader: Path) -> None:
-    content = loader.read_text()
-    if TORCHAUDIO_LOAD not in content and content.count(SOUNDFILE_LOAD) == 1:
-        return
-    if content.count(TORCHAUDIO_LOAD) != 1:
-        raise RuntimeError("fish-speech's torchaudio.load call has changed")
-
-    loader.write_text(content.replace(TORCHAUDIO_LOAD, SOUNDFILE_LOAD))
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("source", type=Path)
-    parser.add_argument("--cuda-major")
-    args = parser.parse_args()
-
-    if args.cuda_major == "13":
-        patch_cuda13_dependencies(args.source / "pyproject.toml")
-    patch_reference_loader(
-        args.source / "fish_speech/inference_engine/reference_loader.py"
-    )
-
-
-if __name__ == "__main__":
-    main()

backend/python/fish-speech/prepare_upstream_test.py

@@ -1,136 +0,0 @@
-# SPDX-License-Identifier: MIT
-
-import importlib.util
-import sys
-import tempfile
-import types
-import unittest
-from pathlib import Path
-
-
-MODULE_PATH = Path(__file__).with_name("prepare_upstream.py")
-
-
-def load_prepare_upstream():
-    if not MODULE_PATH.exists():
-        raise AssertionError("prepare_upstream.py is missing")
-    spec = importlib.util.spec_from_file_location("prepare_upstream", MODULE_PATH)
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return module
-
-
-class FakeAudioData:
-    @property
-    def T(self):
-        return self
-
-    def copy(self):
-        return "channels-first"
-
-
-class PrepareUpstreamTests(unittest.TestCase):
-    def test_cuda13_dependencies_follow_available_pytorch_wheels(self):
-        prepare_upstream = load_prepare_upstream()
-
-        with tempfile.TemporaryDirectory() as tmp:
-            pyproject = Path(tmp) / "pyproject.toml"
-            pyproject.write_text(
-                'dependencies = [\n    "torch==2.8.0",\n    "torchaudio==2.8.0",\n]\n'
-                'stable = [\n    "torch==2.8.0",\n    "torchaudio",\n]\n'
-            )
-
-            prepare_upstream.patch_cuda13_dependencies(pyproject)
-
-            self.assertEqual(
-                pyproject.read_text(),
-                'dependencies = [\n    "torch==2.9.1",\n    "torchaudio==2.9.1",\n]\n'
-                'stable = [\n    "torch==2.9.1",\n    "torchaudio",\n]\n',
-            )
-
-    def test_reference_audio_uses_soundfile_without_torchcodec(self):
-        prepare_upstream = load_prepare_upstream()
-
-        with tempfile.TemporaryDirectory() as tmp:
-            loader = Path(tmp) / "reference_loader.py"
-            loader.write_text(
-                "class ReferenceLoader:\n"
-                "    def load_audio(self, reference_audio):\n"
-                "        waveform, original_sr = torchaudio.load(reference_audio, backend=self.backend)\n"
-                "        return waveform, original_sr\n"
-            )
-            prepare_upstream.patch_reference_loader(loader)
-
-            calls = []
-            fake_soundfile = types.SimpleNamespace(
-                read=lambda source, **kwargs: (
-                    calls.append((source, kwargs)) or FakeAudioData(),
-                    24000,
-                )
-            )
-            fake_torch = types.SimpleNamespace(
-                from_numpy=lambda data: ("tensor", data),
-            )
-            previous_soundfile = sys.modules.get("soundfile")
-            previous_torch = sys.modules.get("torch")
-            sys.modules["soundfile"] = fake_soundfile
-            sys.modules["torch"] = fake_torch
-            try:
-                namespace = {"torchaudio": None}
-                exec(compile(loader.read_text(), str(loader), "exec"), namespace)
-                instance = namespace["ReferenceLoader"]()
-                instance.backend = "soundfile"
-
-                waveform, sample_rate = instance.load_audio("voice.wav")
-            finally:
-                if previous_soundfile is None:
-                    del sys.modules["soundfile"]
-                else:
-                    sys.modules["soundfile"] = previous_soundfile
-                if previous_torch is None:
-                    del sys.modules["torch"]
-                else:
-                    sys.modules["torch"] = previous_torch
-
-            self.assertEqual(waveform, ("tensor", "channels-first"))
-            self.assertEqual(sample_rate, 24000)
-            self.assertEqual(
-                calls,
-                [("voice.wav", {"dtype": "float32", "always_2d": True})],
-            )
-
-    def test_reference_loader_drift_fails_the_build(self):
-        prepare_upstream = load_prepare_upstream()
-
-        with tempfile.TemporaryDirectory() as tmp:
-            loader = Path(tmp) / "reference_loader.py"
-            loader.write_text("def load_audio():\n    pass\n")
-
-            with self.assertRaisesRegex(RuntimeError, "torchaudio.load call"):
-                prepare_upstream.patch_reference_loader(loader)
-
-    def test_preparation_can_be_repeated(self):
-        prepare_upstream = load_prepare_upstream()
-
-        with tempfile.TemporaryDirectory() as tmp:
-            pyproject = Path(tmp) / "pyproject.toml"
-            pyproject.write_text(
-                'dependencies = ["torch==2.8.0", "torchaudio==2.8.0"]\n'
-            )
-            loader = Path(tmp) / "reference_loader.py"
-            loader.write_text(
-                "def load_audio(reference_audio):\n"
-                "        waveform, original_sr = torchaudio.load(reference_audio, backend=self.backend)\n"
-            )
-
-            prepare_upstream.patch_cuda13_dependencies(pyproject)
-            prepare_upstream.patch_reference_loader(loader)
-            try:
-                prepare_upstream.patch_cuda13_dependencies(pyproject)
-                prepare_upstream.patch_reference_loader(loader)
-            except RuntimeError as err:
-                self.fail(f"preparation is not idempotent: {err}")
-
-
-if __name__ == "__main__":
-    unittest.main()

backend/python/fish-speech/requirements-cublas13.txt

@@ -1,3 +1,3 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
-torch==2.9.1+cu130
-torchaudio==2.9.1
+torch
+torchaudio

backend/python/fish-speech/requirements-l4t13.txt

@@ -1,3 +1,3 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
-torch==2.9.1+cu130
-torchaudio==2.9.1
+torch
+torchaudio

backend/python/fish-speech/run.sh

@@ -1,18 +1,9 @@
 #!/bin/bash
-backend_dir=$(dirname "$(realpath "$0")")
+backend_dir=$(dirname $0)
 if [ -d $backend_dir/common ]; then
     source $backend_dir/common/libbackend.sh
 else
     source $backend_dir/../common/libbackend.sh
 fi
 
-# Editable installs record their build-time absolute source path, which becomes
-# stale when the backend is relocated under /backends at install time.
-export PYTHONPATH="${backend_dir}/fish-speech-src${PYTHONPATH:+:${PYTHONPATH}}"
-
-cuda_home=${CUDA_HOME:-/usr/local/cuda}
-if [ -z "${TRITON_PTXAS_PATH:-}" ] && [ -x "$cuda_home/bin/ptxas" ]; then
-    export TRITON_PTXAS_PATH="$cuda_home/bin/ptxas"
-fi
-
-startBackend "$@"
+startBackend $@

backend/python/fish-speech/run_test.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-SCRIPT_DIR=$(dirname "$(realpath "$0")")
-WORK_DIR=$(mktemp -d)
-trap 'rm -rf "$WORK_DIR"' EXIT
-
-mkdir -p "$WORK_DIR/runtime/common" "$WORK_DIR/runtime/fish-speech-src/fish_speech"
-cp "$SCRIPT_DIR/run.sh" "$WORK_DIR/runtime/run.sh"
-
-cat > "$WORK_DIR/runtime/fish-speech-src/fish_speech/inference_engine.py" <<'PY'
-IMPORT_OK = True
-PY
-
-cat > "$WORK_DIR/runtime/common/libbackend.sh" <<'SH'
-startBackend() {
-    python3 -c 'from fish_speech.inference_engine import IMPORT_OK; assert IMPORT_OK'
-}
-SH
-
-bash "$WORK_DIR/runtime/run.sh"
-
-echo "PASS: runtime launcher imports relocated fish-speech source"

backend/python/funasr/Makefile

@@ -1,23 +0,0 @@
-.PHONY: funasr
-funasr:
-	bash install.sh
-
-.PHONY: run
-run: funasr
-	@echo "Running funasr..."
-	bash run.sh
-	@echo "funasr run."
-
-.PHONY: test
-test: funasr
-	@echo "Testing funasr..."
-	bash test.sh
-	@echo "funasr tested."
-
-.PHONY: protogen-clean
-protogen-clean:
-	$(RM) backend_pb2_grpc.py backend_pb2.py
-
-.PHONY: clean
-clean: protogen-clean
-	rm -rf venv __pycache__

backend/python/funasr/backend.py

@@ -1,152 +0,0 @@
-#!/usr/bin/env python3
-"""
-gRPC backend for LocalAI wrapping FunASR (SenseVoice / Paraformer).
-"""
-from concurrent import futures
-import time
-import argparse
-import signal
-import sys
-import os
-import backend_pb2
-import backend_pb2_grpc
-import torch
-
-import grpc
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
-from grpc_auth import get_auth_interceptors
-
-
-_ONE_DAY_IN_SECONDS = 60 * 60 * 24
-MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
-
-
-class BackendServicer(backend_pb2_grpc.BackendServicer):
-    def Health(self, request, context):
-        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
-
-    def LoadModel(self, request, context):
-        from funasr import AutoModel
-
-        device = "cpu"
-        if request.CUDA and torch.cuda.is_available():
-            device = "cuda"
-        elif hasattr(torch, "xpu") and torch.xpu.is_available():
-            device = "xpu"
-        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
-            device = "mps"
-
-        model_id = request.Model or "iic/SenseVoiceSmall"
-        candidate_devices = [device]
-        if device in ("xpu", "mps"):
-            candidate_devices.append("cpu")
-
-        for candidate_device in candidate_devices:
-            try:
-                print(
-                    f"Loading FunASR model: {model_id} on {candidate_device}",
-                    file=sys.stderr,
-                )
-                self.model = AutoModel(
-                    model=model_id,
-                    vad_model="fsmn-vad",
-                    device=candidate_device,
-                    disable_update=True,
-                )
-                print("FunASR model loaded successfully", file=sys.stderr)
-                break
-            except Exception as err:
-                if candidate_device != candidate_devices[-1]:
-                    print(
-                        f"[WARN] FunASR {candidate_device} initialization failed: "
-                        f"{err}; retrying on cpu",
-                        file=sys.stderr,
-                    )
-                    continue
-                print(f"[ERROR] LoadModel failed: {err}", file=sys.stderr)
-                import traceback
-                traceback.print_exc(file=sys.stderr)
-                return backend_pb2.Result(success=False, message=str(err))
-
-        return backend_pb2.Result(message="Model loaded successfully", success=True)
-
-    def AudioTranscription(self, request, context):
-        from funasr.utils.postprocess_utils import rich_transcription_postprocess
-
-        result_segments = []
-        text = ""
-        try:
-            audio_path = request.dst
-            if not audio_path or not os.path.exists(audio_path):
-                print(f"Error: Audio file not found: {audio_path}", file=sys.stderr)
-                return backend_pb2.TranscriptResult(segments=[], text="")
-
-            language = None
-            if request.language and request.language.strip():
-                language = request.language.strip()
-
-            kwargs = {}
-            if language:
-                kwargs["language"] = language
-
-            results = self.model.generate(input=audio_path, **kwargs)
-
-            if not results:
-                return backend_pb2.TranscriptResult(segments=[], text="")
-
-            for idx, r in enumerate(results):
-                seg_text = r.get("text", "") if isinstance(r, dict) else str(r)
-                seg_text = rich_transcription_postprocess(seg_text)
-                text += seg_text
-                result_segments.append(backend_pb2.TranscriptSegment(
-                    id=idx,
-                    start=0,
-                    end=0,
-                    text=seg_text,
-                ))
-
-        except Exception as err:
-            print(f"Error in AudioTranscription: {err}", file=sys.stderr)
-            import traceback
-            traceback.print_exc(file=sys.stderr)
-            return backend_pb2.TranscriptResult(segments=[], text="")
-
-        return backend_pb2.TranscriptResult(segments=result_segments, text=text)
-
-
-def serve(address):
-    server = grpc.server(
-        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
-        options=[
-            ('grpc.max_message_length', 50 * 1024 * 1024),
-            ('grpc.max_send_message_length', 50 * 1024 * 1024),
-            ('grpc.max_receive_message_length', 50 * 1024 * 1024),
-        ],
-        interceptors=get_auth_interceptors(),
-    )
-    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
-    server.add_insecure_port(address)
-    server.start()
-    print("Server started. Listening on: " + address, file=sys.stderr)
-
-    def signal_handler(sig, frame):
-        print("Received termination signal. Shutting down...")
-        server.stop(0)
-        sys.exit(0)
-
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    try:
-        while True:
-            time.sleep(_ONE_DAY_IN_SECONDS)
-    except KeyboardInterrupt:
-        server.stop(0)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the gRPC server.")
-    parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.")
-    args = parser.parse_args()
-    serve(args.addr)

backend/python/funasr/install.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation"
-
-backend_dir=$(dirname $0)
-if [ -d $backend_dir/common ]; then
-    source $backend_dir/common/libbackend.sh
-else
-    source $backend_dir/../common/libbackend.sh
-fi
-
-if [ "x${BUILD_PROFILE}" == "xintel" ]; then
-    EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
-fi
-
-PYTHON_VERSION="3.12"
-PYTHON_PATCH="12"
-PY_STANDALONE_TAG="20251120"
-
-installRequirements

backend/python/funasr/requirements-cpu.txt

@@ -1,4 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cpu
-torch
-torchaudio
-funasr

backend/python/funasr/requirements-cublas12.txt

@@ -1,4 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu121
-torch
-torchaudio
-funasr

backend/python/funasr/requirements-cublas13.txt

@@ -1,4 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
-funasr

backend/python/funasr/requirements-hipblas.txt

@@ -1,4 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/rocm7.0
-torch
-torchaudio
-funasr

backend/python/funasr/requirements-intel.txt

@@ -1,4 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/xpu
-torch
-torchaudio
-funasr

backend/python/funasr/requirements-l4t12.txt

@@ -1,4 +0,0 @@
---extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu129/
-torch
-torchaudio
-funasr

backend/python/funasr/requirements-l4t13.txt

@@ -1,4 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
-funasr

backend/python/funasr/requirements-mps.txt

@@ -1,3 +0,0 @@
-torch==2.7.1
-torchaudio==2.7.1
-funasr

backend/python/funasr/requirements.txt

@@ -1,5 +0,0 @@
-grpcio==1.71.0
-protobuf
-certifi
-packaging==24.1
-setuptools

backend/python/funasr/run.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-backend_dir=$(dirname $0)
-if [ -d $backend_dir/common ]; then
-    source $backend_dir/common/libbackend.sh
-else
-    source $backend_dir/../common/libbackend.sh
-fi
-
-startBackend $@

backend/python/funasr/test.py

@@ -1,295 +0,0 @@
-import importlib
-import os
-import re
-import sys
-import types
-import unittest
-
-
-class _Reply:
-    def __init__(self, message=b""):
-        self.message = message
-
-
-class _Result:
-    def __init__(self, message="", success=False):
-        self.message = message
-        self.success = success
-
-
-class _TranscriptSegment:
-    def __init__(self, id=0, start=0, end=0, text=""):
-        self.id = id
-        self.start = start
-        self.end = end
-        self.text = text
-
-
-class _TranscriptResult:
-    def __init__(self, segments=None, text=""):
-        self.segments = segments or []
-        self.text = text
-
-
-class _FakeBackendServicer:
-    pass
-
-
-class _FakeTorch:
-    cuda = types.SimpleNamespace(is_available=lambda: False)
-    xpu = types.SimpleNamespace(is_available=lambda: False)
-    backends = types.SimpleNamespace(mps=types.SimpleNamespace(is_available=lambda: False))
-
-
-class _FakeAutoModel:
-    instances = []
-    attempts = []
-    fail_devices = set()
-
-    def __init__(self, **kwargs):
-        _FakeAutoModel.attempts.append(kwargs)
-        if kwargs.get("device") in _FakeAutoModel.fail_devices:
-            raise RuntimeError(f"unsupported device: {kwargs['device']}")
-        self.kwargs = kwargs
-        self.generate_calls = []
-        self.results = [{"text": "hello"}, {"text": " world"}]
-        _FakeAutoModel.instances.append(self)
-
-    def generate(self, **kwargs):
-        self.generate_calls.append(kwargs)
-        return self.results
-
-
-def _install_stubs():
-    sys.modules["backend_pb2"] = types.SimpleNamespace(
-        Reply=_Reply,
-        Result=_Result,
-        TranscriptSegment=_TranscriptSegment,
-        TranscriptResult=_TranscriptResult,
-    )
-    sys.modules["backend_pb2_grpc"] = types.SimpleNamespace(
-        BackendServicer=_FakeBackendServicer,
-        add_BackendServicer_to_server=lambda *args, **kwargs: None,
-    )
-    sys.modules["grpc"] = types.SimpleNamespace(
-        RpcMethodHandler=object,
-        ServerInterceptor=object,
-        StatusCode=types.SimpleNamespace(UNAUTHENTICATED="UNAUTHENTICATED"),
-        aio=types.SimpleNamespace(ServerInterceptor=object),
-        server=lambda *args, **kwargs: None,
-    )
-    sys.modules["torch"] = _FakeTorch
-    funasr = types.ModuleType("funasr")
-    funasr.__path__ = []
-    funasr.AutoModel = _FakeAutoModel
-    funasr_utils = types.ModuleType("funasr.utils")
-    funasr_utils.__path__ = []
-    postprocess_utils = types.ModuleType("funasr.utils.postprocess_utils")
-    postprocess_utils.rich_transcription_postprocess = lambda text: re.sub(
-        r"<\|.*?\|>", "", text
-    )
-    sys.modules["funasr"] = funasr
-    sys.modules["funasr.utils"] = funasr_utils
-    sys.modules["funasr.utils.postprocess_utils"] = postprocess_utils
-
-
-def _load_backend():
-    _install_stubs()
-    sys.modules.pop("backend", None)
-    _FakeAutoModel.instances.clear()
-    _FakeAutoModel.attempts.clear()
-    _FakeAutoModel.fail_devices.clear()
-    return importlib.import_module("backend")
-
-
-class TestFunASRBackend(unittest.TestCase):
-    def test_torch_profiles_install_torchaudio(self):
-        backend_dir = os.path.dirname(__file__)
-
-        for profile in (
-            "cpu",
-            "cublas12",
-            "cublas13",
-            "hipblas",
-            "intel",
-            "l4t12",
-            "l4t13",
-            "mps",
-        ):
-            with self.subTest(profile=profile):
-                requirements_path = os.path.join(
-                    backend_dir, f"requirements-{profile}.txt"
-                )
-                with open(requirements_path, encoding="utf-8") as requirements_file:
-                    requirements = {
-                        line.strip().split("=", 1)[0]
-                        for line in requirements_file
-                        if line.strip() and not line.lstrip().startswith(("#", "--"))
-                    }
-
-                self.assertIn("torchaudio", requirements)
-
-    def test_accelerator_profiles_use_current_pytorch_indexes(self):
-        backend_dir = os.path.dirname(__file__)
-        expected_indexes = {
-            "cublas13": "https://download.pytorch.org/whl/cu130",
-            "hipblas": "https://download.pytorch.org/whl/rocm7.0",
-            "intel": "https://download.pytorch.org/whl/xpu",
-            "l4t12": "https://pypi.jetson-ai-lab.io/jp6/cu129/",
-            "l4t13": "https://download.pytorch.org/whl/cu130",
-        }
-
-        for profile, expected_index in expected_indexes.items():
-            with self.subTest(profile=profile):
-                requirements_path = os.path.join(
-                    backend_dir, f"requirements-{profile}.txt"
-                )
-                with open(requirements_path, encoding="utf-8") as requirements_file:
-                    requirements = requirements_file.read()
-
-                self.assertIn(
-                    f"--extra-index-url {expected_index}\n",
-                    requirements,
-                )
-
-    def test_health_returns_ok(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-
-        reply = servicer.Health(types.SimpleNamespace(), None)
-
-        self.assertEqual(reply.message, b"OK")
-
-    def test_load_model_uses_default_sensevoice_model_on_cpu(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-
-        result = servicer.LoadModel(types.SimpleNamespace(Model="", CUDA=False), None)
-
-        self.assertTrue(result.success, result.message)
-        self.assertEqual(result.message, "Model loaded successfully")
-        self.assertEqual(_FakeAutoModel.instances[0].kwargs["model"], "iic/SenseVoiceSmall")
-        self.assertEqual(_FakeAutoModel.instances[0].kwargs["vad_model"], "fsmn-vad")
-        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
-        self.assertTrue(_FakeAutoModel.instances[0].kwargs["disable_update"])
-
-    def test_load_model_uses_xpu_when_available(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-        original_xpu = _FakeTorch.xpu
-        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
-        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
-
-        result = servicer.LoadModel(
-            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
-        )
-
-        self.assertTrue(result.success, result.message)
-        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "xpu")
-
-    def test_load_model_prefers_requested_cuda_over_xpu_and_mps(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-        original_cuda = _FakeTorch.cuda
-        original_xpu = _FakeTorch.xpu
-        original_mps = _FakeTorch.backends.mps
-        _FakeTorch.cuda = types.SimpleNamespace(is_available=lambda: True)
-        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
-        _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True)
-        self.addCleanup(setattr, _FakeTorch, "cuda", original_cuda)
-        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
-        self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps)
-
-        result = servicer.LoadModel(
-            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=True), None
-        )
-
-        self.assertTrue(result.success, result.message)
-        self.assertEqual(
-            [attempt["device"] for attempt in _FakeAutoModel.attempts], ["cuda"]
-        )
-
-    def test_load_model_retries_cpu_when_xpu_initialization_fails(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-        original_xpu = _FakeTorch.xpu
-        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
-        _FakeAutoModel.fail_devices.add("xpu")
-        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
-
-        result = servicer.LoadModel(
-            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
-        )
-
-        self.assertTrue(result.success, result.message)
-        self.assertEqual(
-            [attempt["device"] for attempt in _FakeAutoModel.attempts],
-            ["xpu", "cpu"],
-        )
-        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
-
-    def test_load_model_retries_cpu_when_mps_initialization_fails(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-        original_mps = _FakeTorch.backends.mps
-        _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True)
-        _FakeAutoModel.fail_devices.add("mps")
-        self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps)
-
-        result = servicer.LoadModel(
-            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
-        )
-
-        self.assertTrue(result.success, result.message)
-        self.assertEqual(
-            [attempt["device"] for attempt in _FakeAutoModel.attempts],
-            ["mps", "cpu"],
-        )
-        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
-
-    def test_audio_transcription_passes_language_and_builds_segments(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-        servicer.model = _FakeAutoModel()
-        audio_path = os.path.abspath(__file__)
-        request = types.SimpleNamespace(dst=audio_path, language=" zh ")
-
-        result = servicer.AudioTranscription(request, None)
-
-        self.assertEqual(result.text, "hello world")
-        self.assertEqual([segment.text for segment in result.segments], ["hello", " world"])
-        self.assertEqual(servicer.model.generate_calls, [{"input": audio_path, "language": "zh"}])
-
-    def test_audio_transcription_cleans_sensevoice_tags(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-        servicer.model = _FakeAutoModel()
-        servicer.model.results = [
-            {"text": "<|zh|><|NEUTRAL|><|Speech|><|woitn|>hello"}
-        ]
-
-        result = servicer.AudioTranscription(
-            types.SimpleNamespace(dst=os.path.abspath(__file__), language=""),
-            None,
-        )
-
-        self.assertEqual(result.text, "hello")
-        self.assertEqual([segment.text for segment in result.segments], ["hello"])
-
-    def test_audio_transcription_missing_file_returns_empty_result(self):
-        backend = _load_backend()
-        servicer = backend.BackendServicer()
-        servicer.model = _FakeAutoModel()
-
-        result = servicer.AudioTranscription(
-            types.SimpleNamespace(dst="/tmp/localai-funasr-missing.wav", language=""),
-            None,
-        )
-
-        self.assertEqual(result.text, "")
-        self.assertEqual(result.segments, [])
-        self.assertEqual(servicer.model.generate_calls, [])
-
-
-if __name__ == "__main__":
-    unittest.main()

backend/python/funasr/test.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-set -e
-
-backend_dir=$(dirname $0)
-if [ -d $backend_dir/common ]; then
-    source $backend_dir/common/libbackend.sh
-else
-    source $backend_dir/../common/libbackend.sh
-fi
-
-runUnittests "$@"

backend/python/mlx-distributed/backend.py

@@ -440,9 +440,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     kwargs["tools"] = json.loads(request.Tools)
                 except json.JSONDecodeError:
                     pass
-            enable_thinking = request.Metadata.get("enable_thinking", "").lower()
-            if enable_thinking in ("true", "false"):
-                kwargs["enable_thinking"] = enable_thinking == "true"
+            if request.Metadata.get("enable_thinking", "").lower() == "true":
+                kwargs["enable_thinking"] = True
             try:
                 return self.tokenizer.apply_chat_template(messages, **kwargs)
             except TypeError:

backend/python/mlx-distributed/test.py

@@ -135,37 +135,3 @@ class TestSharedHelpers(unittest.TestCase):
         self.assertEqual(len(calls), 1)
         self.assertEqual(calls[0]["name"], "get_weather")
         self.assertEqual(calls[0]["arguments"], '{"location": "Paris"}')
-
-
-class TestPreparePrompt(unittest.TestCase):
-    """Server-less tests for BackendServicer._prepare_prompt."""
-
-    def test_forwards_enable_thinking(self):
-        from backend import BackendServicer
-
-        class Tok:
-            def __init__(self):
-                self.kwargs = None
-
-            def apply_chat_template(self, messages, **kwargs):
-                self.kwargs = kwargs
-                return "PROMPT"
-
-        def kwargs_for(metadata):
-            servicer = BackendServicer()
-            servicer.tokenizer = Tok()
-            req = types.SimpleNamespace(
-                Prompt="",
-                UseTokenizerTemplate=True,
-                Messages=[backend_pb2.Message(role="user", content="hi")],
-                Tools="",
-                Metadata=metadata,
-            )
-            self.assertEqual(servicer._prepare_prompt(req), "PROMPT")
-            return servicer.tokenizer.kwargs
-
-        self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True)
-        # "false" used to be dropped, so thinking models kept reasoning
-        self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False)
-        self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False)
-        self.assertNotIn("enable_thinking", kwargs_for({}))

backend/python/mlx-video/.gitignore

@@ -1,5 +0,0 @@
-__pycache__/
-backend_pb2.py
-backend_pb2_grpc.py
-common/
-venv/

backend/python/mlx-video/Makefile

@@ -1,16 +0,0 @@
-.PHONY: all run test protogen-clean clean
-
-all:
-	bash install.sh
-
-run: all
-	bash run.sh
-
-test:
-	bash test.sh
-
-protogen-clean:
-	$(RM) backend_pb2.py backend_pb2_grpc.py
-
-clean: protogen-clean
-	rm -rf __pycache__ venv

backend/python/mlx-video/backend.py

@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-
-import argparse
-import os
-import platform
-import signal
-import subprocess
-import sys
-import traceback
-from concurrent import futures
-
-import grpc
-
-import backend_pb2
-import backend_pb2_grpc
-from mlx_video_utils import build_generation_command, classify_model, parse_options
-
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common"))
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common"))
-from grpc_auth import get_auth_interceptors  # noqa: E402
-
-
-MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
-
-
-class BackendServicer(backend_pb2_grpc.BackendServicer):
-    def __init__(self):
-        self.family = None
-        self.model = None
-        self.options = {}
-
-    def Health(self, request, context):
-        return backend_pb2.Reply(message=b"OK")
-
-    def LoadModel(self, request, context):
-        if platform.system() != "Darwin" or platform.machine() != "arm64":
-            return self._fail(
-                context,
-                grpc.StatusCode.FAILED_PRECONDITION,
-                "mlx-video requires macOS on Apple Silicon",
-            )
-
-        model = (
-            request.ModelFile
-            if request.ModelFile and os.path.isdir(request.ModelFile)
-            else request.Model
-        )
-        family = classify_model(model)
-        if family is None:
-            return self._fail(
-                context,
-                grpc.StatusCode.INVALID_ARGUMENT,
-                "mlx-video supports LTX-2 and converted Wan2.1/Wan2.2 MLX checkpoints",
-            )
-        if family == "wan" and not os.path.isdir(model):
-            return self._fail(
-                context,
-                grpc.StatusCode.INVALID_ARGUMENT,
-                "Wan models must be converted to MLX and configured as a local model directory",
-            )
-
-        self.family = family
-        self.model = model
-        self.options = parse_options(request.Options)
-        return backend_pb2.Result(message="mlx-video model configured", success=True)
-
-    def Free(self, request, context):
-        self.family = None
-        self.model = None
-        self.options = {}
-        return backend_pb2.Result(message="Model released", success=True)
-
-    def GenerateVideo(self, request, context):
-        if not self.family or not self.model:
-            return self._fail(
-                context, grpc.StatusCode.FAILED_PRECONDITION, "model is not loaded"
-            )
-        if not request.prompt.strip():
-            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, "prompt is required")
-        if not request.dst:
-            return self._fail(
-                context,
-                grpc.StatusCode.INVALID_ARGUMENT,
-                "output destination is required",
-            )
-
-        process = None
-        try:
-            command = build_generation_command(
-                family=self.family,
-                model=self.model,
-                prompt=request.prompt,
-                destination=request.dst,
-                negative_prompt=request.negative_prompt,
-                start_image=request.start_image,
-                end_image=request.end_image,
-                audio=request.audio,
-                width=request.width,
-                height=request.height,
-                num_frames=request.num_frames,
-                fps=request.fps,
-                seed=request.seed,
-                steps=request.step,
-                cfg_scale=request.cfg_scale,
-                params={**self.options, **dict(request.params)},
-            )
-            os.makedirs(
-                os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True
-            )
-            process = subprocess.Popen(command)
-            if hasattr(context, "add_callback"):
-                context.add_callback(
-                    lambda: process.poll() is None and process.terminate()
-                )
-            return_code = process.wait()
-            if return_code != 0:
-                raise RuntimeError(
-                    f"upstream generator exited with status {return_code}"
-                )
-            if not os.path.isfile(request.dst) or os.path.getsize(request.dst) == 0:
-                raise RuntimeError("upstream generator did not create the requested video")
-            return backend_pb2.Result(message="Video generated successfully", success=True)
-        except ValueError as err:
-            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err))
-        except Exception as err:
-            print(f"mlx-video generation failed: {err}", file=sys.stderr)
-            traceback.print_exc()
-            return self._fail(
-                context,
-                grpc.StatusCode.INTERNAL,
-                f"mlx-video generation failed: {err}",
-            )
-
-    @staticmethod
-    def _fail(context, code, message):
-        context.set_code(code)
-        context.set_details(message)
-        return backend_pb2.Result(message=message, success=False)
-
-
-def serve(address):
-    server = grpc.server(
-        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
-        options=[
-            ("grpc.max_message_length", 64 * 1024 * 1024),
-            ("grpc.max_send_message_length", 64 * 1024 * 1024),
-            ("grpc.max_receive_message_length", 64 * 1024 * 1024),
-        ],
-        interceptors=get_auth_interceptors(),
-    )
-    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
-    server.add_insecure_port(address)
-    server.start()
-    print(f"MLX Video backend listening on {address}", file=sys.stderr)
-
-    def stop_server(signum, frame):
-        del signum, frame
-        server.stop(0)
-
-    signal.signal(signal.SIGINT, stop_server)
-    signal.signal(signal.SIGTERM, stop_server)
-    server.wait_for_termination()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the MLX Video gRPC backend")
-    parser.add_argument("--addr", default="localhost:50051")
-    serve(parser.parse_args().addr)

backend/python/mlx-video/install.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-# SPDX-License-Identifier: MIT
-set -euo pipefail
-
-USE_PIP=true
-PYTHON_VERSION="3.11"
-PYTHON_PATCH="13"
-backend_dir=$(dirname "$0")
-if [ -d "${backend_dir}/common" ]; then
-    source "${backend_dir}/common/libbackend.sh"
-else
-    source "${backend_dir}/../common/libbackend.sh"
-fi
-installRequirements

backend/python/mlx-video/mlx_video_utils.py

@@ -1,102 +0,0 @@
-# SPDX-License-Identifier: MIT
-
-import os
-import sys
-
-
-def parse_options(options):
-    parsed = {}
-    for option in options:
-        key, separator, value = option.partition(":")
-        parsed[key] = value if separator else "true"
-    return parsed
-
-
-def classify_model(model):
-    normalized = model.rstrip("/").lower()
-    name = os.path.basename(normalized)
-    if "ltx-2" in normalized or "ltx2" in normalized:
-        return "ltx2"
-    if "wan2" in name or "wan_2" in name:
-        return "wan"
-    return None
-
-
-def _append(command, flag, value):
-    if value is not None and value != "":
-        command.extend([flag, str(value)])
-
-
-def build_generation_command(
-    *,
-    family,
-    model,
-    prompt,
-    destination,
-    negative_prompt="",
-    start_image="",
-    end_image="",
-    audio="",
-    width=0,
-    height=0,
-    num_frames=0,
-    fps=0,
-    seed=0,
-    steps=0,
-    cfg_scale=0,
-    params=None,
-):
-    params = params or {}
-    if audio:
-        raise ValueError("audio conditioning is not supported by mlx-video")
-
-    if family == "ltx2":
-        command = [sys.executable, "-m", "mlx_video.models.ltx_2.generate"]
-        _append(command, "--model-repo", model)
-        _append(command, "--prompt", prompt)
-        _append(command, "--output-path", destination)
-        _append(command, "--negative-prompt", negative_prompt)
-        _append(command, "--image", start_image)
-        _append(command, "--end-image", end_image)
-        _append(command, "--width", width or None)
-        _append(command, "--height", height or None)
-        _append(command, "--num-frames", num_frames or None)
-        _append(command, "--fps", fps or None)
-        _append(command, "--seed", seed if seed > 0 else None)
-        _append(command, "--steps", steps or None)
-        _append(command, "--cfg-scale", cfg_scale or None)
-        for key, flag in {
-            "pipeline": "--pipeline",
-            "tiling": "--tiling",
-            "image_strength": "--image-strength",
-            "end_image_strength": "--end-image-strength",
-            "text_encoder_repo": "--text-encoder-repo",
-        }.items():
-            _append(command, flag, params.get(key))
-        return command
-
-    if family == "wan":
-        if end_image:
-            raise ValueError("Wan models do not support end_image conditioning")
-        command = [sys.executable, "-m", "mlx_video.models.wan_2.generate"]
-        _append(command, "--model-dir", model)
-        _append(command, "--prompt", prompt)
-        _append(command, "--output-path", destination)
-        _append(command, "--negative-prompt", negative_prompt)
-        _append(command, "--image", start_image)
-        _append(command, "--width", width or None)
-        _append(command, "--height", height or None)
-        _append(command, "--num-frames", num_frames or None)
-        _append(command, "--seed", seed if seed > 0 else None)
-        _append(command, "--steps", steps or None)
-        _append(command, "--guide-scale", cfg_scale or None)
-        for key, flag in {
-            "scheduler": "--scheduler",
-            "shift": "--shift",
-            "tiling": "--tiling",
-            "trim_first_frames": "--trim-first-frames",
-        }.items():
-            _append(command, flag, params.get(key))
-        return command
-
-    raise ValueError(f"unsupported mlx-video model family: {family}")

backend/python/mlx-video/requirements-mps.txt

@@ -1 +0,0 @@
-git+https://github.com/Blaizzy/mlx-video.git@87db56a51758fefb748a359b90a5283bb8ba4837

backend/python/mlx-video/requirements.txt

@@ -1,4 +0,0 @@
-grpcio==1.71.0
-protobuf
-certifi
-setuptools

backend/python/mlx-video/run.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-# SPDX-License-Identifier: MIT
-set -euo pipefail
-
-backend_dir=$(dirname "$0")
-if [ -d "${backend_dir}/common" ]; then
-    source "${backend_dir}/common/libbackend.sh"
-else
-    source "${backend_dir}/../common/libbackend.sh"
-fi
-startBackend "$@"

backend/python/mlx-video/test.py

@@ -1,86 +0,0 @@
-# SPDX-License-Identifier: MIT
-
-import os
-import sys
-import unittest
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-from mlx_video_utils import (  # noqa: E402
-    build_generation_command,
-    classify_model,
-    parse_options,
-)
-
-
-class MLXVideoUtilsTest(unittest.TestCase):
-    def test_parses_model_options_and_preserves_colons(self):
-        self.assertEqual(
-            parse_options(["pipeline:dev", "tiling:aggressive", "flag"]),
-            {"pipeline": "dev", "tiling": "aggressive", "flag": "true"},
-        )
-
-    def test_classifies_supported_model_families(self):
-        self.assertEqual(classify_model("Lightricks/LTX-2"), "ltx2")
-        self.assertEqual(classify_model("/models/Wan2.2-TI2V-5B-mlx"), "wan")
-        self.assertIsNone(classify_model("some-org/unrelated-model"))
-
-    def test_maps_ltx_request_without_invoking_a_shell(self):
-        command = build_generation_command(
-            family="ltx2",
-            model="prince-canuma/LTX-2.3-distilled-mlx",
-            prompt="waves; touch /tmp/nope",
-            destination="/tmp/out.mp4",
-            negative_prompt="blurry",
-            start_image="/tmp/start.png",
-            end_image="/tmp/end.png",
-            width=768,
-            height=512,
-            num_frames=49,
-            fps=24,
-            seed=7,
-            steps=12,
-            cfg_scale=3.5,
-            params={"pipeline": "dev", "tiling": "aggressive"},
-        )
-
-        self.assertEqual(command[:3], [sys.executable, "-m", "mlx_video.models.ltx_2.generate"])
-        self.assertIn("waves; touch /tmp/nope", command)
-        self.assertEqual(command[command.index("--model-repo") + 1], "prince-canuma/LTX-2.3-distilled-mlx")
-        self.assertEqual(command[command.index("--pipeline") + 1], "dev")
-        self.assertEqual(command[command.index("--cfg-scale") + 1], "3.5")
-        self.assertIn("--end-image", command)
-
-    def test_maps_wan_request_and_rejects_unsupported_audio(self):
-        command = build_generation_command(
-            family="wan",
-            model="/models/wan22-mlx",
-            prompt="a fox",
-            destination="/tmp/out.mp4",
-            width=1280,
-            height=704,
-            num_frames=81,
-            fps=16,
-            seed=9,
-            steps=20,
-            cfg_scale=4.0,
-            params={"scheduler": "unipc", "shift": "5.0"},
-        )
-
-        self.assertEqual(command[:3], [sys.executable, "-m", "mlx_video.models.wan_2.generate"])
-        self.assertEqual(command[command.index("--model-dir") + 1], "/models/wan22-mlx")
-        self.assertEqual(command[command.index("--guide-scale") + 1], "4.0")
-        self.assertNotIn("--fps", command)
-
-        with self.assertRaisesRegex(ValueError, "audio conditioning"):
-            build_generation_command(
-                family="wan",
-                model="/models/wan22-mlx",
-                prompt="a fox",
-                destination="/tmp/out.mp4",
-                audio="/tmp/input.wav",
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

backend/python/mlx-video/test.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-# SPDX-License-Identifier: MIT
-set -euo pipefail
-
-backend_dir=$(dirname "$0")
-if [ -d "${backend_dir}/common" ]; then
-    source "${backend_dir}/common/libbackend.sh"
-else
-    source "${backend_dir}/../common/libbackend.sh"
-fi
-runUnittests

backend/python/mlx-vlm/backend.py

@@ -376,9 +376,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 kwargs["tools"] = json.loads(request.Tools)
             except json.JSONDecodeError:
                 pass
-        enable_thinking = request.Metadata.get("enable_thinking", "").lower()
-        if enable_thinking in ("true", "false"):
-            kwargs["enable_thinking"] = enable_thinking == "true"
+        if request.Metadata.get("enable_thinking", "").lower() == "true":
+            kwargs["enable_thinking"] = True
         return kwargs
 
     def _apply_template(self, request, messages, num_images, num_audios):

backend/python/mlx/backend.py

@@ -432,8 +432,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 except json.JSONDecodeError:
                     pass
             enable_thinking = request.Metadata.get("enable_thinking", "").lower()
-            if enable_thinking in ("true", "false"):
-                kwargs["enable_thinking"] = enable_thinking == "true"
+            if enable_thinking == "true":
+                kwargs["enable_thinking"] = True
 
             try:
                 return self.tokenizer.apply_chat_template(messages, **kwargs)

backend/python/mlx/test.py

@@ -340,40 +340,4 @@ class TestSharedHelpers(unittest.TestCase):
         self.assertNotIn("<tool_call>", remaining)
 
 
-class TestPreparePrompt(unittest.TestCase):
-    """Server-less tests for BackendServicer._prepare_prompt."""
-
-    def test_forwards_enable_thinking(self):
-        from backend import BackendServicer
-
-        class Tok:
-            chat_template = "{{ messages }}"
-
-            def __init__(self):
-                self.kwargs = None
-
-            def apply_chat_template(self, messages, **kwargs):
-                self.kwargs = kwargs
-                return "PROMPT"
-
-        def kwargs_for(metadata):
-            servicer = BackendServicer()
-            servicer.tokenizer = Tok()
-            req = types.SimpleNamespace(
-                Prompt="",
-                UseTokenizerTemplate=True,
-                Messages=[backend_pb2.Message(role="user", content="hi")],
-                Tools="",
-                Metadata=metadata,
-            )
-            self.assertEqual(servicer._prepare_prompt(req), "PROMPT")
-            return servicer.tokenizer.kwargs
-
-        self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True)
-        # "false" used to be dropped, so thinking models kept reasoning
-        self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False)
-        self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False)
-        self.assertNotIn("enable_thinking", kwargs_for({}))
-
-
 # Unit tests for ThreadSafeLRUPromptCache are in test_mlx_cache.py

backend/python/outetts/requirements-cpu.txt

@@ -1,6 +1,6 @@
 torch==2.7.1
-llvmlite==0.49.0
-numba==0.67.0
+llvmlite==0.43.0
+numba==0.60.0
 accelerate
 bitsandbytes
 outetts

backend/python/outetts/requirements-cublas12.txt

@@ -1,7 +1,7 @@
 torch==2.7.1
 accelerate
-llvmlite==0.49.0
-numba==0.67.0
+llvmlite==0.43.0
+numba==0.60.0
 bitsandbytes
 protobuf==6.33.5
 outetts

backend/python/outetts/requirements-cublas13.txt

@@ -1,7 +1,7 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
 torch==2.9.0
-llvmlite==0.49.0
-numba==0.67.0
+llvmlite==0.43.0
+numba==0.60.0
 bitsandbytes
 outetts
 protobuf==6.33.5

backend/python/outetts/requirements-hipblas.txt

@@ -1,8 +1,8 @@
 --extra-index-url https://download.pytorch.org/whl/rocm7.0
 torch==2.10.0+rocm7.0
 accelerate
-llvmlite==0.49.0
-numba==0.67.0
+llvmlite==0.43.0
+numba==0.60.0
 bitsandbytes
 outetts
 protobuf==6.33.5

backend/python/outetts/requirements-intel.txt

@@ -1,8 +1,8 @@
 --extra-index-url https://download.pytorch.org/whl/xpu
 torch
 optimum[openvino]
-llvmlite==0.49.0
-numba==0.67.0
+llvmlite==0.43.0
+numba==0.60.0
 bitsandbytes
 outetts
 protobuf==6.33.5

backend/python/qwen-tts/install.sh

@@ -10,11 +10,4 @@ else
     source $backend_dir/../common/libbackend.sh
 fi
 
-# CUDA 13 has no prebuilt FlashAttention wheel, so the fallback source build
-# exceeds the CI runner's memory when ninja compiles multiple units at once.
-if [ "x${BUILD_PROFILE}" = "xcublas13" ]; then
-    export MAX_JOBS="${MAX_JOBS:-1}"
-    export NVCC_THREADS="${NVCC_THREADS:-1}"
-fi
-
 installRequirements

backend/python/qwen-tts/requirements-cublas13-after.txt

@@ -1,2 +0,0 @@
-ninja
-flash-attn

backend/python/qwen-tts/test.sh

@@ -2,15 +2,6 @@
 set -e
 
 backend_dir=$(dirname $0)
-
-for cuda_version in 12 13; do
-    grep -qx "flash-attn" "$backend_dir/requirements-cublas${cuda_version}-after.txt"
-done
-
-grep -q 'BUILD_PROFILE.*cublas13' "$backend_dir/install.sh"
-grep -q 'MAX_JOBS.*1' "$backend_dir/install.sh"
-grep -q 'NVCC_THREADS.*1' "$backend_dir/install.sh"
-
 if [ -d $backend_dir/common ]; then
     source $backend_dir/common/libbackend.sh
 else

backend/python/transformers/install.sh

@@ -1,9 +1,6 @@
 #!/bin/bash
 set -e
 
-PYTHON_VERSION="3.12"
-PYTHON_PATCH="11"
-
 backend_dir=$(dirname $0)
 if [ -d $backend_dir/common ]; then
     source $backend_dir/common/libbackend.sh

backend/python/transformers/requirements-cpu.txt

@@ -1,8 +1,8 @@
 torch==2.7.1
-llvmlite==0.49.0
-numba==0.67.0
+llvmlite==0.43.0
+numba==0.60.0
 accelerate
-transformers>=5.15.1
+transformers>=5.15.0
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-cublas12.txt

@@ -1,8 +1,8 @@
 torch==2.7.1
 accelerate
-llvmlite==0.49.0
-numba==0.67.0
-transformers>=5.15.1
+llvmlite==0.43.0
+numba==0.60.0
+transformers>=5.15.0
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-cublas13.txt

@@ -1,8 +1,8 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
 torch==2.9.0
-llvmlite==0.49.0
-numba==0.67.0
-transformers>=5.15.1
+llvmlite==0.43.0
+numba==0.60.0
+transformers>=5.15.0
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-hipblas.txt

@@ -1,9 +1,9 @@
 --extra-index-url https://download.pytorch.org/whl/rocm7.0
 torch==2.10.0+rocm7.0
 accelerate
-transformers>=5.15.1
-llvmlite==0.49.0
-numba==0.67.0
+transformers>=5.15.0
+llvmlite==0.43.0
+numba==0.60.0
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-intel.txt

@@ -1,9 +1,9 @@
 --extra-index-url https://download.pytorch.org/whl/xpu
 torch
 optimum[openvino]
-llvmlite==0.49.0
-numba==0.67.0
-transformers>=5.15.1
+llvmlite==0.43.0
+numba==0.60.0
+transformers>=5.15.0
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-mps.txt

@@ -1,8 +1,8 @@
 torch==2.7.1
-llvmlite==0.49.0
-numba==0.67.0
+llvmlite==0.43.0
+numba==0.60.0
 accelerate
-transformers>=5.15.1
+transformers>=5.15.0
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements.txt

@@ -2,5 +2,5 @@ grpcio==1.83.0
 protobuf==7.35.0
 certifi
 setuptools
-scipy==1.18.0
-numpy>=2.5.2
+scipy==1.15.1
+numpy>=2.0.0

backend/python/vllm-omni/backend.py

@@ -489,9 +489,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                         except json.JSONDecodeError:
                             pass
                     try:
-                        enable_thinking = request.Metadata.get("enable_thinking", "").lower()
-                        if enable_thinking in ("true", "false"):
-                            template_kwargs["enable_thinking"] = enable_thinking == "true"
+                        if request.Metadata.get("enable_thinking", "").lower() == "true":
+                            template_kwargs["enable_thinking"] = True
                     except Exception:
                         pass
                     try:

backend/python/vllm/requirements-cublas13-after.txt

@@ -3,8 +3,8 @@
 # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index
 # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match
 # so uv consults this index alongside PyPI.
---extra-index-url https://wheels.vllm.ai/0.29.0/cu130
+--extra-index-url https://wheels.vllm.ai/0.28.0/cu130
 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh),
 # which pins this exact vLLM version. Bumping vllm here means coordinating with a
 # vllm-metal release that supports the new version, or macOS/Metal builds break.
-vllm==0.29.0
+vllm==0.28.0

backend/python/vllm/requirements-l4t13-after.txt

@@ -9,4 +9,4 @@
 # memory architecture crash deterministically with an empty "Engine core init
 # failed" set (mudler/LocalAI#10722). Leaving this unpinned let the L4T image
 # drift onto whatever wheel was latest at build time.
-vllm==0.29.0
+vllm==0.28.0

backend/python/whisper-medusa/Makefile

@@ -1,14 +0,0 @@
-.DEFAULT_GOAL := install
-
-.PHONY: install
-install:
-	bash install.sh
-
-.PHONY: clean
-clean:
-	$(RM) backend_pb2_grpc.py backend_pb2.py
-	rm -rf venv __pycache__
-
-.PHONY: test
-test:
-	python3 -m unittest test_unit.py

backend/python/whisper-medusa/backend.py

@@ -1,167 +0,0 @@
-#!/usr/bin/env python3
-"""LocalAI gRPC backend for aiola Whisper-Medusa speech recognition."""
-
-import argparse
-from concurrent import futures
-import os
-import signal
-import sys
-import time
-
-import backend_pb2
-import backend_pb2_grpc
-import grpc
-
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common"))
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common"))
-from grpc_auth import get_auth_interceptors
-from model_utils import resolve_model_reference
-
-
-SAMPLE_RATE = 16000
-MAX_DURATION_SECONDS = 30
-MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
-_ONE_DAY_IN_SECONDS = 60 * 60 * 24
-
-
-def _parse_options(raw_options):
-    options = {}
-    for option in raw_options:
-        if ":" not in option:
-            continue
-        key, value = option.split(":", 1)
-        try:
-            value = int(value)
-        except ValueError:
-            try:
-                value = float(value)
-            except ValueError:
-                pass
-        options[key] = value
-    return options
-
-
-def _prepare_audio(path, torchaudio):
-    waveform, sample_rate = torchaudio.load(path)
-    if waveform.shape[0] > 1:
-        waveform = waveform.mean(dim=0, keepdim=True)
-    if sample_rate != SAMPLE_RATE:
-        waveform = torchaudio.transforms.Resample(sample_rate, SAMPLE_RATE)(waveform)
-        sample_rate = SAMPLE_RATE
-    return waveform, sample_rate
-
-
-class BackendServicer(backend_pb2_grpc.BackendServicer):
-    def __init__(self):
-        self.model = None
-        self.processor = None
-        self.device = None
-        self.options = {}
-
-    def Health(self, request, context):
-        return backend_pb2.Reply(message=b"OK")
-
-    def LoadModel(self, request, context):
-        try:
-            import torch
-            from transformers import WhisperProcessor
-            from whisper_medusa import WhisperMedusaModel
-
-            if request.CUDA and not torch.cuda.is_available():
-                return backend_pb2.Result(success=False, message="CUDA is not available")
-
-            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-            if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
-                self.device = torch.device("mps")
-
-            self.options = _parse_options(request.Options)
-            model_path, local_only = resolve_model_reference(
-                request, "aiola/whisper-medusa-linear-libri"
-            )
-            self.model = WhisperMedusaModel.from_pretrained(
-                model_path, local_files_only=local_only
-            ).to(self.device)
-            self.model.eval()
-            self.processor = WhisperProcessor.from_pretrained(
-                model_path, local_files_only=local_only
-            )
-        except Exception as err:
-            print(f"Whisper-Medusa model load failed: {err}", file=sys.stderr)
-            return backend_pb2.Result(success=False, message=str(err))
-
-        return backend_pb2.Result(success=True, message="Model loaded successfully")
-
-    def AudioTranscription(self, request, context):
-        if self.model is None or self.processor is None:
-            return backend_pb2.TranscriptResult(segments=[], text="")
-
-        try:
-            import torch
-            import torchaudio
-
-            waveform, sample_rate = _prepare_audio(request.dst, torchaudio)
-            duration = waveform.shape[-1] / sample_rate
-            if duration > MAX_DURATION_SECONDS:
-                raise ValueError(
-                    f"Whisper-Medusa supports audio clips up to {MAX_DURATION_SECONDS} seconds"
-                )
-
-            language = request.language or str(self.options.get("language", "en"))
-            regulation_start = int(self.options.get("regulation_start", 140))
-            regulation_factor = float(self.options.get("regulation_factor", 1.01))
-            features = self.processor(
-                waveform.squeeze(), return_tensors="pt", sampling_rate=sample_rate
-            ).input_features.to(self.device)
-            with torch.inference_mode():
-                output = self.model.generate(
-                    features,
-                    language=language,
-                    exponential_decay_length_penalty=(
-                        regulation_start,
-                        regulation_factor,
-                    ),
-                )
-            text = self.processor.decode(output[0], skip_special_tokens=True).strip()
-            segment = backend_pb2.TranscriptSegment(
-                id=0,
-                start=0,
-                end=int(duration * 1_000_000_000),
-                text=text,
-            )
-            return backend_pb2.TranscriptResult(segments=[segment], text=text)
-        except Exception as err:
-            print(f"Whisper-Medusa transcription failed: {err}", file=sys.stderr)
-            return backend_pb2.TranscriptResult(segments=[], text="")
-
-
-def serve(address):
-    server = grpc.server(
-        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
-        options=[
-            ("grpc.max_send_message_length", 50 * 1024 * 1024),
-            ("grpc.max_receive_message_length", 50 * 1024 * 1024),
-        ],
-        interceptors=get_auth_interceptors(),
-    )
-    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
-    server.add_insecure_port(address)
-    server.start()
-    print(f"Server started. Listening on: {address}", file=sys.stderr)
-
-    def stop_server(_signal, _frame):
-        server.stop(0)
-        sys.exit(0)
-
-    signal.signal(signal.SIGINT, stop_server)
-    signal.signal(signal.SIGTERM, stop_server)
-    try:
-        while True:
-            time.sleep(_ONE_DAY_IN_SECONDS)
-    except KeyboardInterrupt:
-        server.stop(0)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the Whisper-Medusa backend")
-    parser.add_argument("--addr", default="localhost:50051")
-    serve(parser.parse_args().addr)

backend/python/whisper-medusa/install.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-set -e
-
-backend_dir=$(dirname "$0")
-if [ -d "$backend_dir/common" ]; then
-    source "$backend_dir/common/libbackend.sh"
-else
-    source "$backend_dir/../common/libbackend.sh"
-fi
-
-PYTHON_VERSION="3.11"
-PYTHON_PATCH="13"
-installRequirements

backend/python/whisper-medusa/protogen.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-set -e
-python3 -m grpc_tools.protoc -I../../ --python_out=. --grpc_python_out=. ../../backend.proto

backend/python/whisper-medusa/requirements-cpu.txt

@@ -1,3 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cpu
-torch==2.2.2
-torchaudio==2.2.2

backend/python/whisper-medusa/requirements-cublas12.txt

@@ -1,3 +0,0 @@
---extra-index-url https://download.pytorch.org/whl/cu121
-torch==2.2.2
-torchaudio==2.2.2