fix(audio-cpp): forward voice reference transcripts

Saved voices send ref_text, but Fish Audio requires reference_text.
Derive the canonical parameter while preserving explicit overrides.
Both TTS modes use the shared builder.

Add regression cases and document the parameter alias.

Assisted-by: Codex:gpt-6

.github/backend-matrix.yml

@@ -166,6 +166,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-whisper-medusa'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'true'
+    backend: "whisper-medusa"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -354,6 +367,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -626,6 +652,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "1"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-whisper-medusa'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "whisper-medusa"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1069,6 +1108,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1404,6 +1456,19 @@ include:
     backend: "qwen-asr"
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
+  - build-type: 'l4t'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-funasr'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    ubuntu-version: '2404'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
   - build-type: 'l4t'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -2388,6 +2453,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'hipblas'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-rocm-hipblas-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'hipblas'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -2668,6 +2746,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2204'
+  - build-type: 'l4t'
+    cuda-major-version: "12"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-funasr'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+    skip-drivers: 'true'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2204'
   - build-type: 'l4t'
     cuda-major-version: "12"
     cuda-minor-version: "0"
@@ -2877,6 +2968,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'intel'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'intel'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -3208,9 +3312,10 @@ include:
   # consumed. Same reason CUDA needs its toolkit in base-image rather than in a
   # builder image: this is the ds4 shape, not the llama-cpp one.
   #
-  # No ROCm entry: upstream has no HIP configuration. No CUDA arm64 or L4T
-  # entry: upstream documents and validates CUDA on x86 only. Darwin/Metal is in
-  # the includeDarwin matrix below, built by scripts/build/audio-cpp-darwin.sh.
+  # ROCm uses upstream's HIP backend and the project-wide ROCm 7.2.1 base. No
+  # CUDA arm64 or L4T entry: upstream documents and validates CUDA on x86 only.
+  # Darwin/Metal is in the includeDarwin matrix below, built by
+  # scripts/build/audio-cpp-darwin.sh.
   #
   # No vulkan entry either, though Dockerfile.audio-cpp and the backend Makefile
   # both handle BUILD_TYPE=vulkan for local builds. Every other vulkan backend
@@ -3278,6 +3383,19 @@ include:
     dockerfile: "./backend/Dockerfile.audio-cpp"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'hipblas'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-rocm-hipblas-audio-cpp'
+    runs-on: 'ubuntu-latest'
+    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+    skip-drivers: 'false'
+    backend: "audio-cpp"
+    dockerfile: "./backend/Dockerfile.audio-cpp"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -6071,6 +6189,20 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-funasr'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -6085,6 +6217,20 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-funasr'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "funasr"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -6324,6 +6470,9 @@ includeDarwin:
   - backend: "mlx"
     tag-suffix: "-metal-darwin-arm64-mlx"
     build-type: "mps"
+  - backend: "mlx-video"
+    tag-suffix: "-metal-darwin-arm64-mlx-video"
+    build-type: "mps"
   - backend: "chatterbox"
     tag-suffix: "-metal-darwin-arm64-chatterbox"
     build-type: "mps"
@@ -6460,6 +6609,9 @@ includeDarwin:
   - backend: "qwen-asr"
     tag-suffix: "-metal-darwin-arm64-qwen-asr"
     build-type: "mps"
+  - backend: "funasr"
+    tag-suffix: "-metal-darwin-arm64-funasr"
+    build-type: "mps"
   - backend: "nemo"
     tag-suffix: "-metal-darwin-arm64-nemo"
     build-type: "mps"

.github/bump_ctranslate2_rocm_wheel.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Bump the CTranslate2 ROCm Python wheel release pin used by faster-whisper.
+set -xe
+
+REPO=$1   # OpenNMT/CTranslate2
+FILE=$2   # backend/python/faster-whisper/install.sh
+VAR=$3    # CTRANSLATE2_VERSION (used for output file names so the workflow can read them)
+
+if [ -z "$FILE" ] || [ -z "$REPO" ] || [ -z "$VAR" ]; then
+    echo "usage: $0 <repo> <install-script> <var-name>" >&2
+    exit 1
+fi
+
+LATEST_RELEASE=$(curl -sS -H "Accept: application/vnd.github+json" \
+    "https://api.github.com/repos/$REPO/releases/latest")
+LATEST_TAG=$(python3 -c "import json,sys; print(json.load(sys.stdin)['tag_name'])" <<< "$LATEST_RELEASE")
+
+set +e
+CURRENT_VERSION=$(grep -m1 "^${VAR}=" "$FILE" | cut -d= -f2 | sed -E 's/^\$\{[^:]+:-([^}]+)\}$/\1/')
+CTRANSLATE2_ROCM_WHEEL_OS=$(grep -m1 '^CTRANSLATE2_ROCM_WHEEL_OS=' "$FILE" | cut -d= -f2 | sed -E 's/^\$\{[^:]+:-([^}]+)\}$/\1/')
+set -e
+
+if [ -z "$CURRENT_VERSION" ]; then
+    echo "Could not find $VAR in $FILE."
+    exit 0
+fi
+
+if [ -z "$CTRANSLATE2_ROCM_WHEEL_OS" ]; then
+    echo "Could not find CTRANSLATE2_ROCM_WHEEL_OS in $FILE."
+    exit 0
+fi
+
+ASSET_NAME="rocm-python-wheels-${CTRANSLATE2_ROCM_WHEEL_OS}.zip"
+LATEST_RELEASE="$LATEST_RELEASE" python3 - "$ASSET_NAME" <<'PY'
+import json
+import os
+import sys
+
+asset_name = sys.argv[1]
+release = json.loads(os.environ["LATEST_RELEASE"])
+assets = {asset.get("name") for asset in release.get("assets", [])}
+if asset_name not in assets:
+    raise SystemExit(f"Could not find release asset {asset_name!r}")
+PY
+
+sed -i "$FILE" -e "s/^${VAR}=.*/${VAR}=\${${VAR}:-${LATEST_TAG}}/"
+
+echo "Changes: https://github.com/$REPO/compare/${CURRENT_VERSION}...${LATEST_TAG}" >> "${VAR}_message.txt"
+echo "${LATEST_TAG}" >> "${VAR}_commit.txt"

.github/workflows/bump_deps.yaml

@@ -170,6 +170,39 @@ jobs:
           body: ${{ steps.bump.outputs.message }}
           signoff: true
 
+  bump-ctranslate2-rocm-wheel:
+    # CTranslate2's ROCm wheels are published as release assets, so the
+    # faster-whisper hipblas install path pins the release tag used in the URL.
+    if: github.repository == 'mudler/LocalAI'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Bump CTranslate2 ROCm wheel pin 🔧
+        id: bump
+        run: |
+          bash .github/bump_ctranslate2_rocm_wheel.sh OpenNMT/CTranslate2 backend/python/faster-whisper/install.sh CTRANSLATE2_VERSION
+          {
+            echo 'message<<EOF'
+            cat "CTRANSLATE2_VERSION_message.txt"
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+          {
+            echo 'commit<<EOF'
+            cat "CTRANSLATE2_VERSION_commit.txt"
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+          rm -rfv CTRANSLATE2_VERSION_message.txt CTRANSLATE2_VERSION_commit.txt
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v8
+        with:
+          token: ${{ secrets.UPDATE_BOT_TOKEN }}
+          push-to-fork: ci-forks/LocalAI
+          commit-message: ':arrow_up: Update OpenNMT/CTranslate2 ROCm wheel'
+          title: 'chore: :arrow_up: Update OpenNMT/CTranslate2 ROCm wheel to `${{ steps.bump.outputs.commit }}`'
+          branch: "update/CTRANSLATE2_VERSION"
+          body: ${{ steps.bump.outputs.message }}
+          signoff: true
+
   bump-vllm-wheel:
     # vLLM's cu130 wheel comes from a per-tag index URL (no /latest/ alias),
     # so the cublas13 requirements file pins both a URL segment and a version

.github/workflows/secscan.yaml

@@ -31,7 +31,7 @@ jobs:
         if: ${{ !github.repository.fork && github.actor != 'dependabot[bot]' }}
       - name: Run Gosec Security Scanner
         if: ${{ !github.repository.fork && github.actor != 'dependabot[bot]' }}
-        uses: securego/gosec@v2.27.1
+        uses: securego/gosec@v2.29.0
         with:
           # we let the report trigger content trigger a failure using the GitHub Security features.
           # backend/go/supertonic is excluded: it vendors upstream supertone-inc/supertonic

Makefile

@@ -1,5 +1,7 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
+.NOTPARALLEL: backends/whisper-medusa
+.NOTPARALLEL: backends/funasr
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -612,6 +614,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/vllm
 	$(MAKE) -C backend/python/vllm-omni
 	$(MAKE) -C backend/python/longcat-video
+	$(MAKE) -C backend/python/mlx-video
 	$(MAKE) -C backend/python/sglang
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/liquid-audio
@@ -621,9 +624,11 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/fish-speech
 	$(MAKE) -C backend/python/faster-qwen3-tts
 	$(MAKE) -C backend/python/qwen-asr
+	$(MAKE) -C backend/python/funasr
 	$(MAKE) -C backend/python/nemo
 	$(MAKE) -C backend/python/voxcpm
 	$(MAKE) -C backend/python/faster-whisper
+	$(MAKE) -C backend/python/whisper-medusa
 	$(MAKE) -C backend/python/whisperx
 	$(MAKE) -C backend/python/ace-step
 	$(MAKE) -C backend/python/trl
@@ -644,6 +649,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/vllm test
 	$(MAKE) -C backend/python/vllm-omni test
 	$(MAKE) -C backend/python/longcat-video test
+	$(MAKE) -C backend/python/mlx-video test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/liquid-audio test
 	$(MAKE) -C backend/python/moonshine test
@@ -652,9 +658,11 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/fish-speech test
 	$(MAKE) -C backend/python/faster-qwen3-tts test
 	$(MAKE) -C backend/python/qwen-asr test
+	$(MAKE) -C backend/python/funasr test
 	$(MAKE) -C backend/python/nemo test
 	$(MAKE) -C backend/python/voxcpm test
 	$(MAKE) -C backend/python/faster-whisper test
+	$(MAKE) -C backend/python/whisper-medusa test
 	$(MAKE) -C backend/python/whisperx test
 	$(MAKE) -C backend/python/ace-step test
 	$(MAKE) -C backend/python/trl test
@@ -1258,6 +1266,10 @@ backends/mlx:
 	BACKEND=mlx $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx.tar)"
 
+backends/mlx-video:
+	BACKEND=mlx-video $(MAKE) build-darwin-python-backend
+	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx-video.tar)"
+
 backends/diffuser-darwin:
 	BACKEND=diffusers $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/diffusers.tar)"
@@ -1344,6 +1356,7 @@ BACKEND_RERANKERS = rerankers|python|.|false|true
 BACKEND_TRANSFORMERS = transformers|python|.|false|true
 BACKEND_OUTETTS = outetts|python|.|false|true
 BACKEND_FASTER_WHISPER = faster-whisper|python|.|false|true
+BACKEND_WHISPER_MEDUSA = whisper-medusa|python|.|false|true
 BACKEND_COQUI = coqui|python|.|false|true
 BACKEND_RFDETR = rfdetr|python|.|false|true
 BACKEND_INSIGHTFACE = insightface|python|.|false|true
@@ -1365,6 +1378,7 @@ BACKEND_QWEN_TTS = qwen-tts|python|.|false|true
 BACKEND_FISH_SPEECH = fish-speech|python|.|false|true
 BACKEND_FASTER_QWEN3_TTS = faster-qwen3-tts|python|.|false|true
 BACKEND_QWEN_ASR = qwen-asr|python|.|false|true
+BACKEND_FUNASR = funasr|python|.|false|true
 BACKEND_NEMO = nemo|python|.|false|true
 BACKEND_VOXCPM = voxcpm|python|.|false|true
 BACKEND_WHISPERX = whisperx|python|.|false|true
@@ -1435,6 +1449,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_RERANKERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRANSFORMERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_OUTETTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FASTER_WHISPER)))
+$(eval $(call generate-docker-build-target,$(BACKEND_WHISPER_MEDUSA)))
 $(eval $(call generate-docker-build-target,$(BACKEND_COQUI)))
 $(eval $(call generate-docker-build-target,$(BACKEND_RFDETR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_INSIGHTFACE)))
@@ -1456,6 +1471,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FISH_SPEECH)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FASTER_QWEN3_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_ASR)))
+$(eval $(call generate-docker-build-target,$(BACKEND_FUNASR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_NEMO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
@@ -1485,6 +1501,8 @@ docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
 docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-nemo-speech-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-audio-cpp
+docker-build-backends: docker-build-whisper-medusa
+docker-build-backends: docker-build-funasr
 
 ########################################################
 ### Mock Backend for E2E Tests

backend/Dockerfile.audio-cpp

@@ -6,13 +6,14 @@ ARG APT_PORTS_MIRROR=""
 # ASR, VAD, diarization, source separation and music generation, wrapped as a
 # LocalAI gRPC backend.
 #
-# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds, or
-# nvidia/cuda:<ver>-devel-ubuntu24.04 for cublas builds; both ship apt and
-# Ubuntu Noble packages, and the CUDA base additionally provides
-# /usr/local/cuda. BUILD_TYPE selects the engine backend in the Makefile:
-# "" = portable CPU with all ggml CPU variants, "cublas" ->
-# -DENGINE_ENABLE_CUDA=ON, "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin
-# (Metal) builds bypass this Dockerfile entirely.
+# BASE_IMAGE is ubuntu:24.04 for cpu and vulkan builds,
+# nvidia/cuda:<ver>-devel-ubuntu24.04 for cublas builds, or
+# rocm/dev-ubuntu-24.04:<ver> for hipblas builds. All ship apt and Ubuntu Noble
+# packages; the GPU bases also provide their toolkits. BUILD_TYPE selects the
+# engine backend in the Makefile: "" = portable CPU with all ggml CPU variants,
+# "cublas" -> -DENGINE_ENABLE_CUDA=ON, "hipblas" -> -DENGINE_ENABLE_HIP=ON,
+# and "vulkan" -> -DENGINE_ENABLE_VULKAN=ON. Darwin (Metal) builds bypass this
+# Dockerfile entirely.
 #
 # Upstream needs GCC 13 or newer, which ubuntu:24.04 and the CUDA 12/13
 # devel-ubuntu24.04 images all provide.
@@ -62,7 +63,7 @@ ENV BUILD_TYPE=${BUILD_TYPE} \
     APT_MIRROR=${APT_MIRROR} \
     APT_PORTS_MIRROR=${APT_PORTS_MIRROR} \
     DEBIAN_FRONTEND=noninteractive \
-    PATH=/usr/local/cuda/bin:${PATH}
+    PATH=/opt/rocm/bin:/usr/local/cuda/bin:${PATH}
 
 WORKDIR /build
 
@@ -73,7 +74,7 @@ WORKDIR /build
 # fallback of its own.
 #
 # BUILD_TYPE=vulkan additionally needs the loader headers and glslc; both are in
-# Noble. The CUDA toolkit for BUILD_TYPE=cublas comes from BASE_IMAGE.
+# Noble. The CUDA and ROCm toolkits come from their matching BASE_IMAGE.
 RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
     sh /usr/local/sbin/apt-mirror && \
     apt-get update && \
@@ -83,6 +84,9 @@ RUN --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mi
     if [ "${BUILD_TYPE}" = "vulkan" ]; then \
         apt-get install -y --no-install-recommends libvulkan-dev glslc; \
     fi && \
+    if [ "${BUILD_TYPE}" = "hipblas" ]; then \
+        apt-get install -y --no-install-recommends hipblas-dev rocblas-dev; \
+    fi && \
     if [ "${TARGETARCH}" = "arm64" ]; then \
         apt-get install -y --no-install-recommends gcc-14 g++-14; \
     fi && \

backend/README.md

@@ -47,7 +47,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
 - **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation
-- **Audio**: coqui, faster-whisper, kitten-tts
+- **Audio**: coqui, faster-whisper, funasr, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro
 

backend/backend.proto

@@ -499,6 +499,10 @@ message ModelOptions {
   // applied verbatim to the backend's engine constructor (e.g. vLLM AsyncEngineArgs).
   // Unknown keys produce an error at LoadModel time.
   string EngineArgs = 73;
+  string OriginalConfigFile = 77;
+
+  // EnvVars carries environment variables to be passed to the backend process.
+  map<string, string> EnvVars = 76;
 
   // Proxy carries the cloud-proxy backend's per-model configuration.
   // Empty for non-proxy backends.

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=05e508a70e3600b01454c647cdb122133ba8e64c
+AUDIO_CPP_VERSION?=efb04233dab73aeee4b2912042a90e7b36329061
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -77,6 +77,16 @@ endif
 
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DENGINE_ENABLE_CUDA=ON "-DCMAKE_CUDA_ARCHITECTURES=$(CUDA_ARCHITECTURES)"
+else ifeq ($(BUILD_TYPE),hipblas)
+    ROCM_HOME ?= /opt/rocm
+    ROCM_PATH ?= /opt/rocm
+    export CXX=$(ROCM_HOME)/llvm/bin/clang++
+    export CC=$(ROCM_HOME)/llvm/bin/clang
+    AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+    # audio.cpp forwards GPU_TARGETS to CMake's semicolon-delimited HIP list.
+    comma := ,
+    HIP_TARGETS := $(subst $(comma),;,$(AMDGPU_TARGETS))
+    CMAKE_ARGS += -DENGINE_ENABLE_HIP=ON "-DGPU_TARGETS=$(HIP_TARGETS)"
 else ifeq ($(BUILD_TYPE),vulkan)
     CMAKE_ARGS += -DENGINE_ENABLE_VULKAN=ON
 else ifeq ($(UNAME_S),Darwin)

backend/cpp/audio-cpp/generation_request.cpp

@@ -22,7 +22,8 @@ bool voice_is_reference_file(const std::string &voice) {
 
 RequestShape build_tts_shape(const backend::TTSRequest &request) {
     RequestShape shape;
-    shape.has_voice_reference = voice_is_reference_file(request.voice());
+    shape.has_voice_reference = voice_is_reference_file(request.voice()) ||
+                                request.params().find("multi_reference_cond") != request.params().end();
     // !empty() as well as has_instructions(), and it must match the guard in
     // build_tts_request: a request whose instructions are an empty string
     // carries no style condition, so telling routing to prefer VoiceDesign for
@@ -134,6 +135,13 @@ build_tts_request(const backend::TTSRequest &request,
         task.options["language"] = request.language();
     }
 
+    // Saved voice profiles send ref_text; Fish Audio reads reference_text.
+    // Derive the alias before copying params so an explicit canonical key wins.
+    const auto reference_text = request.params().find("ref_text");
+    if (reference_text != request.params().end()) {
+        task.options["reference_text"] = reference_text->second;
+    }
+
     // LAST, so an explicit params entry wins over anything derived above. That
     // matters for "caption": a caller who sets params[caption] has named the
     // exact string they want, and it must not be overwritten by `instructions`.

backend/cpp/audio-cpp/generation_request_ctest.cpp

@@ -128,6 +128,14 @@ static void test_tts_shape() {
         check(!shape.has_voice_reference,
               "shape: a preset name is not a voice reference");
     }
+    {
+        backend::TTSRequest request;
+        (*request.mutable_params())["multi_reference_cond"] =
+            R"([{"audio":"one.wav","text":"one"}])";
+        const auto shape = build_tts_shape(request);
+        check(shape.has_voice_reference,
+              "shape: multi-reference conditioning is a voice reference");
+    }
     {
         backend::TTSRequest request;
         request.set_voice(dir.string());
@@ -367,6 +375,40 @@ static void test_tts_language_and_params() {
           "tts params: an explicit param overrides the derived caption");
 }
 
+static void test_tts_reference_transcript() {
+    backend::TTSRequest request;
+    request.set_text("New speech to generate.");
+    request.set_voice("reference.wav");
+    (*request.mutable_params())["ref_text"] = "The saved voice transcript.";
+
+    const auto task = build_tts_request(request, clip(24000, 1));
+    check(option_or(task.options, "reference_text", "") ==
+              "The saved voice transcript.",
+          "tts reference: saved transcript reaches Fish Audio's option");
+    check(option_or(task.options, "ref_text", "") ==
+              "The saved voice transcript.",
+          "tts reference: original transcript parameter is preserved");
+    check(task.text_input->text == "New speech to generate.",
+          "tts reference: transcript does not replace synthesis text");
+
+    (*request.mutable_params())["reference_text"] = "Explicit transcript.";
+    const auto explicit_task = build_tts_request(request, clip(24000, 1));
+    check(option_or(explicit_task.options, "reference_text", "") ==
+              "Explicit transcript.",
+          "tts reference: explicit canonical parameter wins over alias");
+
+    (*request.mutable_params())["reference_text"] = "";
+    const auto empty_task = build_tts_request(request, clip(24000, 1));
+    check(has_key(empty_task.options, "reference_text") &&
+              empty_task.options.at("reference_text").empty(),
+          "tts reference: explicit empty canonical parameter is preserved");
+
+    request.mutable_params()->clear();
+    const auto missing_task = build_tts_request(request, clip(24000, 1));
+    check(!has_key(missing_task.options, "reference_text"),
+          "tts reference: no transcript is invented when none was supplied");
+}
+
 static void test_sound_generation_minimal() {
     backend::SoundGenerationRequest request;
     request.set_text("a distant thunderstorm");
@@ -554,6 +596,7 @@ int main() {
     test_tts_empty_language_is_not_a_language();
     test_tts_clip_and_instructions();
     test_tts_language_and_params();
+    test_tts_reference_transcript();
     test_sound_generation_minimal();
     test_sound_generation_full();
     test_transform_text_absent();

backend/cpp/audio-cpp/loaded_model.cpp

@@ -103,6 +103,9 @@ engine::core::BackendType parse_backend_type(const std::string &value) {
     if (value == "cuda") {
         return engine::core::BackendType::Cuda;
     }
+    if (value == "hip" || value == "rocm") {
+        return engine::core::BackendType::Hip;
+    }
     if (value == "vulkan") {
         return engine::core::BackendType::Vulkan;
     }
@@ -116,7 +119,7 @@ engine::core::BackendType parse_backend_type(const std::string &value) {
         return engine::core::BackendType::Cpu;
     }
     throw ConfigError("audio-cpp: unknown backend option '" + value +
-                      "'. Known backends: cpu, cuda, vulkan, metal, best");
+                      "'. Known backends: cpu, cuda, hip, rocm, vulkan, metal, best");
 }
 
 std::filesystem::path executable_directory() {

backend/cpp/audio-cpp/model_options.cpp

@@ -59,6 +59,12 @@ bool starts_with(const std::string &value, const std::string &prefix) {
            value.compare(0, prefix.size(), prefix) == 0;
 }
 
+bool is_known_backend(const std::string &value) {
+    return value == "cpu" || value == "cuda" || value == "hip" ||
+           value == "rocm" || value == "vulkan" || value == "metal" ||
+           value == "best";
+}
+
 } // namespace
 
 ParsedOptions parse_model_options(const std::vector<std::string> &entries) {
@@ -107,6 +113,12 @@ ParsedOptions parse_model_options(const std::vector<std::string> &entries) {
         } else if (key == "task") {
             parsed.options.task = value;
         } else if (key == "backend") {
+            if (!is_known_backend(value)) {
+                parsed.error = "audio-cpp: unknown backend option '" + value +
+                               "'. Known backends: cpu, cuda, hip, rocm, "
+                               "vulkan, metal, best";
+                return parsed;
+            }
             parsed.options.backend = value;
         } else if (key == "model_spec_override") {
             parsed.options.model_spec_override = value;

backend/cpp/audio-cpp/model_options.h

@@ -17,7 +17,7 @@ struct ModelOptions {
     std::string family;
     // Pins the audio.cpp task, overriding RPC-based routing. Empty means route.
     std::string task;
-    // ggml backend: cpu, cuda, vulkan, metal, best.
+    // ggml backend: cpu, cuda, hip (or rocm), vulkan, metal, best.
     std::string backend = "cpu";
     int device = 0;
     // True once a `device:` entry has been seen. 0 is both the default and a

backend/cpp/audio-cpp/model_options_test.cpp

@@ -75,6 +75,11 @@ static void test_scalar_options() {
     check(parse_model_options({"live_idle_timeout_ms:0"}).options.live_idle_timeout_ms == 0,
           "an explicit 0 turns the live idle limit off rather than reverting to "
           "the default");
+
+    check(parse_model_options({"backend:hip"}).error.empty(),
+          "HIP backend option is accepted");
+    check(parse_model_options({"backend:rocm"}).error.empty(),
+          "ROCm backend alias is accepted");
 }
 
 // Values containing colons must survive: split on the FIRST colon only.
@@ -124,6 +129,8 @@ static void test_errors() {
           "negative device is rejected");
     check(!parse_model_options({"threads:x"}).error.empty(),
           "non-numeric threads is rejected");
+    check(!parse_model_options({"backend:unknown"}).error.empty(),
+          "unknown compute backend is rejected before model loading");
 
     // Values too large for int must be rejected, not silently wrapped into a
     // negative device index that then reaches the ggml backend selector.

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=1a2a8604a6c6c6413c06bf9adfc2f64329af4366
+IK_LLAMA_VERSION?=3bb386eb68ffee0a5dc7db21da0735d594929eeb
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/CMakeLists.txt

@@ -125,4 +125,15 @@ if(LLAMA_GRPC_BUILD_TESTS)
     target_include_directories(thread_params_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_compile_features(thread_params_test PRIVATE cxx_std_17)
     add_test(NAME thread_params_test COMMAND thread_params_test)
+
+    add_executable(model_load_error_test model_load_error_test.cpp model_load_error.h)
+    target_include_directories(model_load_error_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_compile_features(model_load_error_test PRIVATE cxx_std_17)
+    add_test(NAME model_load_error_test COMMAND model_load_error_test)
+
+    # Dead-stream tracker (standard library only).
+    add_executable(stream_peer_test stream_peer_test.cpp stream_peer.h)
+    target_include_directories(stream_peer_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_compile_features(stream_peer_test PRIVATE cxx_std_17)
+    add_test(NAME stream_peer_test COMMAND stream_peer_test)
 endif()

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=67672dc5b76f8bc17785a19d3dc6d1463fc2902c
+LLAMA_VERSION?=df03399b885831b2a1603b3abb0d8c156808e363
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -53,9 +53,11 @@
 #include "arg.h"
 #include "chat-auto-parser.h"
 #include "llama_compat.h" // fork-skew switches, generated by prepare.sh
+#include "model_load_error.h"
 #include "thread_params.h"
 #include "message_content.h"
 #include "passthrough_options.h"
+#include "stream_peer.h"
 #include "tts_request_options.h"
 #include <getopt.h>
 #include <grpcpp/ext/proto_server_reflection_plugin.h>
@@ -1628,7 +1630,8 @@ public:
             {
                 std::lock_guard<std::mutex> lock(error_capture_data.error_mutex);
                 if (!error_capture_data.captured_error.empty()) {
-                    error_msg += ". Error: " + error_capture_data.captured_error;
+                    error_msg += ". Error: " +
+                                 localai::model_load_error_with_hint(error_capture_data.captured_error);
                 } else {
                     error_msg += ". Model file may not exist or be invalid.";
                 }
@@ -2266,6 +2269,11 @@ public:
         // such concept, so there is nothing to emit — the real tokens arrive in
         // the loop below. Feeding this null into build_reply_from_json would
         // throw (uncaught) and surface as a generic RPC error.
+        // A write that returns false means the peer is gone for good. Track it
+        // so the loop below stops decoding instead of feeding a dead stream —
+        // see stream_peer.h for why that matters to everyone else's requests.
+        llama_grpc::StreamPeer peer;
+
         if (first_res_json.is_null()) {
             // skip the begin-of-stream marker
         } else if (first_res_json.is_array()) {
@@ -2278,17 +2286,21 @@ public:
                 if (!is_role_init) {
                     attach_chat_deltas(reply, first_result.get());
                 }
-                writer->Write(reply);
+                peer.observe_write(writer->Write(reply));
+                if (peer.gone()) {
+                    break;
+                }
             }
         } else {
             auto reply = build_reply_from_json(first_res_json, first_result.get());
             attach_chat_deltas(reply, first_result.get());
-            writer->Write(reply);
+            peer.observe_write(writer->Write(reply));
         }
 
         // Process subsequent results
         while (rd.has_next()) {
-            if (context->IsCancelled()) {
+            peer.observe_cancelled(context->IsCancelled());
+            if (peer.gone()) {
                 break;
             }
 
@@ -2309,17 +2321,22 @@ public:
                     if (!is_role_init) {
                         attach_chat_deltas(reply, result.get());
                     }
-                    writer->Write(reply);
+                    peer.observe_write(writer->Write(reply));
+                    if (peer.gone()) {
+                        break;
+                    }
                 }
             } else {
                 auto reply = build_reply_from_json(res_json, result.get());
                 attach_chat_deltas(reply, result.get());
-                writer->Write(reply);
+                peer.observe_write(writer->Write(reply));
             }
         }
 
-        // Check if context was cancelled during processing
-        if (context->IsCancelled()) {
+        // Returning here is what releases the slot: ~server_response_reader()
+        // posts SERVER_TASK_TYPE_CANCEL for whatever is still decoding.
+        peer.observe_cancelled(context->IsCancelled());
+        if (peer.gone()) {
             return grpc::Status(grpc::StatusCode::CANCELLED, "Request cancelled by client");
         }
 

backend/cpp/llama-cpp/model_load_error.h

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <string>
+
+namespace localai {
+
+inline std::string model_load_error_with_hint(const std::string& error) {
+  const std::string mismatch = "wrong number of tensors; expected ";
+  const std::string got = ", got ";
+  const std::string::size_type mismatch_pos = error.find(mismatch);
+  if (mismatch_pos == std::string::npos ||
+      error.find(got, mismatch_pos + mismatch.size()) == std::string::npos) {
+    return error;
+  }
+
+  return error +
+         " Hint: the model may be incompatible with this llama.cpp backend "
+         "or the GGUF file may be corrupt. Try a newer compatible backend "
+         "and verify or re-download the model file.";
+}
+
+}  // namespace localai

backend/cpp/llama-cpp/model_load_error_test.cpp

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: MIT
+
+#include "model_load_error.h"
+
+#include <cassert>
+#include <string>
+
+int main() {
+  const std::string issue_error =
+      "llama_model_load: error loading model: done_getting_tensors: wrong number of tensors; expected 2131, got 720; "
+      "llama_model_load_from_file_impl: failed to load model";
+  const std::string issue_result = localai::model_load_error_with_hint(issue_error);
+  assert(issue_result.compare(0, issue_error.size(), issue_error) == 0);
+  assert(issue_result.find("incompatible") != std::string::npos);
+  assert(issue_result.find("corrupt") != std::string::npos);
+
+  const std::string generic_error =
+      "wrong number of tensors; expected 42, got 17";
+  const std::string generic_result = localai::model_load_error_with_hint(generic_error);
+  assert(generic_result.compare(0, generic_error.size(), generic_error) == 0);
+  assert(generic_result.size() > generic_error.size());
+
+  const std::string unrelated_error = "failed to open GGUF file";
+  assert(localai::model_load_error_with_hint(unrelated_error) == unrelated_error);
+}

backend/cpp/llama-cpp/prepare.sh

@@ -45,6 +45,10 @@ done
 
 cp -r CMakeLists.txt llama.cpp/tools/grpc-server/
 cp -r grpc-server.cpp llama.cpp/tools/grpc-server/
+# Model-load diagnostics (included by grpc-server.cpp) and their standalone
+# regression test.
+cp -r model_load_error.h llama.cpp/tools/grpc-server/
+cp -r model_load_error_test.cpp llama.cpp/tools/grpc-server/
 # Shared message-reconstruction helpers (included by grpc-server.cpp) and their
 # unit test (compiled only when -DLLAMA_GRPC_BUILD_TESTS=ON).
 cp -r message_content.h llama.cpp/tools/grpc-server/
@@ -63,6 +67,10 @@ cp -r thread_params_test.cpp llama.cpp/tools/grpc-server/
 # test (run via backend/cpp/run-unit-tests.sh; also buildable under ctest).
 cp -r parent_watch.h llama.cpp/tools/grpc-server/
 cp -r parent_watch_test.cpp llama.cpp/tools/grpc-server/
+# Dead-stream tracker (included by grpc-server.cpp) and its standalone unit
+# test (run via backend/cpp/run-unit-tests.sh; also buildable under ctest).
+cp -r stream_peer.h llama.cpp/tools/grpc-server/
+cp -r stream_peer_test.cpp llama.cpp/tools/grpc-server/
 cp -rfv llama.cpp/vendor/nlohmann/json.hpp llama.cpp/tools/grpc-server/
 cp -rfv llama.cpp/vendor/cpp-httplib/httplib.h llama.cpp/tools/grpc-server/
 

backend/cpp/llama-cpp/stream_peer.h

@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+
+namespace llama_grpc {
+
+// Tracks whether a server-streaming RPC still has somewhere to send tokens.
+//
+// grpc::ServerWriter::Write() returns false once the peer is gone, and a
+// stream never recovers afterwards. Ignoring that result is not harmless: the
+// handler goes on draining decoded tokens into a dead stream, so the llama.cpp
+// slot stays busy for the rest of the request's token budget. A model config
+// with no max_tokens and a large context turns that into tens of minutes per
+// abandoned request, and the slots are exactly what every other request queues
+// behind.
+//
+// Returning as soon as the peer is gone is what frees the slot: the handler's
+// server_response_reader then goes out of scope and its destructor posts
+// SERVER_TASK_TYPE_CANCEL for whatever is still decoding.
+class StreamPeer {
+public:
+    // Records the outcome of a Write(). Once a write has failed the peer stays
+    // gone -- a later write cannot succeed on a broken stream.
+    void observe_write(bool ok) noexcept {
+        if (!ok) {
+            gone_ = true;
+        }
+    }
+
+    // Folds in the RPC's own cancellation flag, so callers have a single
+    // predicate to test rather than two that can disagree.
+    void observe_cancelled(bool cancelled) noexcept {
+        if (cancelled) {
+            gone_ = true;
+        }
+    }
+
+    bool gone() const noexcept { return gone_; }
+    bool alive() const noexcept { return !gone_; }
+
+private:
+    bool gone_ = false;
+};
+
+} // namespace llama_grpc

backend/cpp/llama-cpp/stream_peer_test.cpp

@@ -0,0 +1,67 @@
+#include "stream_peer.h"
+
+#include <cstdio>
+
+namespace {
+
+int failures = 0;
+
+void check(bool condition, const char *what) {
+    if (!condition) {
+        std::fprintf(stderr, "FAIL: %s\n", what);
+        ++failures;
+    }
+}
+
+} // namespace
+
+int main() {
+    {
+        llama_grpc::StreamPeer peer;
+        check(peer.alive(), "a fresh peer is alive");
+        check(!peer.gone(), "a fresh peer is not gone");
+    }
+
+    {
+        llama_grpc::StreamPeer peer;
+        peer.observe_write(true);
+        peer.observe_write(true);
+        check(peer.alive(), "successful writes keep the peer alive");
+    }
+
+    {
+        llama_grpc::StreamPeer peer;
+        peer.observe_write(false);
+        check(peer.gone(), "a failed write marks the peer gone");
+    }
+
+    {
+        // The whole point of the guard: a stream never comes back, so a later
+        // success must not resurrect a peer an earlier failure retired.
+        llama_grpc::StreamPeer peer;
+        peer.observe_write(false);
+        peer.observe_write(true);
+        check(peer.gone(), "a failed write is sticky across later writes");
+    }
+
+    {
+        llama_grpc::StreamPeer peer;
+        peer.observe_cancelled(false);
+        check(peer.alive(), "an uncancelled RPC keeps the peer alive");
+        peer.observe_cancelled(true);
+        check(peer.gone(), "cancellation marks the peer gone");
+    }
+
+    {
+        llama_grpc::StreamPeer peer;
+        peer.observe_cancelled(true);
+        peer.observe_cancelled(false);
+        check(peer.gone(), "cancellation is sticky across later checks");
+    }
+
+    if (failures != 0) {
+        std::fprintf(stderr, "%d check(s) failed\n", failures);
+        return 1;
+    }
+    return 0;
+}

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=d04e8950c1ec8d30248cbe996682b3182fb1adf6
+STABLEDIFFUSION_GGML_VERSION?=b68d58624d227682eb4b95ef8bcf569cd1311eb5
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/whisper/addr_test.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Specs for resolveAddr run under the suite bootstrap in gowhisper_test.go
+// (TestWhisper); they need no native library, so they never skip.
+var _ = Describe("resolveAddr", func() {
+	It("prefers an explicitly set -addr over a positional argument", func() {
+		Expect(resolveAddr("127.0.0.1:12345", true, []string{"127.0.0.1:59999"})).To(Equal("127.0.0.1:12345"))
+	})
+
+	It("keeps an explicit -addr equal to the default over a positional argument", func() {
+		Expect(resolveAddr(defaultAddr, true, []string{"127.0.0.1:59999"})).To(Equal(defaultAddr))
+	})
+
+	It("falls back to the positional argument when -addr is unset", func() {
+		Expect(resolveAddr(defaultAddr, false, []string{"127.0.0.1:59999"})).To(Equal("127.0.0.1:59999"))
+	})
+
+	It("keeps the default when -addr is unset and no positional argument is given", func() {
+		Expect(resolveAddr(defaultAddr, false, nil)).To(Equal(defaultAddr))
+	})
+
+	It("uses the first positional argument among several", func() {
+		Expect(resolveAddr(defaultAddr, false, []string{"127.0.0.1:59999", "extra"})).To(Equal("127.0.0.1:59999"))
+	})
+
+	It("treats an explicitly empty -addr as unset", func() {
+		Expect(resolveAddr("", true, []string{"127.0.0.1:59999"})).To(Equal("127.0.0.1:59999"))
+		Expect(resolveAddr("", true, nil)).To(Equal(defaultAddr))
+	})
+})

backend/go/whisper/main.go

@@ -10,10 +10,31 @@ import (
 	grpc "github.com/mudler/LocalAI/pkg/grpc"
 )
 
-var (
-	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+const (
+	defaultAddr = "localhost:50051"
 )
 
+var (
+	addr = flag.String("addr", defaultAddr, "the address to listen on")
+)
+
+// resolveAddr picks the address the gRPC server binds to. An explicitly set
+// -addr always wins. Launchers may hand us the listen address as a bare
+// positional argument, which Go's flag package silently drops — honour it
+// next so the server binds the port its caller actually allocated instead of
+// the default one (#11623). An explicitly empty -addr counts as unset:
+// binding the empty address would listen on an OS-chosen port on every
+// interface instead of the one the caller allocated.
+func resolveAddr(flagAddr string, addrSet bool, args []string) string {
+	if addrSet && flagAddr != "" {
+		return flagAddr
+	}
+	if len(args) > 0 {
+		return args[0]
+	}
+	return defaultAddr
+}
+
 type LibFuncs struct {
 	FuncPtr any
 	Name    string
@@ -61,7 +82,16 @@ func main() {
 
 	flag.Parse()
 
-	if err := grpc.StartServer(*addr, &Whisper{}); err != nil {
+	// flag.Visit reports only flags that were explicitly set, so an -addr
+	// equal to the default is still distinguished from an untouched one.
+	addrSet := false
+	flag.Visit(func(f *flag.Flag) {
+		if f.Name == "addr" {
+			addrSet = true
+		}
+	})
+
+	if err := grpc.StartServer(resolveAddr(*addr, addrSet, flag.Args()), &Whisper{}); err != nil {
 		panic(err)
 	}
 }

backend/index.yaml

@@ -150,6 +150,7 @@
     - audio-transcription
     - CPU
     - CUDA
+    - HIP
     - Metal
   # No vulkan key: the vulkan image would carry a Vulkan loader with no Mesa ICD
   # (see the audio-cpp block in .github/backend-matrix.yml). Pointing a
@@ -161,6 +162,7 @@
     nvidia: "cuda12-audio-cpp"
     nvidia-cuda-12: "cuda12-audio-cpp"
     nvidia-cuda-13: "cuda13-audio-cpp"
+    amd: "rocm-audio-cpp"
     metal: "metal-audio-cpp"
     metal-darwin-arm64: "metal-audio-cpp"
 - &whispercpp
@@ -1060,6 +1062,21 @@
     nvidia-l4t: "nvidia-l4t-mlx"
     nvidia-l4t-cuda-12: "nvidia-l4t-mlx"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-mlx"
+- &mlx-video
+  name: "mlx-video"
+  icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
+  urls:
+    - https://github.com/Blaizzy/mlx-video
+  license: MIT
+  description: |
+      Generate videos with LTX-2 and converted Wan2.1/Wan2.2 checkpoints using
+      MLX on Apple Silicon.
+  tags:
+    - text-to-video
+    - image-to-video
+    - MLX
+  capabilities:
+    metal: "metal-mlx-video"
 - &mlx-vlm
   name: "mlx-vlm"
   icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
@@ -1194,6 +1211,7 @@
   tags:
     - image-generation
     - video-generation
+    - sound-generation
     - diffusion-models
   license: apache-2.0
   alias: "diffusers"
@@ -1518,6 +1536,25 @@
     nvidia-cuda-12: "cuda12-faster-whisper"
     nvidia-l4t: "nvidia-l4t-arm64-faster-whisper"
     nvidia-l4t-cuda-12: "nvidia-l4t-arm64-faster-whisper"
+- &whisper-medusa
+  description: |
+    Whisper-Medusa accelerates Whisper transcription by predicting multiple tokens per decoding step.
+    The upstream checkpoints use custom Transformers generation code and accept audio clips up to 30 seconds.
+  urls:
+    - https://github.com/aiola-lab/whisper-medusa
+    - https://huggingface.co/collections/aiola/whisper-medusa
+  tags:
+    - speech-to-text
+    - transcription
+    - Whisper
+    - Medusa
+  license: MIT
+  name: "whisper-medusa"
+  alias: "whisper-medusa"
+  capabilities:
+    default: "cpu-whisper-medusa"
+    nvidia: "cuda12-whisper-medusa"
+    nvidia-cuda-12: "cuda12-whisper-medusa"
 - &moonshine
   description: |
     Moonshine is a fast, accurate, and efficient speech-to-text transcription model using ONNX Runtime.
@@ -1785,6 +1822,32 @@
     nvidia-l4t-cuda-12: "nvidia-l4t-faster-qwen3-tts"
     nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-faster-qwen3-tts"
   icon: https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png
+- &funasr
+  urls:
+    - https://github.com/modelscope/FunASR
+  description: |
+    FunASR is an industrial-grade speech recognition toolkit supporting 50+ languages.
+    Includes SenseVoice (170x realtime, emotion detection), Paraformer (highest Chinese
+    accuracy), and built-in VAD, punctuation restoration, and speaker diarization.
+  tags:
+    - speech-recognition
+    - ASR
+    - multilingual
+  license: mit
+  name: "funasr"
+  alias: "funasr"
+  capabilities:
+    nvidia: "cuda12-funasr"
+    intel: "intel-funasr"
+    amd: "rocm-funasr"
+    metal: "metal-funasr"
+    default: "cpu-funasr"
+    nvidia-cuda-13: "cuda13-funasr"
+    nvidia-cuda-12: "cuda12-funasr"
+    nvidia-l4t: "nvidia-l4t-funasr"
+    nvidia-l4t-cuda-12: "nvidia-l4t-funasr"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr"
+  icon: https://avatars.githubusercontent.com/u/109454077
 - &qwen-asr
   urls:
     - https://github.com/QwenLM/Qwen3-ASR
@@ -2101,6 +2164,7 @@
     nvidia: "cuda12-audio-cpp-development"
     nvidia-cuda-12: "cuda12-audio-cpp-development"
     nvidia-cuda-13: "cuda13-audio-cpp-development"
+    amd: "rocm-audio-cpp-development"
     metal: "metal-audio-cpp-development"
     metal-darwin-arm64: "metal-audio-cpp-development"
 - !!merge <<: *stablediffusionggml
@@ -2164,6 +2228,20 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx"
   mirrors:
     - localai/localai-backends:latest-metal-darwin-arm64-mlx
+- !!merge <<: *mlx-video
+  name: "metal-mlx-video"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-video"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-mlx-video
+- !!merge <<: *mlx-video
+  name: "metal-mlx-video-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-video"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-mlx-video
+- !!merge <<: *mlx-video
+  name: "mlx-video-development"
+  capabilities:
+    metal: "metal-mlx-video-development"
 - !!merge <<: *mlx
   name: "metal-mlx-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx"
@@ -6368,6 +6446,34 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-faster-qwen3-tts"
   mirrors:
     - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-faster-qwen3-tts
+## whisper-medusa
+- !!merge <<: *whisper-medusa
+  name: "whisper-medusa-development"
+  capabilities:
+    default: "cpu-whisper-medusa-development"
+    nvidia: "cuda12-whisper-medusa-development"
+    nvidia-cuda-12: "cuda12-whisper-medusa-development"
+- !!merge <<: *whisper-medusa
+  name: "cpu-whisper-medusa"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-whisper-medusa"
+  mirrors:
+    - localai/localai-backends:latest-cpu-whisper-medusa
+- !!merge <<: *whisper-medusa
+  name: "cpu-whisper-medusa-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-whisper-medusa"
+  mirrors:
+    - localai/localai-backends:master-cpu-whisper-medusa
+- !!merge <<: *whisper-medusa
+  name: "cuda12-whisper-medusa"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-whisper-medusa"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-whisper-medusa
+- !!merge <<: *whisper-medusa
+  name: "cuda12-whisper-medusa-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-whisper-medusa"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-whisper-medusa
+
 ## qwen-asr
 - !!merge <<: *qwen-asr
   name: "qwen-asr-development"
@@ -7054,6 +7160,16 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-audio-cpp"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-13-audio-cpp
+- !!merge <<: *audiocpp
+  name: "rocm-audio-cpp"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-audio-cpp"
+  mirrors:
+    - localai/localai-backends:latest-gpu-rocm-hipblas-audio-cpp
+- !!merge <<: *audiocpp
+  name: "rocm-audio-cpp-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-audio-cpp"
+  mirrors:
+    - localai/localai-backends:master-gpu-rocm-hipblas-audio-cpp
 - !!merge <<: *audiocpp
   name: "metal-audio-cpp"
   uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-audio-cpp"
@@ -7064,3 +7180,97 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-audio-cpp"
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-audio-cpp
+## funasr
+- !!merge <<: *funasr
+  name: "funasr-development"
+  capabilities:
+    nvidia: "cuda12-funasr-development"
+    intel: "intel-funasr-development"
+    amd: "rocm-funasr-development"
+    nvidia-l4t: "nvidia-l4t-funasr-development"
+    metal: "metal-funasr-development"
+    default: "cpu-funasr-development"
+    nvidia-cuda-13: "cuda13-funasr-development"
+    nvidia-cuda-12: "cuda12-funasr-development"
+    nvidia-l4t-cuda-12: "nvidia-l4t-funasr-development"
+    nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr-development"
+- !!merge <<: *funasr
+  name: "cpu-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-funasr"
+  mirrors:
+    - localai/localai-backends:latest-cpu-funasr
+- !!merge <<: *funasr
+  name: "cpu-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-funasr"
+  mirrors:
+    - localai/localai-backends:master-cpu-funasr
+- !!merge <<: *funasr
+  name: "cuda12-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-12-funasr
+- !!merge <<: *funasr
+  name: "cuda12-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-12-funasr
+- !!merge <<: *funasr
+  name: "cuda13-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-nvidia-cuda-13-funasr
+- !!merge <<: *funasr
+  name: "cuda13-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-nvidia-cuda-13-funasr
+- !!merge <<: *funasr
+  name: "intel-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-intel-funasr
+- !!merge <<: *funasr
+  name: "intel-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-intel-funasr
+- !!merge <<: *funasr
+  name: "rocm-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-funasr"
+  mirrors:
+    - localai/localai-backends:latest-gpu-rocm-hipblas-funasr
+- !!merge <<: *funasr
+  name: "rocm-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-funasr"
+  mirrors:
+    - localai/localai-backends:master-gpu-rocm-hipblas-funasr
+- !!merge <<: *funasr
+  name: "nvidia-l4t-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-funasr"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-funasr
+- !!merge <<: *funasr
+  name: "nvidia-l4t-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-funasr"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-funasr
+- !!merge <<: *funasr
+  name: "cuda13-nvidia-l4t-arm64-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr
+- !!merge <<: *funasr
+  name: "cuda13-nvidia-l4t-arm64-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-funasr
+- !!merge <<: *funasr
+  name: "metal-funasr"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-funasr
+- !!merge <<: *funasr
+  name: "metal-funasr-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-funasr

backend/python/README.md

@@ -20,6 +20,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro
 ### Audio & Speech
 - **coqui** - Coqui TTS models
 - **faster-whisper** - Fast Whisper speech recognition
+- **funasr** - Local multilingual transcription with FunASR and SenseVoice
 - **kitten-tts** - Lightweight TTS
 - **mlx-audio** - Apple Silicon audio processing
 - **chatterbox** - TTS model

backend/python/common/libbackend.sh

@@ -563,6 +563,18 @@ function startBackend() {
         echo "Added ${EDIR}/lib to LD_LIBRARY_PATH for GPU libraries"
     fi
 
+    # XPU wheels carry a matching SYCL/Unified Runtime set. Prefer it over
+    # packaged oneAPI or host libraries, which may expose an older loader ABI.
+    if [ "$(uname -s)" = "Linux" ]; then
+        local sycl_runtime
+        for sycl_runtime in "${EDIR}/venv/lib"/libsycl.so*; do
+            if [ -f "${sycl_runtime}" ]; then
+                export LD_LIBRARY_PATH="${EDIR}/venv/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+                break
+            fi
+        done
+    fi
+
     if [ ! -z "${BACKEND_FILE:-}" ]; then
         exec "${EDIR}/venv/bin/python" "${BACKEND_FILE}" "$@"
     elif [ -e "${MY_DIR}/server.py" ]; then

backend/python/diffusers/audio_utils.py

@@ -0,0 +1,24 @@
+import array
+import sys
+import wave
+
+
+def write_pcm_wav(destination, samples, sampling_rate):
+    """Write normalized floating-point audio samples as mono 16-bit PCM."""
+    pcm = array.array(
+        "h",
+        (
+            max(-32768, min(32767, round(float(sample) * 32768)))
+            for sample in samples
+        ),
+    )
+    if pcm.itemsize != 2:
+        raise RuntimeError("16-bit PCM requires two-byte signed integers")
+    if sys.byteorder != "little":
+        pcm.byteswap()
+
+    with wave.open(destination, "wb") as output:
+        output.setnchannels(1)
+        output.setsampwidth(2)
+        output.setframerate(sampling_rate)
+        output.writeframes(pcm.tobytes())

backend/python/diffusers/backend.py

@@ -26,6 +26,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
 from grpc_auth import get_auth_interceptors
 from model_utils import resolve_model_reference
+from audio_utils import write_pcm_wav
 
 
 # Import dynamic loader for pipeline discovery
@@ -35,6 +36,7 @@ from diffusers_dynamic_loader import (
     get_available_pipelines,
     load_diffusers_pipeline,
 )
+from load_options import single_file_load_kwargs
 
 # Import specific items still needed for special cases and safety checker
 from diffusers import DiffusionPipeline, ControlNetModel
@@ -479,6 +481,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # Build kwargs for dynamic loading
         load_kwargs = {"torch_dtype": torchType}
+        load_kwargs.update(
+            single_file_load_kwargs(request.OriginalConfigFile, from_single_file)
+        )
 
         # Add variant if not loading from single file
         if not from_single_file and variant:
@@ -899,6 +904,49 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         return backend_pb2.Result(message="Media generated", success=True)
 
+    def SoundGeneration(self, request, context):
+        if not request.dst:
+            return backend_pb2.Result(success=False, message="request.dst is required")
+
+        prompt = request.text or request.caption
+        if not prompt:
+            return backend_pb2.Result(success=False, message="request.text is required")
+
+        try:
+            generation_options = dict(self.options)
+            if "num_inference_steps" in generation_options:
+                generation_options["num_inference_steps"] = int(
+                    generation_options["num_inference_steps"]
+                )
+            generation_options["prompt"] = prompt
+            if request.HasField("duration"):
+                generation_options["audio_length_in_s"] = request.duration
+            if request.HasField("temperature"):
+                generation_options["guidance_scale"] = request.temperature
+
+            generated = self.pipe(**generation_options)
+            if not hasattr(generated, "audios") or len(generated.audios) == 0:
+                return backend_pb2.Result(
+                    success=False,
+                    message="The diffusers pipeline returned no audio",
+                )
+
+            samples = generated.audios[0]
+            if hasattr(samples, "reshape"):
+                samples = samples.reshape(-1)
+            if hasattr(samples, "tolist"):
+                samples = samples.tolist()
+
+            sampling_rate = getattr(
+                getattr(getattr(self.pipe, "vae", None), "config", None),
+                "sampling_rate",
+                16000,
+            )
+            write_pcm_wav(request.dst, samples, sampling_rate)
+            return backend_pb2.Result(success=True, message="Sound generated successfully")
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"SoundGeneration error: {err}")
+
     def UpscaleImage(self, request, context):
         try:
             if not request.src:

backend/python/diffusers/load_options.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: MIT
+
+
+def single_file_load_kwargs(original_config_file: str, from_single_file: bool) -> dict:
+    if from_single_file and original_config_file:
+        return {"original_config_file": original_config_file}
+    return {}

backend/python/diffusers/test.py

@@ -4,6 +4,9 @@ A test script to test the gRPC service and dynamic loader
 import unittest
 import subprocess
 import time
+import os
+import tempfile
+import wave
 from unittest.mock import patch, MagicMock
 
 # Import dynamic loader for testing (these don't need gRPC)
@@ -445,3 +448,64 @@ class TestDeviceSelection(unittest.TestCase):
 
     def test_mps_overrides(self):
         self.assertEqual(backend.select_device(False, None, True, False, True), "mps")
+
+
+class TestWritePcmWav(unittest.TestCase):
+    def test_writes_clipped_float_samples_as_mono_pcm(self):
+        from audio_utils import write_pcm_wav
+
+        destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        destination.close()
+
+        try:
+            write_pcm_wav(destination.name, [0.0, 0.5, -0.5, 2.0], 16000)
+
+            with wave.open(destination.name, "rb") as generated:
+                self.assertEqual(generated.getframerate(), 16000)
+                self.assertEqual(generated.getnchannels(), 1)
+                self.assertEqual(generated.getsampwidth(), 2)
+                self.assertEqual(generated.getnframes(), 4)
+                self.assertEqual(
+                    generated.readframes(4),
+                    b"\x00\x00\x00@\x00\xc0\xff\x7f",
+                )
+        finally:
+            os.unlink(destination.name)
+
+
+@unittest.skipUnless(GRPC_AVAILABLE, "gRPC modules not available")
+class TestSoundGeneration(unittest.TestCase):
+    def test_maps_request_options_and_writes_pipeline_audio(self):
+        from backend import BackendServicer
+
+        service = BackendServicer.__new__(BackendServicer)
+        service.options = {"num_inference_steps": 200.0}
+        service.pipe = MagicMock()
+        service.pipe.return_value.audios = [[0.0, 0.5, -0.5]]
+        service.pipe.vae.config.sampling_rate = 16000
+
+        destination = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        destination.close()
+
+        try:
+            request = backend_pb2.SoundGenerationRequest(
+                text="ocean waves",
+                dst=destination.name,
+                duration=2.5,
+                temperature=0,
+            )
+
+            result = service.SoundGeneration(request, context=None)
+
+            self.assertTrue(result.success, result.message)
+            service.pipe.assert_called_once_with(
+                num_inference_steps=200,
+                prompt="ocean waves",
+                audio_length_in_s=2.5,
+                guidance_scale=0,
+            )
+            with wave.open(destination.name, "rb") as generated:
+                self.assertEqual(generated.getframerate(), 16000)
+                self.assertEqual(generated.getnframes(), 3)
+        finally:
+            os.unlink(destination.name)

backend/python/diffusers/test_load_options.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: MIT
+
+import unittest
+
+from backend.python.diffusers.load_options import single_file_load_kwargs
+
+
+class SingleFileLoadKwargsTest(unittest.TestCase):
+    def test_includes_original_config_for_single_file(self):
+        self.assertEqual(
+            single_file_load_kwargs("configs/v1-inference.yaml", True),
+            {"original_config_file": "configs/v1-inference.yaml"},
+        )
+
+    def test_omits_original_config_for_pretrained_model(self):
+        self.assertEqual(
+            single_file_load_kwargs("configs/v1-inference.yaml", False),
+            {},
+        )
+
+    def test_omits_empty_original_config(self):
+        self.assertEqual(single_file_load_kwargs("", True), {})
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/faster-whisper/install.sh

@@ -26,4 +26,17 @@ if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then
     USE_PIP=true
 fi
 
+CTRANSLATE2_VERSION=${CTRANSLATE2_VERSION:-v4.7.1}
+CTRANSLATE2_ROCM_WHEEL_OS=${CTRANSLATE2_ROCM_WHEEL_OS:-Linux}
+CTRANSLATE2_ROCM_WHEEL_ARCHIVE="rocm-python-wheels-${CTRANSLATE2_ROCM_WHEEL_OS}.zip"
+
+if [ "x${BUILD_PROFILE}" == "xhipblas" ]; then
+  ensureVenv
+  mkdir /tmp/ctranslate2-rocm
+  wget -O "/tmp/ctranslate2-rocm/${CTRANSLATE2_ROCM_WHEEL_ARCHIVE}" "https://github.com/OpenNMT/CTranslate2/releases/download/${CTRANSLATE2_VERSION}/${CTRANSLATE2_ROCM_WHEEL_ARCHIVE}"
+  unzip "/tmp/ctranslate2-rocm/${CTRANSLATE2_ROCM_WHEEL_ARCHIVE}" -d /tmp/ctranslate2-rocm/
+  python3 -m ensurepip
+  python3 -m pip install --no-dependencies --no-index --find-links=/tmp/ctranslate2-rocm/temp-linux/ ctranslate2
+fi
+
 installRequirements

backend/python/fish-speech/Makefile

@@ -1,18 +1,26 @@
-.PHONY: fish-speech test-source-preparation
-fish-speech: test-source-preparation
+.PHONY: fish-speech test-source-preparation test-runtime-import
+fish-speech: test-source-preparation test-runtime-import
 	bash install.sh
 
 test-source-preparation:
 	bash prepare-source_test.sh
 
+test-runtime-import:
+	bash run_test.sh
+
 .PHONY: run
 run: fish-speech
 	@echo "Running fish-speech..."
 	bash run.sh
 	@echo "fish-speech run."
 
+.PHONY: test-unit
+test-unit:
+	python3 -m unittest -v prepare_upstream_test.py
+	bash run_test.sh
+
 .PHONY: test
-test: fish-speech
+test: fish-speech test-unit
 	@echo "Testing fish-speech..."
 	bash test.sh
 	@echo "fish-speech tested."

backend/python/fish-speech/backend.py

@@ -326,11 +326,29 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             max_new_tokens = self.options.get("max_new_tokens", 1024)
             chunk_length = self.options.get("chunk_length", 200)
 
-            # Build references list for voice cloning
+            # Build references list for voice cloning. Saved LocalAI
+            # personalities use the same ordered JSON shape as audio.cpp.
             references = []
             voice_name = request.voice if request.voice else None
 
-            if voice_name and os.path.isfile(voice_name):
+            multi_reference_cond = request.params.get("multi_reference_cond", "") if hasattr(request, "params") else ""
+            if multi_reference_cond:
+                reference_entries = json.loads(multi_reference_cond)
+                if not isinstance(reference_entries, list) or not reference_entries:
+                    raise ValueError("multi_reference_cond must be a non-empty JSON array")
+                for entry in reference_entries:
+                    if not isinstance(entry, dict) or not entry.get("audio") or not entry.get("text"):
+                        raise ValueError("multi_reference_cond entries require audio and text")
+                    ref_audio_path = self._get_ref_audio_path(entry["audio"])
+                    with open(ref_audio_path, "rb") as f:
+                        audio_bytes = f.read()
+                    references.append(ServeReferenceAudio(audio=audio_bytes, text=entry["text"]))
+                print(
+                    f"[INFO] Using {len(references)} per-request reference audios",
+                    file=sys.stderr,
+                )
+
+            elif voice_name and os.path.isfile(voice_name):
                 ref_audio_path = self._get_ref_audio_path(voice_name)
                 with open(ref_audio_path, "rb") as f:
                     audio_bytes = f.read()

backend/python/fish-speech/install.sh

@@ -44,6 +44,13 @@ fi
 # editable install. pyaudio is only used by the upstream playback client.
 bash "${backend_dir}/prepare-source.sh" "${BUILD_TYPE:-}" "${FISH_SPEECH_DIR}/pyproject.toml"
 
+# CUDA 13 has no torch 2.8 wheels, so fish-speech's exact upstream pin would
+# make pip select the CPU-only aarch64 wheel from PyPI. Prepare the cloned tree
+# before resolving it, and use soundfile for reference audio because torchcodec
+# does not publish Linux aarch64 wheels.
+python3 "${backend_dir}/prepare_upstream.py" "${FISH_SPEECH_DIR}" \
+    --cuda-major "${CUDA_MAJOR_VERSION:-}"
+
 # Install fish-speech deps from source (without the package itself since we use PYTHONPATH)
 ensureVenv
 if [ "x${USE_PIP}" == "xtrue" ]; then

backend/python/fish-speech/prepare_upstream.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+
+import argparse
+from pathlib import Path
+
+
+TORCH_28 = '"torch==2.8.0"'
+TORCH_29 = '"torch==2.9.1"'
+TORCHAUDIO_28 = '"torchaudio==2.8.0"'
+TORCHAUDIO_29 = '"torchaudio==2.9.1"'
+TORCHAUDIO_LOAD = (
+    "        waveform, original_sr = "
+    "torchaudio.load(reference_audio, backend=self.backend)"
+)
+SOUNDFILE_LOAD = "\n".join(
+    (
+        "        import soundfile as _sf",
+        "        import torch as _torch",
+        "",
+        "        data, original_sr = _sf.read(",
+        '            reference_audio, dtype="float32", always_2d=True',
+        "        )",
+        "        waveform = _torch.from_numpy(data.T.copy())",
+    )
+)
+
+
+def patch_cuda13_dependencies(pyproject: Path) -> None:
+    content = pyproject.read_text()
+    if (
+        TORCH_28 not in content
+        and TORCHAUDIO_28 not in content
+        and TORCH_29 in content
+        and TORCHAUDIO_29 in content
+    ):
+        return
+    if TORCH_28 not in content or TORCHAUDIO_28 not in content:
+        raise RuntimeError("fish-speech's torch 2.8 dependency pins have changed")
+
+    content = content.replace(TORCH_28, TORCH_29)
+    content = content.replace(TORCHAUDIO_28, TORCHAUDIO_29)
+    pyproject.write_text(content)
+
+
+def patch_reference_loader(loader: Path) -> None:
+    content = loader.read_text()
+    if TORCHAUDIO_LOAD not in content and content.count(SOUNDFILE_LOAD) == 1:
+        return
+    if content.count(TORCHAUDIO_LOAD) != 1:
+        raise RuntimeError("fish-speech's torchaudio.load call has changed")
+
+    loader.write_text(content.replace(TORCHAUDIO_LOAD, SOUNDFILE_LOAD))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("source", type=Path)
+    parser.add_argument("--cuda-major")
+    args = parser.parse_args()
+
+    if args.cuda_major == "13":
+        patch_cuda13_dependencies(args.source / "pyproject.toml")
+    patch_reference_loader(
+        args.source / "fish_speech/inference_engine/reference_loader.py"
+    )
+
+
+if __name__ == "__main__":
+    main()

backend/python/fish-speech/prepare_upstream_test.py

@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: MIT
+
+import importlib.util
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+
+
+MODULE_PATH = Path(__file__).with_name("prepare_upstream.py")
+
+
+def load_prepare_upstream():
+    if not MODULE_PATH.exists():
+        raise AssertionError("prepare_upstream.py is missing")
+    spec = importlib.util.spec_from_file_location("prepare_upstream", MODULE_PATH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class FakeAudioData:
+    @property
+    def T(self):
+        return self
+
+    def copy(self):
+        return "channels-first"
+
+
+class PrepareUpstreamTests(unittest.TestCase):
+    def test_cuda13_dependencies_follow_available_pytorch_wheels(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            pyproject = Path(tmp) / "pyproject.toml"
+            pyproject.write_text(
+                'dependencies = [\n    "torch==2.8.0",\n    "torchaudio==2.8.0",\n]\n'
+                'stable = [\n    "torch==2.8.0",\n    "torchaudio",\n]\n'
+            )
+
+            prepare_upstream.patch_cuda13_dependencies(pyproject)
+
+            self.assertEqual(
+                pyproject.read_text(),
+                'dependencies = [\n    "torch==2.9.1",\n    "torchaudio==2.9.1",\n]\n'
+                'stable = [\n    "torch==2.9.1",\n    "torchaudio",\n]\n',
+            )
+
+    def test_reference_audio_uses_soundfile_without_torchcodec(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            loader = Path(tmp) / "reference_loader.py"
+            loader.write_text(
+                "class ReferenceLoader:\n"
+                "    def load_audio(self, reference_audio):\n"
+                "        waveform, original_sr = torchaudio.load(reference_audio, backend=self.backend)\n"
+                "        return waveform, original_sr\n"
+            )
+            prepare_upstream.patch_reference_loader(loader)
+
+            calls = []
+            fake_soundfile = types.SimpleNamespace(
+                read=lambda source, **kwargs: (
+                    calls.append((source, kwargs)) or FakeAudioData(),
+                    24000,
+                )
+            )
+            fake_torch = types.SimpleNamespace(
+                from_numpy=lambda data: ("tensor", data),
+            )
+            previous_soundfile = sys.modules.get("soundfile")
+            previous_torch = sys.modules.get("torch")
+            sys.modules["soundfile"] = fake_soundfile
+            sys.modules["torch"] = fake_torch
+            try:
+                namespace = {"torchaudio": None}
+                exec(compile(loader.read_text(), str(loader), "exec"), namespace)
+                instance = namespace["ReferenceLoader"]()
+                instance.backend = "soundfile"
+
+                waveform, sample_rate = instance.load_audio("voice.wav")
+            finally:
+                if previous_soundfile is None:
+                    del sys.modules["soundfile"]
+                else:
+                    sys.modules["soundfile"] = previous_soundfile
+                if previous_torch is None:
+                    del sys.modules["torch"]
+                else:
+                    sys.modules["torch"] = previous_torch
+
+            self.assertEqual(waveform, ("tensor", "channels-first"))
+            self.assertEqual(sample_rate, 24000)
+            self.assertEqual(
+                calls,
+                [("voice.wav", {"dtype": "float32", "always_2d": True})],
+            )
+
+    def test_reference_loader_drift_fails_the_build(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            loader = Path(tmp) / "reference_loader.py"
+            loader.write_text("def load_audio():\n    pass\n")
+
+            with self.assertRaisesRegex(RuntimeError, "torchaudio.load call"):
+                prepare_upstream.patch_reference_loader(loader)
+
+    def test_preparation_can_be_repeated(self):
+        prepare_upstream = load_prepare_upstream()
+
+        with tempfile.TemporaryDirectory() as tmp:
+            pyproject = Path(tmp) / "pyproject.toml"
+            pyproject.write_text(
+                'dependencies = ["torch==2.8.0", "torchaudio==2.8.0"]\n'
+            )
+            loader = Path(tmp) / "reference_loader.py"
+            loader.write_text(
+                "def load_audio(reference_audio):\n"
+                "        waveform, original_sr = torchaudio.load(reference_audio, backend=self.backend)\n"
+            )
+
+            prepare_upstream.patch_cuda13_dependencies(pyproject)
+            prepare_upstream.patch_reference_loader(loader)
+            try:
+                prepare_upstream.patch_cuda13_dependencies(pyproject)
+                prepare_upstream.patch_reference_loader(loader)
+            except RuntimeError as err:
+                self.fail(f"preparation is not idempotent: {err}")
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/fish-speech/requirements-cublas13.txt

@@ -1,3 +1,3 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
+torch==2.9.1+cu130
+torchaudio==2.9.1

backend/python/fish-speech/requirements-l4t13.txt

@@ -1,3 +1,3 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
-torch
-torchaudio
+torch==2.9.1+cu130
+torchaudio==2.9.1

backend/python/fish-speech/run.sh

@@ -1,9 +1,18 @@
 #!/bin/bash
-backend_dir=$(dirname $0)
+backend_dir=$(dirname "$(realpath "$0")")
 if [ -d $backend_dir/common ]; then
     source $backend_dir/common/libbackend.sh
 else
     source $backend_dir/../common/libbackend.sh
 fi
 
-startBackend $@
+# Editable installs record their build-time absolute source path, which becomes
+# stale when the backend is relocated under /backends at install time.
+export PYTHONPATH="${backend_dir}/fish-speech-src${PYTHONPATH:+:${PYTHONPATH}}"
+
+cuda_home=${CUDA_HOME:-/usr/local/cuda}
+if [ -z "${TRITON_PTXAS_PATH:-}" ] && [ -x "$cuda_home/bin/ptxas" ]; then
+    export TRITON_PTXAS_PATH="$cuda_home/bin/ptxas"
+fi
+
+startBackend "$@"

backend/python/fish-speech/run_test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+WORK_DIR=$(mktemp -d)
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+mkdir -p "$WORK_DIR/runtime/common" "$WORK_DIR/runtime/fish-speech-src/fish_speech"
+cp "$SCRIPT_DIR/run.sh" "$WORK_DIR/runtime/run.sh"
+
+cat > "$WORK_DIR/runtime/fish-speech-src/fish_speech/inference_engine.py" <<'PY'
+IMPORT_OK = True
+PY
+
+cat > "$WORK_DIR/runtime/common/libbackend.sh" <<'SH'
+startBackend() {
+    python3 -c 'from fish_speech.inference_engine import IMPORT_OK; assert IMPORT_OK'
+}
+SH
+
+bash "$WORK_DIR/runtime/run.sh"
+
+echo "PASS: runtime launcher imports relocated fish-speech source"

backend/python/funasr/Makefile

@@ -0,0 +1,23 @@
+.PHONY: funasr
+funasr:
+	bash install.sh
+
+.PHONY: run
+run: funasr
+	@echo "Running funasr..."
+	bash run.sh
+	@echo "funasr run."
+
+.PHONY: test
+test: funasr
+	@echo "Testing funasr..."
+	bash test.sh
+	@echo "funasr tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__

backend/python/funasr/backend.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""
+gRPC backend for LocalAI wrapping FunASR (SenseVoice / Paraformer).
+"""
+from concurrent import futures
+import time
+import argparse
+import signal
+import sys
+import os
+import backend_pb2
+import backend_pb2_grpc
+import torch
+
+import grpc
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
+from grpc_auth import get_auth_interceptors
+
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
+
+    def LoadModel(self, request, context):
+        from funasr import AutoModel
+
+        device = "cpu"
+        if request.CUDA and torch.cuda.is_available():
+            device = "cuda"
+        elif hasattr(torch, "xpu") and torch.xpu.is_available():
+            device = "xpu"
+        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            device = "mps"
+
+        model_id = request.Model or "iic/SenseVoiceSmall"
+        candidate_devices = [device]
+        if device in ("xpu", "mps"):
+            candidate_devices.append("cpu")
+
+        for candidate_device in candidate_devices:
+            try:
+                print(
+                    f"Loading FunASR model: {model_id} on {candidate_device}",
+                    file=sys.stderr,
+                )
+                self.model = AutoModel(
+                    model=model_id,
+                    vad_model="fsmn-vad",
+                    device=candidate_device,
+                    disable_update=True,
+                )
+                print("FunASR model loaded successfully", file=sys.stderr)
+                break
+            except Exception as err:
+                if candidate_device != candidate_devices[-1]:
+                    print(
+                        f"[WARN] FunASR {candidate_device} initialization failed: "
+                        f"{err}; retrying on cpu",
+                        file=sys.stderr,
+                    )
+                    continue
+                print(f"[ERROR] LoadModel failed: {err}", file=sys.stderr)
+                import traceback
+                traceback.print_exc(file=sys.stderr)
+                return backend_pb2.Result(success=False, message=str(err))
+
+        return backend_pb2.Result(message="Model loaded successfully", success=True)
+
+    def AudioTranscription(self, request, context):
+        from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+        result_segments = []
+        text = ""
+        try:
+            audio_path = request.dst
+            if not audio_path or not os.path.exists(audio_path):
+                print(f"Error: Audio file not found: {audio_path}", file=sys.stderr)
+                return backend_pb2.TranscriptResult(segments=[], text="")
+
+            language = None
+            if request.language and request.language.strip():
+                language = request.language.strip()
+
+            kwargs = {}
+            if language:
+                kwargs["language"] = language
+
+            results = self.model.generate(input=audio_path, **kwargs)
+
+            if not results:
+                return backend_pb2.TranscriptResult(segments=[], text="")
+
+            for idx, r in enumerate(results):
+                seg_text = r.get("text", "") if isinstance(r, dict) else str(r)
+                seg_text = rich_transcription_postprocess(seg_text)
+                text += seg_text
+                result_segments.append(backend_pb2.TranscriptSegment(
+                    id=idx,
+                    start=0,
+                    end=0,
+                    text=seg_text,
+                ))
+
+        except Exception as err:
+            print(f"Error in AudioTranscription: {err}", file=sys.stderr)
+            import traceback
+            traceback.print_exc(file=sys.stderr)
+            return backend_pb2.TranscriptResult(segments=[], text="")
+
+        return backend_pb2.TranscriptResult(segments=result_segments, text=text)
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ('grpc.max_message_length', 50 * 1024 * 1024),
+            ('grpc.max_send_message_length', 50 * 1024 * 1024),
+            ('grpc.max_receive_message_length', 50 * 1024 * 1024),
+        ],
+        interceptors=get_auth_interceptors(),
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print("Server started. Listening on: " + address, file=sys.stderr)
+
+    def signal_handler(sig, frame):
+        print("Received termination signal. Shutting down...")
+        server.stop(0)
+        sys.exit(0)
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the gRPC server.")
+    parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.")
+    args = parser.parse_args()
+    serve(args.addr)

backend/python/funasr/install.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation"
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+if [ "x${BUILD_PROFILE}" == "xintel" ]; then
+    EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
+fi
+
+PYTHON_VERSION="3.12"
+PYTHON_PATCH="12"
+PY_STANDALONE_TAG="20251120"
+
+installRequirements

backend/python/funasr/requirements-cpu.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-cublas12.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cu121
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-cublas13.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-hipblas.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-intel.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-l4t12.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu129/
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-l4t13.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-mps.txt

@@ -0,0 +1,3 @@
+torch==2.7.1
+torchaudio==2.7.1
+funasr

backend/python/funasr/requirements.txt

@@ -0,0 +1,5 @@
+grpcio==1.71.0
+protobuf
+certifi
+packaging==24.1
+setuptools

backend/python/funasr/run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+startBackend $@

backend/python/funasr/test.py

@@ -0,0 +1,295 @@
+import importlib
+import os
+import re
+import sys
+import types
+import unittest
+
+
+class _Reply:
+    def __init__(self, message=b""):
+        self.message = message
+
+
+class _Result:
+    def __init__(self, message="", success=False):
+        self.message = message
+        self.success = success
+
+
+class _TranscriptSegment:
+    def __init__(self, id=0, start=0, end=0, text=""):
+        self.id = id
+        self.start = start
+        self.end = end
+        self.text = text
+
+
+class _TranscriptResult:
+    def __init__(self, segments=None, text=""):
+        self.segments = segments or []
+        self.text = text
+
+
+class _FakeBackendServicer:
+    pass
+
+
+class _FakeTorch:
+    cuda = types.SimpleNamespace(is_available=lambda: False)
+    xpu = types.SimpleNamespace(is_available=lambda: False)
+    backends = types.SimpleNamespace(mps=types.SimpleNamespace(is_available=lambda: False))
+
+
+class _FakeAutoModel:
+    instances = []
+    attempts = []
+    fail_devices = set()
+
+    def __init__(self, **kwargs):
+        _FakeAutoModel.attempts.append(kwargs)
+        if kwargs.get("device") in _FakeAutoModel.fail_devices:
+            raise RuntimeError(f"unsupported device: {kwargs['device']}")
+        self.kwargs = kwargs
+        self.generate_calls = []
+        self.results = [{"text": "hello"}, {"text": " world"}]
+        _FakeAutoModel.instances.append(self)
+
+    def generate(self, **kwargs):
+        self.generate_calls.append(kwargs)
+        return self.results
+
+
+def _install_stubs():
+    sys.modules["backend_pb2"] = types.SimpleNamespace(
+        Reply=_Reply,
+        Result=_Result,
+        TranscriptSegment=_TranscriptSegment,
+        TranscriptResult=_TranscriptResult,
+    )
+    sys.modules["backend_pb2_grpc"] = types.SimpleNamespace(
+        BackendServicer=_FakeBackendServicer,
+        add_BackendServicer_to_server=lambda *args, **kwargs: None,
+    )
+    sys.modules["grpc"] = types.SimpleNamespace(
+        RpcMethodHandler=object,
+        ServerInterceptor=object,
+        StatusCode=types.SimpleNamespace(UNAUTHENTICATED="UNAUTHENTICATED"),
+        aio=types.SimpleNamespace(ServerInterceptor=object),
+        server=lambda *args, **kwargs: None,
+    )
+    sys.modules["torch"] = _FakeTorch
+    funasr = types.ModuleType("funasr")
+    funasr.__path__ = []
+    funasr.AutoModel = _FakeAutoModel
+    funasr_utils = types.ModuleType("funasr.utils")
+    funasr_utils.__path__ = []
+    postprocess_utils = types.ModuleType("funasr.utils.postprocess_utils")
+    postprocess_utils.rich_transcription_postprocess = lambda text: re.sub(
+        r"<\|.*?\|>", "", text
+    )
+    sys.modules["funasr"] = funasr
+    sys.modules["funasr.utils"] = funasr_utils
+    sys.modules["funasr.utils.postprocess_utils"] = postprocess_utils
+
+
+def _load_backend():
+    _install_stubs()
+    sys.modules.pop("backend", None)
+    _FakeAutoModel.instances.clear()
+    _FakeAutoModel.attempts.clear()
+    _FakeAutoModel.fail_devices.clear()
+    return importlib.import_module("backend")
+
+
+class TestFunASRBackend(unittest.TestCase):
+    def test_torch_profiles_install_torchaudio(self):
+        backend_dir = os.path.dirname(__file__)
+
+        for profile in (
+            "cpu",
+            "cublas12",
+            "cublas13",
+            "hipblas",
+            "intel",
+            "l4t12",
+            "l4t13",
+            "mps",
+        ):
+            with self.subTest(profile=profile):
+                requirements_path = os.path.join(
+                    backend_dir, f"requirements-{profile}.txt"
+                )
+                with open(requirements_path, encoding="utf-8") as requirements_file:
+                    requirements = {
+                        line.strip().split("=", 1)[0]
+                        for line in requirements_file
+                        if line.strip() and not line.lstrip().startswith(("#", "--"))
+                    }
+
+                self.assertIn("torchaudio", requirements)
+
+    def test_accelerator_profiles_use_current_pytorch_indexes(self):
+        backend_dir = os.path.dirname(__file__)
+        expected_indexes = {
+            "cublas13": "https://download.pytorch.org/whl/cu130",
+            "hipblas": "https://download.pytorch.org/whl/rocm7.0",
+            "intel": "https://download.pytorch.org/whl/xpu",
+            "l4t12": "https://pypi.jetson-ai-lab.io/jp6/cu129/",
+            "l4t13": "https://download.pytorch.org/whl/cu130",
+        }
+
+        for profile, expected_index in expected_indexes.items():
+            with self.subTest(profile=profile):
+                requirements_path = os.path.join(
+                    backend_dir, f"requirements-{profile}.txt"
+                )
+                with open(requirements_path, encoding="utf-8") as requirements_file:
+                    requirements = requirements_file.read()
+
+                self.assertIn(
+                    f"--extra-index-url {expected_index}\n",
+                    requirements,
+                )
+
+    def test_health_returns_ok(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+
+        reply = servicer.Health(types.SimpleNamespace(), None)
+
+        self.assertEqual(reply.message, b"OK")
+
+    def test_load_model_uses_default_sensevoice_model_on_cpu(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+
+        result = servicer.LoadModel(types.SimpleNamespace(Model="", CUDA=False), None)
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(result.message, "Model loaded successfully")
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["model"], "iic/SenseVoiceSmall")
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["vad_model"], "fsmn-vad")
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
+        self.assertTrue(_FakeAutoModel.instances[0].kwargs["disable_update"])
+
+    def test_load_model_uses_xpu_when_available(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_xpu = _FakeTorch.xpu
+        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
+        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "xpu")
+
+    def test_load_model_prefers_requested_cuda_over_xpu_and_mps(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_cuda = _FakeTorch.cuda
+        original_xpu = _FakeTorch.xpu
+        original_mps = _FakeTorch.backends.mps
+        _FakeTorch.cuda = types.SimpleNamespace(is_available=lambda: True)
+        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
+        _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True)
+        self.addCleanup(setattr, _FakeTorch, "cuda", original_cuda)
+        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
+        self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=True), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(
+            [attempt["device"] for attempt in _FakeAutoModel.attempts], ["cuda"]
+        )
+
+    def test_load_model_retries_cpu_when_xpu_initialization_fails(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_xpu = _FakeTorch.xpu
+        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
+        _FakeAutoModel.fail_devices.add("xpu")
+        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(
+            [attempt["device"] for attempt in _FakeAutoModel.attempts],
+            ["xpu", "cpu"],
+        )
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
+
+    def test_load_model_retries_cpu_when_mps_initialization_fails(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_mps = _FakeTorch.backends.mps
+        _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True)
+        _FakeAutoModel.fail_devices.add("mps")
+        self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(
+            [attempt["device"] for attempt in _FakeAutoModel.attempts],
+            ["mps", "cpu"],
+        )
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
+
+    def test_audio_transcription_passes_language_and_builds_segments(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        servicer.model = _FakeAutoModel()
+        audio_path = os.path.abspath(__file__)
+        request = types.SimpleNamespace(dst=audio_path, language=" zh ")
+
+        result = servicer.AudioTranscription(request, None)
+
+        self.assertEqual(result.text, "hello world")
+        self.assertEqual([segment.text for segment in result.segments], ["hello", " world"])
+        self.assertEqual(servicer.model.generate_calls, [{"input": audio_path, "language": "zh"}])
+
+    def test_audio_transcription_cleans_sensevoice_tags(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        servicer.model = _FakeAutoModel()
+        servicer.model.results = [
+            {"text": "<|zh|><|NEUTRAL|><|Speech|><|woitn|>hello"}
+        ]
+
+        result = servicer.AudioTranscription(
+            types.SimpleNamespace(dst=os.path.abspath(__file__), language=""),
+            None,
+        )
+
+        self.assertEqual(result.text, "hello")
+        self.assertEqual([segment.text for segment in result.segments], ["hello"])
+
+    def test_audio_transcription_missing_file_returns_empty_result(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        servicer.model = _FakeAutoModel()
+
+        result = servicer.AudioTranscription(
+            types.SimpleNamespace(dst="/tmp/localai-funasr-missing.wav", language=""),
+            None,
+        )
+
+        self.assertEqual(result.text, "")
+        self.assertEqual(result.segments, [])
+        self.assertEqual(servicer.model.generate_calls, [])
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/funasr/test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+runUnittests "$@"

backend/python/mlx-distributed/backend.py

@@ -440,8 +440,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     kwargs["tools"] = json.loads(request.Tools)
                 except json.JSONDecodeError:
                     pass
-            if request.Metadata.get("enable_thinking", "").lower() == "true":
-                kwargs["enable_thinking"] = True
+            enable_thinking = request.Metadata.get("enable_thinking", "").lower()
+            if enable_thinking in ("true", "false"):
+                kwargs["enable_thinking"] = enable_thinking == "true"
             try:
                 return self.tokenizer.apply_chat_template(messages, **kwargs)
             except TypeError:

backend/python/mlx-distributed/test.py

@@ -135,3 +135,37 @@ class TestSharedHelpers(unittest.TestCase):
         self.assertEqual(len(calls), 1)
         self.assertEqual(calls[0]["name"], "get_weather")
         self.assertEqual(calls[0]["arguments"], '{"location": "Paris"}')
+
+
+class TestPreparePrompt(unittest.TestCase):
+    """Server-less tests for BackendServicer._prepare_prompt."""
+
+    def test_forwards_enable_thinking(self):
+        from backend import BackendServicer
+
+        class Tok:
+            def __init__(self):
+                self.kwargs = None
+
+            def apply_chat_template(self, messages, **kwargs):
+                self.kwargs = kwargs
+                return "PROMPT"
+
+        def kwargs_for(metadata):
+            servicer = BackendServicer()
+            servicer.tokenizer = Tok()
+            req = types.SimpleNamespace(
+                Prompt="",
+                UseTokenizerTemplate=True,
+                Messages=[backend_pb2.Message(role="user", content="hi")],
+                Tools="",
+                Metadata=metadata,
+            )
+            self.assertEqual(servicer._prepare_prompt(req), "PROMPT")
+            return servicer.tokenizer.kwargs
+
+        self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True)
+        # "false" used to be dropped, so thinking models kept reasoning
+        self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False)
+        self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False)
+        self.assertNotIn("enable_thinking", kwargs_for({}))

backend/python/mlx-video/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+backend_pb2.py
+backend_pb2_grpc.py
+common/
+venv/

backend/python/mlx-video/Makefile

@@ -0,0 +1,16 @@
+.PHONY: all run test protogen-clean clean
+
+all:
+	bash install.sh
+
+run: all
+	bash run.sh
+
+test:
+	bash test.sh
+
+protogen-clean:
+	$(RM) backend_pb2.py backend_pb2_grpc.py
+
+clean: protogen-clean
+	rm -rf __pycache__ venv

backend/python/mlx-video/backend.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+
+import argparse
+import os
+import platform
+import signal
+import subprocess
+import sys
+import traceback
+from concurrent import futures
+
+import grpc
+
+import backend_pb2
+import backend_pb2_grpc
+from mlx_video_utils import build_generation_command, classify_model, parse_options
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common"))
+from grpc_auth import get_auth_interceptors  # noqa: E402
+
+
+MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def __init__(self):
+        self.family = None
+        self.model = None
+        self.options = {}
+
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=b"OK")
+
+    def LoadModel(self, request, context):
+        if platform.system() != "Darwin" or platform.machine() != "arm64":
+            return self._fail(
+                context,
+                grpc.StatusCode.FAILED_PRECONDITION,
+                "mlx-video requires macOS on Apple Silicon",
+            )
+
+        model = (
+            request.ModelFile
+            if request.ModelFile and os.path.isdir(request.ModelFile)
+            else request.Model
+        )
+        family = classify_model(model)
+        if family is None:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "mlx-video supports LTX-2 and converted Wan2.1/Wan2.2 MLX checkpoints",
+            )
+        if family == "wan" and not os.path.isdir(model):
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "Wan models must be converted to MLX and configured as a local model directory",
+            )
+
+        self.family = family
+        self.model = model
+        self.options = parse_options(request.Options)
+        return backend_pb2.Result(message="mlx-video model configured", success=True)
+
+    def Free(self, request, context):
+        self.family = None
+        self.model = None
+        self.options = {}
+        return backend_pb2.Result(message="Model released", success=True)
+
+    def GenerateVideo(self, request, context):
+        if not self.family or not self.model:
+            return self._fail(
+                context, grpc.StatusCode.FAILED_PRECONDITION, "model is not loaded"
+            )
+        if not request.prompt.strip():
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, "prompt is required")
+        if not request.dst:
+            return self._fail(
+                context,
+                grpc.StatusCode.INVALID_ARGUMENT,
+                "output destination is required",
+            )
+
+        process = None
+        try:
+            command = build_generation_command(
+                family=self.family,
+                model=self.model,
+                prompt=request.prompt,
+                destination=request.dst,
+                negative_prompt=request.negative_prompt,
+                start_image=request.start_image,
+                end_image=request.end_image,
+                audio=request.audio,
+                width=request.width,
+                height=request.height,
+                num_frames=request.num_frames,
+                fps=request.fps,
+                seed=request.seed,
+                steps=request.step,
+                cfg_scale=request.cfg_scale,
+                params={**self.options, **dict(request.params)},
+            )
+            os.makedirs(
+                os.path.dirname(request.dst) or ".", mode=0o750, exist_ok=True
+            )
+            process = subprocess.Popen(command)
+            if hasattr(context, "add_callback"):
+                context.add_callback(
+                    lambda: process.poll() is None and process.terminate()
+                )
+            return_code = process.wait()
+            if return_code != 0:
+                raise RuntimeError(
+                    f"upstream generator exited with status {return_code}"
+                )
+            if not os.path.isfile(request.dst) or os.path.getsize(request.dst) == 0:
+                raise RuntimeError("upstream generator did not create the requested video")
+            return backend_pb2.Result(message="Video generated successfully", success=True)
+        except ValueError as err:
+            return self._fail(context, grpc.StatusCode.INVALID_ARGUMENT, str(err))
+        except Exception as err:
+            print(f"mlx-video generation failed: {err}", file=sys.stderr)
+            traceback.print_exc()
+            return self._fail(
+                context,
+                grpc.StatusCode.INTERNAL,
+                f"mlx-video generation failed: {err}",
+            )
+
+    @staticmethod
+    def _fail(context, code, message):
+        context.set_code(code)
+        context.set_details(message)
+        return backend_pb2.Result(message=message, success=False)
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ("grpc.max_message_length", 64 * 1024 * 1024),
+            ("grpc.max_send_message_length", 64 * 1024 * 1024),
+            ("grpc.max_receive_message_length", 64 * 1024 * 1024),
+        ],
+        interceptors=get_auth_interceptors(),
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print(f"MLX Video backend listening on {address}", file=sys.stderr)
+
+    def stop_server(signum, frame):
+        del signum, frame
+        server.stop(0)
+
+    signal.signal(signal.SIGINT, stop_server)
+    signal.signal(signal.SIGTERM, stop_server)
+    server.wait_for_termination()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the MLX Video gRPC backend")
+    parser.add_argument("--addr", default="localhost:50051")
+    serve(parser.parse_args().addr)

backend/python/mlx-video/install.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+USE_PIP=true
+PYTHON_VERSION="3.11"
+PYTHON_PATCH="13"
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+installRequirements

backend/python/mlx-video/mlx_video_utils.py

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: MIT
+
+import os
+import sys
+
+
+def parse_options(options):
+    parsed = {}
+    for option in options:
+        key, separator, value = option.partition(":")
+        parsed[key] = value if separator else "true"
+    return parsed
+
+
+def classify_model(model):
+    normalized = model.rstrip("/").lower()
+    name = os.path.basename(normalized)
+    if "ltx-2" in normalized or "ltx2" in normalized:
+        return "ltx2"
+    if "wan2" in name or "wan_2" in name:
+        return "wan"
+    return None
+
+
+def _append(command, flag, value):
+    if value is not None and value != "":
+        command.extend([flag, str(value)])
+
+
+def build_generation_command(
+    *,
+    family,
+    model,
+    prompt,
+    destination,
+    negative_prompt="",
+    start_image="",
+    end_image="",
+    audio="",
+    width=0,
+    height=0,
+    num_frames=0,
+    fps=0,
+    seed=0,
+    steps=0,
+    cfg_scale=0,
+    params=None,
+):
+    params = params or {}
+    if audio:
+        raise ValueError("audio conditioning is not supported by mlx-video")
+
+    if family == "ltx2":
+        command = [sys.executable, "-m", "mlx_video.models.ltx_2.generate"]
+        _append(command, "--model-repo", model)
+        _append(command, "--prompt", prompt)
+        _append(command, "--output-path", destination)
+        _append(command, "--negative-prompt", negative_prompt)
+        _append(command, "--image", start_image)
+        _append(command, "--end-image", end_image)
+        _append(command, "--width", width or None)
+        _append(command, "--height", height or None)
+        _append(command, "--num-frames", num_frames or None)
+        _append(command, "--fps", fps or None)
+        _append(command, "--seed", seed if seed > 0 else None)
+        _append(command, "--steps", steps or None)
+        _append(command, "--cfg-scale", cfg_scale or None)
+        for key, flag in {
+            "pipeline": "--pipeline",
+            "tiling": "--tiling",
+            "image_strength": "--image-strength",
+            "end_image_strength": "--end-image-strength",
+            "text_encoder_repo": "--text-encoder-repo",
+        }.items():
+            _append(command, flag, params.get(key))
+        return command
+
+    if family == "wan":
+        if end_image:
+            raise ValueError("Wan models do not support end_image conditioning")
+        command = [sys.executable, "-m", "mlx_video.models.wan_2.generate"]
+        _append(command, "--model-dir", model)
+        _append(command, "--prompt", prompt)
+        _append(command, "--output-path", destination)
+        _append(command, "--negative-prompt", negative_prompt)
+        _append(command, "--image", start_image)
+        _append(command, "--width", width or None)
+        _append(command, "--height", height or None)
+        _append(command, "--num-frames", num_frames or None)
+        _append(command, "--seed", seed if seed > 0 else None)
+        _append(command, "--steps", steps or None)
+        _append(command, "--guide-scale", cfg_scale or None)
+        for key, flag in {
+            "scheduler": "--scheduler",
+            "shift": "--shift",
+            "tiling": "--tiling",
+            "trim_first_frames": "--trim-first-frames",
+        }.items():
+            _append(command, flag, params.get(key))
+        return command
+
+    raise ValueError(f"unsupported mlx-video model family: {family}")

backend/python/mlx-video/requirements-mps.txt

@@ -0,0 +1 @@
+git+https://github.com/Blaizzy/mlx-video.git@87db56a51758fefb748a359b90a5283bb8ba4837

backend/python/mlx-video/requirements.txt

@@ -0,0 +1,4 @@
+grpcio==1.71.0
+protobuf
+certifi
+setuptools

backend/python/mlx-video/run.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+startBackend "$@"

backend/python/mlx-video/test.py

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: MIT
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from mlx_video_utils import (  # noqa: E402
+    build_generation_command,
+    classify_model,
+    parse_options,
+)
+
+
+class MLXVideoUtilsTest(unittest.TestCase):
+    def test_parses_model_options_and_preserves_colons(self):
+        self.assertEqual(
+            parse_options(["pipeline:dev", "tiling:aggressive", "flag"]),
+            {"pipeline": "dev", "tiling": "aggressive", "flag": "true"},
+        )
+
+    def test_classifies_supported_model_families(self):
+        self.assertEqual(classify_model("Lightricks/LTX-2"), "ltx2")
+        self.assertEqual(classify_model("/models/Wan2.2-TI2V-5B-mlx"), "wan")
+        self.assertIsNone(classify_model("some-org/unrelated-model"))
+
+    def test_maps_ltx_request_without_invoking_a_shell(self):
+        command = build_generation_command(
+            family="ltx2",
+            model="prince-canuma/LTX-2.3-distilled-mlx",
+            prompt="waves; touch /tmp/nope",
+            destination="/tmp/out.mp4",
+            negative_prompt="blurry",
+            start_image="/tmp/start.png",
+            end_image="/tmp/end.png",
+            width=768,
+            height=512,
+            num_frames=49,
+            fps=24,
+            seed=7,
+            steps=12,
+            cfg_scale=3.5,
+            params={"pipeline": "dev", "tiling": "aggressive"},
+        )
+
+        self.assertEqual(command[:3], [sys.executable, "-m", "mlx_video.models.ltx_2.generate"])
+        self.assertIn("waves; touch /tmp/nope", command)
+        self.assertEqual(command[command.index("--model-repo") + 1], "prince-canuma/LTX-2.3-distilled-mlx")
+        self.assertEqual(command[command.index("--pipeline") + 1], "dev")
+        self.assertEqual(command[command.index("--cfg-scale") + 1], "3.5")
+        self.assertIn("--end-image", command)
+
+    def test_maps_wan_request_and_rejects_unsupported_audio(self):
+        command = build_generation_command(
+            family="wan",
+            model="/models/wan22-mlx",
+            prompt="a fox",
+            destination="/tmp/out.mp4",
+            width=1280,
+            height=704,
+            num_frames=81,
+            fps=16,
+            seed=9,
+            steps=20,
+            cfg_scale=4.0,
+            params={"scheduler": "unipc", "shift": "5.0"},
+        )
+
+        self.assertEqual(command[:3], [sys.executable, "-m", "mlx_video.models.wan_2.generate"])
+        self.assertEqual(command[command.index("--model-dir") + 1], "/models/wan22-mlx")
+        self.assertEqual(command[command.index("--guide-scale") + 1], "4.0")
+        self.assertNotIn("--fps", command)
+
+        with self.assertRaisesRegex(ValueError, "audio conditioning"):
+            build_generation_command(
+                family="wan",
+                model="/models/wan22-mlx",
+                prompt="a fox",
+                destination="/tmp/out.mp4",
+                audio="/tmp/input.wav",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/mlx-video/test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+set -euo pipefail
+
+backend_dir=$(dirname "$0")
+if [ -d "${backend_dir}/common" ]; then
+    source "${backend_dir}/common/libbackend.sh"
+else
+    source "${backend_dir}/../common/libbackend.sh"
+fi
+runUnittests

backend/python/mlx-vlm/backend.py

@@ -376,8 +376,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 kwargs["tools"] = json.loads(request.Tools)
             except json.JSONDecodeError:
                 pass
-        if request.Metadata.get("enable_thinking", "").lower() == "true":
-            kwargs["enable_thinking"] = True
+        enable_thinking = request.Metadata.get("enable_thinking", "").lower()
+        if enable_thinking in ("true", "false"):
+            kwargs["enable_thinking"] = enable_thinking == "true"
         return kwargs
 
     def _apply_template(self, request, messages, num_images, num_audios):

backend/python/mlx/backend.py

@@ -432,8 +432,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 except json.JSONDecodeError:
                     pass
             enable_thinking = request.Metadata.get("enable_thinking", "").lower()
-            if enable_thinking == "true":
-                kwargs["enable_thinking"] = True
+            if enable_thinking in ("true", "false"):
+                kwargs["enable_thinking"] = enable_thinking == "true"
 
             try:
                 return self.tokenizer.apply_chat_template(messages, **kwargs)

backend/python/mlx/test.py

@@ -340,4 +340,40 @@ class TestSharedHelpers(unittest.TestCase):
         self.assertNotIn("<tool_call>", remaining)
 
 
+class TestPreparePrompt(unittest.TestCase):
+    """Server-less tests for BackendServicer._prepare_prompt."""
+
+    def test_forwards_enable_thinking(self):
+        from backend import BackendServicer
+
+        class Tok:
+            chat_template = "{{ messages }}"
+
+            def __init__(self):
+                self.kwargs = None
+
+            def apply_chat_template(self, messages, **kwargs):
+                self.kwargs = kwargs
+                return "PROMPT"
+
+        def kwargs_for(metadata):
+            servicer = BackendServicer()
+            servicer.tokenizer = Tok()
+            req = types.SimpleNamespace(
+                Prompt="",
+                UseTokenizerTemplate=True,
+                Messages=[backend_pb2.Message(role="user", content="hi")],
+                Tools="",
+                Metadata=metadata,
+            )
+            self.assertEqual(servicer._prepare_prompt(req), "PROMPT")
+            return servicer.tokenizer.kwargs
+
+        self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True)
+        # "false" used to be dropped, so thinking models kept reasoning
+        self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False)
+        self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False)
+        self.assertNotIn("enable_thinking", kwargs_for({}))
+
+
 # Unit tests for ThreadSafeLRUPromptCache are in test_mlx_cache.py

backend/python/outetts/requirements-cpu.txt

@@ -1,6 +1,6 @@
 torch==2.7.1
-llvmlite==0.43.0
-numba==0.60.0
+llvmlite==0.49.0
+numba==0.67.0
 accelerate
 bitsandbytes
 outetts

backend/python/outetts/requirements-cublas12.txt

@@ -1,7 +1,7 @@
 torch==2.7.1
 accelerate
-llvmlite==0.43.0
-numba==0.60.0
+llvmlite==0.49.0
+numba==0.67.0
 bitsandbytes
 protobuf==6.33.5
 outetts

backend/python/outetts/requirements-cublas13.txt

@@ -1,7 +1,7 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
 torch==2.9.0
-llvmlite==0.43.0
-numba==0.60.0
+llvmlite==0.49.0
+numba==0.67.0
 bitsandbytes
 outetts
 protobuf==6.33.5

backend/python/outetts/requirements-hipblas.txt

@@ -1,8 +1,8 @@
 --extra-index-url https://download.pytorch.org/whl/rocm7.0
 torch==2.10.0+rocm7.0
 accelerate
-llvmlite==0.43.0
-numba==0.60.0
+llvmlite==0.49.0
+numba==0.67.0
 bitsandbytes
 outetts
 protobuf==6.33.5

backend/python/outetts/requirements-intel.txt

@@ -1,8 +1,8 @@
 --extra-index-url https://download.pytorch.org/whl/xpu
 torch
 optimum[openvino]
-llvmlite==0.43.0
-numba==0.60.0
+llvmlite==0.49.0
+numba==0.67.0
 bitsandbytes
 outetts
 protobuf==6.33.5

backend/python/qwen-tts/install.sh

@@ -10,4 +10,11 @@ else
     source $backend_dir/../common/libbackend.sh
 fi
 
+# CUDA 13 has no prebuilt FlashAttention wheel, so the fallback source build
+# exceeds the CI runner's memory when ninja compiles multiple units at once.
+if [ "x${BUILD_PROFILE}" = "xcublas13" ]; then
+    export MAX_JOBS="${MAX_JOBS:-1}"
+    export NVCC_THREADS="${NVCC_THREADS:-1}"
+fi
+
 installRequirements

backend/python/qwen-tts/requirements-cublas13-after.txt

@@ -0,0 +1,2 @@
+ninja
+flash-attn

backend/python/qwen-tts/test.sh

@@ -2,6 +2,15 @@
 set -e
 
 backend_dir=$(dirname $0)
+
+for cuda_version in 12 13; do
+    grep -qx "flash-attn" "$backend_dir/requirements-cublas${cuda_version}-after.txt"
+done
+
+grep -q 'BUILD_PROFILE.*cublas13' "$backend_dir/install.sh"
+grep -q 'MAX_JOBS.*1' "$backend_dir/install.sh"
+grep -q 'NVCC_THREADS.*1' "$backend_dir/install.sh"
+
 if [ -d $backend_dir/common ]; then
     source $backend_dir/common/libbackend.sh
 else

backend/python/transformers/install.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+PYTHON_VERSION="3.12"
+PYTHON_PATCH="11"
+
 backend_dir=$(dirname $0)
 if [ -d $backend_dir/common ]; then
     source $backend_dir/common/libbackend.sh

backend/python/transformers/requirements-cpu.txt

@@ -1,8 +1,8 @@
 torch==2.7.1
-llvmlite==0.43.0
-numba==0.60.0
+llvmlite==0.49.0
+numba==0.67.0
 accelerate
-transformers>=5.15.0
+transformers>=5.15.1
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-cublas12.txt

@@ -1,8 +1,8 @@
 torch==2.7.1
 accelerate
-llvmlite==0.43.0
-numba==0.60.0
-transformers>=5.15.0
+llvmlite==0.49.0
+numba==0.67.0
+transformers>=5.15.1
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-cublas13.txt

@@ -1,8 +1,8 @@
 --extra-index-url https://download.pytorch.org/whl/cu130
 torch==2.9.0
-llvmlite==0.43.0
-numba==0.60.0
-transformers>=5.15.0
+llvmlite==0.49.0
+numba==0.67.0
+transformers>=5.15.1
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-hipblas.txt

@@ -1,9 +1,9 @@
 --extra-index-url https://download.pytorch.org/whl/rocm7.0
 torch==2.10.0+rocm7.0
 accelerate
-transformers>=5.15.0
-llvmlite==0.43.0
-numba==0.60.0
+transformers>=5.15.1
+llvmlite==0.49.0
+numba==0.67.0
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-intel.txt

@@ -1,9 +1,9 @@
 --extra-index-url https://download.pytorch.org/whl/xpu
 torch
 optimum[openvino]
-llvmlite==0.43.0
-numba==0.60.0
-transformers>=5.15.0
+llvmlite==0.49.0
+numba==0.67.0
+transformers>=5.15.1
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements-mps.txt

@@ -1,8 +1,8 @@
 torch==2.7.1
-llvmlite==0.43.0
-numba==0.60.0
+llvmlite==0.49.0
+numba==0.67.0
 accelerate
-transformers>=5.15.0
+transformers>=5.15.1
 bitsandbytes
 sentence-transformers==5.7.0
 diffusers

backend/python/transformers/requirements.txt

@@ -2,5 +2,5 @@ grpcio==1.83.0
 protobuf==7.35.0
 certifi
 setuptools
-scipy==1.15.1
-numpy>=2.0.0
+scipy==1.18.0
+numpy>=2.5.2

backend/python/vllm-omni/backend.py

@@ -489,8 +489,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                         except json.JSONDecodeError:
                             pass
                     try:
-                        if request.Metadata.get("enable_thinking", "").lower() == "true":
-                            template_kwargs["enable_thinking"] = True
+                        enable_thinking = request.Metadata.get("enable_thinking", "").lower()
+                        if enable_thinking in ("true", "false"):
+                            template_kwargs["enable_thinking"] = enable_thinking == "true"
                     except Exception:
                         pass
                     try:

backend/python/vllm/requirements-cublas13-after.txt

@@ -3,8 +3,8 @@
 # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index
 # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match
 # so uv consults this index alongside PyPI.
---extra-index-url https://wheels.vllm.ai/0.28.0/cu130
+--extra-index-url https://wheels.vllm.ai/0.29.0/cu130
 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh),
 # which pins this exact vLLM version. Bumping vllm here means coordinating with a
 # vllm-metal release that supports the new version, or macOS/Metal builds break.
-vllm==0.28.0
+vllm==0.29.0

backend/python/vllm/requirements-l4t13-after.txt

@@ -9,4 +9,4 @@
 # memory architecture crash deterministically with an empty "Engine core init
 # failed" set (mudler/LocalAI#10722). Leaving this unpinned let the L4T image
 # drift onto whatever wheel was latest at build time.
-vllm==0.28.0
+vllm==0.29.0

backend/python/whisper-medusa/Makefile

@@ -0,0 +1,14 @@
+.DEFAULT_GOAL := install
+
+.PHONY: install
+install:
+	bash install.sh
+
+.PHONY: clean
+clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+	rm -rf venv __pycache__
+
+.PHONY: test
+test:
+	python3 -m unittest test_unit.py

backend/python/whisper-medusa/backend.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""LocalAI gRPC backend for aiola Whisper-Medusa speech recognition."""
+
+import argparse
+from concurrent import futures
+import os
+import signal
+import sys
+import time
+
+import backend_pb2
+import backend_pb2_grpc
+import grpc
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "common"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common"))
+from grpc_auth import get_auth_interceptors
+from model_utils import resolve_model_reference
+
+
+SAMPLE_RATE = 16000
+MAX_DURATION_SECONDS = 30
+MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+
+def _parse_options(raw_options):
+    options = {}
+    for option in raw_options:
+        if ":" not in option:
+            continue
+        key, value = option.split(":", 1)
+        try:
+            value = int(value)
+        except ValueError:
+            try:
+                value = float(value)
+            except ValueError:
+                pass
+        options[key] = value
+    return options
+
+
+def _prepare_audio(path, torchaudio):
+    waveform, sample_rate = torchaudio.load(path)
+    if waveform.shape[0] > 1:
+        waveform = waveform.mean(dim=0, keepdim=True)
+    if sample_rate != SAMPLE_RATE:
+        waveform = torchaudio.transforms.Resample(sample_rate, SAMPLE_RATE)(waveform)
+        sample_rate = SAMPLE_RATE
+    return waveform, sample_rate
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def __init__(self):
+        self.model = None
+        self.processor = None
+        self.device = None
+        self.options = {}
+
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=b"OK")
+
+    def LoadModel(self, request, context):
+        try:
+            import torch
+            from transformers import WhisperProcessor
+            from whisper_medusa import WhisperMedusaModel
+
+            if request.CUDA and not torch.cuda.is_available():
+                return backend_pb2.Result(success=False, message="CUDA is not available")
+
+            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+                self.device = torch.device("mps")
+
+            self.options = _parse_options(request.Options)
+            model_path, local_only = resolve_model_reference(
+                request, "aiola/whisper-medusa-linear-libri"
+            )
+            self.model = WhisperMedusaModel.from_pretrained(
+                model_path, local_files_only=local_only
+            ).to(self.device)
+            self.model.eval()
+            self.processor = WhisperProcessor.from_pretrained(
+                model_path, local_files_only=local_only
+            )
+        except Exception as err:
+            print(f"Whisper-Medusa model load failed: {err}", file=sys.stderr)
+            return backend_pb2.Result(success=False, message=str(err))
+
+        return backend_pb2.Result(success=True, message="Model loaded successfully")
+
+    def AudioTranscription(self, request, context):
+        if self.model is None or self.processor is None:
+            return backend_pb2.TranscriptResult(segments=[], text="")
+
+        try:
+            import torch
+            import torchaudio
+
+            waveform, sample_rate = _prepare_audio(request.dst, torchaudio)
+            duration = waveform.shape[-1] / sample_rate
+            if duration > MAX_DURATION_SECONDS:
+                raise ValueError(
+                    f"Whisper-Medusa supports audio clips up to {MAX_DURATION_SECONDS} seconds"
+                )
+
+            language = request.language or str(self.options.get("language", "en"))
+            regulation_start = int(self.options.get("regulation_start", 140))
+            regulation_factor = float(self.options.get("regulation_factor", 1.01))
+            features = self.processor(
+                waveform.squeeze(), return_tensors="pt", sampling_rate=sample_rate
+            ).input_features.to(self.device)
+            with torch.inference_mode():
+                output = self.model.generate(
+                    features,
+                    language=language,
+                    exponential_decay_length_penalty=(
+                        regulation_start,
+                        regulation_factor,
+                    ),
+                )
+            text = self.processor.decode(output[0], skip_special_tokens=True).strip()
+            segment = backend_pb2.TranscriptSegment(
+                id=0,
+                start=0,
+                end=int(duration * 1_000_000_000),
+                text=text,
+            )
+            return backend_pb2.TranscriptResult(segments=[segment], text=text)
+        except Exception as err:
+            print(f"Whisper-Medusa transcription failed: {err}", file=sys.stderr)
+            return backend_pb2.TranscriptResult(segments=[], text="")
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ("grpc.max_send_message_length", 50 * 1024 * 1024),
+            ("grpc.max_receive_message_length", 50 * 1024 * 1024),
+        ],
+        interceptors=get_auth_interceptors(),
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print(f"Server started. Listening on: {address}", file=sys.stderr)
+
+    def stop_server(_signal, _frame):
+        server.stop(0)
+        sys.exit(0)
+
+    signal.signal(signal.SIGINT, stop_server)
+    signal.signal(signal.SIGTERM, stop_server)
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the Whisper-Medusa backend")
+    parser.add_argument("--addr", default="localhost:50051")
+    serve(parser.parse_args().addr)

backend/python/whisper-medusa/install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname "$0")
+if [ -d "$backend_dir/common" ]; then
+    source "$backend_dir/common/libbackend.sh"
+else
+    source "$backend_dir/../common/libbackend.sh"
+fi
+
+PYTHON_VERSION="3.11"
+PYTHON_PATCH="13"
+installRequirements

backend/python/whisper-medusa/protogen.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -e
+python3 -m grpc_tools.protoc -I../../ --python_out=. --grpc_python_out=. ../../backend.proto

backend/python/whisper-medusa/requirements-cpu.txt

@@ -0,0 +1,3 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.2.2
+torchaudio==2.2.2

backend/python/whisper-medusa/requirements-cublas12.txt

@@ -0,0 +1,3 @@
+--extra-index-url https://download.pytorch.org/whl/cu121
+torch==2.2.2
+torchaudio==2.2.2