feat: add FunASR speech recognition backend (#10090)

Adds FunASR/SenseVoice as a Python backend for speech-to-text with
support for CPU, CUDA 12/13, ROCm, Intel SYCL, L4T, and Apple MPS.

Co-authored-by: xingyifeng <xingyifeng@users.noreply.github.com>

backend/python/README.md

@@ -20,6 +20,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro
 ### Audio & Speech
 - **coqui** - Coqui TTS models
 - **faster-whisper** - Fast Whisper speech recognition
+- **funasr** - Local multilingual transcription with FunASR and SenseVoice
 - **kitten-tts** - Lightweight TTS
 - **mlx-audio** - Apple Silicon audio processing
 - **chatterbox** - TTS model

backend/python/funasr/Makefile

@@ -0,0 +1,23 @@
+.PHONY: funasr
+funasr:
+	bash install.sh
+
+.PHONY: run
+run: funasr
+	@echo "Running funasr..."
+	bash run.sh
+	@echo "funasr run."
+
+.PHONY: test
+test: funasr
+	@echo "Testing funasr..."
+	bash test.sh
+	@echo "funasr tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__

backend/python/funasr/backend.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""
+gRPC backend for LocalAI wrapping FunASR (SenseVoice / Paraformer).
+"""
+from concurrent import futures
+import time
+import argparse
+import signal
+import sys
+import os
+import backend_pb2
+import backend_pb2_grpc
+import torch
+
+import grpc
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common'))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common'))
+from grpc_auth import get_auth_interceptors
+
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
+
+    def LoadModel(self, request, context):
+        from funasr import AutoModel
+
+        device = "cpu"
+        if request.CUDA and torch.cuda.is_available():
+            device = "cuda"
+        elif hasattr(torch, "xpu") and torch.xpu.is_available():
+            device = "xpu"
+        elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            device = "mps"
+
+        model_id = request.Model or "iic/SenseVoiceSmall"
+        candidate_devices = [device]
+        if device in ("xpu", "mps"):
+            candidate_devices.append("cpu")
+
+        for candidate_device in candidate_devices:
+            try:
+                print(
+                    f"Loading FunASR model: {model_id} on {candidate_device}",
+                    file=sys.stderr,
+                )
+                self.model = AutoModel(
+                    model=model_id,
+                    vad_model="fsmn-vad",
+                    device=candidate_device,
+                    disable_update=True,
+                )
+                print("FunASR model loaded successfully", file=sys.stderr)
+                break
+            except Exception as err:
+                if candidate_device != candidate_devices[-1]:
+                    print(
+                        f"[WARN] FunASR {candidate_device} initialization failed: "
+                        f"{err}; retrying on cpu",
+                        file=sys.stderr,
+                    )
+                    continue
+                print(f"[ERROR] LoadModel failed: {err}", file=sys.stderr)
+                import traceback
+                traceback.print_exc(file=sys.stderr)
+                return backend_pb2.Result(success=False, message=str(err))
+
+        return backend_pb2.Result(message="Model loaded successfully", success=True)
+
+    def AudioTranscription(self, request, context):
+        from funasr.utils.postprocess_utils import rich_transcription_postprocess
+
+        result_segments = []
+        text = ""
+        try:
+            audio_path = request.dst
+            if not audio_path or not os.path.exists(audio_path):
+                print(f"Error: Audio file not found: {audio_path}", file=sys.stderr)
+                return backend_pb2.TranscriptResult(segments=[], text="")
+
+            language = None
+            if request.language and request.language.strip():
+                language = request.language.strip()
+
+            kwargs = {}
+            if language:
+                kwargs["language"] = language
+
+            results = self.model.generate(input=audio_path, **kwargs)
+
+            if not results:
+                return backend_pb2.TranscriptResult(segments=[], text="")
+
+            for idx, r in enumerate(results):
+                seg_text = r.get("text", "") if isinstance(r, dict) else str(r)
+                seg_text = rich_transcription_postprocess(seg_text)
+                text += seg_text
+                result_segments.append(backend_pb2.TranscriptSegment(
+                    id=idx,
+                    start=0,
+                    end=0,
+                    text=seg_text,
+                ))
+
+        except Exception as err:
+            print(f"Error in AudioTranscription: {err}", file=sys.stderr)
+            import traceback
+            traceback.print_exc(file=sys.stderr)
+            return backend_pb2.TranscriptResult(segments=[], text="")
+
+        return backend_pb2.TranscriptResult(segments=result_segments, text=text)
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ('grpc.max_message_length', 50 * 1024 * 1024),
+            ('grpc.max_send_message_length', 50 * 1024 * 1024),
+            ('grpc.max_receive_message_length', 50 * 1024 * 1024),
+        ],
+        interceptors=get_auth_interceptors(),
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print("Server started. Listening on: " + address, file=sys.stderr)
+
+    def signal_handler(sig, frame):
+        print("Received termination signal. Shutting down...")
+        server.stop(0)
+        sys.exit(0)
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the gRPC server.")
+    parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.")
+    args = parser.parse_args()
+    serve(args.addr)

backend/python/funasr/install.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation"
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+if [ "x${BUILD_PROFILE}" == "xintel" ]; then
+    EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
+fi
+
+PYTHON_VERSION="3.12"
+PYTHON_PATCH="12"
+PY_STANDALONE_TAG="20251120"
+
+installRequirements

backend/python/funasr/requirements-cpu.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-cublas12.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cu121
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-cublas13.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-hipblas.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/rocm7.0
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-intel.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-l4t12.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu129/
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-l4t13.txt

@@ -0,0 +1,4 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch
+torchaudio
+funasr

backend/python/funasr/requirements-mps.txt

@@ -0,0 +1,3 @@
+torch==2.7.1
+torchaudio==2.7.1
+funasr

backend/python/funasr/requirements.txt

@@ -0,0 +1,5 @@
+grpcio==1.71.0
+protobuf
+certifi
+packaging==24.1
+setuptools

backend/python/funasr/run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+startBackend $@

backend/python/funasr/test.py

@@ -0,0 +1,295 @@
+import importlib
+import os
+import re
+import sys
+import types
+import unittest
+
+
+class _Reply:
+    def __init__(self, message=b""):
+        self.message = message
+
+
+class _Result:
+    def __init__(self, message="", success=False):
+        self.message = message
+        self.success = success
+
+
+class _TranscriptSegment:
+    def __init__(self, id=0, start=0, end=0, text=""):
+        self.id = id
+        self.start = start
+        self.end = end
+        self.text = text
+
+
+class _TranscriptResult:
+    def __init__(self, segments=None, text=""):
+        self.segments = segments or []
+        self.text = text
+
+
+class _FakeBackendServicer:
+    pass
+
+
+class _FakeTorch:
+    cuda = types.SimpleNamespace(is_available=lambda: False)
+    xpu = types.SimpleNamespace(is_available=lambda: False)
+    backends = types.SimpleNamespace(mps=types.SimpleNamespace(is_available=lambda: False))
+
+
+class _FakeAutoModel:
+    instances = []
+    attempts = []
+    fail_devices = set()
+
+    def __init__(self, **kwargs):
+        _FakeAutoModel.attempts.append(kwargs)
+        if kwargs.get("device") in _FakeAutoModel.fail_devices:
+            raise RuntimeError(f"unsupported device: {kwargs['device']}")
+        self.kwargs = kwargs
+        self.generate_calls = []
+        self.results = [{"text": "hello"}, {"text": " world"}]
+        _FakeAutoModel.instances.append(self)
+
+    def generate(self, **kwargs):
+        self.generate_calls.append(kwargs)
+        return self.results
+
+
+def _install_stubs():
+    sys.modules["backend_pb2"] = types.SimpleNamespace(
+        Reply=_Reply,
+        Result=_Result,
+        TranscriptSegment=_TranscriptSegment,
+        TranscriptResult=_TranscriptResult,
+    )
+    sys.modules["backend_pb2_grpc"] = types.SimpleNamespace(
+        BackendServicer=_FakeBackendServicer,
+        add_BackendServicer_to_server=lambda *args, **kwargs: None,
+    )
+    sys.modules["grpc"] = types.SimpleNamespace(
+        RpcMethodHandler=object,
+        ServerInterceptor=object,
+        StatusCode=types.SimpleNamespace(UNAUTHENTICATED="UNAUTHENTICATED"),
+        aio=types.SimpleNamespace(ServerInterceptor=object),
+        server=lambda *args, **kwargs: None,
+    )
+    sys.modules["torch"] = _FakeTorch
+    funasr = types.ModuleType("funasr")
+    funasr.__path__ = []
+    funasr.AutoModel = _FakeAutoModel
+    funasr_utils = types.ModuleType("funasr.utils")
+    funasr_utils.__path__ = []
+    postprocess_utils = types.ModuleType("funasr.utils.postprocess_utils")
+    postprocess_utils.rich_transcription_postprocess = lambda text: re.sub(
+        r"<\|.*?\|>", "", text
+    )
+    sys.modules["funasr"] = funasr
+    sys.modules["funasr.utils"] = funasr_utils
+    sys.modules["funasr.utils.postprocess_utils"] = postprocess_utils
+
+
+def _load_backend():
+    _install_stubs()
+    sys.modules.pop("backend", None)
+    _FakeAutoModel.instances.clear()
+    _FakeAutoModel.attempts.clear()
+    _FakeAutoModel.fail_devices.clear()
+    return importlib.import_module("backend")
+
+
+class TestFunASRBackend(unittest.TestCase):
+    def test_torch_profiles_install_torchaudio(self):
+        backend_dir = os.path.dirname(__file__)
+
+        for profile in (
+            "cpu",
+            "cublas12",
+            "cublas13",
+            "hipblas",
+            "intel",
+            "l4t12",
+            "l4t13",
+            "mps",
+        ):
+            with self.subTest(profile=profile):
+                requirements_path = os.path.join(
+                    backend_dir, f"requirements-{profile}.txt"
+                )
+                with open(requirements_path, encoding="utf-8") as requirements_file:
+                    requirements = {
+                        line.strip().split("=", 1)[0]
+                        for line in requirements_file
+                        if line.strip() and not line.lstrip().startswith(("#", "--"))
+                    }
+
+                self.assertIn("torchaudio", requirements)
+
+    def test_accelerator_profiles_use_current_pytorch_indexes(self):
+        backend_dir = os.path.dirname(__file__)
+        expected_indexes = {
+            "cublas13": "https://download.pytorch.org/whl/cu130",
+            "hipblas": "https://download.pytorch.org/whl/rocm7.0",
+            "intel": "https://download.pytorch.org/whl/xpu",
+            "l4t12": "https://pypi.jetson-ai-lab.io/jp6/cu129/",
+            "l4t13": "https://download.pytorch.org/whl/cu130",
+        }
+
+        for profile, expected_index in expected_indexes.items():
+            with self.subTest(profile=profile):
+                requirements_path = os.path.join(
+                    backend_dir, f"requirements-{profile}.txt"
+                )
+                with open(requirements_path, encoding="utf-8") as requirements_file:
+                    requirements = requirements_file.read()
+
+                self.assertIn(
+                    f"--extra-index-url {expected_index}\n",
+                    requirements,
+                )
+
+    def test_health_returns_ok(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+
+        reply = servicer.Health(types.SimpleNamespace(), None)
+
+        self.assertEqual(reply.message, b"OK")
+
+    def test_load_model_uses_default_sensevoice_model_on_cpu(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+
+        result = servicer.LoadModel(types.SimpleNamespace(Model="", CUDA=False), None)
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(result.message, "Model loaded successfully")
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["model"], "iic/SenseVoiceSmall")
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["vad_model"], "fsmn-vad")
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
+        self.assertTrue(_FakeAutoModel.instances[0].kwargs["disable_update"])
+
+    def test_load_model_uses_xpu_when_available(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_xpu = _FakeTorch.xpu
+        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
+        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "xpu")
+
+    def test_load_model_prefers_requested_cuda_over_xpu_and_mps(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_cuda = _FakeTorch.cuda
+        original_xpu = _FakeTorch.xpu
+        original_mps = _FakeTorch.backends.mps
+        _FakeTorch.cuda = types.SimpleNamespace(is_available=lambda: True)
+        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
+        _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True)
+        self.addCleanup(setattr, _FakeTorch, "cuda", original_cuda)
+        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
+        self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=True), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(
+            [attempt["device"] for attempt in _FakeAutoModel.attempts], ["cuda"]
+        )
+
+    def test_load_model_retries_cpu_when_xpu_initialization_fails(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_xpu = _FakeTorch.xpu
+        _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True)
+        _FakeAutoModel.fail_devices.add("xpu")
+        self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(
+            [attempt["device"] for attempt in _FakeAutoModel.attempts],
+            ["xpu", "cpu"],
+        )
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
+
+    def test_load_model_retries_cpu_when_mps_initialization_fails(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        original_mps = _FakeTorch.backends.mps
+        _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True)
+        _FakeAutoModel.fail_devices.add("mps")
+        self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps)
+
+        result = servicer.LoadModel(
+            types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None
+        )
+
+        self.assertTrue(result.success, result.message)
+        self.assertEqual(
+            [attempt["device"] for attempt in _FakeAutoModel.attempts],
+            ["mps", "cpu"],
+        )
+        self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu")
+
+    def test_audio_transcription_passes_language_and_builds_segments(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        servicer.model = _FakeAutoModel()
+        audio_path = os.path.abspath(__file__)
+        request = types.SimpleNamespace(dst=audio_path, language=" zh ")
+
+        result = servicer.AudioTranscription(request, None)
+
+        self.assertEqual(result.text, "hello world")
+        self.assertEqual([segment.text for segment in result.segments], ["hello", " world"])
+        self.assertEqual(servicer.model.generate_calls, [{"input": audio_path, "language": "zh"}])
+
+    def test_audio_transcription_cleans_sensevoice_tags(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        servicer.model = _FakeAutoModel()
+        servicer.model.results = [
+            {"text": "<|zh|><|NEUTRAL|><|Speech|><|woitn|>hello"}
+        ]
+
+        result = servicer.AudioTranscription(
+            types.SimpleNamespace(dst=os.path.abspath(__file__), language=""),
+            None,
+        )
+
+        self.assertEqual(result.text, "hello")
+        self.assertEqual([segment.text for segment in result.segments], ["hello"])
+
+    def test_audio_transcription_missing_file_returns_empty_result(self):
+        backend = _load_backend()
+        servicer = backend.BackendServicer()
+        servicer.model = _FakeAutoModel()
+
+        result = servicer.AudioTranscription(
+            types.SimpleNamespace(dst="/tmp/localai-funasr-missing.wav", language=""),
+            None,
+        )
+
+        self.assertEqual(result.text, "")
+        self.assertEqual(result.segments, [])
+        self.assertEqual(servicer.model.generate_calls, [])
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/funasr/test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+runUnittests "$@"