feat(metal): try to extend support to remaining backends (#8374)

* feat(metal): try to extend support to remaining backends

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* neutts doesn't work

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* split outetts out of transformers

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Remove torch pin to whisperx

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

backend/python/chatterbox/requirements-mps.txt

@@ -0,0 +1,7 @@
+torch
+torchaudio
+accelerate
+numpy>=1.24.0,<1.26.0
+transformers
+# https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
+chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster

backend/python/coqui/requirements-mps.txt

@@ -0,0 +1,4 @@
+torch==2.7.1
+transformers==4.48.3
+accelerate
+coqui-tts

backend/python/faster-whisper/requirements-mps.txt

@@ -0,0 +1,8 @@
+torch==2.7.1
+faster-whisper
+opencv-python
+accelerate
+compel
+peft
+sentencepiece
+optimum-quanto

backend/python/kitten-tts/requirements-mps.txt

@@ -0,0 +1,5 @@
+grpcio==1.71.0
+protobuf
+certifi
+packaging==24.1
+https://github.com/KittenML/KittenTTS/releases/download/0.1/kittentts-0.1.0-py3-none-any.whl

backend/python/kokoro/requirements-mps.txt

@@ -0,0 +1,5 @@
+torch==2.7.1
+transformers
+accelerate
+kokoro
+soundfile

backend/python/moonshine/requirements-mps.txt

@@ -0,0 +1,4 @@
+grpcio==1.71.0
+protobuf
+grpcio-tools
+useful-moonshine-onnx@git+https://git@github.com/moonshine-ai/moonshine.git#subdirectory=moonshine-onnx

backend/python/outetts/Makefile

@@ -0,0 +1,23 @@
+.PHONY: outetts
+outetts:
+	bash install.sh
+
+.PHONY: run
+run: outetts
+	@echo "Running outetts..."
+	bash run.sh
+	@echo "outetts run."
+
+.PHONY: test
+test: outetts
+	@echo "Testing outetts..."
+	bash test.sh
+	@echo "outetts tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__

backend/python/outetts/backend.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""
+gRPC server for OuteTTS (OuteAI TTS) models.
+"""
+from concurrent import futures
+
+import argparse
+import signal
+import sys
+import os
+import asyncio
+
+import backend_pb2
+import backend_pb2_grpc
+
+import grpc
+import outetts
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
+
+    def LoadModel(self, request, context):
+        model_name = request.Model
+        if os.path.exists(request.ModelFile):
+            model_name = request.ModelFile
+
+        self.options = {}
+        for opt in request.Options:
+            if ":" not in opt:
+                continue
+            key, value = opt.split(":", 1)
+            try:
+                if "." in value:
+                    value = float(value)
+                else:
+                    value = int(value)
+            except ValueError:
+                pass
+            self.options[key] = value
+
+        MODELNAME = "OuteAI/OuteTTS-0.3-1B"
+        TOKENIZER = "OuteAI/OuteTTS-0.3-1B"
+        VERSION = "0.3"
+        SPEAKER = "en_male_1"
+        for opt in request.Options:
+            if opt.startswith("tokenizer:"):
+                TOKENIZER = opt.split(":")[1]
+                break
+            if opt.startswith("version:"):
+                VERSION = opt.split(":")[1]
+                break
+            if opt.startswith("speaker:"):
+                SPEAKER = opt.split(":")[1]
+                break
+
+        if model_name != "":
+            MODELNAME = model_name
+
+        try:
+            model_config = outetts.HFModelConfig_v2(
+                model_path=MODELNAME,
+                tokenizer_path=TOKENIZER
+            )
+            self.interface = outetts.InterfaceHF(model_version=VERSION, cfg=model_config)
+
+            self.interface.print_default_speakers()
+            if request.AudioPath:
+                if os.path.isabs(request.AudioPath):
+                    self.AudioPath = request.AudioPath
+                else:
+                    self.AudioPath = os.path.join(request.ModelPath, request.AudioPath)
+                self.speaker = self.interface.create_speaker(audio_path=self.AudioPath)
+            else:
+                self.speaker = self.interface.load_default_speaker(name=SPEAKER)
+
+            if request.ContextSize > 0:
+                self.max_tokens = request.ContextSize
+            else:
+                self.max_tokens = self.options.get("max_new_tokens", 512)
+
+        except Exception as err:
+            print("Error:", err, file=sys.stderr)
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.Result(message="Model loaded successfully", success=True)
+
+    def TTS(self, request, context):
+        try:
+            text = request.text if request.text else "Speech synthesis is the artificial production of human speech."
+            print("[OuteTTS] generating TTS", file=sys.stderr)
+            gen_cfg = outetts.GenerationConfig(
+                text=text,
+                temperature=self.options.get("temperature", 0.1),
+                repetition_penalty=self.options.get("repetition_penalty", 1.1),
+                max_length=self.max_tokens,
+                speaker=self.speaker,
+            )
+            output = self.interface.generate(config=gen_cfg)
+            print("[OuteTTS] Generated TTS", file=sys.stderr)
+            output.save(request.dst)
+            print("[OuteTTS] TTS done", file=sys.stderr)
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.Result(success=True)
+
+
+async def serve(address):
+    server = grpc.aio.server(
+        migration_thread_pool=futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ('grpc.max_message_length', 50 * 1024 * 1024),
+            ('grpc.max_send_message_length', 50 * 1024 * 1024),
+            ('grpc.max_receive_message_length', 50 * 1024 * 1024),
+        ])
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+
+    loop = asyncio.get_event_loop()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        loop.add_signal_handler(
+            sig, lambda: asyncio.ensure_future(server.stop(5))
+        )
+
+    await server.start()
+    print("Server started. Listening on: " + address, file=sys.stderr)
+    await server.wait_for_termination()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the OuteTTS gRPC server.")
+    parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.")
+    args = parser.parse_args()
+    asyncio.run(serve(args.addr))

backend/python/outetts/install.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+installRequirements

backend/python/outetts/requirements-cpu.txt

@@ -0,0 +1,7 @@
+torch==2.7.1
+llvmlite==0.43.0
+numba==0.60.0
+accelerate
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements-cublas12.txt

@@ -0,0 +1,7 @@
+torch==2.7.1
+accelerate
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+protobuf==6.33.5
+outetts

backend/python/outetts/requirements-cublas13.txt

@@ -0,0 +1,7 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch==2.9.0
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements-hipblas.txt

@@ -0,0 +1,8 @@
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
+accelerate
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements-intel.txt

@@ -0,0 +1,8 @@
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+optimum[openvino]
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements.txt

@@ -0,0 +1,6 @@
+grpcio==1.76.0
+protobuf==6.33.5
+certifi
+setuptools
+scipy==1.15.1
+numpy>=2.0.0

backend/python/outetts/run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+startBackend $@

backend/python/outetts/test.py

@@ -0,0 +1,35 @@
+"""
+Test script for the OuteTTS gRPC service.
+"""
+import unittest
+import subprocess
+import time
+import backend_pb2
+import backend_pb2_grpc
+
+import grpc
+
+
+class TestBackendServicer(unittest.TestCase):
+    def setUp(self):
+        self.service = subprocess.Popen(["python3", "backend.py", "--addr", "localhost:50051"])
+        time.sleep(5)
+
+    def tearDown(self):
+        self.service.terminate()
+        self.service.wait()
+
+    def test_health(self):
+        try:
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.Health(backend_pb2.HealthMessage())
+                self.assertEqual(response.message, b'OK')
+        except Exception as err:
+            self.fail(f"Health check failed: {err}")
+        finally:
+            self.tearDown()
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/outetts/test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+runUnittests

backend/python/rerankers/requirements-mps.txt

@@ -0,0 +1,4 @@
+torch==2.7.1
+transformers
+accelerate
+rerankers[transformers]

backend/python/rfdetr/requirements-mps.txt

@@ -0,0 +1,7 @@
+torch==2.7.1
+rfdetr
+opencv-python
+accelerate
+peft
+inference
+optimum-quanto

backend/python/transformers/backend.py

@@ -24,7 +24,6 @@ XPU=os.environ.get("XPU", "0") == "1"
 from transformers import AutoTokenizer, AutoModel, set_seed, TextIteratorStreamer, StoppingCriteriaList, StopStringCriteria, MambaConfig, MambaForCausalLM
 from transformers import AutoProcessor, MusicgenForConditionalGeneration, DiaForConditionalGeneration
 from scipy.io import wavfile
-import outetts
 from sentence_transformers import SentenceTransformer
 
 
@@ -89,7 +88,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         self.CUDA = torch.cuda.is_available()
         self.OV=False
-        self.OuteTTS=False
         self.DiaTTS=False
         self.SentenceTransformer = False
 
@@ -239,45 +237,6 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     self.processor = self.processor.to("cuda")
                 print("DiaForConditionalGeneration loaded", file=sys.stderr)
                 self.DiaTTS = True
-            elif request.Type == "OuteTTS":
-                autoTokenizer = False
-                options = request.Options
-                MODELNAME = "OuteAI/OuteTTS-0.3-1B"
-                TOKENIZER = "OuteAI/OuteTTS-0.3-1B"
-                VERSION = "0.3"
-                SPEAKER = "en_male_1"
-                for opt in options:
-                    if opt.startswith("tokenizer:"):
-                        TOKENIZER = opt.split(":")[1]
-                        break
-                    if opt.startswith("version:"):
-                        VERSION = opt.split(":")[1]
-                        break
-                    if opt.startswith("speaker:"):
-                        SPEAKER = opt.split(":")[1]
-                        break
-                
-                if model_name != "":
-                    MODELNAME = model_name
-
-                # Configure the model
-                model_config = outetts.HFModelConfig_v2(
-                    model_path=MODELNAME,
-                    tokenizer_path=TOKENIZER
-                )
-                # Initialize the interface
-                self.interface = outetts.InterfaceHF(model_version=VERSION, cfg=model_config)
-                self.OuteTTS = True
-
-                self.interface.print_default_speakers()
-                if request.AudioPath:
-                    if os.path.isabs(request.AudioPath):
-                        self.AudioPath = request.AudioPath
-                    else:
-                        self.AudioPath = os.path.join(request.ModelPath, request.AudioPath)
-                    self.speaker = self.interface.create_speaker(audio_path=self.AudioPath)
-                else:
-                    self.speaker = self.interface.load_default_speaker(name=SPEAKER)               
             elif request.Type == "SentenceTransformer":
                 autoTokenizer = False
                 self.model = SentenceTransformer(model_name, trust_remote_code=request.TrustRemoteCode)
@@ -588,30 +547,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         return backend_pb2.Result(success=True)
 
 
-    def CallOuteTTS(self, request, context):
-        try:
-            print("[OuteTTS] generating TTS", file=sys.stderr)
-            gen_cfg = outetts.GenerationConfig(
-                text="Speech synthesis is the artificial production of human speech.",
-                temperature=self.options.get("temperature", 0.1),
-                repetition_penalty=self.options.get("repetition_penalty", 1.1),
-                max_length=self.max_tokens,
-                speaker=self.speaker,
-                # voice_characteristics="upbeat enthusiasm, friendliness, clarity, professionalism, and trustworthiness"
-            )
-            output = self.interface.generate(config=gen_cfg)
-            print("[OuteTTS] Generated TTS", file=sys.stderr)
-            output.save(request.dst)
-            print("[OuteTTS] TTS done", file=sys.stderr)
-        except Exception as err:
-            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
-        return backend_pb2.Result(success=True)
-
 # The TTS endpoint is older, and provides fewer features, but exists for compatibility reasons
     def TTS(self, request, context):
-        if self.OuteTTS:
-            return self.CallOuteTTS(request, context)
-        
         if self.DiaTTS:
             print("DiaTTS", file=sys.stderr)
             return self.CallDiaTTS(request, context)

backend/python/transformers/requirements-cpu.txt

@@ -4,6 +4,5 @@ numba==0.60.0
 accelerate
 transformers
 bitsandbytes
-outetts
 sentence-transformers==5.2.2
 protobuf==6.33.5

backend/python/transformers/requirements-cublas12.txt

@@ -4,6 +4,5 @@ llvmlite==0.43.0
 numba==0.60.0
 transformers
 bitsandbytes
-outetts
 sentence-transformers==5.2.2
 protobuf==6.33.5

backend/python/transformers/requirements-cublas13.txt

@@ -4,6 +4,5 @@ llvmlite==0.43.0
 numba==0.60.0
 transformers
 bitsandbytes
-outetts
 sentence-transformers==5.2.2
 protobuf==6.33.5

backend/python/transformers/requirements-hipblas.txt

@@ -5,7 +5,5 @@ transformers
 llvmlite==0.43.0
 numba==0.60.0
 bitsandbytes
-outetts
-bitsandbytes
 sentence-transformers==5.2.2
 protobuf==6.33.5

backend/python/transformers/requirements-intel.txt

@@ -5,6 +5,5 @@ llvmlite==0.43.0
 numba==0.60.0
 transformers
 bitsandbytes
-outetts
 sentence-transformers==5.2.2
 protobuf==6.33.5

backend/python/transformers/requirements-mps.txt

@@ -0,0 +1,8 @@
+torch==2.7.1
+llvmlite==0.43.0
+numba==0.60.0
+accelerate
+transformers
+bitsandbytes
+sentence-transformers==5.2.2
+protobuf==6.33.5

backend/python/whisperx/install.sh

@@ -8,6 +8,8 @@ else
     source $backend_dir/../common/libbackend.sh
 fi
 
-EXTRA_PIP_INSTALL_FLAGS+=" --index-strategy unsafe-best-match"
+if [ "x${BUILD_PROFILE}" != "xmetal" ] && [ "x${BUILD_PROFILE}" != "xmps" ]; then
+    EXTRA_PIP_INSTALL_FLAGS+=" --index-strategy unsafe-best-match"
+fi
 
 installRequirements

backend/python/whisperx/requirements-mps.txt

@@ -0,0 +1,2 @@
+torch
+whisperx @ git+https://github.com/m-bain/whisperX.git