Drop l4t12

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.github/workflows/deploy-explorer.yaml

@@ -33,7 +33,7 @@ jobs:
         run: |
           CGO_ENABLED=0 make build
       - name: rm
-        uses: appleboy/ssh-action@v1.2.4
+        uses: appleboy/ssh-action@v1.2.5
         with:
             host: ${{ secrets.EXPLORER_SSH_HOST }}
             username: ${{ secrets.EXPLORER_SSH_USERNAME }}
@@ -53,7 +53,7 @@ jobs:
             rm: true
             target: ./local-ai
       - name: restarting
-        uses: appleboy/ssh-action@v1.2.4
+        uses: appleboy/ssh-action@v1.2.5
         with:
             host: ${{ secrets.EXPLORER_SSH_HOST }}
             username: ${{ secrets.EXPLORER_SSH_USERNAME }}

.github/workflows/image-pr.yml

@@ -37,7 +37,7 @@
           include:
             - build-type: 'cublas'
               cuda-major-version: "12"
-              cuda-minor-version: "9"
+              cuda-minor-version: "8"
               platforms: 'linux/amd64'
               tag-latest: 'false'
               tag-suffix: '-gpu-nvidia-cuda-12'

.github/workflows/image.yml

@@ -88,7 +88,7 @@
               ubuntu-codename: 'noble'
             - build-type: 'cublas'
               cuda-major-version: "12"
-              cuda-minor-version: "9"
+              cuda-minor-version: "8"
               platforms: 'linux/amd64'
               tag-latest: 'auto'
               tag-suffix: '-gpu-nvidia-cuda-12'

.github/workflows/test-extra.yml

@@ -238,7 +238,7 @@ jobs:
       - name: Dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install build-essential ffmpeg
+          sudo apt-get install -y build-essential ffmpeg
           sudo apt-get install -y ca-certificates cmake curl patch espeak espeak-ng python3-pip
           # Install UV
           curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -257,7 +257,7 @@ jobs:
       - name: Dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install build-essential ffmpeg
+          sudo apt-get install -y build-essential ffmpeg
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           # Install UV
           curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -276,7 +276,7 @@ jobs:
       - name: Dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install build-essential ffmpeg
+          sudo apt-get install -y build-essential ffmpeg
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           # Install UV
           curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -284,4 +284,61 @@ jobs:
       - name: Test pocket-tts
         run: |
           make --jobs=5 --output-sync=target -C backend/python/pocket-tts
-          make --jobs=5 --output-sync=target -C backend/python/pocket-tts test
+          make --jobs=5 --output-sync=target -C backend/python/pocket-tts test
+  tests-qwen-tts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential ffmpeg
+          sudo apt-get install -y ca-certificates cmake curl patch python3-pip
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          pip install --user --no-cache-dir grpcio-tools==1.64.1
+      - name: Test qwen-tts
+        run: |
+          make --jobs=5 --output-sync=target -C backend/python/qwen-tts
+          make --jobs=5 --output-sync=target -C backend/python/qwen-tts test
+  tests-qwen-asr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential ffmpeg sox
+          sudo apt-get install -y ca-certificates cmake curl patch python3-pip
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          pip install --user --no-cache-dir grpcio-tools==1.64.1
+      - name: Test qwen-asr
+        run: |
+          make --jobs=5 --output-sync=target -C backend/python/qwen-asr
+          make --jobs=5 --output-sync=target -C backend/python/qwen-asr test
+  tests-voxcpm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install build-essential ffmpeg
+          sudo apt-get install -y ca-certificates cmake curl patch python3-pip
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          pip install --user --no-cache-dir grpcio-tools==1.64.1
+      - name: Test voxcpm
+        run: |
+          make --jobs=5 --output-sync=target -C backend/python/voxcpm
+          make --jobs=5 --output-sync=target -C backend/python/voxcpm test

.github/workflows/tests-e2e.yml

@@ -0,0 +1,56 @@
+---
+name: 'E2E Backend Tests'
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+    tags:
+      - '*'
+
+concurrency:
+  group: ci-tests-e2e-backend-${{ github.head_ref || github.ref }}-${{ github.repository }}
+  cancel-in-progress: true
+
+jobs:
+  tests-e2e-backend:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go-version: ['1.25.x']
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Setup Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          cache: false
+      - name: Display Go version
+        run: go version
+      - name: Proto Dependencies
+        run: |
+          # Install protoc
+          curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
+          unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
+          rm protoc.zip
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
+          PATH="$PATH:$HOME/go/bin" make protogen-go
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential
+      - name: Test Backend E2E
+        run: |
+          PATH="$PATH:$HOME/go/bin" make build-mock-backend test-e2e
+      - name: Setup tmate session if tests fail
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3.23
+        with:
+          detached: true
+          connect-timeout-seconds: 180
+          limit-access-to-actor: true

.gitignore

@@ -36,6 +36,8 @@ LocalAI
 models/*
 test-models/
 test-dir/
+tests/e2e-aio/backends
+tests/e2e-aio/models
 
 release/
 

AGENTS.md

@@ -4,13 +4,13 @@ Building and testing the project depends on the components involved and the plat
 
 ## Building a specified backend
 
-Let's say the user wants to build a particular backend for a given platform. For example let's say they want to build bark for ROCM/hipblas
+Let's say the user wants to build a particular backend for a given platform. For example let's say they want to build coqui for ROCM/hipblas
 
-- The Makefile has targets like `docker-build-bark` created with `generate-docker-build-target` at the time of writing. Recently added backends may require a new target.
+- The Makefile has targets like `docker-build-coqui` created with `generate-docker-build-target` at the time of writing. Recently added backends may require a new target.
 - At a minimum we need to set the BUILD_TYPE, BASE_IMAGE build-args
   - Use .github/workflows/backend.yml as a reference it lists the needed args in the `include` job strategy matrix
   - l4t and cublas also requires the CUDA major and minor version
-- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:6.4.4 make docker-build-bark`
+- You can pretty print a command like `DOCKER_MAKEFLAGS=-j$(nproc --ignore=1) BUILD_TYPE=hipblas BASE_IMAGE=rocm/dev-ubuntu-24.04:6.4.4 make docker-build-coqui`
 - Unless the user specifies that they want you to run the command, then just print it because not all agent frontends handle long running jobs well and the output may overflow your context
 - The user may say they want to build AMD or ROCM instead of hipblas, or Intel instead of SYCL or NVIDIA insted of l4t or cublas. Ask for confirmation if there is ambiguity.
 - Sometimes the user may need extra parameters to be added to `docker build` (e.g. `--platform` for cross-platform builds or `--progress` to view the full logs), in which case you can generate the `docker build` command directly.
@@ -95,7 +95,7 @@ test-extra: prepare-test-extra
 
 Add a backend definition variable in the backend definitions section (around line 428-457). The format depends on the backend type:
 
-**For Python backends with root context** (like `faster-whisper`, `bark`):
+**For Python backends with root context** (like `faster-whisper`, `coqui`):
 ```makefile
 BACKEND_<BACKEND_NAME> = <backend-name>|python|.|false|true
 ```
@@ -280,3 +280,11 @@ Always check `llama.cpp` for new model configuration options that should be supp
 - `llama.cpp/common/chat-parser.cpp` - Format presets and model-specific handlers
 - `llama.cpp/common/chat.h` - Format enums and parameter structures
 - `llama.cpp/tools/server/server-context.cpp` - Server configuration options
+
+# Documentation
+
+The project documentation is located in `docs/content`. When adding new features or changing existing functionality, it is crucial to update the documentation to reflect these changes. This helps users understand how to use the new capabilities and ensures the documentation stays relevant.
+
+- **Feature Documentation**: If you add a new feature (like a new backend or API endpoint), create a new markdown file in `docs/content/features/` explaining what it is, how to configure it, and how to use it.
+- **Configuration**: If you modify configuration options, update the relevant sections in `docs/content/`.
+- **Examples**: providing concrete examples (like YAML configuration blocks) is highly encouraged to help users get started quickly.

Dockerfile

@@ -10,7 +10,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         ca-certificates curl wget espeak-ng libgomp1 \
-        ffmpeg libopenblas0 libopenblas-dev && \
+        ffmpeg libopenblas0 libopenblas-dev sox && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/stablediffusion-ggml-darwin backends/vllm backends/moonshine backends/pocket-tts
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/qwen-asr backends/voxcpm backends/whisperx backends/ace-step
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -7,16 +7,14 @@ GOVET=$(GOCMD) vet
 BINARY_NAME=local-ai
 LAUNCHER_BINARY_NAME=local-ai-launcher
 
-CUDA_MAJOR_VERSION?=13
-CUDA_MINOR_VERSION?=0
 UBUNTU_VERSION?=2404
 UBUNTU_CODENAME?=noble
 
 GORELEASER?=
 
 export BUILD_TYPE?=
-export CUDA_MAJOR_VERSION?=12
-export CUDA_MINOR_VERSION?=9
+export CUDA_MAJOR_VERSION?=13
+export CUDA_MINOR_VERSION?=0
 
 GO_TAGS?=
 BUILD_ID?=
@@ -191,9 +189,6 @@ run-e2e-aio: protogen-go
 ########################################################
 
 prepare-e2e:
-	mkdir -p $(TEST_DIR)
-	cp -rfv $(abspath ./tests/e2e-fixtures)/gpu.yaml $(TEST_DIR)/gpu.yaml
-	test -e $(TEST_DIR)/ggllm-test-model.bin || wget -q https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q2_K.gguf -O $(TEST_DIR)/ggllm-test-model.bin
 	docker build \
 		--build-arg IMAGE_TYPE=core \
 		--build-arg BUILD_TYPE=$(BUILD_TYPE) \
@@ -207,14 +202,16 @@ prepare-e2e:
 		-t localai-tests .
 
 run-e2e-image:
-	ls -liah $(abspath ./tests/e2e-fixtures)
-	docker run -p 5390:8080 -e MODELS_PATH=/models -e THREADS=1 -e DEBUG=true -d --rm -v $(TEST_DIR):/models --gpus all --name e2e-tests-$(RANDOM) localai-tests
+	docker run -p 5390:8080 -e MODELS_PATH=/models -e THREADS=1 -e DEBUG=true -d --rm -v $(TEST_DIR):/models --name e2e-tests-$(RANDOM) localai-tests
 
-test-e2e:
+test-e2e: build-mock-backend prepare-e2e run-e2e-image
 	@echo 'Running e2e tests'
 	BUILD_TYPE=$(BUILD_TYPE) \
-	LOCALAI_API=http://$(E2E_BRIDGE_IP):5390/v1 \
+	LOCALAI_API=http://$(E2E_BRIDGE_IP):5390 \
 	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e
+	$(MAKE) clean-mock-backend
+	$(MAKE) teardown-e2e
+	docker rmi localai-tests
 
 teardown-e2e:
 	rm -rf $(TEST_DIR) || true
@@ -311,21 +308,35 @@ protogen-go-clean:
 
 prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/transformers
+	$(MAKE) -C backend/python/outetts
 	$(MAKE) -C backend/python/diffusers
 	$(MAKE) -C backend/python/chatterbox
 	$(MAKE) -C backend/python/vllm
+	$(MAKE) -C backend/python/vllm-omni
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/moonshine
 	$(MAKE) -C backend/python/pocket-tts
+	$(MAKE) -C backend/python/qwen-tts
+	$(MAKE) -C backend/python/qwen-asr
+	$(MAKE) -C backend/python/voxcpm
+	$(MAKE) -C backend/python/whisperx
+	$(MAKE) -C backend/python/ace-step
 
 test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/transformers test
+	$(MAKE) -C backend/python/outetts test
 	$(MAKE) -C backend/python/diffusers test
 	$(MAKE) -C backend/python/chatterbox test
 	$(MAKE) -C backend/python/vllm test
+	$(MAKE) -C backend/python/vllm-omni test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/moonshine test
 	$(MAKE) -C backend/python/pocket-tts test
+	$(MAKE) -C backend/python/qwen-tts test
+	$(MAKE) -C backend/python/qwen-asr test
+	$(MAKE) -C backend/python/voxcpm test
+	$(MAKE) -C backend/python/whisperx test
+	$(MAKE) -C backend/python/ace-step test
 
 DOCKER_IMAGE?=local-ai
 DOCKER_AIO_IMAGE?=local-ai-aio
@@ -434,7 +445,6 @@ backend-images:
 BACKEND_LLAMA_CPP = llama-cpp|llama-cpp|.|false|false
 
 # Golang backends
-BACKEND_BARK_CPP = bark-cpp|golang|.|false|true
 BACKEND_PIPER = piper|golang|.|false|true
 BACKEND_LOCAL_STORE = local-store|golang|.|false|true
 BACKEND_HUGGINGFACE = huggingface|golang|.|false|true
@@ -445,20 +455,25 @@ BACKEND_WHISPER = whisper|golang|.|false|true
 # Python backends with root context
 BACKEND_RERANKERS = rerankers|python|.|false|true
 BACKEND_TRANSFORMERS = transformers|python|.|false|true
+BACKEND_OUTETTS = outetts|python|.|false|true
 BACKEND_FASTER_WHISPER = faster-whisper|python|.|false|true
 BACKEND_COQUI = coqui|python|.|false|true
-BACKEND_BARK = bark|python|.|false|true
-BACKEND_EXLLAMA2 = exllama2|python|.|false|true
 BACKEND_RFDETR = rfdetr|python|.|false|true
 BACKEND_KITTEN_TTS = kitten-tts|python|.|false|true
 BACKEND_NEUTTS = neutts|python|.|false|true
 BACKEND_KOKORO = kokoro|python|.|false|true
 BACKEND_VLLM = vllm|python|.|false|true
+BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true
 BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true
 BACKEND_CHATTERBOX = chatterbox|python|.|false|true
 BACKEND_VIBEVOICE = vibevoice|python|.|--progress=plain|true
 BACKEND_MOONSHINE = moonshine|python|.|false|true
 BACKEND_POCKET_TTS = pocket-tts|python|.|false|true
+BACKEND_QWEN_TTS = qwen-tts|python|.|false|true
+BACKEND_QWEN_ASR = qwen-asr|python|.|false|true
+BACKEND_VOXCPM = voxcpm|python|.|false|true
+BACKEND_WHISPERX = whisperx|python|.|false|true
+BACKEND_ACE_STEP = ace-step|python|.|false|true
 
 # Helper function to build docker image for a backend
 # Usage: $(call docker-build-backend,BACKEND_NAME,DOCKERFILE_TYPE,BUILD_CONTEXT,PROGRESS_FLAG,NEEDS_BACKEND_ARG)
@@ -482,7 +497,6 @@ endef
 
 # Generate all docker-build targets
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
-$(eval $(call generate-docker-build-target,$(BACKEND_BARK_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PIPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE)))
@@ -491,26 +505,41 @@ $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML)))
 $(eval $(call generate-docker-build-target,$(BACKEND_WHISPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_RERANKERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TRANSFORMERS)))
+$(eval $(call generate-docker-build-target,$(BACKEND_OUTETTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_FASTER_WHISPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_COQUI)))
-$(eval $(call generate-docker-build-target,$(BACKEND_BARK)))
-$(eval $(call generate-docker-build-target,$(BACKEND_EXLLAMA2)))
 $(eval $(call generate-docker-build-target,$(BACKEND_RFDETR)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KITTEN_TTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KOKORO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM)))
+$(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VIBEVOICE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_MOONSHINE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_POCKET_TTS)))
+$(eval $(call generate-docker-build-target,$(BACKEND_QWEN_TTS)))
+$(eval $(call generate-docker-build-target,$(BACKEND_QWEN_ASR)))
+$(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
+$(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
+$(eval $(call generate-docker-build-target,$(BACKEND_ACE_STEP)))
 
 # Pattern rule for docker-save targets
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-transformers docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-bark docker-build-chatterbox docker-build-vibevoice docker-build-exllama2 docker-build-moonshine docker-build-pocket-tts
+docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-qwen-asr docker-build-voxcpm docker-build-whisperx docker-build-ace-step
+
+########################################################
+### Mock Backend for E2E Tests
+########################################################
+
+build-mock-backend: protogen-go
+	$(GOCMD) build -o tests/e2e/mock-backend/mock-backend ./tests/e2e/mock-backend
+
+clean-mock-backend:
+	rm -f tests/e2e/mock-backend/mock-backend
 
 ########################################################
 ### END Backends

README.md

@@ -51,34 +51,16 @@
 **LocalAI** is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that's compatible with OpenAI (Elevenlabs, Anthropic... ) API specifications for local AI inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families. Does not require GPU. It is created and maintained by [Ettore Di Giacinto](https://github.com/mudler).
 
 
-## 📚🆕 Local Stack Family
+## Local Stack Family
 
-🆕 LocalAI is now part of a comprehensive suite of AI tools designed to work together:
+Liking LocalAI? LocalAI is part of an integrated suite of AI infrastructure tools, you might also like:
+
+- **[LocalAGI](https://github.com/mudler/LocalAGI)** - AI agent orchestration platform with OpenAI Responses API compatibility and advanced agentic capabilities
+- **[LocalRecall](https://github.com/mudler/LocalRecall)** - MCP/REST API knowledge base system providing persistent memory and storage for AI agents
+- 🆕 **[Cogito](https://github.com/mudler/cogito)** - Go library for building intelligent, co-operative agentic software and LLM-powered workflows, focusing on improving results for small, open source language models that scales to any LLM. Powers LocalAGI and LocalAI MCP/Agentic capabilities
+- 🆕 **[Wiz](https://github.com/mudler/wiz)** - Terminal-based AI agent accessible via Ctrl+Space keybinding. Portable, local-LLM friendly shell assistant with TUI/CLI modes, tool execution with approval, MCP protocol support, and multi-shell compatibility (zsh, bash, fish)
+- 🆕 **[SkillServer](https://github.com/mudler/skillserver)** - Simple, centralized skills database for AI agents via MCP. Manages skills as Markdown files with MCP server integration, web UI for editing, Git synchronization, and full-text search capabilities
 
-<table>
-  <tr>
-    <td width="50%" valign="top">
-      <a href="https://github.com/mudler/LocalAGI">
-        <img src="https://raw.githubusercontent.com/mudler/LocalAGI/refs/heads/main/webui/react-ui/public/logo_2.png" width="300" alt="LocalAGI Logo">
-      </a>
-    </td>
-    <td width="50%" valign="top">
-      <h3><a href="https://github.com/mudler/LocalAGI">LocalAGI</a></h3>
-      <p>A powerful Local AI agent management platform that serves as a drop-in replacement for OpenAI's Responses API, enhanced with advanced agentic capabilities.</p>
-    </td>
-  </tr>
-  <tr>
-    <td width="50%" valign="top">
-      <a href="https://github.com/mudler/LocalRecall">
-        <img src="https://raw.githubusercontent.com/mudler/LocalRecall/refs/heads/main/static/localrecall_horizontal.png" width="300" alt="LocalRecall Logo">
-      </a>
-    </td>
-    <td width="50%" valign="top">
-      <h3><a href="https://github.com/mudler/LocalRecall">LocalRecall</a></h3>
-      <p>A REST-ful API and knowledge base management system that provides persistent memory and storage capabilities for AI agents.</p>
-    </td>
-  </tr>
-</table>
 
 ## Screenshots / Video
 
@@ -221,7 +203,8 @@ local-ai run oci://localai/phi-2:latest
 For more information, see [💻 Getting started](https://localai.io/basics/getting_started/index.html), if you are interested in our roadmap items and future enhancements, you can see the [Issues labeled as Roadmap here](https://github.com/mudler/LocalAI/issues?q=is%3Aissue+is%3Aopen+label%3Aroadmap)
 
 ## 📰 Latest project news
-
+- February 2026: [Realtime API for audio-to-audio with tool calling](https://github.com/mudler/LocalAI/pull/6245), [ACE-Step 1.5 support](https://github.com/mudler/LocalAI/pull/8396)
+- January 2026: **LocalAI 3.10.0** - Major release with Anthropic API support, Open Responses API for stateful agents, video & image generation suite (LTX-2), unified GPU backends, tool streaming & XML parsing, system-aware backend gallery, crash fixes for AVX-only CPUs and AMD VRAM reporting, request tracing, and new backends: **Moonshine** (ultra-fast transcription), **Pocket-TTS** (lightweight TTS). Vulkan arm64 builds now available. [Release notes](https://github.com/mudler/LocalAI/releases/tag/v3.10.0).
 - December 2025: [Dynamic Memory Resource reclaimer](https://github.com/mudler/LocalAI/pull/7583), [Automatic fitting of models to multiple GPUS(llama.cpp)](https://github.com/mudler/LocalAI/pull/7584), [Added Vibevoice backend](https://github.com/mudler/LocalAI/pull/7494)
 - November 2025: Major improvements to the UX. Among these: [Import models via URL](https://github.com/mudler/LocalAI/pull/7245) and [Multiple chats and history](https://github.com/mudler/LocalAI/pull/7325)
 - October 2025: 🔌 [Model Context Protocol (MCP)](https://localai.io/docs/features/mcp/) support added for agentic capabilities with external tools
@@ -257,6 +240,7 @@ Roadmap items: [List of issues](https://github.com/mudler/LocalAI/issues?q=is%3A
 - 🔈 [Audio to Text](https://localai.io/features/audio-to-text/) (Audio transcription with `whisper.cpp`)
 - 🎨 [Image generation](https://localai.io/features/image-generation)
 - 🔥 [OpenAI-alike tools API](https://localai.io/features/openai-functions/) 
+- ⚡ [Realtime API](https://localai.io/features/openai-realtime/) (Speech-to-speech) 
 - 🧠 [Embeddings generation for vector databases](https://localai.io/features/embeddings/)
 - ✍️ [Constrained grammars](https://localai.io/features/constrained_grammars/)
 - 🖼️ [Download Models directly from Huggingface ](https://localai.io/models/)
@@ -278,7 +262,6 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration
 | **llama.cpp** | LLM inference in C/C++ | CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, CPU |
 | **vLLM** | Fast LLM inference with PagedAttention | CUDA 12/13, ROCm, Intel |
 | **transformers** | HuggingFace transformers framework | CUDA 12/13, ROCm, Intel, CPU |
-| **exllama2** | GPTQ inference library | CUDA 12/13 |
 | **MLX** | Apple Silicon LLM inference | Metal (M1/M2/M3+) |
 | **MLX-VLM** | Apple Silicon Vision-Language Models | Metal (M1/M2/M3+) |
 
@@ -287,8 +270,7 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration
 |---------|-------------|---------------------|
 | **whisper.cpp** | OpenAI Whisper in C/C++ | CUDA 12/13, ROCm, Intel SYCL, Vulkan, CPU |
 | **faster-whisper** | Fast Whisper with CTranslate2 | CUDA 12/13, ROCm, Intel, CPU |
-| **bark** | Text-to-audio generation | CUDA 12/13, ROCm, Intel |
-| **bark-cpp** | C++ implementation of Bark | CUDA, Metal, CPU |
+| **moonshine** | Ultra-fast transcription engine for low-end devices | CUDA 12/13, Metal, CPU |
 | **coqui** | Advanced TTS with 1100+ languages | CUDA 12/13, ROCm, Intel, CPU |
 | **kokoro** | Lightweight TTS model | CUDA 12/13, ROCm, Intel, CPU |
 | **chatterbox** | Production-grade TTS | CUDA 12/13, CPU |
@@ -298,6 +280,8 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration
 | **neutts** | Text-to-speech with voice cloning | CUDA 12/13, ROCm, CPU |
 | **vibevoice** | Real-time TTS with voice cloning | CUDA 12/13, ROCm, Intel, CPU |
 | **pocket-tts** | Lightweight CPU-based TTS | CUDA 12/13, ROCm, Intel, CPU |
+| **qwen-tts** | High-quality TTS with custom voice, voice design, and voice cloning | CUDA 12/13, ROCm, Intel, CPU |
+| **ace-step** | Music generation from text descriptions, lyrics, or audio samples | CUDA 12/13, ROCm, Intel, Metal, CPU |
 
 ### Image & Video Generation
 | Backend | Description | Acceleration Support |
@@ -319,11 +303,11 @@ LocalAI supports a comprehensive range of AI backends with multiple acceleration
 |-------------------|-------------------|------------------|
 | **NVIDIA CUDA 12** | All CUDA-compatible backends | Nvidia hardware |
 | **NVIDIA CUDA 13** | All CUDA-compatible backends | Nvidia hardware |
-| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, bark, neutts, vibevoice, pocket-tts | AMD Graphics |
-| **Intel oneAPI** | llama.cpp, whisper, stablediffusion, vllm, transformers, diffusers, rfdetr, rerankers, exllama2, coqui, kokoro, bark, vibevoice, pocket-tts | Intel Arc, Intel iGPUs |
-| **Apple Metal** | llama.cpp, whisper, diffusers, MLX, MLX-VLM, bark-cpp | Apple M1/M2/M3+ |
+| **AMD ROCm** | llama.cpp, whisper, vllm, transformers, diffusers, rerankers, coqui, kokoro, neutts, vibevoice, pocket-tts, qwen-tts, ace-step | AMD Graphics |
+| **Intel oneAPI** | llama.cpp, whisper, stablediffusion, vllm, transformers, diffusers, rfdetr, rerankers, coqui, kokoro, vibevoice, pocket-tts, qwen-tts, ace-step | Intel Arc, Intel iGPUs |
+| **Apple Metal** | llama.cpp, whisper, diffusers, MLX, MLX-VLM, moonshine, ace-step | Apple M1/M2/M3+ |
 | **Vulkan** | llama.cpp, whisper, stablediffusion | Cross-platform GPUs |
-| **NVIDIA Jetson (CUDA 12)** | llama.cpp, whisper, stablediffusion, diffusers, rfdetr | ARM64 embedded AI (AGX Orin, etc.) |
+| **NVIDIA Jetson (CUDA 12)** | llama.cpp, whisper, stablediffusion, diffusers, rfdetr, ace-step | ARM64 embedded AI (AGX Orin, etc.) |
 | **NVIDIA Jetson (CUDA 13)** | llama.cpp, whisper, stablediffusion, diffusers, rfdetr | ARM64 embedded AI (DGX Spark) |
 | **CPU Optimized** | All backends | AVX/AVX2/AVX512, quantization support |
 
@@ -343,6 +327,10 @@ Agentic Libraries:
 MCPs:
 - https://github.com/mudler/MCPs
 
+OS Assistant:
+
+- https://github.com/mudler/Keygeist - Keygeist is an AI-powered keyboard operator that listens for key combinations and responds with AI-generated text typed directly into your Linux box.
+
 Model galleries
 - https://github.com/go-skynet/model-gallery
 

backend/README.md

@@ -46,7 +46,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **vllm**: High-performance LLM inference
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
-- **Audio**: bark, coqui, faster-whisper, kitten-tts
+- **Audio**: coqui, faster-whisper, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro
 
@@ -55,7 +55,6 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **stablediffusion-ggml**: Stable Diffusion in Go with GGML Cpp backend
 - **huggingface**: Hugging Face model integration
 - **piper**: Text-to-speech synthesis Golang with C bindings using rhaspy/piper
-- **bark-cpp**: Bark TTS models Golang with Cpp bindings
 - **local-store**: Vector storage backend
 
 #### C++ Backends (`cpp/`)

backend/backend.proto

@@ -17,6 +17,7 @@ service Backend {
   rpc GenerateVideo(GenerateVideoRequest) returns (Result) {}
   rpc AudioTranscription(TranscriptRequest) returns (TranscriptResult) {}
   rpc TTS(TTSRequest) returns (Result) {}
+  rpc TTSStream(TTSRequest) returns (stream Reply) {}
   rpc SoundGeneration(SoundGenerationRequest) returns (Result) {}
   rpc TokenizeString(PredictOptions) returns (TokenizationResponse) {}
   rpc Status(HealthMessage) returns (StatusResponse) {}
@@ -32,6 +33,8 @@ service Backend {
   rpc GetMetrics(MetricsRequest) returns (MetricsResponse);
 
   rpc VAD(VADRequest) returns (VADResponse) {}
+
+  rpc ModelMetadata(ModelOptions) returns (ModelMetadataResponse) {}
 }
 
 // Define the empty request
@@ -296,6 +299,7 @@ message TranscriptSegment {
   int64 end = 3;
   string text = 4;
   repeated int32 tokens = 5;
+  string speaker = 6;
 }
 
 message GenerateImageRequest {
@@ -361,6 +365,14 @@ message SoundGenerationRequest {
   optional bool sample = 6;
   optional string src = 7;
   optional int32 src_divisor = 8;
+  optional bool think = 9;
+  optional string caption = 10;
+  optional string lyrics = 11;
+  optional int32 bpm = 12;
+  optional string keyscale = 13;
+  optional string language = 14;
+  optional string timesignature = 15;
+  optional bool instrumental = 17;
 }
 
 message TokenizationResponse {
@@ -410,3 +422,8 @@ message Detection {
 message DetectResponse {
   repeated Detection Detections = 1;
 }
+
+message ModelMetadataResponse {
+  bool supports_thinking = 1;
+  string rendered_template = 2;  // The rendered chat template with enable_thinking=true (empty if not applicable)
+}

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=2fbde785bc106ae1c4102b0e82b9b41d9c466579
+LLAMA_VERSION?=b536eb023368701fe3564210440e2df6151c3e65
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -83,8 +83,8 @@ static void start_llama_server(server_context& ctx_server) {
 
     // print sample chat example to make it clear which template is used
     // LOG_INF("%s: chat template, chat_template: %s, example_format: '%s'\n", __func__,
-    //     common_chat_templates_source(ctx_server.impl->chat_templates.get()),
-    //     common_chat_format_example(ctx_server.impl->chat_templates.get(), ctx_server.impl->params_base.use_jinja).c_str(), ctx_server.impl->params_base.default_template_kwargs);
+    //     common_chat_templates_source(ctx_server.impl->chat_params.tmpls.get()),
+    //     common_chat_format_example(ctx_server.impl->chat_params.tmpls.get(), ctx_server.impl->params_base.use_jinja).c_str(), ctx_server.impl->params_base.default_template_kwargs);
 
     // Keep the chat templates initialized in load_model() so they can be used when UseTokenizerTemplate is enabled
     // Templates will only be used conditionally in Predict/PredictStream when UseTokenizerTemplate is true and Messages are provided
@@ -778,8 +778,8 @@ public:
             if (!params.mmproj.path.empty()) {
                 error_msg += " (with mmproj: " + params.mmproj.path + ")";
             }
-            if (params.has_speculative() && !params.speculative.model.path.empty()) {
-                error_msg += " (with draft model: " + params.speculative.model.path + ")";
+            if (params.speculative.has_dft() && !params.speculative.mparams_dft.path.empty()) {
+                error_msg += " (with draft model: " + params.speculative.mparams_dft.path + ")";
             }
             
             // Add captured error details if available
@@ -882,7 +882,7 @@ public:
             std::string prompt_str;
             std::vector<raw_buffer> files; // Declare files early so it's accessible in both branches
             // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided
-            if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_templates != nullptr) {
+            if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) {
                 // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse
                 json body_json;
                 json messages_json = json::array();
@@ -1261,12 +1261,7 @@ public:
                 // Use the same approach as server.cpp: call oaicompat_chat_params_parse
                 // This handles all template application, grammar merging, etc. automatically
                 // Files extracted from multimodal content in messages will be added to the files vector
-                // Create parser options with current chat_templates to ensure tmpls is not null
-                oaicompat_parser_options parser_opt = ctx_server.impl->oai_parser_opt;
-                parser_opt.tmpls = ctx_server.impl->chat_templates.get(); // Ensure tmpls is set to current chat_templates
-                // Update allow_image and allow_audio based on current mctx state
-                parser_opt.allow_image = ctx_server.impl->mctx ? mtmd_support_vision(ctx_server.impl->mctx) : false;
-                parser_opt.allow_audio = ctx_server.impl->mctx ? mtmd_support_audio(ctx_server.impl->mctx) : false;
+                // chat_params already contains tmpls, allow_image, and allow_audio set during model loading
 
                 // Debug: Log tools before template processing
                 if (body_json.contains("tools")) {
@@ -1312,7 +1307,7 @@ public:
                     }
                 }
 
-                json parsed_data = oaicompat_chat_params_parse(body_json, parser_opt, files);
+                json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files);
 
                 // Debug: Log tools after template processing
                 if (parsed_data.contains("tools")) {
@@ -1365,7 +1360,7 @@ public:
 
             // If not using chat templates, extract files from image_data/audio_data fields
             // (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
-            if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_templates == nullptr) {
+            if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
                 const auto &images_data = data.find("image_data");
                 if (images_data != data.end() && images_data->is_array())
                 {
@@ -1593,7 +1588,7 @@ public:
             std::string prompt_str;
             std::vector<raw_buffer> files; // Declare files early so it's accessible in both branches
             // Handle chat templates when UseTokenizerTemplate is enabled and Messages are provided
-            if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_templates != nullptr) {
+            if (request->usetokenizertemplate() && request->messages_size() > 0 && ctx_server.impl->chat_params.tmpls != nullptr) {
                 // Convert proto Messages to JSON format compatible with oaicompat_chat_params_parse
                 json body_json;
                 json messages_json = json::array();
@@ -1997,12 +1992,7 @@ public:
                 // Use the same approach as server.cpp: call oaicompat_chat_params_parse
                 // This handles all template application, grammar merging, etc. automatically
                 // Files extracted from multimodal content in messages will be added to the files vector
-                // Create parser options with current chat_templates to ensure tmpls is not null
-                oaicompat_parser_options parser_opt = ctx_server.impl->oai_parser_opt;
-                parser_opt.tmpls = ctx_server.impl->chat_templates.get(); // Ensure tmpls is set to current chat_templates
-                // Update allow_image and allow_audio based on current mctx state
-                parser_opt.allow_image = ctx_server.impl->mctx ? mtmd_support_vision(ctx_server.impl->mctx) : false;
-                parser_opt.allow_audio = ctx_server.impl->mctx ? mtmd_support_audio(ctx_server.impl->mctx) : false;
+                // chat_params already contains tmpls, allow_image, and allow_audio set during model loading
 
                 // Debug: Log tools before template processing
                 if (body_json.contains("tools")) {
@@ -2048,7 +2038,7 @@ public:
                     }
                 }
 
-                json parsed_data = oaicompat_chat_params_parse(body_json, parser_opt, files);
+                json parsed_data = oaicompat_chat_params_parse(body_json, ctx_server.impl->chat_params, files);
 
                 // Debug: Log tools after template processing
                 if (parsed_data.contains("tools")) {
@@ -2101,7 +2091,7 @@ public:
 
             // If not using chat templates, extract files from image_data/audio_data fields
             // (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
-            if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_templates == nullptr) {
+            if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
                 const auto &images_data = data.find("image_data");
                 if (images_data != data.end() && images_data->is_array())
                 {
@@ -2486,6 +2476,47 @@ public:
         response->set_prompt_tokens_processed(res_metrics->n_prompt_tokens_processed_total);
 
 
+        return grpc::Status::OK;
+    }
+
+    grpc::Status ModelMetadata(ServerContext* /*context*/, const backend::ModelOptions* /*request*/, backend::ModelMetadataResponse* response) override {
+        // Check if model is loaded
+        if (params_base.model.path.empty()) {
+            return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
+        }
+
+        // Check if chat templates are initialized
+        if (ctx_server.impl->chat_params.tmpls == nullptr) {
+            // If templates are not initialized, we can't detect thinking support
+            // Return false as default
+            response->set_supports_thinking(false);
+            response->set_rendered_template("");
+            return grpc::Status::OK;
+        }
+
+        // Detect thinking support using llama.cpp's function
+        bool supports_thinking = common_chat_templates_support_enable_thinking(ctx_server.impl->chat_params.tmpls.get());
+        response->set_supports_thinking(supports_thinking);
+
+        // Render the template with enable_thinking=true so Go code can detect thinking tokens
+        // This allows reusing existing detection functions in Go
+        std::string rendered_template = "";
+        if (params_base.use_jinja) {
+            // Render the template with enable_thinking=true to see what the actual prompt looks like
+            common_chat_templates_inputs dummy_inputs;
+            common_chat_msg msg;
+            msg.role = "user";
+            msg.content = "test";
+            dummy_inputs.messages = {msg};
+            dummy_inputs.enable_thinking = true;
+            dummy_inputs.use_jinja = params_base.use_jinja;
+            
+            const auto rendered = common_chat_templates_apply(ctx_server.impl->chat_params.tmpls.get(), dummy_inputs);
+            rendered_template = rendered.prompt;
+        }
+        
+        response->set_rendered_template(rendered_template);
+
         return grpc::Status::OK;
     }
 };

backend/go/bark-cpp/Makefile

@@ -1,51 +0,0 @@
-INCLUDE_PATH := $(abspath ./)
-LIBRARY_PATH := $(abspath ./)
-
-AR?=ar
-
-CMAKE_ARGS?=-DGGML_NATIVE=OFF
-BUILD_TYPE?=
-GOCMD=go
-# keep standard at C11 and C++11
-CXXFLAGS = -I. -I$(INCLUDE_PATH)/sources/bark.cpp/examples -I$(INCLUDE_PATH)/sources/bark.cpp/encodec.cpp/ggml/include -I$(INCLUDE_PATH)/sources/bark.cpp/spm-headers -I$(INCLUDE_PATH)/sources/bark.cpp -O3 -DNDEBUG -std=c++17 -fPIC
-LDFLAGS  = -L$(LIBRARY_PATH) -L$(LIBRARY_PATH)/sources/bark.cpp/build/examples -lbark -lstdc++ -lm
-
-# bark.cpp
-BARKCPP_REPO?=https://github.com/PABannier/bark.cpp.git
-BARKCPP_VERSION?=5d5be84f089ab9ea53b7a793f088d3fbf7247495
-
-# warnings
-CXXFLAGS += -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function
-
-## bark.cpp
-sources/bark.cpp:
-	git clone --recursive $(BARKCPP_REPO) sources/bark.cpp && \
-	cd sources/bark.cpp && \
-	git checkout $(BARKCPP_VERSION) && \
-	git submodule update --init --recursive --depth 1 --single-branch
-
-sources/bark.cpp/build/libbark.a: sources/bark.cpp
-	cd sources/bark.cpp && \
-	mkdir -p build && \
-	cd build && \
-	cmake $(CMAKE_ARGS) .. && \
-	cmake --build . --config Release
-
-gobark.o:
-	$(CXX) $(CXXFLAGS) gobark.cpp -o gobark.o -c $(LDFLAGS)
-
-libbark.a: sources/bark.cpp/build/libbark.a gobark.o
-	cp $(INCLUDE_PATH)/sources/bark.cpp/build/libbark.a ./
-	$(AR) rcs libbark.a gobark.o
-
-bark-cpp: libbark.a
-	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH="$(CURDIR)" LIBRARY_PATH=$(CURDIR) \
-	$(GOCMD) build -v -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o bark-cpp ./
-
-package:
-	bash package.sh
-
-build: bark-cpp package
-
-clean:
-	rm -f gobark.o libbark.a

backend/go/bark-cpp/gobark.cpp

@@ -1,85 +0,0 @@
-#include <iostream>
-#include <tuple>
-
-#include "bark.h"
-#include "gobark.h"
-#include "common.h"
-#include "ggml.h"
-
-struct bark_context *c;
-
-void bark_print_progress_callback(struct bark_context *bctx, enum bark_encoding_step step, int progress, void *user_data) {
-    if (step == bark_encoding_step::SEMANTIC) {
-        printf("\rGenerating semantic tokens... %d%%", progress);
-    } else if (step == bark_encoding_step::COARSE) {
-        printf("\rGenerating coarse tokens... %d%%", progress);
-    } else if (step == bark_encoding_step::FINE) {
-        printf("\rGenerating fine tokens... %d%%", progress);
-    }
-    fflush(stdout);
-}
-
-int load_model(char *model) {
-    // initialize bark context
-    struct bark_context_params ctx_params = bark_context_default_params();
-    bark_params params;
-
-    params.model_path = model;
-
-   // ctx_params.verbosity = verbosity;
-    ctx_params.progress_callback = bark_print_progress_callback;
-    ctx_params.progress_callback_user_data = nullptr;
-
-    struct bark_context *bctx = bark_load_model(params.model_path.c_str(), ctx_params, params.seed);
-    if (!bctx) {
-        fprintf(stderr, "%s: Could not load model\n", __func__);
-        return 1;
-    }
-
-    c = bctx;
-
-    return 0;
-}
-
-int tts(char *text,int  threads, char *dst ) {
-
-    ggml_time_init();
-    const int64_t t_main_start_us = ggml_time_us();
-
-    // generate audio
-    if (!bark_generate_audio(c, text, threads)) {
-        fprintf(stderr, "%s: An error occurred. If the problem persists, feel free to open an issue to report it.\n", __func__);
-        return 1;
-    }
-
-    const float *audio_data = bark_get_audio_data(c);
-    if (audio_data == NULL) {
-        fprintf(stderr, "%s: Could not get audio data\n", __func__);
-        return 1;
-    }
-
-    const int audio_arr_size = bark_get_audio_data_size(c);
-
-    std::vector<float> audio_arr(audio_data, audio_data + audio_arr_size);
-
-    write_wav_on_disk(audio_arr, dst);
-
-    // report timing
-    {
-        const int64_t t_main_end_us = ggml_time_us();
-        const int64_t t_load_us = bark_get_load_time(c);
-        const int64_t t_eval_us = bark_get_eval_time(c);
-
-        printf("\n\n");
-        printf("%s:     load time = %8.2f ms\n", __func__, t_load_us / 1000.0f);
-        printf("%s:     eval time = %8.2f ms\n", __func__, t_eval_us / 1000.0f);
-        printf("%s:    total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us) / 1000.0f);
-    }
-    
-    return 0;
-}
-
-int unload() {
-    bark_free(c);
-}
-

backend/go/bark-cpp/gobark.go

@@ -1,52 +0,0 @@
-package main
-
-// #cgo CXXFLAGS: -I${SRCDIR}/sources/bark.cpp/ -I${SRCDIR}/sources/bark.cpp/encodec.cpp -I${SRCDIR}/sources/bark.cpp/encodec.cpp/ggml/include -I${SRCDIR}/sources/bark.cpp/examples -I${SRCDIR}/sources/bark.cpp/spm-headers
-// #cgo LDFLAGS: -L${SRCDIR}/ -L${SRCDIR}/sources/bark.cpp/build/examples -L${SRCDIR}/sources/bark.cpp/build/encodec.cpp/ggml/src/ -L${SRCDIR}/sources/bark.cpp/build/encodec.cpp/ -lbark -lencodec -lcommon -lggml -lgomp
-// #include <gobark.h>
-// #include <stdlib.h>
-import "C"
-
-import (
-	"fmt"
-	"unsafe"
-
-	"github.com/mudler/LocalAI/pkg/grpc/base"
-	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
-)
-
-type Bark struct {
-	base.SingleThread
-	threads int
-}
-
-func (sd *Bark) Load(opts *pb.ModelOptions) error {
-
-	sd.threads = int(opts.Threads)
-
-	modelFile := C.CString(opts.ModelFile)
-	defer C.free(unsafe.Pointer(modelFile))
-
-	ret := C.load_model(modelFile)
-	if ret != 0 {
-		return fmt.Errorf("inference failed")
-	}
-
-	return nil
-}
-
-func (sd *Bark) TTS(opts *pb.TTSRequest) error {
-	t := C.CString(opts.Text)
-	defer C.free(unsafe.Pointer(t))
-
-	dst := C.CString(opts.Dst)
-	defer C.free(unsafe.Pointer(dst))
-
-	threads := C.int(sd.threads)
-
-	ret := C.tts(t, threads, dst)
-	if ret != 0 {
-		return fmt.Errorf("inference failed")
-	}
-
-	return nil
-}

backend/go/bark-cpp/gobark.h

@@ -1,8 +0,0 @@
-#ifdef __cplusplus
-extern "C" {
-#endif
-int load_model(char *model);
-int tts(char *text,int  threads, char *dst );
-#ifdef __cplusplus
-}
-#endif

backend/go/bark-cpp/main.go

@@ -1,20 +0,0 @@
-package main
-
-// Note: this is started internally by LocalAI and a server is allocated for each model
-import (
-	"flag"
-
-	grpc "github.com/mudler/LocalAI/pkg/grpc"
-)
-
-var (
-	addr = flag.String("addr", "localhost:50051", "the address to connect to")
-)
-
-func main() {
-	flag.Parse()
-
-	if err := grpc.StartServer(*addr, &Bark{}); err != nil {
-		panic(err)
-	}
-}

backend/go/bark-cpp/package.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-# Script to copy the appropriate libraries based on architecture
-# This script is used in the final stage of the Dockerfile
-
-set -e
-
-CURDIR=$(dirname "$(realpath $0)")
-
-# Create lib directory
-mkdir -p $CURDIR/package/lib
-cp -avrf $CURDIR/bark-cpp $CURDIR/package/
-cp -rfv $CURDIR/run.sh $CURDIR/package/
-
-# Detect architecture and copy appropriate libraries
-if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
-    # x86_64 architecture
-    echo "Detected x86_64 architecture, copying x86_64 libraries..."
-    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
-    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
-    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
-elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
-    # ARM64 architecture
-    echo "Detected ARM64 architecture, copying ARM64 libraries..."
-    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
-    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
-    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
-    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
-else
-    echo "Error: Could not detect architecture"
-    exit 1
-fi
-
-echo "Packaging completed successfully" 
-ls -liah $CURDIR/package/
-ls -liah $CURDIR/package/lib/

backend/go/bark-cpp/run.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-set -ex
-
-CURDIR=$(dirname "$(realpath $0)")
-export LD_LIBRARY_PATH=$CURDIR/lib:$LD_LIBRARY_PATH
-
-# If there is a lib/ld.so, use it
-if [ -f $CURDIR/lib/ld.so ]; then
-	echo "Using lib/ld.so"
-	exec $CURDIR/lib/ld.so $CURDIR/bark-cpp "$@"
-fi
-
-exec $CURDIR/bark-cpp "$@"

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=9565c7f6bd5fcff124c589147b2621244f2c4aa1
+STABLEDIFFUSION_GGML_VERSION?=e411520407663e1ddf8ff2e5ed4ff3a116fbbc97
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=f53dc74843e97f19f94a79241357f74ad5b691a6
+WHISPER_CPP_VERSION?=941bdabbe4561bc6de68981aea01bc5ab05781c5
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

backend/go/whisper/gowhisper.go

@@ -130,8 +130,9 @@ func (w *Whisper) AudioTranscription(opts *pb.TranscriptRequest) (pb.TranscriptR
 	segments := []*pb.TranscriptSegment{}
 	text := ""
 	for i := range int(segsLen) {
-		s := CppGetSegmentStart(i)
-		t := CppGetSegmentEnd(i)
+		// segment start/end conversion factor taken from https://github.com/ggml-org/whisper.cpp/blob/master/examples/cli/cli.cpp#L895
+		s := CppGetSegmentStart(i) * (10000000)
+		t := CppGetSegmentEnd(i) * (10000000)
 		txt := strings.Clone(CppGetSegmentText(i))
 		tokens := make([]int32, CppNTokens(i))
 

backend/python/README.md

@@ -16,10 +16,8 @@ The Python backends use a unified build system based on `libbackend.sh` that pro
 - **transformers** - Hugging Face Transformers framework (PyTorch-based)
 - **vllm** - High-performance LLM inference engine
 - **mlx** - Apple Silicon optimized ML framework
-- **exllama2** - ExLlama2 quantized models
 
 ### Audio & Speech
-- **bark** - Text-to-speech synthesis
 - **coqui** - Coqui TTS models
 - **faster-whisper** - Fast Whisper speech recognition
 - **kitten-tts** - Lightweight TTS

backend/python/ace-step/Makefile

@@ -0,0 +1,16 @@
+.DEFAULT_GOAL := install
+
+.PHONY: install
+install:
+	bash install.sh
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__
+
+test: install
+	bash test.sh

backend/python/ace-step/backend.py

@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+"""
+LocalAI ACE-Step Backend
+
+gRPC backend for ACE-Step 1.5 music generation. Aligns with upstream acestep API:
+- LoadModel: initializes AceStepHandler (DiT) and LLMHandler, parses Options.
+- SoundGeneration: uses create_sample (simple mode), format_sample (optional), then
+  generate_music from acestep.inference. Writes first output to request.dst.
+- Fail hard: no fallback WAV on error; exceptions propagate to gRPC.
+"""
+from concurrent import futures
+import argparse
+import shutil
+import signal
+import sys
+import os
+import tempfile
+
+import backend_pb2
+import backend_pb2_grpc
+import grpc
+from acestep.inference import (
+    GenerationParams,
+    GenerationConfig,
+    generate_music,
+    create_sample,
+    format_sample,
+)
+from acestep.handler import AceStepHandler
+from acestep.llm_inference import LLMHandler
+from acestep.model_downloader import ensure_lm_model
+
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+MAX_WORKERS = int(os.environ.get("PYTHON_GRPC_MAX_WORKERS", "1"))
+
+# Model name -> HuggingFace/ModelScope repo (from upstream api_server.py)
+MODEL_REPO_MAPPING = {
+    "acestep-v15-turbo": "ACE-Step/Ace-Step1.5",
+    "acestep-5Hz-lm-0.6B": "ACE-Step/Ace-Step1.5",
+    "acestep-5Hz-lm-1.7B": "ACE-Step/Ace-Step1.5",
+    "vae": "ACE-Step/Ace-Step1.5",
+    "Qwen3-Embedding-0.6B": "ACE-Step/Ace-Step1.5",
+    "acestep-v15-base": "ACE-Step/acestep-v15-base",
+    "acestep-v15-sft": "ACE-Step/acestep-v15-sft",
+    "acestep-v15-turbo-shift3": "ACE-Step/acestep-v15-turbo-shift3",
+    "acestep-5Hz-lm-4B": "ACE-Step/acestep-5Hz-lm-4B",
+}
+DEFAULT_REPO_ID = "ACE-Step/Ace-Step1.5"
+
+def _is_float(s):
+    try:
+        float(s)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def _is_int(s):
+    try:
+        int(s)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def _parse_timesteps(s):
+    if s is None or (isinstance(s, str) and not s.strip()):
+        return None
+    if isinstance(s, (list, tuple)):
+        return [float(x) for x in s]
+    try:
+        return [float(x.strip()) for x in str(s).split(",") if x.strip()]
+    except (ValueError, TypeError):
+        return None
+
+
+def _parse_options(opts_list):
+    """Parse repeated 'key:value' options into a dict. Coerce numeric and bool."""
+    out = {}
+    for opt in opts_list or []:
+        if ":" not in opt:
+            continue
+        key, value = opt.split(":", 1)
+        key = key.strip()
+        value = value.strip()
+        if _is_int(value):
+            out[key] = int(value)
+        elif _is_float(value):
+            out[key] = float(value)
+        elif value.lower() in ("true", "false"):
+            out[key] = value.lower() == "true"
+        else:
+            out[key] = value
+    return out
+
+
+def _generate_audio_sync(servicer, payload, dst_path):
+    """
+    Run full ACE-Step pipeline using acestep.inference:
+    - If sample_mode/sample_query: create_sample() for caption/lyrics/metadata.
+    - If use_format and caption/lyrics: format_sample().
+    - Build GenerationParams and GenerationConfig, then generate_music().
+    Writes the first generated audio to dst_path. Raises on failure.
+    """
+
+    opts = servicer.options
+    dit_handler = servicer.dit_handler
+    llm_handler = servicer.llm_handler
+
+    for key, value in opts.items():
+        if key not in payload:
+            payload[key] = value
+
+    def _opt(name, default):
+        return opts.get(name, default)
+
+    lm_temperature = _opt("temperature", 0.85)
+    lm_cfg_scale = _opt("lm_cfg_scale", _opt("cfg_scale", 2.0))
+    lm_top_k = opts.get("top_k")
+    lm_top_p = _opt("top_p", 0.9)
+    if lm_top_p is not None and lm_top_p >= 1.0:
+        lm_top_p = None
+    inference_steps = _opt("inference_steps", 8)
+    guidance_scale = _opt("guidance_scale", 7.0)
+    batch_size = max(1, int(_opt("batch_size", 1)))
+
+    use_simple = bool(payload.get("sample_query") or payload.get("text"))
+    sample_mode = use_simple and (payload.get("thinking") or payload.get("sample_mode"))
+    sample_query = (payload.get("sample_query") or payload.get("text") or "").strip()
+    use_format = bool(payload.get("use_format"))
+    caption = (payload.get("prompt") or payload.get("caption") or "").strip()
+    lyrics = (payload.get("lyrics") or "").strip()
+    vocal_language = (payload.get("vocal_language") or "en").strip()
+    instrumental = bool(payload.get("instrumental"))
+    bpm = payload.get("bpm")
+    key_scale = (payload.get("key_scale") or "").strip()
+    time_signature = (payload.get("time_signature") or "").strip()
+    audio_duration = payload.get("audio_duration")
+    if audio_duration is not None:
+        try:
+            audio_duration = float(audio_duration)
+        except (TypeError, ValueError):
+            audio_duration = None
+
+    if sample_mode and llm_handler and getattr(llm_handler, "llm_initialized", False):
+        parsed_language = None
+        if sample_query:
+            for hint in ("english", "en", "chinese", "zh", "japanese", "ja"):
+                if hint in sample_query.lower():
+                    parsed_language = "en" if hint == "english" or hint == "en" else hint
+                    break
+        vocal_lang = vocal_language if vocal_language and vocal_language != "unknown" else parsed_language
+        sample_result = create_sample(
+            llm_handler=llm_handler,
+            query=sample_query or "NO USER INPUT",
+            instrumental=instrumental,
+            vocal_language=vocal_lang,
+            temperature=lm_temperature,
+            top_k=lm_top_k,
+            top_p=lm_top_p,
+            use_constrained_decoding=True,
+        )
+        if not sample_result.success:
+            raise RuntimeError(f"create_sample failed: {sample_result.error or sample_result.status_message}")
+        caption = sample_result.caption or caption
+        lyrics = sample_result.lyrics or lyrics
+        bpm = sample_result.bpm
+        key_scale = sample_result.keyscale or key_scale
+        time_signature = sample_result.timesignature or time_signature
+        if sample_result.duration is not None:
+            audio_duration = sample_result.duration
+        if getattr(sample_result, "language", None):
+            vocal_language = sample_result.language
+
+    if use_format and (caption or lyrics) and llm_handler and getattr(llm_handler, "llm_initialized", False):
+        user_metadata = {}
+        if bpm is not None:
+            user_metadata["bpm"] = bpm
+        if audio_duration is not None and float(audio_duration) > 0:
+            user_metadata["duration"] = int(audio_duration)
+        if key_scale:
+            user_metadata["keyscale"] = key_scale
+        if time_signature:
+            user_metadata["timesignature"] = time_signature
+        if vocal_language and vocal_language != "unknown":
+            user_metadata["language"] = vocal_language
+        format_result = format_sample(
+            llm_handler=llm_handler,
+            caption=caption,
+            lyrics=lyrics,
+            user_metadata=user_metadata if user_metadata else None,
+            temperature=lm_temperature,
+            top_k=lm_top_k,
+            top_p=lm_top_p,
+            use_constrained_decoding=True,
+        )
+        if format_result.success:
+            caption = format_result.caption or caption
+            lyrics = format_result.lyrics or lyrics
+            if format_result.duration is not None:
+                audio_duration = format_result.duration
+            if format_result.bpm is not None:
+                bpm = format_result.bpm
+            if format_result.keyscale:
+                key_scale = format_result.keyscale
+            if format_result.timesignature:
+                time_signature = format_result.timesignature
+            if getattr(format_result, "language", None):
+                vocal_language = format_result.language
+
+    thinking = bool(payload.get("thinking"))
+    use_cot_metas = not sample_mode
+    params = GenerationParams(
+        task_type=payload.get("task_type", "text2music"),
+        instruction=payload.get("instruction", "Fill the audio semantic mask based on the given conditions:"),
+        reference_audio=payload.get("reference_audio_path"),
+        src_audio=payload.get("src_audio_path"),
+        audio_codes=payload.get("audio_code_string", ""),
+        caption=caption,
+        lyrics=lyrics,
+        instrumental=instrumental or (not lyrics or str(lyrics).strip().lower() in ("[inst]", "[instrumental]")),
+        vocal_language=vocal_language or "unknown",
+        bpm=bpm,
+        keyscale=key_scale,
+        timesignature=time_signature,
+        duration=float(audio_duration) if audio_duration and float(audio_duration) > 0 else -1.0,
+        inference_steps=inference_steps,
+        seed=int(payload.get("seed", -1)),
+        guidance_scale=guidance_scale,
+        use_adg=bool(payload.get("use_adg")),
+        cfg_interval_start=float(payload.get("cfg_interval_start", 0.0)),
+        cfg_interval_end=float(payload.get("cfg_interval_end", 1.0)),
+        shift=float(payload.get("shift", 1.0)),
+        infer_method=(payload.get("infer_method") or "ode").strip(),
+        timesteps=_parse_timesteps(payload.get("timesteps")),
+        repainting_start=float(payload.get("repainting_start", 0.0)),
+        repainting_end=float(payload.get("repainting_end", -1)) if payload.get("repainting_end") is not None else -1,
+        audio_cover_strength=float(payload.get("audio_cover_strength", 1.0)),
+        thinking=thinking,
+        lm_temperature=lm_temperature,
+        lm_cfg_scale=lm_cfg_scale,
+        lm_top_k=lm_top_k or 0,
+        lm_top_p=lm_top_p if lm_top_p is not None and lm_top_p < 1.0 else 0.9,
+        lm_negative_prompt=payload.get("lm_negative_prompt", "NO USER INPUT"),
+        use_cot_metas=use_cot_metas,
+        use_cot_caption=bool(payload.get("use_cot_caption", True)),
+        use_cot_language=bool(payload.get("use_cot_language", True)),
+        use_constrained_decoding=True,
+    )
+
+    config = GenerationConfig(
+        batch_size=batch_size,
+        allow_lm_batch=bool(payload.get("allow_lm_batch", False)),
+        use_random_seed=bool(payload.get("use_random_seed", True)),
+        seeds=payload.get("seeds"),
+        lm_batch_chunk_size=max(1, int(payload.get("lm_batch_chunk_size", 8))),
+        constrained_decoding_debug=bool(payload.get("constrained_decoding_debug")),
+        audio_format=(payload.get("audio_format") or "flac").strip() or "flac",
+    )
+
+    save_dir = tempfile.mkdtemp(prefix="ace_step_")
+    try:
+        result = generate_music(
+            dit_handler=dit_handler,
+            llm_handler=llm_handler if (llm_handler and getattr(llm_handler, "llm_initialized", False)) else None,
+            params=params,
+            config=config,
+            save_dir=save_dir,
+            progress=None,
+        )
+        if not result.success:
+            raise RuntimeError(result.error or result.status_message or "generate_music failed")
+
+        audios = result.audios or []
+        if not audios:
+            raise RuntimeError("generate_music returned no audio")
+
+        first_path = audios[0].get("path") or ""
+        if not first_path or not os.path.isfile(first_path):
+            raise RuntimeError("first generated audio path missing or not a file")
+
+        shutil.copy2(first_path, dst_path)
+    finally:
+        try:
+            shutil.rmtree(save_dir, ignore_errors=True)
+        except Exception:
+            pass
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def __init__(self):
+        self.model_path = None
+        self.model_dir = None
+        self.checkpoint_dir = None
+        self.project_root = None
+        self.options = {}
+        self.dit_handler = None
+        self.llm_handler = None
+
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=b"OK")
+
+    def LoadModel(self, request, context):
+        try:
+            self.options = _parse_options(list(getattr(request, "Options", []) or []))
+            model_path = getattr(request, "ModelPath", None) or ""
+            model_name = (request.Model or "").strip()
+            model_file = (getattr(request, "ModelFile", None) or "").strip()
+
+            # Model dir: where we store checkpoints (always under LocalAI models path, never backend dir)
+            if model_path and model_name:
+                model_dir = os.path.join(model_path, model_name)
+            elif model_file:
+                model_dir = model_file
+            else:
+                model_dir = os.path.abspath(model_name or ".")
+            self.model_dir = model_dir
+            self.checkpoint_dir = os.path.join(model_dir, "checkpoints")
+            self.project_root = model_dir
+            self.model_path = os.path.join(self.checkpoint_dir, model_name or os.path.basename(model_dir.rstrip("/\\")))
+
+            config_path = model_name or os.path.basename(model_dir.rstrip("/\\"))
+            os.makedirs(self.checkpoint_dir, exist_ok=True)
+
+            self.dit_handler = AceStepHandler()
+            # Patch handler so it uses our model dir instead of site-packages/checkpoints
+            self.dit_handler._get_project_root = lambda: self.project_root
+            device = self.options.get("device", "auto")
+            use_flash = self.options.get("use_flash_attention", True)
+            if isinstance(use_flash, str):
+                use_flash = str(use_flash).lower() in ("1", "true", "yes")
+            offload = self.options.get("offload_to_cpu", False)
+            if isinstance(offload, str):
+                offload = str(offload).lower() in ("1", "true", "yes")
+            status_msg, ok = self.dit_handler.initialize_service(
+                project_root=self.project_root,
+                config_path=config_path,
+                device=device,
+                use_flash_attention=use_flash,
+                compile_model=False,
+                offload_to_cpu=offload,
+                offload_dit_to_cpu=bool(self.options.get("offload_dit_to_cpu", False)),
+            )
+            if not ok:
+                return backend_pb2.Result(success=False, message=f"DiT init failed: {status_msg}")
+
+            self.llm_handler = None
+            if self.options.get("init_lm", True):
+                lm_model = self.options.get("lm_model_path", "acestep-5Hz-lm-0.6B")
+                
+                # Ensure LM model is downloaded before initializing
+                try:
+                    from pathlib import Path
+                    lm_success, lm_msg = ensure_lm_model(
+                        model_name=lm_model,
+                        checkpoints_dir=Path(self.checkpoint_dir),
+                        prefer_source=None,  # Auto-detect HuggingFace vs ModelScope
+                    )
+                    if not lm_success:
+                        print(f"[ace-step] Warning: LM model download failed: {lm_msg}", file=sys.stderr)
+                        # Continue anyway - LLM initialization will fail gracefully
+                    else:
+                        print(f"[ace-step] LM model ready: {lm_msg}", file=sys.stderr)
+                except Exception as e:
+                    print(f"[ace-step] Warning: LM model download check failed: {e}", file=sys.stderr)
+                    # Continue anyway - LLM initialization will fail gracefully
+                
+                self.llm_handler = LLMHandler()
+                lm_backend = (self.options.get("lm_backend") or "vllm").strip().lower()
+                if lm_backend not in ("vllm", "pt"):
+                    lm_backend = "vllm"
+                lm_status, lm_ok = self.llm_handler.initialize(
+                    checkpoint_dir=self.checkpoint_dir,
+                    lm_model_path=lm_model,
+                    backend=lm_backend,
+                    device=device,
+                    offload_to_cpu=offload,
+                    dtype=getattr(self.dit_handler, "dtype", None),
+                )
+                if not lm_ok:
+                    self.llm_handler = None
+                    print(f"[ace-step] LM init failed (optional): {lm_status}", file=sys.stderr)
+
+            print(f"[ace-step] LoadModel: model={self.model_path}, options={list(self.options.keys())}", file=sys.stderr)
+            return backend_pb2.Result(success=True, message="Model loaded successfully")
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"LoadModel error: {err}")
+
+    def SoundGeneration(self, request, context):
+        if not request.dst:
+            return backend_pb2.Result(success=False, message="request.dst is required")
+
+        use_simple = bool(request.text)
+        if use_simple:
+            payload = {
+                "sample_query": request.text or "",
+                "sample_mode": True,
+                "thinking": True,
+                "vocal_language": request.language or request.GetLanguage() or "en",
+                "instrumental": request.instrumental if request.HasField("instrumental") else False,
+            }
+        else:
+            caption = request.caption or request.GetCaption() or request.text
+            payload = {
+                "prompt": caption,
+                "lyrics": request.lyrics or request.lyrics or "",
+                "thinking": request.think if request.HasField("think") else False,
+                "vocal_language": request.language or request.GetLanguage() or "en",
+            }
+            if request.HasField("bpm"):
+                payload["bpm"] = request.bpm
+            if request.HasField("keyscale") and request.keyscale:
+                payload["key_scale"] = request.keyscale
+            if request.HasField("timesignature") and request.timesignature:
+                payload["time_signature"] = request.timesignature
+            if request.HasField("duration") and request.duration:
+                payload["audio_duration"] = int(request.duration) if request.duration else None
+            if request.src:
+                payload["src_audio_path"] = request.src
+
+        _generate_audio_sync(self, payload, request.dst)
+        return backend_pb2.Result(success=True, message="Sound generated successfully")
+
+    def TTS(self, request, context):
+        if not request.dst:
+            return backend_pb2.Result(success=False, message="request.dst is required")
+        payload = {
+            "sample_query": request.text,
+            "sample_mode": True,
+            "thinking": False,
+            "vocal_language": (request.language if request.language else "") or "en",
+            "instrumental": False,
+        }
+        _generate_audio_sync(self, payload, request.dst)
+        return backend_pb2.Result(success=True, message="TTS (music fallback) generated successfully")
+
+
+def serve(address):
+    server = grpc.server(
+        futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ("grpc.max_message_length", 50 * 1024 * 1024),
+            ("grpc.max_send_message_length", 50 * 1024 * 1024),
+            ("grpc.max_receive_message_length", 50 * 1024 * 1024),
+        ],
+    )
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+    server.start()
+    print(f"[ace-step] Server listening on {address}", file=sys.stderr)
+
+    def shutdown(sig, frame):
+        server.stop(0)
+        sys.exit(0)
+
+    signal.signal(signal.SIGINT, shutdown)
+    signal.signal(signal.SIGTERM, shutdown)
+
+    try:
+        while True:
+            import time
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except KeyboardInterrupt:
+        server.stop(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--addr", default="localhost:50051", help="Listen address")
+    args = parser.parse_args()
+    serve(args.addr)

backend/python/ace-step/install.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+PYTHON_VERSION="3.11"
+PYTHON_PATCH="14"
+PY_STANDALONE_TAG="20260203"
+
+installRequirements
+
+if [ ! -d ACE-Step-1.5 ]; then
+    git clone https://github.com/ace-step/ACE-Step-1.5
+    cd ACE-Step-1.5/
+    if [ "x${USE_PIP}" == "xtrue" ]; then
+        pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps .
+    else
+        uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --no-deps .
+    fi
+fi
+

backend/python/ace-step/requirements-cpu.txt

@@ -0,0 +1,22 @@
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch
+torchaudio
+torchvision
+
+# Core dependencies
+transformers>=4.51.0,<4.58.0
+diffusers
+gradio
+matplotlib>=3.7.5
+scipy>=1.10.1
+soundfile>=0.13.1
+loguru>=0.7.3
+einops>=0.8.1
+accelerate>=1.12.0
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+numba>=0.63.1
+vector-quantize-pytorch>=1.27.15
+torchcodec>=0.9.1
+torchao
+modelscope

backend/python/ace-step/requirements-cublas12.txt

@@ -0,0 +1,30 @@
+--extra-index-url https://download.pytorch.org/whl/cu121
+torch
+torchaudio
+torchvision
+
+# Core dependencies
+transformers>=4.51.0,<4.58.0
+diffusers
+gradio>=6.5.1
+matplotlib>=3.7.5
+scipy>=1.10.1
+soundfile>=0.13.1
+loguru>=0.7.3
+einops>=0.8.1
+accelerate>=1.12.0
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+numba>=0.63.1
+vector-quantize-pytorch>=1.27.15
+torchcodec>=0.9.1; sys_platform != 'darwin' or platform_machine == 'arm64'
+torchao
+modelscope
+
+# LoRA Training dependencies (optional)
+peft>=0.7.0
+lightning>=2.0.0
+
+triton>=3.0.0
+flash-attn
+xxhash

backend/python/ace-step/requirements-cublas13.txt

@@ -0,0 +1,30 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch==2.7.1
+torchaudio==2.7.1
+torchvision
+
+# Core dependencies
+transformers>=4.51.0,<4.58.0
+diffusers
+gradio>=6.5.1
+matplotlib>=3.7.5
+scipy>=1.10.1
+soundfile>=0.13.1
+loguru>=0.7.3
+einops>=0.8.1
+accelerate>=1.12.0
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+numba>=0.63.1
+vector-quantize-pytorch>=1.27.15
+torchcodec>=0.9.1; sys_platform != 'darwin' or platform_machine == 'arm64'
+torchao
+modelscope
+
+# LoRA Training dependencies (optional)
+peft>=0.7.0
+lightning>=2.0.0
+
+triton>=3.0.0
+flash-attn
+xxhash

backend/python/ace-step/requirements-hipblas.txt

@@ -0,0 +1,22 @@
+--extra-index-url https://download.pytorch.org/whl/rocm6.4
+torch==2.8.0+rocm6.4
+torchaudio
+torchvision
+
+# Core dependencies
+transformers>=4.51.0,<4.58.0
+diffusers
+gradio>=6.5.1
+matplotlib>=3.7.5
+scipy>=1.10.1
+soundfile>=0.13.1
+loguru>=0.7.3
+einops>=0.8.1
+accelerate>=1.12.0
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+numba>=0.63.1
+vector-quantize-pytorch>=1.27.15
+torchcodec>=0.9.1; sys_platform != 'darwin' or platform_machine == 'arm64'
+torchao
+modelscope

backend/python/ace-step/requirements-intel.txt

@@ -0,0 +1,26 @@
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+torchaudio
+torchvision
+
+# Core dependencies
+transformers>=4.51.0,<4.58.0
+diffusers
+gradio
+matplotlib>=3.7.5
+scipy>=1.10.1
+soundfile>=0.13.1
+loguru>=0.7.3
+einops>=0.8.1
+accelerate>=1.12.0
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+numba>=0.63.1
+vector-quantize-pytorch>=1.27.15
+torchcodec>=0.9.1
+torchao
+modelscope
+
+# LoRA Training dependencies (optional)
+peft>=0.7.0
+lightning>=2.0.0

backend/python/ace-step/requirements-l4t13.txt

@@ -0,0 +1,29 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch
+torchaudio
+torchvision
+# Core dependencies
+transformers>=4.51.0,<4.58.0
+diffusers
+gradio>=6.5.1
+matplotlib>=3.7.5
+scipy>=1.10.1
+soundfile>=0.13.1
+loguru>=0.7.3
+einops>=0.8.1
+accelerate>=1.12.0
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+numba>=0.63.1
+vector-quantize-pytorch>=1.27.15
+torchcodec>=0.9.1; sys_platform != 'darwin' or platform_machine == 'arm64'
+torchao
+modelscope
+
+# LoRA Training dependencies (optional)
+peft>=0.7.0
+lightning>=2.0.0
+
+triton>=3.0.0
+#flash-attn
+xxhash

backend/python/ace-step/requirements-mps.txt

@@ -0,0 +1,25 @@
+torch
+torchaudio
+torchvision
+
+# Core dependencies
+transformers>=4.51.0,<4.58.0
+diffusers
+gradio
+matplotlib>=3.7.5
+scipy>=1.10.1
+soundfile>=0.13.1
+loguru>=0.7.3
+einops>=0.8.1
+accelerate>=1.12.0
+fastapi>=0.110.0
+uvicorn[standard]>=0.27.0
+numba>=0.63.1
+vector-quantize-pytorch>=1.27.15
+torchcodec>=0.9.1
+torchao
+modelscope
+
+# LoRA Training dependencies (optional)
+peft>=0.7.0
+lightning>=2.0.0

backend/python/ace-step/requirements.txt

@@ -1,5 +1,4 @@
+setuptools
 grpcio==1.76.0
 protobuf
 certifi
-wheel
-setuptools

backend/python/ace-step/run.sh

@@ -6,4 +6,4 @@ else
     source $backend_dir/../common/libbackend.sh
 fi
 
-startBackend $@
+startBackend $@

backend/python/ace-step/test.py

@@ -0,0 +1,53 @@
+"""
+Tests for the ACE-Step gRPC backend.
+"""
+import os
+import tempfile
+import unittest
+
+import backend_pb2
+import backend_pb2_grpc
+import grpc
+
+
+class TestACEStepBackend(unittest.TestCase):
+    """Test Health, LoadModel, and SoundGeneration (minimal; no real model required)."""
+
+    @classmethod
+    def setUpClass(cls):
+        port = os.environ.get("BACKEND_PORT", "50051")
+        cls.channel = grpc.insecure_channel(f"localhost:{port}")
+        cls.stub = backend_pb2_grpc.BackendStub(cls.channel)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.channel.close()
+
+    def test_health(self):
+        response = self.stub.Health(backend_pb2.HealthMessage())
+        self.assertEqual(response.message, b"OK")
+
+    def test_load_model(self):
+        response = self.stub.LoadModel(backend_pb2.ModelOptions(Model="ace-step-test"))
+        self.assertTrue(response.success, response.message)
+
+    def test_sound_generation_minimal(self):
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+            dst = f.name
+        try:
+            req = backend_pb2.SoundGenerationRequest(
+                text="upbeat pop song",
+                model="ace-step-test",
+                dst=dst,
+            )
+            response = self.stub.SoundGeneration(req)
+            self.assertTrue(response.success, response.message)
+            self.assertTrue(os.path.exists(dst), f"Output file not created: {dst}")
+            self.assertGreater(os.path.getsize(dst), 0)
+        finally:
+            if os.path.exists(dst):
+                os.unlink(dst)
+
+
+if __name__ == "__main__":
+    unittest.main()

backend/python/ace-step/test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+# Start backend in background (use env to avoid port conflict in parallel tests)
+export PYTHONUNBUFFERED=1
+BACKEND_PORT=${BACKEND_PORT:-50051}
+python backend.py --addr "localhost:${BACKEND_PORT}" &
+BACKEND_PID=$!
+trap "kill $BACKEND_PID 2>/dev/null || true" EXIT
+sleep 3
+export BACKEND_PORT
+runUnittests

backend/python/bark/README.md

@@ -1,16 +0,0 @@
-# Creating a separate environment for ttsbark project
-
-```
-make ttsbark
-```
-
-# Testing the gRPC server
-
-```
-<The path of your python interpreter> -m unittest test_ttsbark.py
-```
-
-For example
-```
-/opt/conda/envs/bark/bin/python -m unittest extra/grpc/bark/test_ttsbark.py
-``````

backend/python/bark/backend.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-"""
-This is an extra gRPC server of LocalAI for Bark TTS
-"""
-from concurrent import futures
-import time
-import argparse
-import signal
-import sys
-import os
-from scipy.io.wavfile import write as write_wav
-
-import backend_pb2
-import backend_pb2_grpc
-from bark import SAMPLE_RATE, generate_audio, preload_models
-
-import grpc
-
-
-_ONE_DAY_IN_SECONDS = 60 * 60 * 24
-
-# If MAX_WORKERS are specified in the environment use it, otherwise default to 1
-MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
-
-# Implement the BackendServicer class with the service methods
-class BackendServicer(backend_pb2_grpc.BackendServicer):
-    """
-    BackendServicer is the class that implements the gRPC service
-    """
-    def Health(self, request, context):
-        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
-    def LoadModel(self, request, context):
-        model_name = request.Model
-        try:
-            print("Preparing models, please wait", file=sys.stderr)
-            # download and load all models
-            preload_models()
-        except Exception as err:
-            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
-        # Implement your logic here for the LoadModel service
-        # Replace this with your desired response
-        return backend_pb2.Result(message="Model loaded successfully", success=True)
-
-    def TTS(self, request, context):
-        model = request.model
-        print(request, file=sys.stderr)
-        try:
-            audio_array = None
-            if model != "":
-                audio_array = generate_audio(request.text, history_prompt=model)
-            else:
-                audio_array = generate_audio(request.text)
-            print("saving to", request.dst, file=sys.stderr)
-            # save audio to disk
-            write_wav(request.dst, SAMPLE_RATE, audio_array)
-            print("saved to", request.dst, file=sys.stderr)
-            print("tts for", file=sys.stderr)
-            print(request, file=sys.stderr)
-        except Exception as err:
-            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
-        return backend_pb2.Result(success=True)
-
-def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
-        options=[
-            ('grpc.max_message_length', 50 * 1024 * 1024),  # 50MB
-            ('grpc.max_send_message_length', 50 * 1024 * 1024),  # 50MB
-            ('grpc.max_receive_message_length', 50 * 1024 * 1024),  # 50MB
-        ])
-    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
-    server.add_insecure_port(address)
-    server.start()
-    print("Server started. Listening on: " + address, file=sys.stderr)
-
-    # Define the signal handler function
-    def signal_handler(sig, frame):
-        print("Received termination signal. Shutting down...")
-        server.stop(0)
-        sys.exit(0)
-
-    # Set the signal handlers for SIGINT and SIGTERM
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    try:
-        while True:
-            time.sleep(_ONE_DAY_IN_SECONDS)
-    except KeyboardInterrupt:
-        server.stop(0)
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the gRPC server.")
-    parser.add_argument(
-        "--addr", default="localhost:50051", help="The address to bind the server to."
-    )
-    args = parser.parse_args()
-
-    serve(args.addr)

backend/python/bark/install.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-set -e
-
-backend_dir=$(dirname $0)
-if [ -d $backend_dir/common ]; then
-    source $backend_dir/common/libbackend.sh
-else
-    source $backend_dir/../common/libbackend.sh
-fi
-
-# This is here because the Intel pip index is broken and returns 200 status codes for every package name, it just doesn't return any package links.
-# This makes uv think that the package exists in the Intel pip index, and by default it stops looking at other pip indexes once it finds a match.
-# We need uv to continue falling through to the pypi default index to find optimum[openvino] in the pypi index
-# the --upgrade actually allows us to *downgrade* torch to the version provided in the Intel pip index
-if [ "x${BUILD_PROFILE}" == "xintel" ]; then
-    EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match"
-fi
-
-installRequirements

backend/python/bark/requirements-cpu.txt

@@ -1,4 +0,0 @@
-transformers
-accelerate
-torch==2.4.1
-torchaudio==2.4.1

backend/python/bark/requirements-cublas12.txt

@@ -1,4 +0,0 @@
-torch==2.4.1
-torchaudio==2.4.1
-transformers
-accelerate

backend/python/bark/requirements-intel.txt

@@ -1,9 +0,0 @@
---extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-intel-extension-for-pytorch==2.8.10+xpu
-torch==2.3.1+cxx11.abi
-torchaudio==2.3.1+cxx11.abi
-oneccl_bind_pt==2.3.100+xpu
-optimum[openvino]
-setuptools
-transformers
-accelerate

backend/python/bark/requirements.txt

@@ -1,4 +0,0 @@
-bark==0.1.5
-grpcio==1.76.0
-protobuf
-certifi

backend/python/chatterbox/requirements-intel.txt

@@ -1,7 +1,6 @@
---extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-intel-extension-for-pytorch==2.3.110+xpu
-torch==2.3.1+cxx11.abi
-torchaudio==2.3.1+cxx11.abi
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+torchaudio
 transformers
 numpy>=1.24.0,<1.26.0
 # https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289

backend/python/chatterbox/requirements-mps.txt

@@ -0,0 +1,7 @@
+torch
+torchaudio
+accelerate
+numpy>=1.24.0,<1.26.0
+transformers
+# https://github.com/mudler/LocalAI/pull/6240#issuecomment-3329518289
+chatterbox-tts@git+https://git@github.com/mudler/chatterbox.git@faster

backend/python/common/libbackend.sh

@@ -398,7 +398,7 @@ function runProtogen() {
 # NOTE: for BUILD_PROFILE==intel, this function does NOT automatically use the Intel python package index.
 # you may want to add the following line to a requirements-intel.txt if you use one:
 #
-# --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
+# --index-url https://download.pytorch.org/whl/xpu
 #
 # If you need to add extra flags into the pip install command you can do so by setting the variable EXTRA_PIP_INSTALL_FLAGS
 # before calling installRequirements.  For example:

backend/python/common/template/requirements-intel.txt

@@ -1,5 +1,4 @@
---extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-intel-extension-for-pytorch==2.8.10+xpu
+--extra-index-url https://download.pytorch.org/whl/xpu
 torch==2.8.0
 oneccl_bind_pt==2.8.0+xpu
 optimum[openvino]

backend/python/coqui/README.md

@@ -1,4 +1,4 @@
-# Creating a separate environment for ttsbark project
+# Creating a separate environment for coqui project
 
 ```
 make coqui

backend/python/coqui/backend.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-This is an extra gRPC server of LocalAI for Bark TTS
+This is an extra gRPC server of LocalAI for Coqui TTS
 """
 from concurrent import futures
 import time

backend/python/coqui/requirements-intel.txt

@@ -1,8 +1,6 @@
---extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-intel-extension-for-pytorch==2.3.110+xpu
-torch==2.3.1+cxx11.abi
-torchaudio==2.3.1+cxx11.abi
-oneccl_bind_pt==2.3.100+xpu
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch==2.8.0+xpu
+torchaudio==2.8.0+xpu
 optimum[openvino]
 setuptools
 transformers==4.48.3

backend/python/coqui/requirements-mps.txt

@@ -0,0 +1,4 @@
+torch==2.7.1
+transformers==4.48.3
+accelerate
+coqui-tts

backend/python/diffusers/backend.py

@@ -42,12 +42,8 @@ from transformers import T5EncoderModel
 from safetensors.torch import load_file
 
 # Import LTX-2 specific utilities
-try:
-    from diffusers.pipelines.ltx2.export_utils import encode_video as ltx2_encode_video
-    LTX2_AVAILABLE = True
-except ImportError:
-    LTX2_AVAILABLE = False
-    ltx2_encode_video = None
+from diffusers.pipelines.ltx2.export_utils import encode_video as ltx2_encode_video
+from diffusers import LTX2VideoTransformer3DModel, GGUFQuantizationConfig
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 COMPEL = os.environ.get("COMPEL", "0") == "1"
@@ -302,12 +298,96 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         if pipeline_type == "LTX2ImageToVideoPipeline":
             self.img2vid = True
             self.ltx2_pipeline = True
-            pipe = load_diffusers_pipeline(
-                class_name="LTX2ImageToVideoPipeline",
-                model_id=request.Model,
-                torch_dtype=torchType,
-                variant=variant
-            )
+            
+            # Check if loading from single file (GGUF)
+            if fromSingleFile and LTX2VideoTransformer3DModel is not None:
+                _, single_file_ext = os.path.splitext(modelFile)
+                if single_file_ext == ".gguf":
+                    # Load transformer from single GGUF file with quantization
+                    transformer_kwargs = {}
+                    quantization_config = GGUFQuantizationConfig(compute_dtype=torchType)
+                    transformer_kwargs["quantization_config"] = quantization_config
+                    
+                    transformer = LTX2VideoTransformer3DModel.from_single_file(
+                        modelFile,
+                        config=request.Model,  # Use request.Model as the config/model_id
+                        subfolder="transformer",
+                        **transformer_kwargs,
+                    )
+                    
+                    # Load pipeline with custom transformer
+                    pipe = load_diffusers_pipeline(
+                        class_name="LTX2ImageToVideoPipeline",
+                        model_id=request.Model,
+                        transformer=transformer,
+                        torch_dtype=torchType,
+                    )
+                else:
+                    # Single file but not GGUF - use standard single file loading
+                    pipe = load_diffusers_pipeline(
+                        class_name="LTX2ImageToVideoPipeline",
+                        model_id=modelFile,
+                        from_single_file=True,
+                        torch_dtype=torchType,
+                    )
+            else:
+                # Standard loading from pretrained
+                pipe = load_diffusers_pipeline(
+                    class_name="LTX2ImageToVideoPipeline",
+                    model_id=request.Model,
+                    torch_dtype=torchType,
+                    variant=variant
+                )
+            
+            if not DISABLE_CPU_OFFLOAD:
+                pipe.enable_model_cpu_offload()
+            return pipe
+
+        # LTX2Pipeline - text-to-video pipeline, needs txt2vid flag, CPU offload, and special handling
+        if pipeline_type == "LTX2Pipeline":
+            self.txt2vid = True
+            self.ltx2_pipeline = True
+            
+            # Check if loading from single file (GGUF)
+            if fromSingleFile and LTX2VideoTransformer3DModel is not None:
+                _, single_file_ext = os.path.splitext(modelFile)
+                if single_file_ext == ".gguf":
+                    # Load transformer from single GGUF file with quantization
+                    transformer_kwargs = {}
+                    quantization_config = GGUFQuantizationConfig(compute_dtype=torchType)
+                    transformer_kwargs["quantization_config"] = quantization_config
+                    
+                    transformer = LTX2VideoTransformer3DModel.from_single_file(
+                        modelFile,
+                        config=request.Model,  # Use request.Model as the config/model_id
+                        subfolder="transformer",
+                        **transformer_kwargs,
+                    )
+                    
+                    # Load pipeline with custom transformer
+                    pipe = load_diffusers_pipeline(
+                        class_name="LTX2Pipeline",
+                        model_id=request.Model,
+                        transformer=transformer,
+                        torch_dtype=torchType,
+                    )
+                else:
+                    # Single file but not GGUF - use standard single file loading
+                    pipe = load_diffusers_pipeline(
+                        class_name="LTX2Pipeline",
+                        model_id=modelFile,
+                        from_single_file=True,
+                        torch_dtype=torchType,
+                    )
+            else:
+                # Standard loading from pretrained
+                pipe = load_diffusers_pipeline(
+                    class_name="LTX2Pipeline",
+                    model_id=request.Model,
+                    torch_dtype=torchType,
+                    variant=variant
+                )
+            
             if not DISABLE_CPU_OFFLOAD:
                 pipe.enable_model_cpu_offload()
             return pipe
@@ -428,6 +508,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             self.txt2vid = False
             self.ltx2_pipeline = False
 
+            print(f"LoadModel: PipelineType from request: {request.PipelineType}", file=sys.stderr)
+
             # Load pipeline using dynamic loader
             # Special cases that require custom initialization are handled first
             self.pipe = self._load_pipeline(
@@ -437,6 +519,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 torchType=torchType,
                 variant=variant
             )
+            
+            print(f"LoadModel: After loading - ltx2_pipeline: {self.ltx2_pipeline}, img2vid: {self.img2vid}, txt2vid: {self.txt2vid}, PipelineType: {self.PipelineType}", file=sys.stderr)
 
             if CLIPSKIP and request.CLIPSkip != 0:
                 self.clip_skip = request.CLIPSkip
@@ -674,14 +758,20 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         try:
             prompt = request.prompt
             if not prompt:
+                print(f"GenerateVideo: No prompt provided for video generation.", file=sys.stderr)
                 return backend_pb2.Result(success=False, message="No prompt provided for video generation")
 
+            # Debug: Print raw request values
+            print(f"GenerateVideo: Raw request values - num_frames: {request.num_frames}, fps: {request.fps}, cfg_scale: {request.cfg_scale}, step: {request.step}", file=sys.stderr)
+
             # Set default values from request or use defaults
             num_frames = request.num_frames if request.num_frames > 0 else 81
             fps = request.fps if request.fps > 0 else 16
             cfg_scale = request.cfg_scale if request.cfg_scale > 0 else 4.0
             num_inference_steps = request.step if request.step > 0 else 40
             
+            print(f"GenerateVideo: Using values - num_frames: {num_frames}, fps: {fps}, cfg_scale: {cfg_scale}, num_inference_steps: {num_inference_steps}", file=sys.stderr)
+            
             # Prepare generation parameters
             kwargs = {
                 "prompt": prompt,
@@ -707,19 +797,34 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 kwargs["end_image"] = load_image(request.end_image)
 
             print(f"Generating video with {kwargs=}", file=sys.stderr)
+            print(f"GenerateVideo: Pipeline type: {self.PipelineType}, ltx2_pipeline flag: {self.ltx2_pipeline}", file=sys.stderr)
 
             # Generate video frames based on pipeline type
-            if self.ltx2_pipeline or self.PipelineType == "LTX2ImageToVideoPipeline":
-                # LTX-2 image-to-video generation with audio
-                if not LTX2_AVAILABLE:
-                    return backend_pb2.Result(success=False, message="LTX-2 pipeline requires diffusers.pipelines.ltx2.export_utils")
+            if self.ltx2_pipeline or self.PipelineType in ["LTX2Pipeline", "LTX2ImageToVideoPipeline"]:
+                # LTX-2 generation with audio (supports both text-to-video and image-to-video)
+                # Determine if this is text-to-video (no image) or image-to-video (has image)
+                has_image = bool(request.start_image)
                 
-                # LTX-2 uses 'image' parameter instead of 'start_image'
-                if request.start_image:
-                    image = load_image(request.start_image)
-                    kwargs["image"] = image
-                    # Remove start_image if it was added
-                    kwargs.pop("start_image", None)
+                # Remove image-related parameters that might have been added earlier
+                kwargs.pop("start_image", None)
+                kwargs.pop("end_image", None)
+                
+                # LTX2ImageToVideoPipeline uses 'image' parameter for image-to-video
+                # LTX2Pipeline (text-to-video) doesn't need an image parameter
+                if has_image:
+                    # Image-to-video: use 'image' parameter
+                    if self.PipelineType == "LTX2ImageToVideoPipeline":
+                        image = load_image(request.start_image)
+                        kwargs["image"] = image
+                        print(f"LTX-2: Using image-to-video mode with image", file=sys.stderr)
+                    else:
+                        # If pipeline type is LTX2Pipeline but we have an image, we can't do image-to-video
+                        return backend_pb2.Result(success=False, message="LTX2Pipeline does not support image-to-video. Use LTX2ImageToVideoPipeline for image-to-video generation.")
+                else:
+                    # Text-to-video: no image parameter needed
+                    # Ensure no image-related kwargs are present
+                    kwargs.pop("image", None)
+                    print(f"LTX-2: Using text-to-video mode (no image)", file=sys.stderr)
                 
                 # LTX-2 uses 'frame_rate' instead of 'fps'
                 frame_rate = float(fps)
@@ -730,20 +835,45 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 kwargs["return_dict"] = False
                 
                 # Generate video and audio
-                video, audio = self.pipe(**kwargs)
+                print(f"LTX-2: Generating with kwargs: {kwargs}", file=sys.stderr)
+                try:
+                    video, audio = self.pipe(**kwargs)
+                    print(f"LTX-2: Generated video shape: {video.shape}, audio shape: {audio.shape}", file=sys.stderr)
+                except Exception as e:
+                    print(f"LTX-2: Error during pipe() call: {e}", file=sys.stderr)
+                    traceback.print_exc()
+                    return backend_pb2.Result(success=False, message=f"Error generating video with LTX-2 pipeline: {e}")
                 
                 # Convert video to uint8 format
                 video = (video * 255).round().astype("uint8")
                 video = torch.from_numpy(video)
                 
+                print(f"LTX-2: Converting video, shape after conversion: {video.shape}", file=sys.stderr)
+                print(f"LTX-2: Audio sample rate: {self.pipe.vocoder.config.output_sampling_rate}", file=sys.stderr)
+                print(f"LTX-2: Output path: {request.dst}", file=sys.stderr)
+                
                 # Use LTX-2's encode_video function which handles audio
-                ltx2_encode_video(
-                    video[0],
-                    fps=frame_rate,
-                    audio=audio[0].float().cpu(),
-                    audio_sample_rate=self.pipe.vocoder.config.output_sampling_rate,
-                    output_path=request.dst,
-                )
+                try:
+                    ltx2_encode_video(
+                        video[0],
+                        fps=frame_rate,
+                        audio=audio[0].float().cpu(),
+                        audio_sample_rate=self.pipe.vocoder.config.output_sampling_rate,
+                        output_path=request.dst,
+                    )
+                    # Verify file was created and has content
+                    import os
+                    if os.path.exists(request.dst):
+                        file_size = os.path.getsize(request.dst)
+                        print(f"LTX-2: Video file created successfully, size: {file_size} bytes", file=sys.stderr)
+                        if file_size == 0:
+                            return backend_pb2.Result(success=False, message=f"Video file was created but is empty (0 bytes). Check LTX-2 encode_video function.")
+                    else:
+                        return backend_pb2.Result(success=False, message=f"Video file was not created at {request.dst}")
+                except Exception as e:
+                    print(f"LTX-2: Error encoding video: {e}", file=sys.stderr)
+                    traceback.print_exc()
+                    return backend_pb2.Result(success=False, message=f"Error encoding video: {e}")
                 
                 return backend_pb2.Result(message="Video generated successfully", success=True)
             elif self.PipelineType == "WanPipeline":
@@ -785,11 +915,23 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 output = self.pipe(**kwargs)
                 frames = output.frames[0]
             else:
+                print(f"GenerateVideo: Pipeline {self.PipelineType} does not match any known video pipeline handler", file=sys.stderr)
                 return backend_pb2.Result(success=False, message=f"Pipeline {self.PipelineType} does not support video generation")
 
             # Export video (for non-LTX-2 pipelines)
+            print(f"GenerateVideo: Exporting video to {request.dst} with fps={fps}", file=sys.stderr)
             export_to_video(frames, request.dst, fps=fps)
             
+            # Verify file was created
+            import os
+            if os.path.exists(request.dst):
+                file_size = os.path.getsize(request.dst)
+                print(f"GenerateVideo: Video file created, size: {file_size} bytes", file=sys.stderr)
+                if file_size == 0:
+                    return backend_pb2.Result(success=False, message=f"Video file was created but is empty (0 bytes)")
+            else:
+                return backend_pb2.Result(success=False, message=f"Video file was not created at {request.dst}")
+            
             return backend_pb2.Result(message="Video generated successfully", success=True)
 
         except Exception as err:

backend/python/diffusers/requirements-intel.txt

@@ -1,8 +1,6 @@
---extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-intel-extension-for-pytorch==2.3.110+xpu
-torch==2.5.1+cxx11.abi
-torchvision==0.20.1+cxx11.abi
-oneccl_bind_pt==2.8.0+xpu
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+torchvision
 optimum[openvino]
 setuptools
 git+https://github.com/huggingface/diffusers

backend/python/diffusers/requirements.txt

@@ -3,3 +3,4 @@ grpcio==1.76.0
 pillow
 protobuf
 certifi
+av

backend/python/exllama2/.gitignore

@@ -1 +0,0 @@
-source

backend/python/exllama2/Makefile

@@ -1,17 +0,0 @@
-.PHONY: exllama2
-exllama2:
-	bash install.sh
-
-.PHONY: run
-run: exllama2
-	@echo "Running exllama2..."
-	bash run.sh
-	@echo "exllama2 run."
-
-.PHONY: protogen-clean
-protogen-clean:
-	$(RM) backend_pb2_grpc.py backend_pb2.py
-
-.PHONY: clean
-clean: protogen-clean
-	$(RM) -r venv source __pycache__

backend/python/exllama2/backend.py

@@ -1,143 +0,0 @@
-#!/usr/bin/env python3
-import grpc
-from concurrent import futures
-import time
-import backend_pb2
-import backend_pb2_grpc
-import argparse
-import signal
-import sys
-import os
-import glob
-
-from pathlib import Path
-import torch
-import torch.nn.functional as F
-from torch import version as torch_version
-
-
-from exllamav2.generator import (
-    ExLlamaV2BaseGenerator,
-    ExLlamaV2Sampler
-)
-
-
-from exllamav2 import (
-    ExLlamaV2,
-    ExLlamaV2Config,
-    ExLlamaV2Cache,
-    ExLlamaV2Cache_8bit,
-    ExLlamaV2Tokenizer,
-    model_init,
-)
-
-
-_ONE_DAY_IN_SECONDS = 60 * 60 * 24
-
-# If MAX_WORKERS are specified in the environment use it, otherwise default to 1
-MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
-
-# Implement the BackendServicer class with the service methods
-class BackendServicer(backend_pb2_grpc.BackendServicer):
-    def Health(self, request, context):
-        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
-
-    def LoadModel(self, request, context):
-        try:
-            model_directory = request.ModelFile
-
-            config = ExLlamaV2Config()
-            config.model_dir = model_directory
-            config.prepare()
-
-            model = ExLlamaV2(config)
-
-            cache = ExLlamaV2Cache(model, lazy=True)
-            model.load_autosplit(cache)
-
-            tokenizer = ExLlamaV2Tokenizer(config)
-
-            # Initialize generator
-
-            generator = ExLlamaV2BaseGenerator(model, cache, tokenizer)
-
-            self.generator = generator
-
-            generator.warmup()
-            self.model = model
-            self.tokenizer = tokenizer
-            self.cache = cache
-        except Exception as err:
-            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
-        return backend_pb2.Result(message="Model loaded successfully", success=True)
-
-    def Predict(self, request, context):
-
-        penalty = 1.15
-        if request.Penalty != 0.0:
-            penalty = request.Penalty
-
-        settings = ExLlamaV2Sampler.Settings()
-        settings.temperature = request.Temperature
-        settings.top_k = request.TopK
-        settings.top_p = request.TopP
-        settings.token_repetition_penalty = penalty
-        settings.disallow_tokens(self.tokenizer, [self.tokenizer.eos_token_id])
-        tokens = 512
-
-        if request.Tokens != 0:
-            tokens = request.Tokens
-        output = self.generator.generate_simple(
-            request.Prompt, settings, tokens)
-
-        # Remove prompt from response if present
-        if request.Prompt in output:
-            output = output.replace(request.Prompt, "")
-
-        return backend_pb2.Result(message=bytes(output, encoding='utf-8'))
-
-    def PredictStream(self, request, context):
-        # Implement PredictStream RPC
-        # for reply in some_data_generator():
-        #    yield reply
-        # Not implemented yet
-        return self.Predict(request, context)
-
-
-def serve(address):
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
-        options=[
-            ('grpc.max_message_length', 50 * 1024 * 1024),  # 50MB
-            ('grpc.max_send_message_length', 50 * 1024 * 1024),  # 50MB
-            ('grpc.max_receive_message_length', 50 * 1024 * 1024),  # 50MB
-        ])
-    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
-    server.add_insecure_port(address)
-    server.start()
-    print("Server started. Listening on: " + address, file=sys.stderr)
-
-    # Define the signal handler function
-    def signal_handler(sig, frame):
-        print("Received termination signal. Shutting down...")
-        server.stop(0)
-        sys.exit(0)
-
-    # Set the signal handlers for SIGINT and SIGTERM
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    try:
-        while True:
-            time.sleep(_ONE_DAY_IN_SECONDS)
-    except KeyboardInterrupt:
-        server.stop(0)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the gRPC server.")
-    parser.add_argument(
-        "--addr", default="localhost:50051", help="The address to bind the server to."
-    )
-    args = parser.parse_args()
-
-    serve(args.addr)

backend/python/exllama2/install.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-set -e
-
-LIMIT_TARGETS="cublas"
-EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation"
-EXLLAMA2_VERSION=c0ddebaaaf8ffd1b3529c2bb654e650bce2f790f
-
-backend_dir=$(dirname $0)
-if [ -d $backend_dir/common ]; then
-    source $backend_dir/common/libbackend.sh
-else
-    source $backend_dir/../common/libbackend.sh
-fi
-
-installRequirements
-
-git clone https://github.com/turboderp/exllamav2 $MY_DIR/source
-pushd ${MY_DIR}/source && git checkout -b build ${EXLLAMA2_VERSION} && popd
-
-# This installs exllamav2 in JIT mode so it will compile the appropriate torch extension at runtime
-EXLLAMA_NOCOMPILE= uv pip install ${EXTRA_PIP_INSTALL_FLAGS} ${MY_DIR}/source/

backend/python/exllama2/requirements-cpu.txt

@@ -1,3 +0,0 @@
-transformers
-accelerate
-torch==2.4.1

backend/python/exllama2/requirements-cublas12.txt

@@ -1,3 +0,0 @@
-torch==2.4.1
-transformers
-accelerate

backend/python/exllama2/requirements-install.txt

@@ -1,4 +0,0 @@
-# This is here to trigger the install script to add --no-build-isolation to the uv pip install commands
-# exllama2 does not specify it's build requirements per PEP517, so we need to provide some things ourselves
-wheel
-setuptools

backend/python/faster-whisper/backend.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-This is an extra gRPC server of LocalAI for Bark TTS
+This is an extra gRPC server of LocalAI for Faster Whisper TTS
 """
 from concurrent import futures
 import time
@@ -40,7 +40,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             device = "mps"
         try:
             print("Preparing models, please wait", file=sys.stderr)
-            self.model = WhisperModel(request.Model, device=device, compute_type="float16")
+            self.model = WhisperModel(request.Model, device=device, compute_type="default")
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
         # Implement your logic here for the LoadModel service
@@ -55,11 +55,12 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             id = 0
             for segment in segments:
                 print("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
-                resultSegments.append(backend_pb2.TranscriptSegment(id=id, start=segment.start, end=segment.end, text=segment.text))
+                resultSegments.append(backend_pb2.TranscriptSegment(id=id, start=int(segment.start)*1e9, end=int(segment.end)*1e9, text=segment.text))
                 text += segment.text
-                id += 1            
+                id += 1
         except Exception as err:
             print(f"Unexpected {err=}, {type(err)=}", file=sys.stderr)
+            raise err
 
         return backend_pb2.TranscriptResult(segments=resultSegments, text=text)
 

backend/python/faster-whisper/requirements-intel.txt

@@ -1,6 +1,4 @@
---extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-intel-extension-for-pytorch==2.3.110+xpu
-torch==2.3.1+cxx11.abi
-oneccl_bind_pt==2.3.100+xpu
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
 optimum[openvino]
 faster-whisper

backend/python/faster-whisper/requirements-mps.txt

@@ -0,0 +1,8 @@
+torch==2.7.1
+faster-whisper
+opencv-python
+accelerate
+compel
+peft
+sentencepiece
+optimum-quanto

backend/python/kitten-tts/requirements-mps.txt

@@ -0,0 +1,5 @@
+grpcio==1.71.0
+protobuf
+certifi
+packaging==24.1
+https://github.com/KittenML/KittenTTS/releases/download/0.1/kittentts-0.1.0-py3-none-any.whl

backend/python/kokoro/requirements-intel.txt

@@ -1,8 +1,6 @@
---extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-intel-extension-for-pytorch==2.8.10+xpu
-torch==2.5.1+cxx11.abi
-oneccl_bind_pt==2.8.0+xpu
-torchaudio==2.5.1+cxx11.abi
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+torchaudio
 optimum[openvino]
 setuptools
 transformers==4.48.3

backend/python/kokoro/requirements-mps.txt

@@ -0,0 +1,5 @@
+torch==2.7.1
+transformers
+accelerate
+kokoro
+soundfile

backend/python/mlx-audio/requirements-cpu.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cpu]

backend/python/mlx-audio/requirements-cublas12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda12]

backend/python/mlx-audio/requirements-cublas13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda13]

backend/python/mlx-audio/requirements-l4t12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda12]

backend/python/mlx-audio/requirements-l4t13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-audio
+mlx[cuda13]

backend/python/mlx-vlm/requirements-cpu.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cpu]

backend/python/mlx-vlm/requirements-cublas12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda12]

backend/python/mlx-vlm/requirements-cublas13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda13]

backend/python/mlx-vlm/requirements-l4t12.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda12]

backend/python/mlx-vlm/requirements-l4t13.txt

@@ -0,0 +1,2 @@
+git+https://github.com/Blaizzy/mlx-vlm
+mlx[cuda13]

backend/python/mlx/requirements-cpu.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cpu]

backend/python/mlx/requirements-cublas12.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda12]

backend/python/mlx/requirements-cublas13.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda13]

backend/python/mlx/requirements-l4t12.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda12]

backend/python/mlx/requirements-l4t13.txt

@@ -0,0 +1,2 @@
+mlx-lm
+mlx[cuda13]

backend/python/moonshine/requirements-mps.txt

@@ -0,0 +1,4 @@
+grpcio==1.71.0
+protobuf
+grpcio-tools
+useful-moonshine-onnx@git+https://git@github.com/moonshine-ai/moonshine.git#subdirectory=moonshine-onnx

backend/python/outetts/Makefile

@@ -0,0 +1,23 @@
+.PHONY: outetts
+outetts:
+	bash install.sh
+
+.PHONY: run
+run: outetts
+	@echo "Running outetts..."
+	bash run.sh
+	@echo "outetts run."
+
+.PHONY: test
+test: outetts
+	@echo "Testing outetts..."
+	bash test.sh
+	@echo "outetts tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__

backend/python/outetts/backend.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""
+gRPC server for OuteTTS (OuteAI TTS) models.
+"""
+from concurrent import futures
+
+import argparse
+import signal
+import sys
+import os
+import asyncio
+
+import backend_pb2
+import backend_pb2_grpc
+
+import grpc
+import outetts
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
+
+
+class BackendServicer(backend_pb2_grpc.BackendServicer):
+    def Health(self, request, context):
+        return backend_pb2.Reply(message=bytes("OK", 'utf-8'))
+
+    def LoadModel(self, request, context):
+        model_name = request.Model
+        if os.path.exists(request.ModelFile):
+            model_name = request.ModelFile
+
+        self.options = {}
+        for opt in request.Options:
+            if ":" not in opt:
+                continue
+            key, value = opt.split(":", 1)
+            try:
+                if "." in value:
+                    value = float(value)
+                else:
+                    value = int(value)
+            except ValueError:
+                pass
+            self.options[key] = value
+
+        MODELNAME = "OuteAI/OuteTTS-0.3-1B"
+        TOKENIZER = "OuteAI/OuteTTS-0.3-1B"
+        VERSION = "0.3"
+        SPEAKER = "en_male_1"
+        for opt in request.Options:
+            if opt.startswith("tokenizer:"):
+                TOKENIZER = opt.split(":")[1]
+                break
+            if opt.startswith("version:"):
+                VERSION = opt.split(":")[1]
+                break
+            if opt.startswith("speaker:"):
+                SPEAKER = opt.split(":")[1]
+                break
+
+        if model_name != "":
+            MODELNAME = model_name
+
+        try:
+            model_config = outetts.HFModelConfig_v2(
+                model_path=MODELNAME,
+                tokenizer_path=TOKENIZER
+            )
+            self.interface = outetts.InterfaceHF(model_version=VERSION, cfg=model_config)
+
+            self.interface.print_default_speakers()
+            if request.AudioPath:
+                if os.path.isabs(request.AudioPath):
+                    self.AudioPath = request.AudioPath
+                else:
+                    self.AudioPath = os.path.join(request.ModelPath, request.AudioPath)
+                self.speaker = self.interface.create_speaker(audio_path=self.AudioPath)
+            else:
+                self.speaker = self.interface.load_default_speaker(name=SPEAKER)
+
+            if request.ContextSize > 0:
+                self.max_tokens = request.ContextSize
+            else:
+                self.max_tokens = self.options.get("max_new_tokens", 512)
+
+        except Exception as err:
+            print("Error:", err, file=sys.stderr)
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.Result(message="Model loaded successfully", success=True)
+
+    def TTS(self, request, context):
+        try:
+            text = request.text if request.text else "Speech synthesis is the artificial production of human speech."
+            print("[OuteTTS] generating TTS", file=sys.stderr)
+            gen_cfg = outetts.GenerationConfig(
+                text=text,
+                temperature=self.options.get("temperature", 0.1),
+                repetition_penalty=self.options.get("repetition_penalty", 1.1),
+                max_length=self.max_tokens,
+                speaker=self.speaker,
+            )
+            output = self.interface.generate(config=gen_cfg)
+            print("[OuteTTS] Generated TTS", file=sys.stderr)
+            output.save(request.dst)
+            print("[OuteTTS] TTS done", file=sys.stderr)
+        except Exception as err:
+            return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
+        return backend_pb2.Result(success=True)
+
+
+async def serve(address):
+    server = grpc.aio.server(
+        migration_thread_pool=futures.ThreadPoolExecutor(max_workers=MAX_WORKERS),
+        options=[
+            ('grpc.max_message_length', 50 * 1024 * 1024),
+            ('grpc.max_send_message_length', 50 * 1024 * 1024),
+            ('grpc.max_receive_message_length', 50 * 1024 * 1024),
+        ])
+    backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server)
+    server.add_insecure_port(address)
+
+    loop = asyncio.get_event_loop()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        loop.add_signal_handler(
+            sig, lambda: asyncio.ensure_future(server.stop(5))
+        )
+
+    await server.start()
+    print("Server started. Listening on: " + address, file=sys.stderr)
+    await server.wait_for_termination()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the OuteTTS gRPC server.")
+    parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.")
+    args = parser.parse_args()
+    asyncio.run(serve(args.addr))

backend/python/outetts/install.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+installRequirements

backend/python/outetts/requirements-cpu.txt

@@ -0,0 +1,7 @@
+torch==2.7.1
+llvmlite==0.43.0
+numba==0.60.0
+accelerate
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements-cublas12.txt

@@ -0,0 +1,7 @@
+torch==2.7.1
+accelerate
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+protobuf==6.33.5
+outetts

backend/python/outetts/requirements-cublas13.txt

@@ -0,0 +1,7 @@
+--extra-index-url https://download.pytorch.org/whl/cu130
+torch==2.9.0
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements-hipblas.txt

@@ -1,5 +1,8 @@
 --extra-index-url https://download.pytorch.org/whl/rocm6.4
 torch==2.8.0+rocm6.4
-torchaudio==2.8.0+rocm6.4
-transformers
-accelerate
+accelerate
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements-intel.txt

@@ -0,0 +1,8 @@
+--extra-index-url https://download.pytorch.org/whl/xpu
+torch
+optimum[openvino]
+llvmlite==0.43.0
+numba==0.60.0
+bitsandbytes
+outetts
+protobuf==6.33.5

backend/python/outetts/requirements.txt

@@ -0,0 +1,6 @@
+grpcio==1.76.0
+protobuf==6.33.5
+certifi
+setuptools
+scipy==1.15.1
+numpy>=2.0.0

backend/python/outetts/run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+backend_dir=$(dirname $0)
+if [ -d $backend_dir/common ]; then
+    source $backend_dir/common/libbackend.sh
+else
+    source $backend_dir/../common/libbackend.sh
+fi
+
+startBackend $@

backend/python/outetts/test.py

@@ -0,0 +1,35 @@
+"""
+Test script for the OuteTTS gRPC service.
+"""
+import unittest
+import subprocess
+import time
+import backend_pb2
+import backend_pb2_grpc
+
+import grpc
+
+
+class TestBackendServicer(unittest.TestCase):
+    def setUp(self):
+        self.service = subprocess.Popen(["python3", "backend.py", "--addr", "localhost:50051"])
+        time.sleep(5)
+
+    def tearDown(self):
+        self.service.terminate()
+        self.service.wait()
+
+    def test_health(self):
+        try:
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.Health(backend_pb2.HealthMessage())
+                self.assertEqual(response.message, b'OK')
+        except Exception as err:
+            self.fail(f"Health check failed: {err}")
+        finally:
+            self.tearDown()
+
+
+if __name__ == "__main__":
+    unittest.main()