diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 5d4163f60..e72d26902 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -871,6 +871,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-magpie-tts-cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1935,6 +1948,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-magpie-tts-cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -2000,6 +2026,19 @@ include: backend: "moss-tts-cpp" dockerfile: "./backend/Dockerfile.golang" context: "./" + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-cuda-13-arm64-magpie-tts-cpp' + base-image: "ubuntu:24.04" + ubuntu-version: '2404' + runs-on: 'ubuntu-24.04-arm' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -4583,6 +4622,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-magpie-tts-cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" @@ -4598,6 +4651,20 @@ include: context: "./" ubuntu-version: '2404' # omnivoice-cpp + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-magpie-tts-cpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" @@ -4652,6 +4719,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'sycl_f32' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-intel-sycl-f32-magpie-tts-cpp' + runs-on: 'ubuntu-latest' + base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'sycl_f32' cuda-major-version: "" cuda-minor-version: "" @@ -4691,6 +4771,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'sycl_f16' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-intel-sycl-f16-magpie-tts-cpp' + runs-on: 'ubuntu-latest' + base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'sycl_f16' cuda-major-version: "" cuda-minor-version: "" @@ -4732,6 +4825,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-magpie-tts-cpp' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'vulkan' cuda-major-version: "" cuda-minor-version: "" @@ -4774,6 +4881,20 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'vulkan' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-gpu-vulkan-magpie-tts-cpp' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'vulkan' cuda-major-version: "" cuda-minor-version: "" @@ -4814,6 +4935,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2204' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/arm64' + skip-drivers: 'false' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-arm64-magpie-tts-cpp' + base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + runs-on: 'ubuntu-24.04-arm' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2204' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "0" @@ -4853,6 +4987,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-magpie-tts-cpp' + base-image: "rocm/dev-ubuntu-24.04:6.4.4" + runs-on: 'ubuntu-latest' + skip-drivers: 'false' + backend: "magpie-tts-cpp" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'hipblas' cuda-major-version: "" cuda-minor-version: "" @@ -5774,6 +5921,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-moss-tts-cpp" build-type: "metal" lang: "go" + - backend: "magpie-tts-cpp" + tag-suffix: "-metal-darwin-arm64-magpie-tts-cpp" + build-type: "metal" + lang: "go" - backend: "omnivoice-cpp" tag-suffix: "-metal-darwin-arm64-omnivoice-cpp" build-type: "metal" diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml index 93ab8dd9c..b5dc8c31d 100644 --- a/.github/workflows/bump_deps.yaml +++ b/.github/workflows/bump_deps.yaml @@ -110,6 +110,10 @@ jobs: variable: "VIBEVOICE_CPP_VERSION" branch: "master" file: "backend/go/vibevoice-cpp/Makefile" + - repository: "mudler/magpie-tts.cpp" + variable: "MAGPIETTS_CPP_VERSION" + branch: "main" + file: "backend/go/magpie-tts-cpp/Makefile" runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/test-extra.yml b/.github/workflows/test-extra.yml index 262196a8c..899b98e60 100644 --- a/.github/workflows/test-extra.yml +++ b/.github/workflows/test-extra.yml @@ -37,6 +37,7 @@ jobs: sglang: ${{ steps.detect.outputs.sglang }} acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }} qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }} + magpie-tts-cpp: ${{ steps.detect.outputs.magpie-tts-cpp }} rfdetr-cpp: ${{ steps.detect.outputs.rfdetr-cpp }} locate-anything-cpp: ${{ steps.detect.outputs.locate-anything-cpp }} vibevoice-cpp: ${{ steps.detect.outputs.vibevoice-cpp }} @@ -866,6 +867,38 @@ jobs: - name: Test qwen3-tts-cpp run: | make --jobs=5 --output-sync=target -C backend/go/qwen3-tts-cpp test + tests-magpie-tts-cpp: + needs: detect-changes + if: needs.detect-changes.outputs.magpie-tts-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true' + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v7 + with: + submodules: true + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake curl libopenblas-dev ffmpeg + - name: Setup Go + uses: actions/setup-go@v5 + - name: Display Go version + run: go version + - name: Proto Dependencies + run: | + # Install protoc + curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \ + unzip -j -d /usr/local/bin protoc.zip bin/protoc && \ + rm protoc.zip + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + PATH="$PATH:$HOME/go/bin" make protogen-go + - name: Build magpie-tts-cpp + run: | + make --jobs=5 --output-sync=target -C backend/go/magpie-tts-cpp + - name: Test magpie-tts-cpp + run: | + make --jobs=5 --output-sync=target -C backend/go/magpie-tts-cpp test # Per-backend smoke for rfdetr-cpp: builds the .so + Go binary and runs # `make -C backend/go/rfdetr-cpp test`. test.sh fetches the small (~20 MB) # rfdetr-nano-q8_0 GGUF from the published mudler/rfdetr-cpp-nano HF repo diff --git a/Makefile b/Makefile index 992c4c05d..6ac015876 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -1257,6 +1257,7 @@ BACKEND_VOXTRAL = voxtral|golang|.|false|true BACKEND_ACESTEP_CPP = acestep-cpp|golang|.|false|true BACKEND_QWEN3_TTS_CPP = qwen3-tts-cpp|golang|.|false|true BACKEND_MOSS_TTS_CPP = moss-tts-cpp|golang|.|false|true +BACKEND_MAGPIE_TTS_CPP = magpie-tts-cpp|golang|.|false|true BACKEND_OMNIVOICE_CPP = omnivoice-cpp|golang|.|false|true BACKEND_VIBEVOICE_CPP = vibevoice-cpp|golang|.|false|true BACKEND_LOCALVQE = localvqe|golang|.|false|true @@ -1384,6 +1385,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_ACE_STEP))) $(eval $(call generate-docker-build-target,$(BACKEND_ACESTEP_CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_QWEN3_TTS_CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_MOSS_TTS_CPP))) +$(eval $(call generate-docker-build-target,$(BACKEND_MAGPIE_TTS_CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_OMNIVOICE_CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_VIBEVOICE_CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_LOCALVQE))) @@ -1403,7 +1405,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter ######################################################## ### Mock Backend for E2E Tests diff --git a/README.md b/README.md index 5c356d0fc..dbde8079f 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,7 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native | [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet ASR (tdt/ctc/rnnt/hybrid), with cache-aware streaming transcription | | [moss-transcribe.cpp](https://github.com/localai-org/moss-transcribe.cpp) | C++/GGML port of OpenMOSS MOSS-Transcribe-Diarize: joint long-form transcription, speaker diarization and timestamping in a single pass | | [moss-tts.cpp](https://github.com/mudler/moss-tts.cpp) | C++/GGML port of the OpenMOSS MOSS-TTS family: text-to-speech (MOSS-TTS-Local v1.5, 48 kHz stereo) with reference-audio voice cloning, through the MOSS-Audio-Tokenizer neural codec | +| [magpie-tts.cpp](https://github.com/mudler/magpie-tts.cpp) | C++/GGML port of NVIDIA's Magpie TTS Multilingual 357M: 22.05 kHz mono text-to-speech in 5 voices and 9+ languages, with the NanoCodec neural codec and tokenizer/G2P embedded in a single GGUF | | [ced.cpp](https://github.com/localai-org/ced.cpp) | C++/GGML port of the CED audio-tagging models: sound-event classification (527-class AudioSet) over REST and the realtime API for live recognition | | [voice-detect.cpp](https://github.com/localai-org/voice-detect.cpp) | Speaker recognition and voice analysis (ECAPA-TDNN, WeSpeaker, ERes2Net, CAM++, wav2vec2 age/gender/emotion), replacing the Python speaker-recognition backend | | [voxtral-tts.c](https://github.com/mudler/voxtral-tts.c) | Voxtral Realtime 4B speech-to-text in pure C | diff --git a/backend/go/magpie-tts-cpp/.gitignore b/backend/go/magpie-tts-cpp/.gitignore new file mode 100644 index 000000000..d738faca8 --- /dev/null +++ b/backend/go/magpie-tts-cpp/.gitignore @@ -0,0 +1,6 @@ +magpie-tts-cpp +*.so +*.dylib +sources/ +package/ +magpie-tts-models/ diff --git a/backend/go/magpie-tts-cpp/CMakeLists.txt b/backend/go/magpie-tts-cpp/CMakeLists.txt new file mode 100644 index 000000000..7e2f4bfe5 --- /dev/null +++ b/backend/go/magpie-tts-cpp/CMakeLists.txt @@ -0,0 +1,35 @@ +cmake_minimum_required(VERSION 3.16) +project(gomagpiettscpp LANGUAGES C CXX) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(MAGPIE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/sources/magpie-tts.cpp) + +# Override upstream's CMAKE_CUDA_ARCHITECTURES before add_subdirectory. +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real") +endif() + +# The magpie-tts C-API is exported directly by the upstream shared library +# (magpie_tts_capi_* in libmagpie-tts.so, ggml statically linked inside), so +# unlike qwen3-tts-cpp / moss-tts-cpp no local C shim is needed -- this wrapper +# only configures the upstream project as a purego-loadable shared library. +set(MAGPIE_SHARED ON CACHE BOOL "" FORCE) +set(MAGPIE_BUILD_CLI OFF CACHE BOOL "" FORCE) +set(MAGPIE_BUILD_TESTS OFF CACHE BOOL "" FORCE) + +# Upstream FORCE-overwrites GGML_CUDA / GGML_METAL / GGML_VULKAN / GGML_HIP from +# its own MAGPIE_GGML_* toggles, which would silently discard the -DGGML_*=ON +# flags the LocalAI Makefile passes for GPU BUILD_TYPEs. Translate them into the +# MAGPIE_GGML_* vocabulary before add_subdirectory. (GGML_SYCL / GGML_BLAS and +# the CPU ISA flags are not touched by upstream and reach ggml unchanged.) +foreach(_be CUDA METAL VULKAN HIP) + if(GGML_${_be}) + set(MAGPIE_GGML_${_be} ON CACHE BOOL "" FORCE) + endif() +endforeach() + +add_subdirectory(${MAGPIE_DIR} magpie EXCLUDE_FROM_ALL) + +# Place libmagpie-tts.so at the build root, where the Makefile picks it up. +set_target_properties(magpie-tts PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/backend/go/magpie-tts-cpp/Makefile b/backend/go/magpie-tts-cpp/Makefile new file mode 100644 index 000000000..ea4f623cb --- /dev/null +++ b/backend/go/magpie-tts-cpp/Makefile @@ -0,0 +1,140 @@ +CMAKE_ARGS?= +BUILD_TYPE?= +NATIVE?=false + +GOCMD?=go +GO_TAGS?= +JOBS?=$(shell nproc --ignore=1) + +# magpie-tts.cpp version +MAGPIETTS_REPO?=https://github.com/mudler/magpie-tts.cpp +MAGPIETTS_CPP_VERSION?=6f7696cffbfa1e85c17bc42eca24f5cbdb82c949 +SO_TARGET?=libgomagpiettscpp.so + +CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF + +ifeq ($(NATIVE),false) + CMAKE_ARGS+=-DGGML_NATIVE=OFF +endif + +ifeq ($(BUILD_TYPE),cublas) + CMAKE_ARGS+=-DGGML_CUDA=ON +else ifeq ($(BUILD_TYPE),openblas) + CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS +else ifeq ($(BUILD_TYPE),clblas) + CMAKE_ARGS+=-DGGML_CLBLAST=ON -DCLBlast_DIR=/some/path +else ifeq ($(BUILD_TYPE),hipblas) + # This ggml only understands GGML_HIP (GGML_HIPBLAS was removed upstream), + # so passing GGML_HIPBLAS silently produced a CPU-only build (see #10666). + ROCM_HOME ?= /opt/rocm + ROCM_PATH ?= /opt/rocm + export CXX=$(ROCM_HOME)/llvm/bin/clang++ + export CC=$(ROCM_HOME)/llvm/bin/clang + AMDGPU_TARGETS ?= gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201 + CMAKE_ARGS+=-DGGML_HIP=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) +else ifeq ($(BUILD_TYPE),vulkan) + CMAKE_ARGS+=-DGGML_VULKAN=ON +else ifeq ($(OS),Darwin) + ifneq ($(BUILD_TYPE),metal) + CMAKE_ARGS+=-DGGML_METAL=OFF + else + CMAKE_ARGS+=-DGGML_METAL=ON + CMAKE_ARGS+=-DGGML_METAL_EMBED_LIBRARY=ON + endif +endif + +ifeq ($(BUILD_TYPE),sycl_f16) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DGGML_SYCL_F16=ON +endif + +ifeq ($(BUILD_TYPE),sycl_f32) + CMAKE_ARGS+=-DGGML_SYCL=ON \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx +endif + +sources/magpie-tts.cpp: + mkdir -p sources/magpie-tts.cpp + cd sources/magpie-tts.cpp && \ + git init && \ + git remote add origin $(MAGPIETTS_REPO) && \ + git fetch origin && \ + git checkout $(MAGPIETTS_CPP_VERSION) && \ + git submodule update --init --recursive --depth 1 --single-branch + +# Detect OS +UNAME_S := $(shell uname -s) + +# Only build CPU variants on Linux +ifeq ($(UNAME_S),Linux) + VARIANT_TARGETS = libgomagpiettscpp-avx.so libgomagpiettscpp-avx2.so libgomagpiettscpp-avx512.so libgomagpiettscpp-fallback.so +else + # On non-Linux (e.g., Darwin), build only fallback variant (as a dylib) + VARIANT_TARGETS = libgomagpiettscpp-fallback.dylib +endif + +magpie-tts-cpp: main.go gomagpiettscpp.go $(VARIANT_TARGETS) + CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o magpie-tts-cpp ./ + +package: magpie-tts-cpp + bash package.sh + +build: package + +clean: purge + rm -rf libgomagpiettscpp*.so libgomagpiettscpp*.dylib package sources/magpie-tts.cpp magpie-tts-cpp + +purge: + rm -rf build* + +# Variants must build sequentially +.NOTPARALLEL: + +# Build all variants (Linux only) +ifeq ($(UNAME_S),Linux) +libgomagpiettscpp-avx.so: sources/magpie-tts.cpp + $(info ${GREEN}I magpie-tts-cpp build info:avx${RESET}) + SO_TARGET=libgomagpiettscpp-avx.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgomagpiettscpp-custom + rm -rf build-libgomagpiettscpp-avx.so + +libgomagpiettscpp-avx2.so: sources/magpie-tts.cpp + $(info ${GREEN}I magpie-tts-cpp build info:avx2${RESET}) + SO_TARGET=libgomagpiettscpp-avx2.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgomagpiettscpp-custom + rm -rf build-libgomagpiettscpp-avx2.so + +libgomagpiettscpp-avx512.so: sources/magpie-tts.cpp + $(info ${GREEN}I magpie-tts-cpp build info:avx512${RESET}) + SO_TARGET=libgomagpiettscpp-avx512.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on -DGGML_BMI2=on" $(MAKE) libgomagpiettscpp-custom + rm -rf build-libgomagpiettscpp-avx512.so +endif + +# Build fallback variant (all platforms) +libgomagpiettscpp-fallback.so: sources/magpie-tts.cpp + $(info ${GREEN}I magpie-tts-cpp build info:fallback${RESET}) + SO_TARGET=libgomagpiettscpp-fallback.so CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgomagpiettscpp-custom + rm -rf build-libgomagpiettscpp-fallback.so + +# Build fallback variant as a dylib (Darwin) +libgomagpiettscpp-fallback.dylib: sources/magpie-tts.cpp + $(info ${GREEN}I magpie-tts-cpp build info:fallback (dylib)${RESET}) + SO_TARGET=libgomagpiettscpp-fallback.dylib CMAKE_ARGS="$(CMAKE_ARGS) -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off" $(MAKE) libgomagpiettscpp-custom + rm -rf build-libgomagpiettscpp-fallback.dylib + +libgomagpiettscpp-custom: CMakeLists.txt + mkdir -p build-$(SO_TARGET) && \ + cd build-$(SO_TARGET) && \ + cmake .. $(CMAKE_ARGS) && \ + cmake --build . --config Release -j$(JOBS) --target magpie-tts && \ + cd .. && \ + (mv build-$(SO_TARGET)/libmagpie-tts.so ./$(SO_TARGET) 2>/dev/null || \ + mv build-$(SO_TARGET)/libmagpie-tts.dylib ./$(SO_TARGET) 2>/dev/null) + +test: magpie-tts-cpp + @echo "Running magpie-tts-cpp tests..." + bash test.sh + @echo "magpie-tts-cpp tests completed." + +all: magpie-tts-cpp package diff --git a/backend/go/magpie-tts-cpp/README.md b/backend/go/magpie-tts-cpp/README.md new file mode 100644 index 000000000..a6cacaab9 --- /dev/null +++ b/backend/go/magpie-tts-cpp/README.md @@ -0,0 +1,69 @@ +# Magpie TTS C++ backend + +This backend runs NVIDIA's **Magpie TTS Multilingual 357M** GGUF through +[magpie-tts.cpp](https://github.com/mudler/magpie-tts.cpp), a from-scratch +C++/ggml port (model + NanoCodec + tokenizer + G2P dictionaries in one +self-contained GGUF, no Python at inference time). It generates **22.05 kHz +mono** speech in 5 baked voices across 9+ languages. + +The library is loaded via purego (cgo-less `dlopen`) exactly like +`qwen3-tts-cpp` / `moss-tts-cpp`; the flat C-API (`magpie_tts_capi_*`) is +exported directly by the upstream shared library, so there is no local C shim. + +## Model configuration + +The model path points at the single GGUF: + +```yaml +name: magpie-tts-cpp +backend: magpie-tts-cpp +parameters: + model: magpie-tts-multilingual-357m-q8_0.gguf +known_usecases: + - tts +options: + - "speaker:Aria" # optional default voice (Aria, Jason, John, Leo, Sofia, or 0-4) + - "language:en" # optional default language +``` + +GGUFs live at +[mudler/magpie-tts.cpp-gguf](https://huggingface.co/mudler/magpie-tts.cpp-gguf) +(q8_0 recommended: near-lossless, ~624 MB, fastest decode). + +## Voices and languages + +Magpie has 5 baked speakers - `Aria`, `Jason`, `John`, `Leo`, `Sofia` - and no +voice cloning. The request `voice` accepts the names case-insensitively or the +indices `0`-`4`; empty selects Aria. Languages: `en`, `es`, `de`, `fr`, `it`, +`pt-BR`, `hi`, `vi`, `ko`, `ar-AE`, `ar-SA`, `ar-MSA` (case-insensitive; +default `en`). + +## API example + +```bash +curl http://localhost:8080/v1/audio/speech \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "magpie-tts-cpp", + "input": "Hello world, this is a test of the text to speech system.", + "voice": "sofia", + "language": "en" + }' \ + --output speech.wav +``` + +## Native end-to-end test + +The labeled test loads a real GGUF, synthesizes WAVs (verifying rate, layout +and non-silence), and exercises the streaming path: + +```bash +make -C backend/go/magpie-tts-cpp magpie-tts-cpp + +MAGPIETTS_MODEL=/path/to/magpie-tts-multilingual-357m-q8_0.gguf \ +MAGPIETTS_LIBRARY=backend/go/magpie-tts-cpp/libgomagpiettscpp-fallback.so \ + go test ./backend/go/magpie-tts-cpp -ginkgo.label-filter=e2e +``` + +`bash test.sh` does the same and auto-downloads the q8_0 GGUF when +`MAGPIETTS_MODEL` is unset. diff --git a/backend/go/magpie-tts-cpp/audio.go b/backend/go/magpie-tts-cpp/audio.go new file mode 100644 index 000000000..24f3cdf2f --- /dev/null +++ b/backend/go/magpie-tts-cpp/audio.go @@ -0,0 +1,93 @@ +package main + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + + "github.com/go-audio/audio" + "github.com/go-audio/wav" +) + +// magpieSampleRate is the fixed Magpie TTS Multilingual (NanoCodec) output +// rate: 22.05 kHz. +const magpieSampleRate = 22050 + +// magpieChannels is the fixed output layout: mono. +const magpieChannels = 1 + +// wavHeaderMono returns a 44-byte WAV header for a streaming 16-bit mono PCM +// stream at 22050 Hz, with placeholder (0xFFFFFFFF) sizes since the total +// length is unknown up front. Emitted as the first chunk of TTSStream so the +// HTTP layer receives a self-describing WAV. +func wavHeaderMono() []byte { + const blockAlign = magpieChannels * 2 // 16-bit mono + var buf bytes.Buffer + w := func(v any) { _ = binary.Write(&buf, binary.LittleEndian, v) } + buf.WriteString("RIFF") + w(uint32(0xFFFFFFFF)) + buf.WriteString("WAVE") + buf.WriteString("fmt ") + w(uint32(16)) // Subchunk1Size + w(uint16(1)) // PCM + w(uint16(magpieChannels)) // mono + w(uint32(magpieSampleRate)) // sample rate + w(uint32(magpieSampleRate * blockAlign)) // byte rate = SR * blockAlign + w(uint16(blockAlign)) // block align + w(uint16(16)) // bits per sample + buf.WriteString("data") + w(uint32(0xFFFFFFFF)) + return buf.Bytes() +} + +// floatToPCM16LE clamps each sample to [-1,1] and encodes it as little-endian +// signed 16-bit PCM. +func floatToPCM16LE(samples []float32) []byte { + out := make([]byte, len(samples)*2) + for i, s := range samples { + if s > 1 { + s = 1 + } else if s < -1 { + s = -1 + } + v := int16(s * 32767) + out[i*2] = byte(v) // #nosec G115 -- intentional little-endian split of a clamped int16 + out[i*2+1] = byte(v >> 8) // #nosec G115 -- high byte of the same clamped int16 + } + return out +} + +// writeWAVMono writes float samples as a finalized 16-bit mono WAV at +// 22050 Hz. +func writeWAVMono(dst string, samples []float32) error { + f, err := os.Create(dst) // #nosec G304 -- dst is the server-chosen output path from the TTS request, not user-traversable + if err != nil { + return fmt.Errorf("magpie-tts: create %q: %w", dst, err) + } + enc := wav.NewEncoder(f, magpieSampleRate, 16, magpieChannels, 1) + ints := make([]int, len(samples)) + for i, s := range samples { + if s > 1 { + s = 1 + } else if s < -1 { + s = -1 + } + ints[i] = int(s * 32767) + } + b := &audio.IntBuffer{ + Format: &audio.Format{NumChannels: magpieChannels, SampleRate: magpieSampleRate}, + Data: ints, + SourceBitDepth: 16, + } + if err := enc.Write(b); err != nil { + _ = enc.Close() + _ = f.Close() + return fmt.Errorf("magpie-tts: encode WAV: %w", err) + } + if err := enc.Close(); err != nil { + _ = f.Close() + return fmt.Errorf("magpie-tts: finalize WAV: %w", err) + } + return f.Close() +} diff --git a/backend/go/magpie-tts-cpp/e2e_test.go b/backend/go/magpie-tts-cpp/e2e_test.go new file mode 100644 index 000000000..6df281378 --- /dev/null +++ b/backend/go/magpie-tts-cpp/e2e_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "encoding/binary" + "math" + "os" + + "github.com/ebitengine/purego" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func ttsReq(text, voice, lang, dst string) *pb.TTSRequest { + r := &pb.TTSRequest{Text: text, Voice: voice, Dst: dst} + if lang != "" { + r.Language = &lang + } + return r +} + +// wavRMS parses a 16-bit PCM WAV file and returns (sampleRate, channels, RMS +// of the normalized samples). +func wavRMS(path string) (int, int, float64) { + data, err := os.ReadFile(path) // #nosec G304 -- test-owned temp file + Expect(err).ToNot(HaveOccurred()) + Expect(len(data)).To(BeNumerically(">", 44)) + Expect(string(data[0:4])).To(Equal("RIFF")) + Expect(string(data[8:12])).To(Equal("WAVE")) + channels := int(binary.LittleEndian.Uint16(data[22:24])) + rate := int(binary.LittleEndian.Uint32(data[24:28])) + // Find the data chunk (go-audio writes fmt first, data after). + off := 12 + for off+8 <= len(data) { + id := string(data[off : off+4]) + sz := int(binary.LittleEndian.Uint32(data[off+4 : off+8])) + if id == "data" { + pcm := data[off+8:] + if sz < len(pcm) { + pcm = pcm[:sz] + } + var sum float64 + n := len(pcm) / 2 + for i := 0; i < n; i++ { + s := float64(int16(binary.LittleEndian.Uint16(pcm[i*2:]))) / 32768.0 + sum += s * s + } + Expect(n).To(BeNumerically(">", 0)) + return rate, channels, math.Sqrt(sum / float64(n)) + } + off += 8 + sz + } + Fail("no data chunk found in " + path) + return 0, 0, 0 +} + +var _ = Describe("magpie-tts-cpp e2e", Label("e2e"), func() { + var ( + loaded bool + backend *MagpieTtsCpp + ) + + BeforeEach(func() { + modelPath := os.Getenv("MAGPIETTS_MODEL") + if modelPath == "" { + Skip("MAGPIETTS_MODEL not set; skipping e2e") + } + if !loaded { + lib := os.Getenv("MAGPIETTS_LIBRARY") + if lib == "" { + lib = "./libgomagpiettscpp-fallback.so" + } + h, err := purego.Dlopen(lib, purego.RTLD_NOW|purego.RTLD_GLOBAL) + Expect(err).ToNot(HaveOccurred()) + purego.RegisterLibFunc(&CppAbiVersion, h, "magpie_tts_capi_abi_version") + purego.RegisterLibFunc(&CppLoad, h, "magpie_tts_capi_load") + purego.RegisterLibFunc(&CppFree, h, "magpie_tts_capi_free") + purego.RegisterLibFunc(&CppSynthesize, h, "magpie_tts_capi_synthesize") + purego.RegisterLibFunc(&CppFreeAudio, h, "magpie_tts_capi_free_audio") + purego.RegisterLibFunc(&CppLastError, h, "magpie_tts_capi_last_error") + + backend = &MagpieTtsCpp{} + Expect(backend.Load(&pb.ModelOptions{ModelFile: modelPath})).To(Succeed()) + loaded = true + } + }) + + It("synthesizes a non-silent 22.05 kHz mono WAV via TTS", func() { + dst := GinkgoT().TempDir() + "/out.wav" + Expect(backend.TTS(ttsReq("Hello world, this is a test.", "Aria", "en", dst))).To(Succeed()) + rate, channels, rms := wavRMS(dst) + Expect(rate).To(Equal(22050)) + Expect(channels).To(Equal(1)) + Expect(rms).To(BeNumerically(">", 0.01), "audio should not be silent") + }) + + It("accepts a case-insensitive voice and a speaker index", func() { + dst := GinkgoT().TempDir() + "/out.wav" + Expect(backend.TTS(ttsReq("Short test.", "sofia", "en", dst))).To(Succeed()) + Expect(backend.TTS(ttsReq("Short test.", "1", "en", dst))).To(Succeed()) + }) + + It("rejects an unknown voice before reaching the engine", func() { + dst := GinkgoT().TempDir() + "/out.wav" + err := backend.TTS(ttsReq("Short test.", "not-a-speaker", "en", dst)) + Expect(err).To(MatchError(ContainSubstring("unknown voice"))) + }) + + It("streams a self-describing WAV via TTSStream", func() { + results := make(chan []byte, 4096) + done := make(chan error, 1) + go func() { done <- backend.TTSStream(ttsReq("Hello there, streaming test.", "", "", ""), results) }() + + var chunks int + var first []byte + for c := range results { + if chunks == 0 { + first = c + } + chunks++ + } + Expect(<-done).ToNot(HaveOccurred()) + Expect(chunks).To(BeNumerically(">=", 2)) + Expect(string(first[0:4])).To(Equal("RIFF")) + Expect(string(first[8:12])).To(Equal("WAVE")) + }) +}) diff --git a/backend/go/magpie-tts-cpp/gomagpiettscpp.go b/backend/go/magpie-tts-cpp/gomagpiettscpp.go new file mode 100644 index 000000000..6c94b9e3d --- /dev/null +++ b/backend/go/magpie-tts-cpp/gomagpiettscpp.go @@ -0,0 +1,147 @@ +package main + +import ( + "fmt" + "path/filepath" + "unsafe" + + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/xlog" +) + +// capiABIVersion is the magpie_tts_capi.h surface this backend binds. Bumped +// upstream on any breaking signature/semantics change; refuse to run on a +// mismatch instead of crashing inside a miscompiled call. +const capiABIVersion = 1 + +var ( + // magpie_tts_capi_abi_version() int + CppAbiVersion func() int + // magpie_tts_capi_load(gguf_path) -> ctx (NULL on failure) + CppLoad func(path string) uintptr + // magpie_tts_capi_free(ctx) + CppFree func(ctx uintptr) + // magpie_tts_capi_synthesize(ctx, text, language, speaker, out_n) -> float* + // 22050 Hz mono f32 PCM in [-1,1]; NULL on failure (see last_error). + CppSynthesize func(ctx uintptr, text, language, speaker string, outN unsafe.Pointer) uintptr + // magpie_tts_capi_free_audio(ptr) + CppFreeAudio func(ptr uintptr) + // magpie_tts_capi_last_error(ctx) -> const char* (ctx-owned, "" if none) + CppLastError func(ctx uintptr) string +) + +// MagpieTtsCpp serves the Magpie TTS Multilingual 357M GGUF through the +// magpie-tts.cpp C-API. The context is stateful (per-call error buffer, reused +// graph allocator) and NOT safe for concurrent synthesize calls, so +// base.SingleThread serializes everything behind the server-level lock. +type MagpieTtsCpp struct { + base.SingleThread + ctx uintptr + opts loadOptions +} + +func (m *MagpieTtsCpp) Load(opts *pb.ModelOptions) error { + model := opts.ModelFile + if model == "" { + model = opts.ModelPath + } + if !filepath.IsAbs(model) && opts.ModelPath != "" { + model = filepath.Join(opts.ModelPath, model) + } + + m.opts = parseOptions(opts.Options) + + if abi := CppAbiVersion(); abi != capiABIVersion { + return fmt.Errorf("magpie-tts: C-API ABI mismatch: library reports v%d, backend built for v%d", abi, capiABIVersion) + } + + xlog.Info("[magpie-tts-cpp] Load", "model", model) + + ctx := CppLoad(model) + if ctx == 0 { + // Load failures have no context to query last_error on; the C side + // logs the reason to stderr. + return fmt.Errorf("magpie-tts: failed to load model %q", model) + } + m.ctx = ctx + return nil +} + +// lastError surfaces the context's last error, falling back to a generic +// message when the C side left it empty. +func (m *MagpieTtsCpp) lastError() string { + if m.ctx == 0 { + return "no model loaded" + } + if e := CppLastError(m.ctx); e != "" { + return e + } + return "unknown error" +} + +// synthesize runs one C-API synthesis and copies the PCM out of C memory. +func (m *MagpieTtsCpp) synthesize(req *pb.TTSRequest) ([]float32, error) { + if m.ctx == 0 { + return nil, fmt.Errorf("magpie-tts: no model loaded") + } + if req.Text == "" { + return nil, fmt.Errorf("magpie-tts: TTS requires text") + } + speaker, err := resolveSpeaker(req.Voice, m.opts.speaker) + if err != nil { + return nil, err + } + lang := resolveLanguage(req.Language, m.opts.language) + + var n int32 + ptr := CppSynthesize(m.ctx, req.Text, lang, speaker, unsafe.Pointer(&n)) // #nosec G103 -- out-param for the purego-bound C-API + if ptr == 0 { + return nil, fmt.Errorf("magpie-tts: synthesis failed: %s", m.lastError()) + } + // Register the free as soon as we own a non-null buffer, so the n<=0 guard + // below cannot leak it (defensive: the C contract returns NULL on failure). + defer CppFreeAudio(ptr) + if n <= 0 { + return nil, fmt.Errorf("magpie-tts: synthesis produced no samples") + } + //nolint:govet // C-allocated PCM, copied out before free + src := unsafe.Slice((*float32)(unsafe.Pointer(ptr)), int(n)) // #nosec G103 -- C-allocated PCM, copied out before free + out := make([]float32, int(n)) + copy(out, src) + return out, nil +} + +func (m *MagpieTtsCpp) TTS(req *pb.TTSRequest) error { + if req.Dst == "" { + return fmt.Errorf("magpie-tts: TTS requires a destination path") + } + samples, err := m.synthesize(req) + if err != nil { + return err + } + return writeWAVMono(req.Dst, samples) +} + +// TTSStream synthesizes one-shot (the magpie C-API has no streaming call) and +// then emits a self-describing mono WAV: a header chunk followed by the PCM in +// fixed-size slices, so the HTTP layer still receives a streamed WAV (the gRPC +// TTSStream path never sets Message, so the backend owns the header - see +// core/backend/tts.go:ModelTTSStream). +func (m *MagpieTtsCpp) TTSStream(req *pb.TTSRequest, results chan []byte) error { + defer close(results) + samples, err := m.synthesize(req) + if err != nil { + return err + } + results <- wavHeaderMono() + const sampleChunk = 4096 // mono samples per emitted chunk + for off := 0; off < len(samples); off += sampleChunk { + end := off + sampleChunk + if end > len(samples) { + end = len(samples) + } + results <- floatToPCM16LE(samples[off:end]) + } + return nil +} diff --git a/backend/go/magpie-tts-cpp/magpiettscpp_test.go b/backend/go/magpie-tts-cpp/magpiettscpp_test.go new file mode 100644 index 000000000..b4bb19f2c --- /dev/null +++ b/backend/go/magpie-tts-cpp/magpiettscpp_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMagpieTtsCpp(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "magpie-tts-cpp suite") +} + +var _ = Describe("resolveSpeaker", func() { + It("canonicalizes case-insensitive names", func() { + for in, want := range map[string]string{ + "aria": "Aria", "JASON": "Jason", "john": "John", + "Leo": "Leo", "sofia": "Sofia", + } { + got, err := resolveSpeaker(in, "") + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(want)) + } + }) + It("accepts indices 0-4", func() { + for in, want := range map[string]string{ + "0": "Aria", "1": "Jason", "2": "John", "3": "Leo", "4": "Sofia", + } { + got, err := resolveSpeaker(in, "") + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(want)) + } + }) + It("selects the engine default on empty", func() { + got, err := resolveSpeaker("", "") + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeEmpty()) + }) + It("falls back to the model-level default speaker", func() { + got, err := resolveSpeaker("", "sofia") + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal("Sofia")) + }) + It("prefers the request voice over the fallback", func() { + got, err := resolveSpeaker("leo", "sofia") + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal("Leo")) + }) + It("rejects out-of-range indices", func() { + _, err := resolveSpeaker("5", "") + Expect(err).To(HaveOccurred()) + _, err = resolveSpeaker("-1", "") + Expect(err).To(HaveOccurred()) + }) + It("rejects unknown names with the valid choices", func() { + _, err := resolveSpeaker("serena", "") + Expect(err).To(MatchError(ContainSubstring("Aria, Jason, John, Leo, Sofia"))) + }) +}) + +var _ = Describe("resolveLanguage", func() { + strp := func(s string) *string { return &s } + + It("defaults to empty (engine picks en)", func() { + Expect(resolveLanguage(nil, "")).To(BeEmpty()) + }) + It("canonicalizes case for known codes", func() { + Expect(resolveLanguage(strp("EN"), "")).To(Equal("en")) + Expect(resolveLanguage(strp("pt-br"), "")).To(Equal("pt-BR")) + Expect(resolveLanguage(strp("AR-MSA"), "")).To(Equal("ar-MSA")) + }) + It("falls back to the model-level default language", func() { + Expect(resolveLanguage(nil, "de")).To(Equal("de")) + Expect(resolveLanguage(strp(""), "PT-BR")).To(Equal("pt-BR")) + }) + It("prefers the request language over the fallback", func() { + Expect(resolveLanguage(strp("it"), "de")).To(Equal("it")) + }) + It("passes unknown codes through verbatim", func() { + Expect(resolveLanguage(strp("zh"), "")).To(Equal("zh")) + }) +}) + +var _ = Describe("parseOptions", func() { + It("reads speaker and language defaults", func() { + o := parseOptions([]string{"speaker:Jason", "language:de"}) + Expect(o.speaker).To(Equal("Jason")) + Expect(o.language).To(Equal("de")) + }) + It("accepts the voice/lang aliases and ignores unknown keys", func() { + o := parseOptions([]string{"voice: sofia ", "lang: pt-BR", "bogus:1", "novalue"}) + Expect(o.speaker).To(Equal("sofia")) + Expect(o.language).To(Equal("pt-BR")) + }) +}) + +var _ = Describe("audio encoding", func() { + It("emits a well-formed streaming mono WAV header", func() { + h := wavHeaderMono() + Expect(h).To(HaveLen(44)) + Expect(string(h[0:4])).To(Equal("RIFF")) + Expect(string(h[8:12])).To(Equal("WAVE")) + // channels (offset 22) == 1, sample rate (offset 24) == 22050 + Expect(int(h[22]) | int(h[23])<<8).To(Equal(1)) + Expect(int(h[24]) | int(h[25])<<8 | int(h[26])<<16 | int(h[27])<<24).To(Equal(22050)) + }) + It("clamps float PCM to int16", func() { + b := floatToPCM16LE([]float32{2, -2, 0}) + Expect(b).To(HaveLen(6)) + Expect(int16(uint16(b[0]) | uint16(b[1])<<8)).To(Equal(int16(32767))) + Expect(int16(uint16(b[2]) | uint16(b[3])<<8)).To(Equal(int16(-32767))) + Expect(int16(uint16(b[4]) | uint16(b[5])<<8)).To(Equal(int16(0))) + }) +}) diff --git a/backend/go/magpie-tts-cpp/main.go b/backend/go/magpie-tts-cpp/main.go new file mode 100644 index 000000000..386c49dfd --- /dev/null +++ b/backend/go/magpie-tts-cpp/main.go @@ -0,0 +1,54 @@ +package main + +// Note: this is started internally by LocalAI and a server is allocated for each model +import ( + "flag" + "os" + "runtime" + + "github.com/ebitengine/purego" + grpc "github.com/mudler/LocalAI/pkg/grpc" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +type LibFuncs struct { + FuncPtr any + Name string +} + +func main() { + libName := os.Getenv("MAGPIETTS_LIBRARY") + if libName == "" { + if runtime.GOOS == "darwin" { + libName = "./libgomagpiettscpp-fallback.dylib" + } else { + libName = "./libgomagpiettscpp-fallback.so" + } + } + + lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + panic(err) + } + + libFuncs := []LibFuncs{ + {&CppAbiVersion, "magpie_tts_capi_abi_version"}, + {&CppLoad, "magpie_tts_capi_load"}, + {&CppFree, "magpie_tts_capi_free"}, + {&CppSynthesize, "magpie_tts_capi_synthesize"}, + {&CppFreeAudio, "magpie_tts_capi_free_audio"}, + {&CppLastError, "magpie_tts_capi_last_error"}, + } + for _, lf := range libFuncs { + purego.RegisterLibFunc(lf.FuncPtr, lib, lf.Name) + } + + flag.Parse() + + if err := grpc.StartServer(*addr, &MagpieTtsCpp{}); err != nil { + panic(err) + } +} diff --git a/backend/go/magpie-tts-cpp/options.go b/backend/go/magpie-tts-cpp/options.go new file mode 100644 index 000000000..ca264a9d3 --- /dev/null +++ b/backend/go/magpie-tts-cpp/options.go @@ -0,0 +1,108 @@ +package main + +import ( + "fmt" + "strconv" + "strings" +) + +// loadOptions holds the parsed model-level options. Magpie is a single +// self-contained GGUF (model + codec + tokenizer + G2P dictionaries), so the +// options only cover synthesis defaults. +type loadOptions struct { + // speaker is the default baked speaker when a request has no voice. + speaker string + // language is the default language when a request has none ("" = engine + // default, which is "en"). + language string +} + +func splitOption(o string) (key, value string, ok bool) { + i := strings.Index(o, ":") + if i < 0 { + return "", "", false + } + return strings.TrimSpace(o[:i]), strings.TrimSpace(o[i+1:]), true +} + +// parseOptions reads the backend "key:value" option slice. Unknown keys are +// ignored. +func parseOptions(opts []string) loadOptions { + var o loadOptions + for _, oo := range opts { + key, value, ok := splitOption(oo) + if !ok { + continue + } + switch key { + case "speaker", "voice": + o.speaker = value + case "language", "lang": + o.language = value + } + } + return o +} + +// magpieSpeakers are the baked speakers of Magpie TTS Multilingual 357M, in +// index order (the engine matches names exactly, so the Go side canonicalizes +// case-insensitive names and 0-4 indices to these strings). +var magpieSpeakers = []string{"Aria", "Jason", "John", "Leo", "Sofia"} + +// resolveSpeaker maps the request voice (falling back to the model-level +// default) onto a canonical baked speaker name. Accepted forms: +// case-insensitive names (aria, JASON, ...) and indices 0-4. Empty selects the +// engine default (speaker 0, Aria). Anything else is rejected with the valid +// choices, instead of surfacing the engine's late error. +func resolveSpeaker(voice, fallback string) (string, error) { + v := strings.TrimSpace(voice) + if v == "" { + v = strings.TrimSpace(fallback) + } + if v == "" { + return "", nil + } + if idx, err := strconv.Atoi(v); err == nil { + if idx < 0 || idx >= len(magpieSpeakers) { + return "", fmt.Errorf("magpie-tts: speaker index %d out of range 0-%d", idx, len(magpieSpeakers)-1) + } + return magpieSpeakers[idx], nil + } + for _, s := range magpieSpeakers { + if strings.EqualFold(s, v) { + return s, nil + } + } + return "", fmt.Errorf("magpie-tts: unknown voice %q (valid: %s, or 0-%d)", + voice, strings.Join(magpieSpeakers, ", "), len(magpieSpeakers)-1) +} + +// magpieLanguages are the canonical language codes the tokenizer's language +// map knows (exact-match on the C side), keyed by their lowercase form so +// requests can be case-insensitive. +var magpieLanguages = map[string]string{ + "en": "en", "es": "es", "de": "de", "fr": "fr", "it": "it", + "pt-br": "pt-BR", "hi": "hi", "vi": "vi", "ko": "ko", + "ar-ae": "ar-AE", "ar-sa": "ar-SA", "ar-msa": "ar-MSA", +} + +// resolveLanguage picks the request language, else the model-level default, +// else "" (the engine defaults to "en"), canonicalizing case for the known +// codes. Unknown codes pass through verbatim so the engine reports them with +// its own exact-vocabulary error. +func resolveLanguage(reqLang *string, fallback string) string { + l := "" + if reqLang != nil { + l = strings.TrimSpace(*reqLang) + } + if l == "" { + l = strings.TrimSpace(fallback) + } + if l == "" { + return "" + } + if canon, ok := magpieLanguages[strings.ToLower(l)]; ok { + return canon + } + return l +} diff --git a/backend/go/magpie-tts-cpp/package.sh b/backend/go/magpie-tts-cpp/package.sh new file mode 100755 index 000000000..06bd00112 --- /dev/null +++ b/backend/go/magpie-tts-cpp/package.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Script to copy the appropriate libraries based on architecture +# This script is used in the final stage of the Dockerfile + +set -e + +CURDIR=$(dirname "$(realpath $0)") +REPO_ROOT="${CURDIR}/../../.." + +# Create lib directory +mkdir -p $CURDIR/package/lib + +cp -avf $CURDIR/magpie-tts-cpp $CURDIR/package/ +cp -fv $CURDIR/libgomagpiettscpp-*.so $CURDIR/package/ 2>/dev/null || true +cp -fv $CURDIR/libgomagpiettscpp-*.dylib $CURDIR/package/ 2>/dev/null || true +cp -fv $CURDIR/run.sh $CURDIR/package/ + +# Detect architecture and copy appropriate libraries +if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then + # x86_64 architecture + echo "Detected x86_64 architecture, copying x86_64 libraries..." + cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so + cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then + # ARM64 architecture + echo "Detected ARM64 architecture, copying ARM64 libraries..." + cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so + cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6 + cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2 + cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1 + cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 +elif [ $(uname -s) = "Darwin" ]; then + echo "Detected Darwin" +else + echo "Error: Could not detect architecture" + exit 1 +fi + +# Package GPU libraries based on BUILD_TYPE +GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh" +if [ -f "$GPU_LIB_SCRIPT" ]; then + echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..." + source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib" + package_gpu_libs +fi + +echo "Packaging completed successfully" +ls -liah $CURDIR/package/ +ls -liah $CURDIR/package/lib/ diff --git a/backend/go/magpie-tts-cpp/run.sh b/backend/go/magpie-tts-cpp/run.sh new file mode 100755 index 000000000..841faa04d --- /dev/null +++ b/backend/go/magpie-tts-cpp/run.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -ex + +# Get the absolute current dir where the script is located +CURDIR=$(dirname "$(realpath "$0")") + +cd / + +echo "CPU info:" +if [ "$(uname)" != "Darwin" ]; then + grep -e "model\sname" /proc/cpuinfo | head -1 + grep -e "flags" /proc/cpuinfo | head -1 +fi + +if [ "$(uname)" = "Darwin" ]; then + # macOS: single dylib variant (Metal or Accelerate) + LIBRARY="$CURDIR/libgomagpiettscpp-fallback.dylib" + export DYLD_LIBRARY_PATH="$CURDIR"/lib:$DYLD_LIBRARY_PATH +else + LIBRARY="$CURDIR/libgomagpiettscpp-fallback.so" + + if grep -q -e "\savx\s" /proc/cpuinfo ; then + echo "CPU: AVX found OK" + if [ -e "$CURDIR"/libgomagpiettscpp-avx.so ]; then + LIBRARY="$CURDIR/libgomagpiettscpp-avx.so" + fi + fi + + if grep -q -e "\savx2\s" /proc/cpuinfo ; then + echo "CPU: AVX2 found OK" + if [ -e "$CURDIR"/libgomagpiettscpp-avx2.so ]; then + LIBRARY="$CURDIR/libgomagpiettscpp-avx2.so" + fi + fi + + # Check avx 512 + if grep -q -e "\savx512f\s" /proc/cpuinfo ; then + echo "CPU: AVX512F found OK" + if [ -e "$CURDIR"/libgomagpiettscpp-avx512.so ]; then + LIBRARY="$CURDIR/libgomagpiettscpp-avx512.so" + fi + fi + + export LD_LIBRARY_PATH="$CURDIR"/lib:$LD_LIBRARY_PATH +fi + +export MAGPIETTS_LIBRARY=$LIBRARY + +# If there is a lib/ld.so, use it +if [ -f "$CURDIR"/lib/ld.so ]; then + echo "Using lib/ld.so" + echo "Using library: $LIBRARY" + exec "$CURDIR"/lib/ld.so "$CURDIR"/magpie-tts-cpp "$@" +fi + +echo "Using library: $LIBRARY" +exec "$CURDIR"/magpie-tts-cpp "$@" diff --git a/backend/go/magpie-tts-cpp/test.sh b/backend/go/magpie-tts-cpp/test.sh new file mode 100755 index 000000000..9e4ec06d5 --- /dev/null +++ b/backend/go/magpie-tts-cpp/test.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +CURDIR=$(dirname "$(realpath $0)") +cd "$CURDIR" + +echo "Running magpie-tts-cpp backend tests..." + +# Auto-download the q8_0 GGUF only when MAGPIETTS_MODEL is not set. +if [ -z "$MAGPIETTS_MODEL" ]; then + MODEL_DIR="./magpie-tts-models" + mkdir -p "$MODEL_DIR" + REPO_ID="mudler/magpie-tts.cpp-gguf" + BASE_URL="https://huggingface.co/${REPO_ID}/resolve/main" + FILE="magpie-tts-multilingual-357m-q8_0.gguf" + dest="${MODEL_DIR}/${FILE}" + if [ -f "${dest}" ]; then + echo " [skip] ${FILE}" + else + echo " [download] ${FILE}..." + curl -L -o "${dest}" "${BASE_URL}/${FILE}" --progress-bar + fi + export MAGPIETTS_MODEL="${dest}" +fi + +go test -v -timeout 1200s . + +echo "All magpie-tts-cpp tests passed." diff --git a/backend/index.yaml b/backend/index.yaml index 5d6eb8d50..bf898e40e 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -1171,6 +1171,48 @@ nvidia-l4t: "nvidia-l4t-arm64-moss-tts-cpp-development" nvidia-l4t-cuda-12: "nvidia-l4t-arm64-moss-tts-cpp-development" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-moss-tts-cpp-development" +- &magpiettscpp + name: "magpie-tts-cpp" + description: | + Magpie TTS C++ backend using GGML (magpie-tts.cpp). Native C++ + text-to-speech for NVIDIA's Magpie TTS Multilingual 357M model (encoder + + autoregressive decoder over NanoCodec tokens), running from a single + self-contained GGUF (model, codec, tokenizer, G2P dictionaries) with no + Python at inference time. 22.05kHz mono output, 5 baked voices (Aria, + Jason, John, Leo, Sofia), 9+ languages. + urls: + - https://github.com/mudler/magpie-tts.cpp + - https://huggingface.co/mudler/magpie-tts.cpp-gguf + tags: + - text-to-speech + - tts + alias: "magpie-tts-cpp" + capabilities: + default: "cpu-magpie-tts-cpp" + nvidia: "cuda12-magpie-tts-cpp" + nvidia-cuda-13: "cuda13-magpie-tts-cpp" + nvidia-cuda-12: "cuda12-magpie-tts-cpp" + intel: "intel-sycl-f16-magpie-tts-cpp" + metal: "metal-magpie-tts-cpp" + amd: "rocm-magpie-tts-cpp" + vulkan: "vulkan-magpie-tts-cpp" + nvidia-l4t: "nvidia-l4t-arm64-magpie-tts-cpp" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-magpie-tts-cpp" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-magpie-tts-cpp" +- !!merge <<: *magpiettscpp + name: "magpie-tts-cpp-development" + capabilities: + default: "cpu-magpie-tts-cpp-development" + nvidia: "cuda12-magpie-tts-cpp-development" + nvidia-cuda-13: "cuda13-magpie-tts-cpp-development" + nvidia-cuda-12: "cuda12-magpie-tts-cpp-development" + intel: "intel-sycl-f16-magpie-tts-cpp-development" + metal: "metal-magpie-tts-cpp-development" + amd: "rocm-magpie-tts-cpp-development" + vulkan: "vulkan-magpie-tts-cpp-development" + nvidia-l4t: "nvidia-l4t-arm64-magpie-tts-cpp-development" + nvidia-l4t-cuda-12: "nvidia-l4t-arm64-magpie-tts-cpp-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-magpie-tts-cpp-development" - &omnivoicecpp name: "omnivoice-cpp" description: | @@ -4754,6 +4796,107 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-moss-tts-cpp" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-13-moss-tts-cpp +## magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "nvidia-l4t-arm64-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-arm64-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "nvidia-l4t-arm64-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-nvidia-l4t-arm64-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cuda13-nvidia-l4t-arm64-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cuda13-nvidia-l4t-arm64-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cpu-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-cpu-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "metal-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "metal-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cpu-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-cpu-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cuda12-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "rocm-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "intel-sycl-f32-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-gpu-intel-sycl-f32-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "intel-sycl-f16-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-gpu-intel-sycl-f16-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "vulkan-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-gpu-vulkan-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "vulkan-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-gpu-vulkan-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cuda12-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "rocm-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "intel-sycl-f32-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-gpu-intel-sycl-f32-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "intel-sycl-f16-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-gpu-intel-sycl-f16-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cuda13-magpie-tts-cpp" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-magpie-tts-cpp" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-magpie-tts-cpp +- !!merge <<: *magpiettscpp + name: "cuda13-magpie-tts-cpp-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-magpie-tts-cpp" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-magpie-tts-cpp ## omnivoice-cpp - !!merge <<: *omnivoicecpp name: "omnivoice-cpp-development" diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 6605bcf39..79687c302 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -466,6 +466,12 @@ var BackendCapabilities = map[string]BackendCapability{ VoiceCloning: referenceVoiceCloning(), Description: "Qwen3 TTS C++ - text-to-speech with streaming, named speakers, voice design and cloning (qwentts.cpp / GGML)", }, + "magpie-tts-cpp": { + GRPCMethods: []GRPCMethod{MethodTTS, MethodTTSStream}, + PossibleUsecases: []string{UsecaseTTS}, + DefaultUsecases: []string{UsecaseTTS}, + Description: "Magpie TTS C++ - NVIDIA Magpie TTS Multilingual 357M with 5 baked voices and 9+ languages (magpie-tts.cpp / GGML)", + }, "faster-qwen3-tts": { GRPCMethods: []GRPCMethod{MethodTTS}, PossibleUsecases: []string{UsecaseTTS}, diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index f699c7d3b..286ba27c9 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -41,6 +41,7 @@ var knownPrefOnlyBackends = []schema.KnownBackend{ {Name: "kokoros", Modality: "tts", AutoDetect: false, Description: "Kokoros TTS (preference-only)"}, {Name: "qwen-tts", Modality: "tts", AutoDetect: false, Description: "Qwen TTS (preference-only)"}, {Name: "qwen3-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Qwen3 TTS C++ (preference-only)"}, + {Name: "magpie-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Magpie TTS Multilingual C++ (preference-only)"}, {Name: "omnivoice-cpp", Modality: "tts", AutoDetect: false, Description: "OmniVoice C++ TTS with voice cloning and voice design (preference-only)"}, {Name: "faster-qwen3-tts", Modality: "tts", AutoDetect: false, Description: "Faster Qwen3 TTS (preference-only)"}, {Name: "supertonic", Modality: "tts", AutoDetect: false, Description: "Supertonic multilingual ONNX TTS (preference-only)"}, diff --git a/core/http/endpoints/localai/backend_test.go b/core/http/endpoints/localai/backend_test.go index 996732fb8..dc6d39bea 100644 --- a/core/http/endpoints/localai/backend_test.go +++ b/core/http/endpoints/localai/backend_test.go @@ -157,6 +157,7 @@ var _ = Describe("Backend Endpoints", func() { expectPrefOnly("kokoros", "tts") expectPrefOnly("qwen-tts", "tts") expectPrefOnly("qwen3-tts-cpp", "tts") + expectPrefOnly("magpie-tts-cpp", "tts") expectPrefOnly("faster-qwen3-tts", "tts") expectPrefOnly("supertonic", "tts") expectPrefOnly("sam3-cpp", "detection") diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index e7e9a2c7d..47ee22c6c 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -60,6 +60,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [VibeVoice](https://github.com/microsoft/VibeVoice) | Real-time TTS with voice cloning | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T | | [vibevoice.cpp](https://github.com/mudler/vibevoice.cpp) | Native C++/GGML port of VibeVoice for TTS (voice cloning) and long-form ASR with diarization | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | | [moss-tts.cpp](https://github.com/mudler/moss-tts.cpp) | Native C++/GGML port of OpenMOSS MOSS-TTS-Local v1.5: 48 kHz stereo TTS with reference-audio voice cloning | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | +| [magpie-tts.cpp](https://github.com/mudler/magpie-tts.cpp) | Native C++/GGML port of NVIDIA Magpie TTS Multilingual 357M: 22.05 kHz mono TTS, 5 baked voices, 9+ languages | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | | [Qwen3-TTS](https://github.com/QwenLM/Qwen3-TTS) | TTS with custom voice, voice design, and voice cloning | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T | | [qwentts.cpp](https://github.com/ServeurpersoCom/qwentts.cpp) | Native C++/GGML Qwen3-TTS with streaming, named speakers, and voice design | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | | [OmniVoice](https://github.com/ServeurpersoCom/omnivoice.cpp) | Native C++/GGML TTS with voice cloning, voice design, and streaming | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | diff --git a/gallery/index.yaml b/gallery/index.yaml index 4f921b9df..1c710a7c3 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -6699,6 +6699,60 @@ - filename: moss-tts-cpp-v1_5-f16/moss-tokenizer-v1_5.gguf sha256: d0f4f371f80ff32ba222b1b39c192df8285c7b88986a55f954454db69618479e uri: huggingface://mudler/MOSS-TTS-Local-Transformer-v1.5-GGUF/moss-tokenizer-v1_5.gguf +- &magpiettscpp_gallery + name: magpie-tts-cpp-357m-q8_0 + variants: + - model: magpie-tts-cpp-357m-f16 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/magpie-tts.cpp-gguf + - https://github.com/mudler/magpie-tts.cpp + - https://huggingface.co/nvidia/magpie_tts_multilingual_357m + description: | + Magpie TTS Multilingual 357M (C++/ggml, magpie-tts.cpp), Q8_0 (~624 MB), near-lossless + and the fastest decode. Native C++ text-to-speech for NVIDIA's Magpie TTS Multilingual + (encoder + autoregressive decoder over NanoCodec tokens) from one self-contained GGUF + (model, codec, tokenizer, G2P dictionaries). 22.05kHz mono, 5 baked voices (set `voice` + to Aria, Jason, John, Leo, Sofia or 0-4), 9+ languages (en, es, de, fr, it, pt-BR, hi, + vi, ko, plus Arabic variants). Parity-gated per component against the NeMo reference. + license: other + tags: + - magpie-tts + - magpie-tts-cpp + - tts + - text-to-speech + - multilingual + - gguf + - ggml + last_checked: "2026-07-24" + overrides: + backend: magpie-tts-cpp + known_usecases: + - tts + name: magpie-tts-cpp-357m-q8_0 + parameters: + model: magpie-tts-cpp-357m-q8_0/magpie-tts-multilingual-357m-q8_0.gguf + files: + - filename: magpie-tts-cpp-357m-q8_0/magpie-tts-multilingual-357m-q8_0.gguf + sha256: 0f26f552c82b45a176bd7b7de261581821d5e81acc99cedb4450fd20a10ad259 + uri: huggingface://mudler/magpie-tts.cpp-gguf/magpie-tts-multilingual-357m-q8_0.gguf +- !!merge <<: *magpiettscpp_gallery + name: magpie-tts-cpp-357m-f16 + variants: [] + description: | + Magpie TTS Multilingual 357M (C++/ggml, magpie-tts.cpp), F16 (~784 MB). 22.05kHz mono + text-to-speech with 5 baked voices and 9+ languages. + overrides: + backend: magpie-tts-cpp + known_usecases: + - tts + name: magpie-tts-cpp-357m-f16 + parameters: + model: magpie-tts-cpp-357m-f16/magpie-tts-multilingual-357m-f16.gguf + files: + - filename: magpie-tts-cpp-357m-f16/magpie-tts-multilingual-357m-f16.gguf + sha256: cf1782f26703cac58b97ed453e42320e974a13be7b592f83be14e58f955b97ec + uri: huggingface://mudler/magpie-tts.cpp-gguf/magpie-tts-multilingual-357m-f16.gguf - name: omnivoice-cpp variants: - model: omnivoice-cpp-hq diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs index 1054b0d2f..50edd1726 100644 --- a/scripts/lib/backend-filter.mjs +++ b/scripts/lib/backend-filter.mjs @@ -42,6 +42,14 @@ export function inferBackendPath(item) { if (item.backend === "moss-tts-cpp") { return `backend/go/moss-tts-cpp/`; } + // magpie-tts-cpp is a Go backend (Dockerfile.golang) wrapping the + // magpie-tts.cpp ggml port via purego, living in backend/go/magpie-tts-cpp/. + // Same explicit-branch rationale as its siblings above: the generic golang + // fallthrough would also resolve it, but this documents the mapping and + // guards a future dockerfile-suffix change. + if (item.backend === "magpie-tts-cpp") { + return `backend/go/magpie-tts-cpp/`; + } if (item.dockerfile.endsWith("golang")) { return `backend/go/${item.backend}/`; }