diff --git a/backend/cpp/audio-cpp/family_gate.cpp b/backend/cpp/audio-cpp/family_gate.cpp index 83ccd42fa..2515c7abb 100644 --- a/backend/cpp/audio-cpp/family_gate.cpp +++ b/backend/cpp/audio-cpp/family_gate.cpp @@ -73,9 +73,12 @@ namespace { // supertonic: upstream's docs/gguf.md:90 records its 16-bit GGUF column as // "---", i.e. NOT TESTED, and its q8_0 as "No (unsupported weight dtype)". Only // the `orig` package is marked Pass, and its 698 weight tensors are f32 while -// its 72 index and shape constants are i64. Attributed rather than assumed: the -// abort is identical through the unary TTS RPC and through TTSStream, so it is -// the packaging and not the streaming path. +// its 72 index and shape constants are i64. The f16 abort is a LOCAL +// OBSERVATION rather than an upstream claim, and it is attributed rather than +// assumed: it is identical through the unary TTS RPC and through TTSStream, so +// it is the packaging and not the streaming path. q8_0 was never run here and is +// refused on upstream's "unsupported weight dtype" alone, which is the weaker of +// the two claims. See the header for why keeping them apart matters. // // TO REMOVE AN ENTRY: bump AUDIO_CPP_VERSION past a fix, load a package in the // refused dtype, and synthesise. If audio comes out, delete the entry. No test diff --git a/backend/cpp/audio-cpp/family_gate.h b/backend/cpp/audio-cpp/family_gate.h index 7e8210fd2..e516ca54b 100644 --- a/backend/cpp/audio-cpp/family_gate.h +++ b/backend/cpp/audio-cpp/family_gate.h @@ -36,12 +36,27 @@ FamilyDecision decide_family(bool path_is_gguf, const std::string &embedded_fami // string a TensorMetadata carries ("f32", "f16", "q8_0", "i64", ...). // // This is a LIST OF FAMILIES THAT CRASH THE PROCESS, not a list of families that -// perform badly. It exists because the failure is not an exception: loading a -// supertonic package whose weights are f16 or q8_0 reaches ggml_concat with one -// f16 operand and one f32 one, and ggml_abort takes the backend down with -// SIGABRT on the FIRST request. Nothing upstream of the load can catch that, so -// an operator sees a model that loaded successfully and a backend that dies on -// every request with no status and no message. +// perform badly. It exists because the failure is not an exception: loading the +// supertonic f16 GGUF package reaches ggml_concat with one f16 operand and one +// f32 one, GGML_ASSERT(a->type == b->type) fails (external/ggml/src/ggml.c:2595) +// and ggml_abort takes the backend down with SIGABRT on the FIRST request. +// Nothing upstream of the load can catch that, so an operator sees a model that +// loaded successfully and a backend that dies on every request with no status +// and no message. +// +// EVIDENCE, per dtype, because the two are not equally attested: +// - f16 was OBSERVED to abort here, identically through the unary TTS RPC and +// through TTSStream, so it is the packaging and not the streaming path. +// Upstream's docs/gguf.md:90 has supertonic's 16-bit column as "---", which +// its own legend (:53) defines as not tested, so upstream neither confirms +// nor contradicts it. +// - q8_0 was NOT run here. Upstream records it as "No (unsupported weight +// dtype)" in the same row, which is a weaker claim than the f16 abort: it +// says the format is unusable, not that it takes the process down. +// Both are refused, because the allow list is what the family CAN run (f32 for +// weights, i64 for the shape and index constants) rather than a list of the +// dtypes that fail, and a format upstream calls unusable has no business being +// loaded either way. // // A family with no entry is unrestricted, which is every family but one. // diff --git a/backend/cpp/audio-cpp/streaming_driver_ctest.cpp b/backend/cpp/audio-cpp/streaming_driver_ctest.cpp index 95b49c0ef..e25616419 100644 --- a/backend/cpp/audio-cpp/streaming_driver_ctest.cpp +++ b/backend/cpp/audio-cpp/streaming_driver_ctest.cpp @@ -1,5 +1,6 @@ // Tests for the streaming drivers in loaded_model: begin_stream, -// run_streaming_pull, run_streaming_audio and run_streaming_live. +// run_streaming_pull, run_streaming_audio and run_streaming_live, plus +// resolve_model_path, which lives in the same engine-linked unit. // // Engine-linked, so this runs through ctest rather than // backend/cpp/run-unit-tests.sh. It builds no model and loads no file: a @@ -865,7 +866,83 @@ static void test_live_refuses_a_non_streaming_session() { "run_streaming_live refuses a session with no streaming half"); } +// -------------------------------------------------------------------------- +// resolve_model_path +// -------------------------------------------------------------------------- +// +// It lives in loaded_model.cpp and is a pure (dir, file, name) -> string, so it +// is tested here rather than in a standalone unit: that file cannot compile +// without the engine headers. +// +// It is tested at all because it shipped a bug no test could have caught. THE +// SHAPES BELOW ARE THE PRODUCTION SHAPES, not convenient ones, and that +// distinction is the entire point. Task 15 verified the bundled: form with a +// hand-written LoadModel that left ModelFile empty, which is the one shape the +// server never produces: pkg/model/loader.go's LoadModelWithFile always fills +// ModelFile with filepath.Join(ModelPath, model), and core/backend/options.go +// only overrides it for a managed artifact. The first case below is therefore +// the regression test; the other three are what it must not have broken. + +// std::string::ends_with is C++20 and this target is C++17. +static bool ends_with(const std::string &value, const std::string &suffix) { + return value.size() >= suffix.size() && + value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +// THE REGRESSION CASE. What a model YAML saying `model: bundled:silero_vad` +// actually arrives as: Model intact, ModelFile joined onto the models directory. +static void test_bundled_in_model_survives_a_joined_model_file() { + const std::string resolved = audiocpp_backend::resolve_model_path( + "/models", "/models/bundled:silero_vad", "bundled:silero_vad"); + + check(ends_with(resolved, "/assets/silero_vad"), + "bundled: in Model resolves under the package assets dir (got \"" + + resolved + "\")"); + // Checked separately from the suffix because this is the failure that + // shipped: the joined ModelFile came back verbatim and the load died on + // "model path does not exist: /models/bundled:silero_vad". + check(resolved.find("/models/") == std::string::npos, + "bundled: in Model is not resolved against the models directory (got \"" + + resolved + "\")"); +} + +// Task 15's shape: the form in ModelFile with Model empty. It worked before the +// fix and must keep working. +static void test_bundled_in_model_file_still_resolves() { + const std::string resolved = + audiocpp_backend::resolve_model_path("", "bundled:marblenet_vad", ""); + + check(ends_with(resolved, "/assets/marblenet_vad"), + "bundled: in ModelFile still resolves under the package assets dir (got \"" + + resolved + "\")"); +} + +// The ordinary case, and the one the bundled: lookup must not capture: a real +// artifact path in ModelFile with a plain name in Model. +static void test_a_plain_name_resolves_to_the_model_file() { + const std::string resolved = audiocpp_backend::resolve_model_path( + "/models", "/models/chatterbox-q8_0.gguf", "chatterbox-q8_0.gguf"); + + check_eq(resolved, "/models/chatterbox-q8_0.gguf", + "a plain name resolves to the absolute ModelFile"); +} + +// A relative ModelFile is still joined onto ModelPath. The fix does not touch +// this branch, which is why it is pinned: the bundled: lookup now runs before it +// and has to fall through for every non-bundled input. +static void test_a_relative_model_file_joins_the_model_path() { + const std::string resolved = audiocpp_backend::resolve_model_path( + "/models", "sub/nemotron-asr-q8_0.gguf", "nemotron-asr"); + + check_eq(resolved, "/models/sub/nemotron-asr-q8_0.gguf", + "a relative ModelFile joins the models directory"); +} + int main() { + test_bundled_in_model_survives_a_joined_model_file(); + test_bundled_in_model_file_still_resolves(); + test_a_plain_name_resolves_to_the_model_file(); + test_a_relative_model_file_joins_the_model_path(); test_begin_stream_prepares_then_starts(); test_base_start_stream_resets(); test_begin_stream_refuses_a_non_streaming_session(); diff --git a/core/gallery/importers/importers_test.go b/core/gallery/importers/importers_test.go index 70da1d01e..9d0ea89c3 100644 --- a/core/gallery/importers/importers_test.go +++ b/core/gallery/importers/importers_test.go @@ -466,6 +466,14 @@ var _ = Describe("audio-cpp importer registration", func() { // GGUF repo hosts 30-odd families in one place, so repo-level matching // would be a coin flip. An importer that matched .gguf would additionally // capture every llama.cpp repo it saw first. + // + // THIS IS A TRIPWIRE, NOT COVERAGE. It exercises no audio-cpp behaviour and + // cannot go red for anything but the one act it is aimed at: someone adding + // an AudioCpp*Importer to the registry, which would silently turn the + // backend into an auto-detect candidate for every GGUF repo. Do not read a + // green here as the registration being tested; that assertion lives in + // core/http/endpoints/localai/backend_test.go, against the /backends/known + // payload that actually reaches the import form. It("registers no importer that would auto-match GGUF repositories", func() { for _, importer := range importers.Registry() { Expect(fmt.Sprintf("%T", importer)).ToNot(ContainSubstring("AudioCpp"), diff --git a/docs/content/features/audio-cpp.md b/docs/content/features/audio-cpp.md index 5204b7ab6..edeb76aee 100644 --- a/docs/content/features/audio-cpp.md +++ b/docs/content/features/audio-cpp.md @@ -160,7 +160,8 @@ Then call it like any other VAD model, as described in ## Source separation stems -Separation families such as `demucs` and `roformer` produce several named outputs from one +Separation families such as `htdemucs` and `mel_band_roformer` produce several named +outputs from one run. Running inference once per stem would cost a full separation each time, so the backend runs **once**, writes every stem to a sibling file next to the main output, and puts the selected one in the response body. `params[stem]` picks which one: @@ -181,9 +182,13 @@ ignored. ## Family notes -- **Supertonic**: use the `orig` GGUF package. Its f16 and q8_0 packages mix weight dtypes - in a way that reaches `ggml_abort` and takes the backend process down rather than - returning an error, so the backend refuses those dtypes at load time with a clear message. +- **Supertonic**: use the `orig` GGUF package, whose weights are f32. The f16 package was + observed to reach `ggml_concat` with mismatched operand types and take the backend + process down with `SIGABRT` on the first request, rather than returning an error. + Upstream's `docs/gguf.md` leaves supertonic's 16-bit column untested and records its + q8_0 as "No (unsupported weight dtype)", which says the format is unusable rather than + that it crashes. The backend refuses both at load time with a message naming the + offending tensor, so neither reaches that abort. - **Live transcription latency**: `AudioTranscriptionLive` needs a family advertising `asr` in streaming mode. At the pinned upstream revision that is `nemotron_asr`, `higgs_audio_stt` and `voxtral_realtime`. `higgs_audio_stt` and `voxtral_realtime` decode diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 3c285bdc9..6f315ffbc 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -14,7 +14,7 @@ The transcription endpoint allows to convert audio files to text. The endpoint s - **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). - **llama-cpp**: Route transcription to any multimodal-audio GGUF model served by the `llama-cpp` backend (e.g. [Qwen3-ASR](https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF), Voxtral, Qwen2-Audio). Under the hood the request is converted into a chat completion with the audio attached via the model's audio encoder - the same path the upstream llama.cpp server uses. Set `backend: llama-cpp` in the model YAML and point `mmproj` at the matching audio encoder. - **voxtral**: Voxtral-family models served by a dedicated backend -- **[audio.cpp](https://github.com/0xShug0/audio.cpp)**: Multi-family GGML audio engine. Serves transcription and forced alignment from families such as nemotron, qwen3-asr, citrinet, higgs-audio-stt and voxtral-realtime, and covers diarization, VAD, TTS and source separation from the same backend. See the [audio.cpp backend]({{%relref "features/audio-cpp" %}}) page for the model options. +- **[audio.cpp](https://github.com/0xShug0/audio.cpp)**: Multi-family GGML audio engine. Serves transcription and forced alignment from families such as `nemotron_asr`, `qwen3_asr`, `citrinet_asr`, `higgs_audio_stt` and `voxtral_realtime`, and covers diarization, VAD, TTS and source separation from the same backend. See the [audio.cpp backend]({{%relref "features/audio-cpp" %}}) page for the model options. The endpoint input supports all the audio formats supported by `ffmpeg`. diff --git a/docs/content/features/audio-transform.md b/docs/content/features/audio-transform.md index 4b3b5a88a..4f2a32dc4 100644 --- a/docs/content/features/audio-transform.md +++ b/docs/content/features/audio-transform.md @@ -19,9 +19,10 @@ a 1.3 M-parameter GGML-based model that performs joint AEC + noise suppression is a derivative of the Microsoft DeepVQE paper. Source separation and voice conversion are served by the -[audio.cpp backend]({{%relref "features/audio-cpp" %}}): its `demucs` and -`roformer` families produce the named stems described below, and `seed_vc`, -`vevo2` and `miocodec` do voice conversion against a reference speaker. +[audio.cpp backend]({{%relref "features/audio-cpp" %}}): its `htdemucs` and +`mel_band_roformer` families produce the named stems described below, and +`seed_vc`, `vevo2` and `miocodec` do voice conversion against a reference +speaker. ## The mental model diff --git a/docs/content/features/text-to-audio.md b/docs/content/features/text-to-audio.md index ae428b041..a2485f3bf 100644 --- a/docs/content/features/text-to-audio.md +++ b/docs/content/features/text-to-audio.md @@ -750,7 +750,7 @@ curl -L http://localhost:8080/tts \ [audio.cpp](https://github.com/0xShug0/audio.cpp) is a multi-family GGML audio engine, so one installed backend covers TTS (`supertonic`, `vibevoice`, `voxcpm2`, `fish_audio`, `pocket_tts`, `omnivoice`, `higgs_audio_tts`), voice cloning (`chatterbox`, `index_tts2`, -`irodori_tts`, `moss`) and voice design (`qwen3_tts`, `irodori_tts`), alongside ASR, VAD, +`irodori_tts`, `moss_tts_local`, `moss_tts_nano`) and voice design (`qwen3_tts`, `irodori_tts`), alongside ASR, VAD, diarization and separation. ```yaml diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index 7eab3ab81..de1cd3302 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -48,7 +48,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [Qwen3-ASR](https://github.com/QwenLM/Qwen3-ASR) | Qwen3 automatic speech recognition | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T | | [NeMo](https://github.com/NVIDIA/NeMo) | NVIDIA NeMo ASR toolkit | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal | | [sherpa-onnx](https://k2-fsa.github.io/sherpa/onnx/) | Sherpa-ONNX ASR (Whisper, Paraformer, SenseVoice) and TTS | CPU, CUDA 12, Metal | -| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: transcription, forced alignment and speaker diarization (nemotron, qwen3-asr, voxtral-realtime, sortformer and more). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: transcription, forced alignment and speaker diarization (`nemotron_asr`, `qwen3_asr`, `voxtral_realtime`, `sortformer_diar` and more). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Text-to-Speech @@ -76,7 +76,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [Supertonic](https://github.com/supertone-inc/supertonic) | Lightning-fast on-device multilingual TTS via ONNX | CPU | | [MLX-Audio](https://github.com/Blaizzy/mlx-audio) | Audio models on Apple Silicon | CPU, CUDA 12/13, Metal, Jetson L4T | | [liquid-audio](https://github.com/Liquid4All/liquid-audio) | LFM2 end-to-end speech-to-speech, ASR, and TTS | CPU, CUDA 12/13, ROCm, Intel SYCL, Jetson L4T | -| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: TTS, voice cloning and voice design (chatterbox, supertonic, qwen3-tts, vibevoice and more). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: TTS, voice cloning and voice design (`supertonic`, `vibevoice`, `qwen3_tts`, `chatterbox` and more). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Music & Sound Generation @@ -84,7 +84,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ |---------|-------------|-------------| | [ACE-Step](https://github.com/ace-step/ACE-Step-1.5) | Music generation from text descriptions, lyrics, or audio | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal | | [acestep.cpp](https://github.com/ace-step/acestep.cpp) | ACE-Step 1.5 C++ backend using GGML | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | -| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: music and sound generation (ace-step, stable-audio, heartmula). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: music and sound generation (`ace_step`, `stable_audio`, `heartmula`). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Image & Video Generation @@ -115,7 +115,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [Silero VAD](https://github.com/snakers4/silero-vad) | Voice Activity Detection | CPU, Metal | | [LocalVQE](https://github.com/localai-org/LocalVQE) | Joint acoustic echo cancellation, noise suppression, and dereverberation in C/C++ using GGML | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Jetson L4T | | [Opus](https://opus-codec.org/) | Audio codec for WebRTC / Realtime API | CPU, Metal | -| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: voice activity detection (bundled silero-vad and marblenet-vad, no download), source separation into named stems, and voice conversion. See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: voice activity detection (bundled `silero_vad` and `marblenet_vad`, no download), source separation into named stems, and voice conversion. See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Utilities & Other