diff --git a/core/gallery/importers/importers_test.go b/core/gallery/importers/importers_test.go index 7e34b7b3e..70da1d01e 100644 --- a/core/gallery/importers/importers_test.go +++ b/core/gallery/importers/importers_test.go @@ -458,3 +458,18 @@ invalid: yaml: content: [unclosed bracket }) }) }) + +var _ = Describe("audio-cpp importer registration", func() { + // audio-cpp stays preference-only on purpose. Its only reliable signal is + // the audiocpp.model_spec.family key embedded INSIDE the GGUF, which an + // importer cannot read from a remote HuggingFace repo, and the upstream + // GGUF repo hosts 30-odd families in one place, so repo-level matching + // would be a coin flip. An importer that matched .gguf would additionally + // capture every llama.cpp repo it saw first. + It("registers no importer that would auto-match GGUF repositories", func() { + for _, importer := range importers.Registry() { + Expect(fmt.Sprintf("%T", importer)).ToNot(ContainSubstring("AudioCpp"), + "audio-cpp must stay preference-only; a GGUF auto-matcher would capture llama.cpp repos") + } + }) +}) diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index 286ba27c9..3d4a25b33 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -45,6 +45,16 @@ var knownPrefOnlyBackends = []schema.KnownBackend{ {Name: "omnivoice-cpp", Modality: "tts", AutoDetect: false, Description: "OmniVoice C++ TTS with voice cloning and voice design (preference-only)"}, {Name: "faster-qwen3-tts", Modality: "tts", AutoDetect: false, Description: "Faster Qwen3 TTS (preference-only)"}, {Name: "supertonic", Modality: "tts", AutoDetect: false, Description: "Supertonic multilingual ONNX TTS (preference-only)"}, + // audio-cpp spans far more than TTS: ASR, forced alignment, VAD, speaker + // diarization, source separation, voice conversion and music generation. + // KnownBackend.Modality is a single string and the import form only chips + // on a fixed key set (core/http/react-ui/src/components/ModalityChips.jsx), + // so the extra modalities live in the description rather than in an + // invented modality key the UI would bucket as "other". + // No importer: the only reliable signal, the audiocpp.model_spec.family + // GGUF metadata key, is not readable from a remote HuggingFace repo, and + // audio-cpp/audio.cpp-gguf hosts 30-odd families in one repository. + {Name: "audio-cpp", Modality: "tts", AutoDetect: false, Description: "audio.cpp multi-family audio engine: TTS, voice cloning, ASR, alignment, VAD, diarization, separation, music generation (preference-only)"}, // Detection {Name: "sam3-cpp", Modality: "detection", AutoDetect: false, Description: "SAM3 C++ object detection (preference-only)"}, // Audio transform (audio-in / audio-out, optional reference signal) diff --git a/core/http/endpoints/localai/backend_test.go b/core/http/endpoints/localai/backend_test.go index dc6d39bea..3c43fb60c 100644 --- a/core/http/endpoints/localai/backend_test.go +++ b/core/http/endpoints/localai/backend_test.go @@ -268,6 +268,34 @@ var _ = Describe("Backend Endpoints", func() { Expect(entry.AutoDetect).To(BeFalse()) }) + It("advertises audio-cpp as a preference-only backend naming its other modalities", func() { + req := httptest.NewRequest(http.MethodGet, "/backends/known", nil) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + + var payload []schema.KnownBackend + Expect(json.Unmarshal(rec.Body.Bytes(), &payload)).To(Succeed()) + + byName := map[string]schema.KnownBackend{} + for _, b := range payload { + byName[b.Name] = b + } + + entry, ok := byName["audio-cpp"] + Expect(ok).To(BeTrue(), "audio-cpp must appear in the import form dropdown") + // AutoDetect=false is the honest answer, not an omission: the only + // reliable signal is the audiocpp.model_spec.family key inside the + // GGUF, which no remote-repo probe can read. + Expect(entry.AutoDetect).To(BeFalse(), + "audio-cpp has no remote-detectable signal: the family lives inside the GGUF") + // Modality is a single string and the import form chips on a fixed + // key set, so the modalities it cannot carry have to be named in + // the description instead. + Expect(entry.Modality).To(Equal("tts")) + Expect(entry.Description).To(ContainSubstring("ASR"), + "the description must name the modalities the single Modality field cannot") + }) + It("is sorted by Modality then Name", func() { req := httptest.NewRequest(http.MethodGet, "/backends/known", nil) rec := httptest.NewRecorder() diff --git a/docs/content/features/audio-cpp.md b/docs/content/features/audio-cpp.md new file mode 100644 index 000000000..5204b7ab6 --- /dev/null +++ b/docs/content/features/audio-cpp.md @@ -0,0 +1,212 @@ ++++ +disableToc = false +title = "audio.cpp backend" +weight = 38 +url = "/features/audio-cpp/" ++++ + +[audio.cpp](https://github.com/0xShug0/audio.cpp) is a GGML audio inference framework +covering roughly thirty model families in one engine. LocalAI exposes it through the +native C++ `audio-cpp` backend, so a single installed backend can serve text to speech, +voice cloning, transcription, forced alignment, voice activity detection, speaker +diarization, source separation, voice conversion and music generation, depending only on +which model you point it at. + +audio.cpp is developed by ShugoAI LLC, not by the LocalAI project. Model packages live in +[huggingface.co/audio-cpp/audio.cpp-gguf](https://huggingface.co/audio-cpp/audio.cpp-gguf). + +## Installing + +```bash +local-ai backends install audio-cpp +``` + +Or install it from the **Backends** page in the web UI. `audio-cpp` is a +preference-only backend: LocalAI never picks it automatically during model import, +because the only reliable signal for an audio.cpp model is the `audiocpp.model_spec.family` +key stored *inside* the GGUF, which cannot be read from a remote repository, and the +upstream GGUF repository hosts every family in one place. Set `backend: audio-cpp` +in the model YAML, or select it explicitly in the import form. + +## What it serves + +One model serves one family, and a family advertises the tasks it can perform. The +backend routes each LocalAI endpoint onto an audio.cpp task and refuses, with the +family's real capability list in the message, when the loaded model cannot serve it. + +| LocalAI endpoint | gRPC RPC | audio.cpp tasks tried | +|---|---|---| +| `/v1/audio/speech`, `/tts` | `TTS`, `TTSStream` | `tts`, `clon` (voice cloning), `vdes` (voice design) | +| `/v1/audio/transcriptions` | `AudioTranscription`, `AudioTranscriptionStream` | `asr`, `align` (forced alignment, when a prompt is supplied) | +| `/v1/realtime` live transcription | `AudioTranscriptionLive` | `asr` in streaming mode | +| `/v1/vad` | `VAD` | `vad` | +| `/v1/audio/diarization` | `Diarize` | `diar` | +| `/v1/sound-generation` | `SoundGeneration` | `gen` | +| `/audio/transform`, `/audio/transformations` | `AudioTransform` | `sep` (separation), `vc` (voice conversion), `svc`, `s2s` | + +A supplied speaker clip routes TTS to `clon`; free-form instructions route it to `vdes`. +Singing voice conversion (`svc`) is never chosen automatically, because no request signal +means "this input is singing"; pin it with `task:svc`. + +### Not served, and why + +These RPCs return `UNIMPLEMENTED` with an explanation rather than a degraded answer: + +- **`AudioEncode` / `AudioDecode`**: audio.cpp has no codec task kind, so no family can be + asked to turn PCM into codec frames or back. +- **`AudioTransformStream`**: no family advertises streaming for any task + `AudioTransform` routes to (`sep`, `vc`, `svc`, `s2s`). Upstream advertises streaming for + `tts`, `asr` and `vad` only, so a streaming transform here would be a buffered offline + call in disguise. +- **`AudioToAudioStream`**: LocalAI's contract is an OpenAI-Realtime shaped audio + conversation with a system prompt and a tool loop. audio.cpp's `s2s` is offline + clip-to-clip processing against a target voice. +- **`VoiceEmbed`**: no family advertises the `spk` (speaker recognition) task, so nothing in + the engine can produce a speaker embedding. Use the + [voice recognition]({{%relref "features/voice-recognition" %}}) backends instead. + +## Model YAML + +None of the option names are guessable, so here is the full shape: + +```yaml +name: supertonic +backend: audio-cpp +parameters: + model: supertonic-3-orig.gguf +known_usecases: + - FLAG_TTS +options: + # Optional for an audio.cpp GGUF: the family is read from the file's + # audiocpp.model_spec.family metadata. REQUIRED for a safetensors file or a + # package directory, which carry no such key. + - family:supertonic + # Optional: pins the audio.cpp task instead of routing it from the RPC. + - task:tts + # ggml compute backend inside audio.cpp (not the LocalAI backend above). + - backend:cuda + - device:0 + - threads:8 + # Give up on a queued request instead of piling up behind a wedged run. + - busy_timeout_ms:120000 + # Per-family knobs, passed through to the upstream loader and session. + - load.vibevoice.lora:/models/my-adapter + - session.seed_vc.weight_type:q8_0 +``` + +Every entry is `key:value`, split on the **first** colon so values may contain more +(paths, for instance). An unknown key fails the load with `INVALID_ARGUMENT` and a message +listing the keys that exist, rather than being silently ignored. + +### Option reference + +| Option | Default | Meaning | +|---|---|---| +| `family:` | read from the GGUF | The audio.cpp family. Optional for a standalone audio.cpp GGUF, which embeds `audiocpp.model_spec.family`. Required for a safetensors file or a package directory. Setting it explicitly also overrides the embedded value. | +| `task:` | routed from the RPC | Pins the task. One of `gen`, `tts`, `clon`, `vc`, `svc`, `s2s`, `asr`, `align`, `vad`, `diar`, `sep`, `vdes`, `spk`. A pinned task is honoured exactly: if the family cannot serve it, the request is refused rather than rerouted. | +| `backend:` | `cpu` | ggml compute backend: `cpu`, `cuda`, `vulkan`, `metal`, `best`. Must match the backend image you installed (a `cuda` value needs the CUDA image). | +| `device:` | `0` | GPU index for the selected compute backend. Non-negative integer. | +| `threads:` | runtime default | CPU threads. `0` leaves the choice to the runtime. | +| `busy_timeout_ms:` | `0` (unbounded) | Bounds the wait for the model's single inference lane. A request that arrives while a run has already been in flight longer than this fails immediately with `UNAVAILABLE` instead of queueing, and a request that waits this long without the lane freeing up fails the same way. `0` waits indefinitely. | +| `live_idle_timeout_ms:` | `30000` | How long `AudioTranscriptionLive` waits for the next audio frame before cancelling the stream and releasing the lane. That RPC holds the lane for the whole stream, so a peer that stops sending without closing would otherwise block every other request against this model. `0` disables the limit. | +| `model_spec_override:` | unset | A model spec JSON file, or a directory containing `.json`, used instead of the spec embedded in the GGUF. The path must exist at load time. | +| `load.:` | none | Forwarded to the upstream loader with the `load.` prefix stripped. | +| `session.:` | none | Forwarded to the upstream session with the `session.` prefix stripped. | + +Numeric options are validated at load time: a non-integer or negative value is refused +before the model is read, not after. + +### The `load.` and `session.` namespaces + +Upstream families advertise their own options in two buckets, and the prefix decides which +bucket an entry lands in. The key that follows the prefix is passed through verbatim, and +upstream names those keys after the family, so both segments are needed: + +```yaml +options: + # Loader options (upstream "load options"): + - load.ace_step.dit_model_path:acestep-v15-turbo + - load.vevo2.whisper_model_path:/models/whisper-dir + - load.vibevoice.lora:/models/my-adapter + # Session options (upstream "session options"): + - session.seed_vc.weight_type:q8_0 +``` + +The names each family accepts are declared upstream in its loader, and audio.cpp's own CLI +prints them under "Model session options" and "Model load options" when asked for a model's +help. Options that upstream lists as *request* options are not configurable here: they come +from the API request, not from the model YAML. + +## Voice activity detection with no download + +The backend package ships upstream's `silero_vad` and `marblenet_vad` assets, so VAD works +with nothing to fetch. `bundled:` resolves to the asset directory inside the installed +backend. Those assets are directories rather than GGUF files, so the family cannot be +inferred and `family:` is required: + +```yaml +name: silero-vad +backend: audio-cpp +parameters: + model: bundled:silero_vad +known_usecases: + - FLAG_VAD +options: + - family:silero_vad +``` + +Then call it like any other VAD model, as described in +[Voice Activity Detection]({{%relref "features/voice-activity-detection" %}}). + +## Source separation stems + +Separation families such as `demucs` and `roformer` produce several named outputs from one +run. Running inference once per stem would cost a full separation each time, so the backend +runs **once**, writes every stem to a sibling file next to the main output, and puts the +selected one in the response body. `params[stem]` picks which one: + +```bash +curl http://localhost:8080/audio/transformations \ + -F model=htdemucs -F audio=@song.wav -F 'params[stem]=drums' -o drums.wav +``` + +Without `params[stem]` the body carries `vocals` when the model produces it, and the first +output otherwise. An unknown stem name is refused with a message listing the stems the model +really has, rather than quietly returning a different one. Every stem, including the one in +the body, is listed in the `X-Audio-Stems` response header. See +[Audio Transform]({{%relref "features/audio-transform" %}}) for the full endpoint contract. + +`params[stem]` against a model that produces a single unnamed output is refused rather than +ignored. + +## Family notes + +- **Supertonic**: use the `orig` GGUF package. Its f16 and q8_0 packages mix weight dtypes + in a way that reaches `ggml_abort` and takes the backend process down rather than + returning an error, so the backend refuses those dtypes at load time with a clear message. +- **Live transcription latency**: `AudioTranscriptionLive` needs a family advertising `asr` + in streaming mode. At the pinned upstream revision that is `nemotron_asr`, + `higgs_audio_stt` and `voxtral_realtime`. `higgs_audio_stt` and `voxtral_realtime` decode + incrementally and emit partial text as chunks arrive. `nemotron_asr` only buffers in its + chunk handler and decodes everything in `finalize()`, so with that family every delta + arrives after the client closes its send side. Pick accordingly if you need partials + during speech. +- **Voice conversion vs singing**: `seed_vc` and `vevo2` advertise both `vc` and `svc`, and + automatic routing always picks `vc`. Pin `task:svc` for singing voice conversion. +- **Cloning-only families**: `chatterbox` advertises `clon` and `vc`, and no plain `tts` at + all, so every `/v1/audio/speech` request against it must carry a reference clip in + `voice`. Without one the request is refused with `INVALID_ARGUMENT` naming that field, + rather than failing deep inside the model. + +## Acceleration + +| Variant | Platform | +|---|---| +| CPU | linux/amd64, linux/arm64 | +| CUDA 12 | linux/amd64 | +| CUDA 13 | linux/amd64 | +| Metal | darwin/arm64 | + +There is no ROCm image, because upstream has no HIP build configuration, and no Vulkan +image: it would ship a Vulkan loader with no ICD inside the container. `BUILD_TYPE=vulkan` +still works when building the backend yourself. diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index 6b3aeda89..b424a358a 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -9,10 +9,11 @@ url = "/features/audio-diarization/" Speaker diarization answers the question **"who spoke when?"** - given an audio clip with multiple speakers, it returns time-stamped segments labelled with a stable speaker ID (`SPEAKER_00`, `SPEAKER_01`, …). -LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Two backends are supported today: +LocalAI exposes this through the `/v1/audio/diarization` endpoint, modelled after `/v1/audio/transcriptions`. Three backends are supported today: - **[sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx)** - pyannote-3.0 segmentation + a speaker-embedding extractor (3D-Speaker, NeMo, WeSpeaker) + fast clustering. Pure diarization - no transcription cost. Recommended when you only need speaker turns. - **[vibevoice.cpp](https://github.com/microsoft/VibeVoice)** - produces speaker-labelled segments as a by-product of its long-form ASR pass, so you can optionally get a transcript per segment for free. +- **[audio.cpp](https://github.com/0xShug0/audio.cpp)** - the `sortformer_diar` family, served by the multi-modality [audio.cpp backend]({{%relref "features/audio-cpp" %}}). Because diarization is exposed as a regular OpenAI-compatible endpoint, any HTTP client works. There is no Python dependency on pyannote or NeMo on the consumer side. diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index d1487690e..3c285bdc9 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -14,6 +14,7 @@ The transcription endpoint allows to convert audio files to text. The endpoint s - **[parakeet-cpp](https://github.com/mudler/parakeet.cpp)**: A C++/ggml port of NVIDIA NeMo Parakeet (FastConformer TDT/CTC/RNNT/hybrid). Runs quantized GGUFs on CPU or GPU, emits word-level timestamps, and supports cache-aware streaming (the `realtime_eou` model surfaces end-of-utterance events). - **llama-cpp**: Route transcription to any multimodal-audio GGUF model served by the `llama-cpp` backend (e.g. [Qwen3-ASR](https://huggingface.co/ggml-org/Qwen3-ASR-0.6B-GGUF), Voxtral, Qwen2-Audio). Under the hood the request is converted into a chat completion with the audio attached via the model's audio encoder - the same path the upstream llama.cpp server uses. Set `backend: llama-cpp` in the model YAML and point `mmproj` at the matching audio encoder. - **voxtral**: Voxtral-family models served by a dedicated backend +- **[audio.cpp](https://github.com/0xShug0/audio.cpp)**: Multi-family GGML audio engine. Serves transcription and forced alignment from families such as nemotron, qwen3-asr, citrinet, higgs-audio-stt and voxtral-realtime, and covers diarization, VAD, TTS and source separation from the same backend. See the [audio.cpp backend]({{%relref "features/audio-cpp" %}}) page for the model options. The endpoint input supports all the audio formats supported by `ffmpeg`. diff --git a/docs/content/features/audio-transform.md b/docs/content/features/audio-transform.md index d183348f8..4b3b5a88a 100644 --- a/docs/content/features/audio-transform.md +++ b/docs/content/features/audio-transform.md @@ -18,6 +18,11 @@ a 1.3 M-parameter GGML-based model that performs joint AEC + noise suppression + dereverberation on 16 kHz mono speech, ~9.6× realtime on a desktop CPU. It is a derivative of the Microsoft DeepVQE paper. +Source separation and voice conversion are served by the +[audio.cpp backend]({{%relref "features/audio-cpp" %}}): its `demucs` and +`roformer` families produce the named stems described below, and `seed_vc`, +`vevo2` and `miocodec` do voice conversion against a reference speaker. + ## The mental model Every audio-transform request carries: diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 2ed52cf06..0e7c960a6 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -130,14 +130,15 @@ For getting started, see the available backends in LocalAI here: https://github. LocalAI supports various types of backends: - **LLM Backends**: For running language models (e.g., llama.cpp, vLLM, vllm.cpp, SGLang, transformers, MLX) -- **Speech-to-Text Backends**: For transcription and speaker diarization (e.g., whisper.cpp, parakeet.cpp, moss-transcribe.cpp, faster-whisper, NeMo) -- **Text-to-Speech Backends**: For speech synthesis (e.g., piper, Kokoro, VibeVoice, Qwen3-TTS) -- **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step) +- **Speech-to-Text Backends**: For transcription, forced alignment and speaker diarization (e.g., whisper.cpp, parakeet.cpp, moss-transcribe.cpp, faster-whisper, NeMo, [audio.cpp]({{%relref "features/audio-cpp" %}})) +- **Text-to-Speech Backends**: For speech synthesis (e.g., piper, Kokoro, VibeVoice, Qwen3-TTS, [audio.cpp]({{%relref "features/audio-cpp" %}})) +- **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step, [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Sound Classification Backends**: For sound-event classification / audio tagging - identifying everyday sounds like baby cry, glass breaking, alarms (e.g., ced.cpp) - **Image & Video Generation Backends**: For diffusion and audio-conditioned avatar models (e.g., stable-diffusion.cpp, diffusers, vLLM-Omni, [LongCat-Video]({{%relref "features/video-generation" %}})) - **3D Generation Backends**: For image-to-3D mesh generation ([trellis2.cpp]({{%relref "features/3d-generation" %}}) — Microsoft TRELLIS.2, producing GLB assets with PBR textures) - **Vision & Detection Backends**: For object detection, segmentation, depth, and face/voice recognition (e.g., rf-detr.cpp, locate-anything.cpp, sam3.cpp, insightface) -- **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE) +- **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE, [audio.cpp]({{%relref "features/audio-cpp" %}})) +- **Source Separation & Voice Conversion Backends**: For splitting a mix into named stems (vocals, drums, bass) and for converting speech or singing to a target voice (e.g., [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Utility Backends**: For reranking, PII/NER token classification, fine-tuning, quantization, and vector storage (e.g., rerankers, privacy-filter.cpp, TRL, local-store, valkey-store) See the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}) for the full catalog. diff --git a/docs/content/features/text-to-audio.md b/docs/content/features/text-to-audio.md index ff71ef691..ae428b041 100644 --- a/docs/content/features/text-to-audio.md +++ b/docs/content/features/text-to-audio.md @@ -745,6 +745,35 @@ curl -L http://localhost:8080/tts \ }' | aplay ``` +### audio.cpp + +[audio.cpp](https://github.com/0xShug0/audio.cpp) is a multi-family GGML audio engine, so +one installed backend covers TTS (`supertonic`, `vibevoice`, `voxcpm2`, `fish_audio`, +`pocket_tts`, `omnivoice`, `higgs_audio_tts`), voice cloning (`chatterbox`, `index_tts2`, +`irodori_tts`, `moss`) and voice design (`qwen3_tts`, `irodori_tts`), alongside ASR, VAD, +diarization and separation. + +```yaml +name: supertonic +backend: audio-cpp +parameters: + model: supertonic-3-orig.gguf +known_usecases: + - FLAG_TTS +options: + - backend:cuda + - device:0 +``` + +The family is read from the GGUF's own metadata, so `family:` is only needed for a +safetensors file or a package directory. Sending a speaker reference clip routes the +request to the family's voice-cloning task automatically, and supplying instructions routes +it to voice design, where the family supports them. Cloning-only families such as +`chatterbox` advertise no plain TTS task, so every request to them must carry a reference +clip. See the +[audio.cpp backend]({{%relref "features/audio-cpp" %}}) page for the full option list, +including the `load.` and `session.` namespaces and the supertonic packaging caveat. + ## Response format To provide some compatibility with OpenAI API regarding `response_format`, ffmpeg must be installed (or a docker image including ffmpeg used) to leverage converting the generated wav file before the api provide its response. diff --git a/docs/content/features/voice-activity-detection.md b/docs/content/features/voice-activity-detection.md index 9e58be415..2fc3f6e5b 100644 --- a/docs/content/features/voice-activity-detection.md +++ b/docs/content/features/voice-activity-detection.md @@ -7,6 +7,8 @@ url = "/features/voice-activity-detection/" Voice Activity Detection (VAD) identifies segments of speech in audio data. LocalAI provides a `/v1/vad` endpoint powered by the [Silero VAD](https://github.com/snakers4/silero-vad) backend. +The [audio.cpp backend]({{%relref "features/audio-cpp" %}}) also serves this endpoint, and ships the `silero_vad` and `marblenet_vad` assets inside its own package, so VAD works there with nothing to download (`model: bundled:silero_vad` plus the `family:silero_vad` option). + ## API - **Method:** `POST` diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index 8e739f93e..7eab3ab81 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -48,6 +48,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [Qwen3-ASR](https://github.com/QwenLM/Qwen3-ASR) | Qwen3 automatic speech recognition | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal, Jetson L4T | | [NeMo](https://github.com/NVIDIA/NeMo) | NVIDIA NeMo ASR toolkit | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal | | [sherpa-onnx](https://k2-fsa.github.io/sherpa/onnx/) | Sherpa-ONNX ASR (Whisper, Paraformer, SenseVoice) and TTS | CPU, CUDA 12, Metal | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: transcription, forced alignment and speaker diarization (nemotron, qwen3-asr, voxtral-realtime, sortformer and more). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Text-to-Speech @@ -75,6 +76,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [Supertonic](https://github.com/supertone-inc/supertonic) | Lightning-fast on-device multilingual TTS via ONNX | CPU | | [MLX-Audio](https://github.com/Blaizzy/mlx-audio) | Audio models on Apple Silicon | CPU, CUDA 12/13, Metal, Jetson L4T | | [liquid-audio](https://github.com/Liquid4All/liquid-audio) | LFM2 end-to-end speech-to-speech, ASR, and TTS | CPU, CUDA 12/13, ROCm, Intel SYCL, Jetson L4T | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: TTS, voice cloning and voice design (chatterbox, supertonic, qwen3-tts, vibevoice and more). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Music & Sound Generation @@ -82,6 +84,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ |---------|-------------|-------------| | [ACE-Step](https://github.com/ace-step/ACE-Step-1.5) | Music generation from text descriptions, lyrics, or audio | CPU, CUDA 12/13, ROCm, Intel SYCL, Metal | | [acestep.cpp](https://github.com/ace-step/acestep.cpp) | ACE-Step 1.5 C++ backend using GGML | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: music and sound generation (ace-step, stable-audio, heartmula). See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Image & Video Generation @@ -112,6 +115,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [Silero VAD](https://github.com/snakers4/silero-vad) | Voice Activity Detection | CPU, Metal | | [LocalVQE](https://github.com/localai-org/LocalVQE) | Joint acoustic echo cancellation, noise suppression, and dereverberation in C/C++ using GGML | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Jetson L4T | | [Opus](https://opus-codec.org/) | Audio codec for WebRTC / Realtime API | CPU, Metal | +| [audio.cpp](https://github.com/0xShug0/audio.cpp) | Multi-family GGML audio engine: voice activity detection (bundled silero-vad and marblenet-vad, no download), source separation into named stems, and voice conversion. See [audio.cpp backend](/features/audio-cpp/) | CPU, CUDA 12/13, Metal | ## Utilities & Other