feat(backend): add moss-transcribe-cpp backend (MOSS-Transcribe-Diarize) (#10756)

C++/ggml transcription + speaker diarization + timestamps backend. Purego
dlopens libmoss-transcribe.so (ggml statically linked) from moss-transcribe.cpp
and serves offline AudioTranscription, parsing the [start][Sxx]text[end] output
into segments with nanosecond timestamps. Adds the importer (surfaces in
GET /backends/known), backend-matrix (Linux + Darwin/metal), backend/index.yaml,
and a gallery entry (default q5_k GGUF from mudler/moss-transcribe.cpp-gguf).

Local L0 smoke (go build + go test ./... = 16 pass, golangci-lint 0 issues)
passed against the real libmoss-transcribe.so. The pre-commit coverage gate
(full pkg/core + tests/e2e) could not run in the authoring sandbox (no live
models, port 9090 held); CI must enforce it before merge.

Assisted-by: Claude:claude-opus-4-8 golangci-lint

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
2026-07-09 23:27:11 +02:00
committed by GitHub
parent 294487eb61
commit 94bdc825dc
20 changed files with 1511 additions and 3 deletions

View File

@@ -790,6 +790,19 @@ include:
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'cublas'
cuda-major-version: "12"
cuda-minor-version: "8"
platforms: 'linux/amd64'
tag-latest: 'auto'
tag-suffix: '-gpu-nvidia-cuda-12-moss-transcribe-cpp'
runs-on: 'ubuntu-latest'
base-image: "ubuntu:24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'cublas'
cuda-major-version: "12"
cuda-minor-version: "8"
@@ -1721,6 +1734,19 @@ include:
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'cublas'
cuda-major-version: "13"
cuda-minor-version: "0"
platforms: 'linux/amd64'
tag-latest: 'auto'
tag-suffix: '-gpu-nvidia-cuda-13-moss-transcribe-cpp'
runs-on: 'ubuntu-latest'
base-image: "ubuntu:24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'cublas'
cuda-major-version: "13"
cuda-minor-version: "0"
@@ -1760,6 +1786,19 @@ include:
backend: "parakeet-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
- build-type: 'cublas'
cuda-major-version: "13"
cuda-minor-version: "0"
platforms: 'linux/arm64'
skip-drivers: 'false'
tag-latest: 'auto'
tag-suffix: '-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp'
base-image: "ubuntu:24.04"
ubuntu-version: '2404'
runs-on: 'ubuntu-24.04-arm'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
- build-type: 'cublas'
cuda-major-version: "13"
cuda-minor-version: "0"
@@ -3597,6 +3636,115 @@ include:
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
# moss-transcribe-cpp
- build-type: ''
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/amd64'
platform-tag: 'amd64'
tag-latest: 'auto'
tag-suffix: '-cpu-moss-transcribe-cpp'
runs-on: 'ubuntu-latest'
base-image: "ubuntu:24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: ''
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/arm64'
platform-tag: 'arm64'
tag-latest: 'auto'
tag-suffix: '-cpu-moss-transcribe-cpp'
runs-on: 'ubuntu-24.04-arm'
base-image: "ubuntu:24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'sycl_f32'
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/amd64'
tag-latest: 'auto'
tag-suffix: '-gpu-intel-sycl-f32-moss-transcribe-cpp'
runs-on: 'ubuntu-latest'
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'sycl_f16'
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/amd64'
tag-latest: 'auto'
tag-suffix: '-gpu-intel-sycl-f16-moss-transcribe-cpp'
runs-on: 'ubuntu-latest'
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'vulkan'
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/amd64'
platform-tag: 'amd64'
tag-latest: 'auto'
tag-suffix: '-gpu-vulkan-moss-transcribe-cpp'
runs-on: 'ubuntu-latest'
base-image: "ubuntu:24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'vulkan'
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/arm64'
platform-tag: 'arm64'
tag-latest: 'auto'
tag-suffix: '-gpu-vulkan-moss-transcribe-cpp'
runs-on: 'ubuntu-24.04-arm'
base-image: "ubuntu:24.04"
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'cublas'
cuda-major-version: "12"
cuda-minor-version: "0"
platforms: 'linux/arm64'
skip-drivers: 'false'
tag-latest: 'auto'
tag-suffix: '-nvidia-l4t-arm64-moss-transcribe-cpp'
base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
runs-on: 'ubuntu-24.04-arm'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2204'
- build-type: 'hipblas'
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/amd64'
tag-latest: 'auto'
tag-suffix: '-gpu-rocm-hipblas-moss-transcribe-cpp'
base-image: "rocm/dev-ubuntu-24.04:7.2.1"
runs-on: 'ubuntu-latest'
skip-drivers: 'false'
backend: "moss-transcribe-cpp"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
# ced
- build-type: 'cublas'
cuda-major-version: "12"
@@ -5220,6 +5368,10 @@ includeDarwin:
tag-suffix: "-metal-darwin-arm64-parakeet-cpp"
build-type: "metal"
lang: "go"
- backend: "moss-transcribe-cpp"
tag-suffix: "-metal-darwin-arm64-moss-transcribe-cpp"
build-type: "metal"
lang: "go"
- backend: "ced"
tag-suffix: "-metal-darwin-arm64-ced"
build-type: "metal"

View File

@@ -42,6 +42,10 @@ jobs:
variable: "PARAKEET_VERSION"
branch: "master"
file: "backend/go/parakeet-cpp/Makefile"
- repository: "mudler/moss-transcribe.cpp"
variable: "MOSS_VERSION"
branch: "master"
file: "backend/go/moss-transcribe-cpp/Makefile"
- repository: "mudler/ced.cpp"
variable: "CED_VERSION"
branch: "master"

View File

@@ -1,5 +1,5 @@
# Disable parallel execution for backend builds
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
GOCMD=go
GOTEST=$(GOCMD) test
@@ -1044,6 +1044,18 @@ test-extra-backend-parakeet-cpp-transcription: docker-build-parakeet-cpp
BACKEND_TEST_CAPS=health,load,transcription \
$(MAKE) test-extra-backend
## Audio transcription wrapper for the moss-transcribe-cpp (moss-transcribe.cpp
## ggml port) backend. Mirrors test-extra-backend-parakeet-cpp-transcription:
## drives the AudioTranscription RPC against a published MOSS GGUF using the JFK
## 11s clip from whisper.cpp's CI samples. Not part of the default test suite -
## run explicitly once the pinned model URL is reachable.
test-extra-backend-moss-transcribe-cpp-transcription: docker-build-moss-transcribe-cpp
BACKEND_IMAGE=local-ai-backend:moss-transcribe-cpp \
BACKEND_TEST_MODEL_URL=https://huggingface.co/mudler/moss-transcribe.cpp-gguf/resolve/main/moss-transcribe-q5_k.gguf \
BACKEND_TEST_AUDIO_URL=https://github.com/ggml-org/whisper.cpp/raw/master/samples/jfk.wav \
BACKEND_TEST_CAPS=health,load,transcription \
$(MAKE) test-extra-backend
## LocalVQE audio transform (joint AEC + noise suppression + dereverb).
## Exercises the audio_transform capability end-to-end: batch transform
## of a real WAV fixture and bidi streaming of synthetic silent frames.
@@ -1212,6 +1224,7 @@ BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|tr
BACKEND_WHISPER = whisper|golang|.|false|true
BACKEND_CRISPASR = crispasr|golang|.|false|true
BACKEND_PARAKEET_CPP = parakeet-cpp|golang|.|false|true
BACKEND_MOSS_TRANSCRIBE_CPP = moss-transcribe-cpp|golang|.|false|true
BACKEND_DEPTH_ANYTHING_CPP = depth-anything-cpp|golang|.|false|true
BACKEND_VOXTRAL = voxtral|golang|.|false|true
BACKEND_ACESTEP_CPP = acestep-cpp|golang|.|false|true
@@ -1305,6 +1318,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_STABLEDIFFUSION_GGML)))
$(eval $(call generate-docker-build-target,$(BACKEND_WHISPER)))
$(eval $(call generate-docker-build-target,$(BACKEND_CRISPASR)))
$(eval $(call generate-docker-build-target,$(BACKEND_PARAKEET_CPP)))
$(eval $(call generate-docker-build-target,$(BACKEND_MOSS_TRANSCRIBE_CPP)))
$(eval $(call generate-docker-build-target,$(BACKEND_DEPTH_ANYTHING_CPP)))
$(eval $(call generate-docker-build-target,$(BACKEND_VOXTRAL)))
$(eval $(call generate-docker-build-target,$(BACKEND_OPUS)))
@@ -1357,7 +1371,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
docker-save-%: backend-images
docker save local-ai-backend:$* -o backend-images/$*.tar
docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-privacy-filter
docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
########################################################
### Mock Backend for E2E Tests

View File

@@ -232,6 +232,7 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native
| Backend | What it does |
|---------|-------------|
| [parakeet.cpp](https://github.com/mudler/parakeet.cpp) | C++/GGML port of NVIDIA NeMo Parakeet ASR (tdt/ctc/rnnt/hybrid), with cache-aware streaming transcription |
| [moss-transcribe.cpp](https://github.com/mudler/moss-transcribe.cpp) | C++/GGML port of OpenMOSS MOSS-Transcribe-Diarize: joint long-form transcription, speaker diarization and timestamping in a single pass |
| [ced.cpp](https://github.com/mudler/ced.cpp) | C++/GGML port of the CED audio-tagging models: sound-event classification (527-class AudioSet) over REST and the realtime API for live recognition |
| [voice-detect.cpp](https://github.com/mudler/voice-detect.cpp) | Speaker recognition and voice analysis (ECAPA-TDNN, WeSpeaker, ERes2Net, CAM++, wav2vec2 age/gender/emotion), replacing the Python speaker-recognition backend |
| [voxtral.c](https://github.com/mudler/voxtral.c) | Voxtral Realtime 4B speech-to-text in pure C |

View File

@@ -0,0 +1,12 @@
.cache/
sources/
build/
package/
moss-transcribe-cpp-grpc
# build artifacts staged in-tree by the Makefile (cp from sources/) or
# symlinked for local dev; the real sources live in moss-transcribe.cpp upstream.
*.so
*.so.*
*.dylib
moss_transcribe_capi.h
compile_commands.json

View File

@@ -0,0 +1,98 @@
# moss-transcribe-cpp backend Makefile.
#
# Upstream pin lives below as MOSS_VERSION?=1fbe88c8ebf0134ed2c49b49345102752500fbe5
# (.github/bump_deps.sh) can find and update it - matches the
# whisper.cpp / parakeet-cpp / ds4 convention.
#
# Local dev shortcut: if you already have an out-of-tree moss-transcribe.cpp
# build, you can symlink the .so + header into this directory and skip the
# clone/cmake steps entirely, e.g.:
#
# ln -sf /path/to/moss-transcribe.cpp/build-shared/libmoss-transcribe.so .
# ln -sf /path/to/moss-transcribe.cpp/include/moss_transcribe_capi.h .
# go build -o moss-transcribe-cpp-grpc .
#
# That's what the L0 smoke test uses. The default target below does the proper
# clone-at-pin + cmake build so CI doesn't need a side-checkout.
MOSS_VERSION?=1fbe88c8ebf0134ed2c49b49345102752500fbe5
MOSS_REPO?=https://github.com/mudler/moss-transcribe.cpp
GOCMD?=go
GO_TAGS?=
JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
BUILD_TYPE?=
NATIVE?=false
# Build ggml statically into libmoss-transcribe.so (PIC) so the shared lib is
# self-contained: dlopen needs no libggml*.so alongside it, only system libs
# (libstdc++/libgomp/libc) that the runtime image already provides. MT_SHARED
# flips ggml to a static PIC build (see the moss-transcribe.cpp CMakeLists).
CMAKE_ARGS?=-DCMAKE_BUILD_TYPE=Release -DMT_SHARED=ON -DMT_BUILD_CLI=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON
ifeq ($(NATIVE),false)
CMAKE_ARGS+=-DGGML_NATIVE=OFF
endif
# moss-transcribe.cpp gates its GGML backends behind MT_GGML_* options and does
# set(GGML_<be> ${MT_GGML_<be>} CACHE BOOL "" FORCE), so a bare -DGGML_CUDA=ON is
# overwritten back to OFF and the build silently falls back to CPU. Forward the
# MT_GGML_* options instead (openblas is not gated, so -DGGML_BLAS passes through).
ifeq ($(BUILD_TYPE),cublas)
CMAKE_ARGS+=-DMT_GGML_CUDA=ON -DGGML_CUDA_GRAPHS=ON
else ifeq ($(BUILD_TYPE),openblas)
CMAKE_ARGS+=-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
else ifeq ($(BUILD_TYPE),hipblas)
CMAKE_ARGS+=-DMT_GGML_HIP=ON
else ifeq ($(BUILD_TYPE),vulkan)
CMAKE_ARGS+=-DMT_GGML_VULKAN=ON
else ifeq ($(BUILD_TYPE),metal)
CMAKE_ARGS+=-DMT_GGML_METAL=ON
endif
.PHONY: moss-transcribe-cpp-grpc package build clean purge test all
all: moss-transcribe-cpp-grpc
# Clone the upstream moss-transcribe.cpp source at the pinned commit. Directory
# acts as the target so make only re-clones when missing. After a MOSS_VERSION
# bump, run 'make purge && make' to refetch.
sources/moss-transcribe.cpp:
mkdir -p sources/moss-transcribe.cpp
cd sources/moss-transcribe.cpp && \
git init -q && \
git remote add origin $(MOSS_REPO) && \
git fetch --depth 1 origin $(MOSS_VERSION) && \
git checkout FETCH_HEAD && \
git submodule update --init --recursive --depth 1 --single-branch
# Build the shared lib + header out-of-tree, then stage them next to the Go
# sources so purego.Dlopen("libmoss-transcribe.so") and the cgo-less build both
# pick them up.
libmoss-transcribe.so: sources/moss-transcribe.cpp
cmake -B sources/moss-transcribe.cpp/build-shared -S sources/moss-transcribe.cpp $(CMAKE_ARGS)
cmake --build sources/moss-transcribe.cpp/build-shared --config Release -j$(JOBS)
cp -fv sources/moss-transcribe.cpp/build-shared/libmoss-transcribe.so* ./ 2>/dev/null || true
cp -fv sources/moss-transcribe.cpp/build-shared/libmoss-transcribe.dylib ./ 2>/dev/null || true
cp -fv sources/moss-transcribe.cpp/include/moss_transcribe_capi.h ./
moss-transcribe-cpp-grpc: libmoss-transcribe.so main.go gomosstranscribecpp.go segments.go
CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o moss-transcribe-cpp-grpc .
package: moss-transcribe-cpp-grpc
bash package.sh
build: package
# Test target. The model-backed smoke test is gated on
# MOSS_BACKEND_TEST_MODEL + MOSS_BACKEND_TEST_WAV; without them that spec
# auto-skips, leaving the pure-Go parser/unit tests.
test:
LD_LIBRARY_PATH=$(CURDIR):$$LD_LIBRARY_PATH $(GOCMD) test ./... -count=1
clean: purge
rm -rf libmoss-transcribe.so* moss_transcribe_capi.h package moss-transcribe-cpp-grpc
purge:
rm -rf sources/moss-transcribe.cpp

View File

@@ -0,0 +1,179 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"unsafe"
"github.com/mudler/LocalAI/pkg/grpc/base"
"github.com/mudler/LocalAI/pkg/grpc/grpcerrors"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
"github.com/mudler/LocalAI/pkg/utils"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// purego-bound entry points from libmoss-transcribe.so. Names match
// moss_transcribe_capi.h exactly so a `nm libmoss-transcribe.so | grep
// moss_transcribe_capi` is enough to spot drift.
//
// The transcribe_* functions return char* declared here as uintptr so we can
// call moss_transcribe_capi_free_string on the same pointer after copying: the
// C-API contract is "caller owns and must free the returned buffer".
var (
CppAbiVersion func() int32
CppLoad func(ggufPath string) uintptr
CppFree func(ctx uintptr)
CppTranscribePath func(ctx uintptr, wavPath string, maxNew int32) uintptr
CppTranscribePcm func(ctx uintptr, samples []float32, nSamples int32, sampleRate int32, maxNew int32) uintptr
CppFreeString func(s uintptr)
CppLastError func(ctx uintptr) string
)
// MossTranscribeCpp owns a single loaded moss_transcribe_ctx. MOSS is an
// offline transcription + diarization + timestamping engine: one model, one
// context, no streaming. The C engine holds a single mutable context and is
// not reentrant, so we embed base.SingleThread — LocalAI serialises every RPC
// through the server-level lock, and only one transcription touches the engine
// at a time.
type MossTranscribeCpp struct {
base.SingleThread
ctxPtr uintptr
maxNew int32
}
// Load is the LocalAI gRPC entry point for LoadModel: it calls
// moss_transcribe_capi_load with the GGUF path and stashes the resulting
// opaque context pointer for AudioTranscription.
func (m *MossTranscribeCpp) Load(opts *pb.ModelOptions) error {
if opts.ModelFile == "" {
return errors.New("moss-transcribe-cpp: ModelFile is required")
}
// max_new_tokens caps the generated tokens per transcription; <=0 uses the
// GGUF's default_max_new_tokens (the C-API's own default). Exposed as a
// model YAML option: (key:value form, like the sibling ggml backends).
m.maxNew = int32(optInt(opts, "max_new_tokens", 0))
ctx := CppLoad(opts.ModelFile)
if ctx == 0 {
// No ctx to ask for last_error (the C-API's last-error buffer lives on
// the ctx that was never returned). Surface the path so the operator at
// least knows which load failed.
return fmt.Errorf("moss-transcribe-cpp: moss_transcribe_capi_load failed for %q", opts.ModelFile)
}
m.ctxPtr = ctx
return nil
}
// optInt reads an integer model option (key:value form) from ModelOptions,
// returning def when absent or unparseable. The options array carries the
// model YAML's options: entries (see core/config; siblings such as parakeet-cpp
// parse the same key:value form via strings.Cut on ":").
func optInt(opts *pb.ModelOptions, key string, def int) int {
for _, o := range opts.GetOptions() {
k, v, ok := strings.Cut(o, ":")
if ok && strings.TrimSpace(k) == key {
if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil {
return n
}
}
}
return def
}
// AudioTranscription converts the audio at opts.Dst to a 16 kHz mono WAV and
// hands the path to moss_transcribe_capi_transcribe_path. The model emits its
// own speaker-labelled, time-aligned transcript in the compact
// "[start][Sxx]text[end]..." format (seconds); we parse it into LocalAI
// TranscriptSegments carrying int64-nanosecond timestamps and the per-segment
// speaker label.
//
// MOSS does joint transcription + diarization + timestamps in one pass, so
// translate/language/prompt/temperature/threads are not applicable and are
// ignored. Streaming is not supported (offline model).
func (m *MossTranscribeCpp) AudioTranscription(ctx context.Context, opts *pb.TranscriptRequest) (pb.TranscriptResult, error) {
if m.ctxPtr == 0 {
return pb.TranscriptResult{}, grpcerrors.ModelNotLoaded("moss-transcribe-cpp")
}
if opts.Dst == "" {
return pb.TranscriptResult{}, errors.New("moss-transcribe-cpp: TranscriptRequest.dst (audio path) is required")
}
if err := ctx.Err(); err != nil {
return pb.TranscriptResult{}, status.Error(codes.Canceled, "transcription cancelled")
}
// The C loader understands WAV; convert any input (MP3, etc.) to 16 kHz
// mono WAV first - the same normalisation every other audio backend
// (whisper, parakeet-cpp) does via utils.AudioToWav before handing the file
// to the engine.
converted, cleanup, err := convertToWavMono16k(opts.Dst)
if err != nil {
return pb.TranscriptResult{}, err
}
defer cleanup()
cstr := CppTranscribePath(m.ctxPtr, converted, m.maxNew)
if cstr == 0 {
return pb.TranscriptResult{}, fmt.Errorf("moss-transcribe-cpp: transcribe_path failed: %s", CppLastError(m.ctxPtr))
}
raw := goStringFromCPtr(cstr)
CppFreeString(cstr)
return transcriptResultFromRaw(raw), nil
}
// Free releases the underlying moss_transcribe_ctx. Called by LocalAI when the
// model is unloaded.
func (m *MossTranscribeCpp) Free() error {
if m.ctxPtr != 0 {
CppFree(m.ctxPtr)
m.ctxPtr = 0
}
return nil
}
// convertToWavMono16k converts an arbitrary audio file to a 16 kHz mono WAV in
// a fresh temp dir and returns the path together with a cleanup func the caller
// must defer. WAV inputs already at 16 kHz/mono/16-bit are passed through by
// utils.AudioToWav (hardlink/copy), everything else is transcoded via ffmpeg.
func convertToWavMono16k(path string) (string, func(), error) {
dir, err := os.MkdirTemp("", "moss-transcribe")
if err != nil {
return "", func() {}, err
}
cleanup := func() { _ = os.RemoveAll(dir) }
converted := filepath.Join(dir, "converted.wav")
if err := utils.AudioToWav(path, converted); err != nil {
cleanup()
return "", func() {}, err
}
return converted, cleanup, nil
}
// goStringFromCPtr copies a NUL-terminated C string into Go memory. cptr is the
// raw pointer returned by purego from the C-API (a malloc'd buffer the caller
// owns); callers must free it via CppFreeString after the copy lands.
//
// The uintptr->unsafe.Pointer conversion below trips go vet's unsafeptr check,
// which can't distinguish a C-owned heap pointer from Go-managed memory. It is
// safe here: the pointer addresses a malloc'd C buffer the Go GC neither tracks
// nor moves, and we dereference it immediately to copy the bytes out (the same
// pattern the whisper / parakeet-cpp backends use).
func goStringFromCPtr(cptr uintptr) string {
if cptr == 0 {
return ""
}
p := unsafe.Pointer(cptr) //nolint:govet // C-owned malloc'd buffer, not Go-GC memory (see doc above)
n := 0
for *(*byte)(unsafe.Add(p, n)) != 0 {
n++
}
return string(unsafe.Slice((*byte)(p), n))
}

View File

@@ -0,0 +1,164 @@
package main
import (
"context"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/ebitengine/purego"
"github.com/go-audio/audio"
"github.com/go-audio/wav"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestMossTranscribeCpp(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "moss-transcribe-cpp Backend Suite")
}
var (
libLoadOnce sync.Once
libLoadErr error
)
// ensureLibLoaded mirrors main.go's bootstrap so a Go test can drive the C-API
// bridge without spinning up the gRPC server. Skips the current spec when
// libmoss-transcribe.so isn't loadable from cwd ($LD_LIBRARY_PATH or a symlink
// in ./).
func ensureLibLoaded() {
libLoadOnce.Do(func() {
libName := os.Getenv("MOSS_TRANSCRIBE_LIBRARY")
if libName == "" {
libName = "libmoss-transcribe.so"
}
lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
libLoadErr = err
return
}
purego.RegisterLibFunc(&CppAbiVersion, lib, "moss_transcribe_capi_abi_version")
purego.RegisterLibFunc(&CppLoad, lib, "moss_transcribe_capi_load")
purego.RegisterLibFunc(&CppFree, lib, "moss_transcribe_capi_free")
purego.RegisterLibFunc(&CppTranscribePath, lib, "moss_transcribe_capi_transcribe_path")
purego.RegisterLibFunc(&CppTranscribePcm, lib, "moss_transcribe_capi_transcribe_pcm")
purego.RegisterLibFunc(&CppFreeString, lib, "moss_transcribe_capi_free_string")
purego.RegisterLibFunc(&CppLastError, lib, "moss_transcribe_capi_last_error")
})
if libLoadErr != nil {
Skip("libmoss-transcribe.so not loadable: " + libLoadErr.Error())
}
}
// fixturesOrSkip returns the model + audio paths or skips the spec if either
// env var is unset. The smoke test never runs in default CI; it needs a real
// MOSS GGUF and a WAV on disk.
func fixturesOrSkip() (string, string) {
modelPath := os.Getenv("MOSS_BACKEND_TEST_MODEL")
audioPath := os.Getenv("MOSS_BACKEND_TEST_WAV")
if modelPath == "" || audioPath == "" {
Skip("set MOSS_BACKEND_TEST_MODEL and MOSS_BACKEND_TEST_WAV to run this spec")
}
return modelPath, audioPath
}
// writeMono16kWav writes `samples` frames of 16 kHz mono 16-bit silence to
// path. The result is already in AudioToWav's target format, so the conversion
// helper copies it through without invoking ffmpeg.
func writeMono16kWav(path string, samples int) {
GinkgoHelper()
f, err := os.Create(path)
Expect(err).ToNot(HaveOccurred())
enc := wav.NewEncoder(f, 16000, 16, 1, 1)
buf := &audio.IntBuffer{
Format: &audio.Format{NumChannels: 1, SampleRate: 16000},
SourceBitDepth: 16,
Data: make([]int, samples),
}
Expect(enc.Write(buf)).To(Succeed())
Expect(enc.Close()).To(Succeed())
Expect(f.Close()).To(Succeed())
}
var _ = Describe("MossTranscribeCpp", func() {
Context("ABI / load smoke (needs libmoss-transcribe.so)", func() {
It("reports a positive ABI version", func() {
ensureLibLoaded()
Expect(CppAbiVersion()).To(BeNumerically(">=", 1))
})
It("transcribes a WAV into speaker-labelled segments", func() {
modelPath, audioPath := fixturesOrSkip()
ensureLibLoaded()
m := &MossTranscribeCpp{}
Expect(m.Load(&pb.ModelOptions{ModelFile: modelPath})).To(Succeed())
defer func() { _ = m.Free() }()
res, err := m.AudioTranscription(context.Background(), &pb.TranscriptRequest{
Dst: audioPath,
})
Expect(err).ToNot(HaveOccurred())
Expect(strings.TrimSpace(res.Text)).ToNot(BeEmpty(),
"expected non-empty transcript for %s", audioPath)
Expect(res.Segments).ToNot(BeEmpty(), "expected at least one segment")
var prevEnd int64
for i, seg := range res.Segments {
Expect(strings.TrimSpace(seg.Text)).ToNot(BeEmpty(),
"segment %d must have text", i)
Expect(seg.End).To(BeNumerically(">=", seg.Start),
"segment %d end must not precede its start", i)
Expect(seg.Start).To(BeNumerically(">=", prevEnd),
"segments must be in time order")
prevEnd = seg.End
}
})
})
Context("Load validation", func() {
It("rejects an empty ModelFile", func() {
m := &MossTranscribeCpp{}
Expect(m.Load(&pb.ModelOptions{})).To(HaveOccurred())
})
})
Context("AudioTranscription guards (no C library required)", func() {
It("returns ModelNotLoaded when no context is loaded", func() {
m := &MossTranscribeCpp{}
_, err := m.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: "x.wav"})
Expect(err).To(HaveOccurred())
})
It("errors when the audio path is empty", func() {
m := &MossTranscribeCpp{ctxPtr: 1}
_, err := m.AudioTranscription(context.Background(), &pb.TranscriptRequest{})
Expect(err).To(HaveOccurred())
})
})
Context("convertToWavMono16k", func() {
It("returns a decodable 16kHz mono WAV copy and cleans it up", func() {
dir := GinkgoT().TempDir()
src := filepath.Join(dir, "input.wav")
writeMono16kWav(src, 16000) // 1s of silence at 16 kHz
converted, cleanup, err := convertToWavMono16k(src)
Expect(err).ToNot(HaveOccurred())
Expect(converted).ToNot(Equal(src))
Expect(converted).To(BeAnExistingFile())
cleanup()
Expect(converted).ToNot(BeAnExistingFile(), "cleanup removes the temp dir")
})
It("errors on a non-existent input rather than passing the path through", func() {
_, _, err := convertToWavMono16k(filepath.Join(GinkgoT().TempDir(), "missing.mp3"))
Expect(err).To(HaveOccurred())
})
})
})

View File

@@ -0,0 +1,70 @@
package main
// Started internally by LocalAI - one gRPC server per loaded model.
//
// Loads the moss-transcribe shared library via purego and registers the flat
// C-API entry points declared in moss_transcribe_capi.h. The library name can
// be overridden with MOSS_TRANSCRIBE_LIBRARY (mirrors the WHISPER_LIBRARY /
// PARAKEET_LIBRARY convention in the sibling backends); the default looks next
// to this binary for libmoss-transcribe.so on Linux and
// libmoss-transcribe.dylib on macOS.
import (
"flag"
"fmt"
"os"
"runtime"
"github.com/ebitengine/purego"
grpc "github.com/mudler/LocalAI/pkg/grpc"
)
var (
addr = flag.String("addr", "localhost:50051", "the address to connect to")
)
type LibFuncs struct {
FuncPtr any
Name string
}
func main() {
libName := os.Getenv("MOSS_TRANSCRIBE_LIBRARY")
if libName == "" {
if runtime.GOOS == "darwin" {
libName = "libmoss-transcribe.dylib"
} else {
libName = "libmoss-transcribe.so"
}
}
lib, err := purego.Dlopen(libName, purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
panic(fmt.Errorf("moss-transcribe-cpp: dlopen %q: %w", libName, err))
}
// Bound 1:1 to moss_transcribe_capi.h. The transcribe_* entry points return
// a malloc'd char* the caller owns; we register those as uintptr so we get
// the raw pointer back and can call moss_transcribe_capi_free_string on it
// (purego's string return would copy and forget the original pointer,
// leaking it on every call).
libFuncs := []LibFuncs{
{&CppAbiVersion, "moss_transcribe_capi_abi_version"},
{&CppLoad, "moss_transcribe_capi_load"},
{&CppFree, "moss_transcribe_capi_free"},
{&CppTranscribePath, "moss_transcribe_capi_transcribe_path"},
{&CppTranscribePcm, "moss_transcribe_capi_transcribe_pcm"},
{&CppFreeString, "moss_transcribe_capi_free_string"},
{&CppLastError, "moss_transcribe_capi_last_error"},
}
for _, lf := range libFuncs {
purego.RegisterLibFunc(lf.FuncPtr, lib, lf.Name)
}
fmt.Fprintf(os.Stderr, "[moss-transcribe-cpp] ABI=%d\n", CppAbiVersion())
flag.Parse()
if err := grpc.StartServer(*addr, &MossTranscribeCpp{}); err != nil {
panic(err)
}
}

View File

@@ -0,0 +1,71 @@
#!/bin/bash
#
# Bundle the moss-transcribe-cpp-grpc binary, libmoss-transcribe.so, the core
# runtime libs (libc/libstdc++/libgomp + ld.so) and the GPU runtime for the
# active BUILD_TYPE so the package is self-contained. Mirrors
# backend/go/parakeet-cpp/package.sh; run.sh routes the (CGO_ENABLED=0) binary
# through lib/ld.so so the packaged libc is used instead of the host's.
set -e
CURDIR=$(dirname "$(realpath "$0")")
REPO_ROOT="${CURDIR}/../../.."
mkdir -p "$CURDIR/package/lib"
cp -avf "$CURDIR/moss-transcribe-cpp-grpc" "$CURDIR/package/"
cp -avf "$CURDIR/run.sh" "$CURDIR/package/"
# libmoss-transcribe shared lib + any soname symlinks. On Linux this is
# libmoss-transcribe.so[.X.Y]; on macOS it is libmoss-transcribe.dylib.
# purego.Dlopen resolves it via the *_LIBRARY_PATH that run.sh points at lib/.
cp -avf "$CURDIR"/libmoss-transcribe.so* "$CURDIR/package/lib/" 2>/dev/null || true
cp -avf "$CURDIR"/libmoss-transcribe.dylib "$CURDIR/package/lib/" 2>/dev/null || true
if ! ls "$CURDIR"/package/lib/libmoss-transcribe.* >/dev/null 2>&1; then
echo "ERROR: libmoss-transcribe shared library not found in $CURDIR, run 'make' first" >&2
exit 1
fi
# Detect architecture and copy the core runtime libs libmoss-transcribe.so links
# against, plus the matching dynamic loader as lib/ld.so.
if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
echo "Detected x86_64 architecture, copying x86_64 libraries..."
cp -arfLv /lib64/ld-linux-x86-64.so.2 "$CURDIR/package/lib/ld.so"
cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 "$CURDIR/package/lib/libc.so.6"
cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 "$CURDIR/package/lib/libgcc_s.so.1"
cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 "$CURDIR/package/lib/libstdc++.so.6"
cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 "$CURDIR/package/lib/libm.so.6"
cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 "$CURDIR/package/lib/libgomp.so.1"
cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 "$CURDIR/package/lib/libdl.so.2"
cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 "$CURDIR/package/lib/librt.so.1"
cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 "$CURDIR/package/lib/libpthread.so.0"
elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
echo "Detected ARM64 architecture, copying ARM64 libraries..."
cp -arfLv /lib/ld-linux-aarch64.so.1 "$CURDIR/package/lib/ld.so"
cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 "$CURDIR/package/lib/libc.so.6"
cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 "$CURDIR/package/lib/libgcc_s.so.1"
cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 "$CURDIR/package/lib/libstdc++.so.6"
cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 "$CURDIR/package/lib/libm.so.6"
cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 "$CURDIR/package/lib/libgomp.so.1"
cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 "$CURDIR/package/lib/libdl.so.2"
cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 "$CURDIR/package/lib/librt.so.1"
cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 "$CURDIR/package/lib/libpthread.so.0"
elif [ "$(uname -s)" = "Darwin" ]; then
echo "Detected Darwin — system frameworks linked dynamically, no bundled libs needed"
else
echo "Error: Could not detect architecture"
exit 1
fi
# Package GPU libraries (CUDA/ROCm/Intel/Vulkan loader + ICDs + drivers) based
# on BUILD_TYPE so the backend can reach the GPU without the runtime base image
# shipping those drivers.
GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
if [ -f "$GPU_LIB_SCRIPT" ]; then
echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
package_gpu_libs
fi
echo "Packaging completed successfully"
ls -liah "$CURDIR/package/" "$CURDIR/package/lib/"

View File

@@ -0,0 +1,22 @@
#!/bin/bash
set -e
CURDIR=$(dirname "$(realpath "$0")")
if [ "$(uname)" = "Darwin" ]; then
export DYLD_LIBRARY_PATH="$CURDIR/lib:"$CURDIR":${DYLD_LIBRARY_PATH:-}"
export MOSS_TRANSCRIBE_LIBRARY="$CURDIR/lib/libmoss-transcribe.dylib"
else
export LD_LIBRARY_PATH="$CURDIR/lib:"$CURDIR":${LD_LIBRARY_PATH:-}"
export MOSS_TRANSCRIBE_LIBRARY="$CURDIR/lib/libmoss-transcribe.so"
fi
# If a self-contained ld.so was packaged, route through it so the packaged libc
# / libstdc++ are used instead of the host's (matches the whisper / parakeet-cpp
# backends' runtime layout). Linux only.
if [ -f "$CURDIR/lib/ld.so" ]; then
echo "Using lib/ld.so"
exec "$CURDIR/lib/ld.so" "$CURDIR/moss-transcribe-cpp-grpc" "$@"
fi
exec "$CURDIR/moss-transcribe-cpp-grpc" "$@"

View File

@@ -0,0 +1,156 @@
package main
import (
"regexp"
"strconv"
"strings"
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
)
// mossSegment is one parsed unit of the MOSS transcript: a speaker-labelled
// span with fractional-second start/end times, straight from the model's own
// output. Timestamps stay in seconds here; secondsToNanos converts them at the
// TranscriptSegment boundary.
type mossSegment struct {
Start float64
End float64
Speaker string
Text string
}
// bracketRe matches one "[...]" token (timestamp or speaker tag). The MOSS
// transcript is a concatenation of "[start][Sxx]text[end]" segments, e.g.
//
// [0.28][S01] And so, my fellow Americans,[7.71][8.12][S02] ask ...[10.59]
//
// so the bracketed tokens carry all the structure and the free text lives
// between a speaker tag and the following (end) timestamp.
var bracketRe = regexp.MustCompile(`\[([^\]]*)\]`)
// speakerRe matches a speaker tag: an 'S' (any case) followed by one or more
// digits, e.g. "S01", "S12". Anything else in brackets that isn't a speaker tag
// is treated as a timestamp candidate.
var speakerRe = regexp.MustCompile(`^[Ss][0-9]+$`)
// bracketToken is a "[...]" token plus the free text that follows it up to the
// next token (or end of string). For a speaker tag that trailing text is the
// segment's transcript.
type bracketToken struct {
content string
textAfter string
}
// parseTranscript parses the compact "[start][Sxx]text[end]..." MOSS transcript
// into structured segments. It walks the bracket tokens looking for a
// start-timestamp, a speaker tag, and an end-timestamp, taking the text that
// follows the speaker tag as the segment transcript. Tokens that don't fit the
// grammar are skipped rather than aborting the parse, so a slightly malformed
// stream still yields the segments it can.
func parseTranscript(raw string) []mossSegment {
locs := bracketRe.FindAllStringSubmatchIndex(raw, -1)
toks := make([]bracketToken, len(locs))
for i, m := range locs {
// m[2]:m[3] is the capture group (the content inside the brackets);
// m[1] is the byte just past the closing ']'.
nextStart := len(raw)
if i+1 < len(locs) {
nextStart = locs[i+1][0]
}
toks[i] = bracketToken{
content: raw[m[2]:m[3]],
textAfter: raw[m[1]:nextStart],
}
}
var segs []mossSegment
i := 0
for i < len(toks) {
start, ok := parseTimestamp(toks[i].content)
if !ok {
i++
continue
}
// A start timestamp must be followed by a speaker tag; otherwise this
// isn't a segment head, so skip it.
if i+1 >= len(toks) || !isSpeakerTag(toks[i+1].content) {
i++
continue
}
speaker := strings.ToUpper(strings.TrimSpace(toks[i+1].content))
text := strings.TrimSpace(toks[i+1].textAfter)
// The end timestamp is the next token if present and numeric; a segment
// that runs to the end of the stream without a closing timestamp falls
// back to start==end.
end := start
consumed := 2
if i+2 < len(toks) {
if e, ok := parseTimestamp(toks[i+2].content); ok {
end = e
consumed = 3
}
}
segs = append(segs, mossSegment{Start: start, End: end, Speaker: speaker, Text: text})
i += consumed
}
return segs
}
// isSpeakerTag reports whether the bracket content is a MOSS speaker tag ("Sxx").
func isSpeakerTag(content string) bool {
return speakerRe.MatchString(strings.TrimSpace(content))
}
// parseTimestamp parses a bracket content as a fractional-second timestamp. A
// speaker tag ("S01") never parses as a float, so this cleanly distinguishes
// the two token kinds.
func parseTimestamp(content string) (float64, bool) {
f, err := strconv.ParseFloat(strings.TrimSpace(content), 64)
if err != nil {
return 0, false
}
return f, true
}
// secondsToNanos converts the transcript's fractional-second timestamps into
// the int64 nanoseconds LocalAI carries on TranscriptSegment, the same
// nanosecond convention the whisper / parakeet-cpp backends use.
func secondsToNanos(sec float64) int64 {
return int64(sec * 1e9)
}
// transcriptResultFromRaw parses the raw MOSS transcript and shapes it into a
// TranscriptResult. Each parsed segment becomes a TranscriptSegment with
// nanosecond start/end and the model's own speaker label; Text is the segments
// joined with single spaces. When nothing parses (no bracket structure) it
// falls back to a single whole-clip text segment so callers always get a
// transcript.
func transcriptResultFromRaw(raw string) pb.TranscriptResult {
segs := parseTranscript(raw)
if len(segs) == 0 {
text := strings.TrimSpace(raw)
return pb.TranscriptResult{
Text: text,
Segments: []*pb.TranscriptSegment{{Id: 0, Text: text}},
}
}
var full strings.Builder
pbSegs := make([]*pb.TranscriptSegment, 0, len(segs))
for id, s := range segs {
if id > 0 && s.Text != "" {
full.WriteString(" ")
}
full.WriteString(s.Text)
pbSegs = append(pbSegs, &pb.TranscriptSegment{
Id: int32(id),
Start: secondsToNanos(s.Start),
End: secondsToNanos(s.End),
Text: s.Text,
Speaker: s.Speaker,
})
}
return pb.TranscriptResult{Text: strings.TrimSpace(full.String()), Segments: pbSegs}
}

View File

@@ -0,0 +1,104 @@
package main
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("parseTranscript ([start][Sxx]text[end] grammar)", func() {
It("parses the README two-speaker example into ordered segments", func() {
raw := "[0.28][S01] And so, my fellow Americans, ask not what your country can do for you,[7.71][8.12][S02] ask what you can do for your country.[10.59]"
segs := parseTranscript(raw)
Expect(segs).To(HaveLen(2))
Expect(segs[0].Speaker).To(Equal("S01"))
Expect(segs[0].Start).To(BeNumerically("~", 0.28, 1e-9))
Expect(segs[0].End).To(BeNumerically("~", 7.71, 1e-9))
Expect(segs[0].Text).To(Equal("And so, my fellow Americans, ask not what your country can do for you,"))
Expect(segs[1].Speaker).To(Equal("S02"))
Expect(segs[1].Start).To(BeNumerically("~", 8.12, 1e-9))
Expect(segs[1].End).To(BeNumerically("~", 10.59, 1e-9))
Expect(segs[1].Text).To(Equal("ask what you can do for your country."))
})
It("parses a single segment", func() {
segs := parseTranscript("[1.50][S01] hello world[3.25]")
Expect(segs).To(HaveLen(1))
Expect(segs[0].Start).To(BeNumerically("~", 1.50, 1e-9))
Expect(segs[0].End).To(BeNumerically("~", 3.25, 1e-9))
Expect(segs[0].Speaker).To(Equal("S01"))
Expect(segs[0].Text).To(Equal("hello world"))
})
It("uppercases a lowercase speaker tag", func() {
segs := parseTranscript("[0.00][s03] hi[0.90]")
Expect(segs).To(HaveLen(1))
Expect(segs[0].Speaker).To(Equal("S03"))
})
It("falls back to start==end when the closing timestamp is missing", func() {
segs := parseTranscript("[2.00][S01] trailing words with no end")
Expect(segs).To(HaveLen(1))
Expect(segs[0].Start).To(BeNumerically("~", 2.00, 1e-9))
Expect(segs[0].End).To(Equal(segs[0].Start))
Expect(segs[0].Text).To(Equal("trailing words with no end"))
})
It("handles multi-digit speaker ids and higher speaker counts", func() {
segs := parseTranscript("[0.0][S01] a[1.0][1.0][S12] b[2.0]")
Expect(segs).To(HaveLen(2))
Expect(segs[0].Speaker).To(Equal("S01"))
Expect(segs[1].Speaker).To(Equal("S12"))
})
It("returns no segments for text without bracket structure", func() {
Expect(parseTranscript("just some plain text")).To(BeEmpty())
Expect(parseTranscript("")).To(BeEmpty())
})
It("skips a start timestamp not followed by a speaker tag", func() {
// A stray timestamp pair without a speaker tag is not a segment head.
segs := parseTranscript("[1.0][2.0][S01] real[3.0]")
Expect(segs).To(HaveLen(1))
Expect(segs[0].Speaker).To(Equal("S01"))
Expect(segs[0].Text).To(Equal("real"))
})
})
var _ = Describe("transcriptResultFromRaw", func() {
It("shapes parsed segments into a TranscriptResult with nanosecond times and speakers", func() {
raw := "[0.28][S01] first part,[7.71][8.12][S02] second part.[10.59]"
res := transcriptResultFromRaw(raw)
Expect(res.Segments).To(HaveLen(2))
Expect(res.Segments[0].Id).To(Equal(int32(0)))
Expect(res.Segments[0].Start).To(Equal(secondsToNanos(0.28)))
Expect(res.Segments[0].End).To(Equal(secondsToNanos(7.71)))
Expect(res.Segments[0].Speaker).To(Equal("S01"))
Expect(res.Segments[0].Text).To(Equal("first part,"))
Expect(res.Segments[1].Id).To(Equal(int32(1)))
Expect(res.Segments[1].Speaker).To(Equal("S02"))
Expect(res.Segments[1].Start).To(Equal(secondsToNanos(8.12)))
// Full text is the segments joined with single spaces.
Expect(res.Text).To(Equal("first part, second part."))
})
It("falls back to a single whole-clip text segment when nothing parses", func() {
res := transcriptResultFromRaw("plain transcript with no markers")
Expect(res.Segments).To(HaveLen(1))
Expect(res.Segments[0].Text).To(Equal("plain transcript with no markers"))
Expect(res.Segments[0].Start).To(Equal(int64(0)))
Expect(res.Text).To(Equal("plain transcript with no markers"))
})
})
var _ = Describe("secondsToNanos", func() {
It("converts fractional seconds to int64 nanoseconds", func() {
Expect(secondsToNanos(0)).To(Equal(int64(0)))
Expect(secondsToNanos(1)).To(Equal(int64(1e9)))
Expect(secondsToNanos(0.28)).To(Equal(int64(0.28 * 1e9)))
})
})

View File

@@ -178,6 +178,35 @@
nvidia-cuda-12: "cuda12-parakeet-cpp"
nvidia-l4t-cuda-12: "nvidia-l4t-arm64-parakeet-cpp"
nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-parakeet-cpp"
- &mosstranscribecpp
name: "moss-transcribe-cpp"
alias: "moss-transcribe-cpp"
license: mit
icon: https://avatars.githubusercontent.com/u/95302084
description: |
moss-transcribe.cpp is a C++/ggml port of OpenMOSS MOSS-Transcribe-Diarize, an end-to-end audio understanding model.
It does joint long-form transcription, speaker diarization and timestamping in a single pass, emitting a speaker-labelled,
time-aligned transcript, and runs on CPU, NVIDIA CUDA, AMD ROCm/HIP, Intel SYCL, Vulkan, Apple Metal and NVIDIA Jetson (L4T) targets.
urls:
- https://github.com/mudler/moss-transcribe.cpp
tags:
- audio-transcription
- CPU
- GPU
- CUDA
- HIP
capabilities:
default: "cpu-moss-transcribe-cpp"
nvidia: "cuda12-moss-transcribe-cpp"
intel: "intel-sycl-f16-moss-transcribe-cpp"
metal: "metal-moss-transcribe-cpp"
amd: "rocm-moss-transcribe-cpp"
vulkan: "vulkan-moss-transcribe-cpp"
nvidia-l4t: "nvidia-l4t-arm64-moss-transcribe-cpp"
nvidia-cuda-13: "cuda13-moss-transcribe-cpp"
nvidia-cuda-12: "cuda12-moss-transcribe-cpp"
nvidia-l4t-cuda-12: "nvidia-l4t-arm64-moss-transcribe-cpp"
nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp"
- &ced
name: "ced"
alias: "ced"
@@ -2784,6 +2813,121 @@
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-parakeet-cpp"
mirrors:
- localai/localai-backends:master-gpu-nvidia-cuda-13-parakeet-cpp
## moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "moss-transcribe-cpp-development"
capabilities:
default: "cpu-moss-transcribe-cpp-development"
nvidia: "cuda12-moss-transcribe-cpp-development"
intel: "intel-sycl-f16-moss-transcribe-cpp-development"
metal: "metal-moss-transcribe-cpp-development"
amd: "rocm-moss-transcribe-cpp-development"
vulkan: "vulkan-moss-transcribe-cpp-development"
nvidia-l4t: "nvidia-l4t-arm64-moss-transcribe-cpp-development"
nvidia-cuda-13: "cuda13-moss-transcribe-cpp-development"
nvidia-cuda-12: "cuda12-moss-transcribe-cpp-development"
nvidia-l4t-cuda-12: "nvidia-l4t-arm64-moss-transcribe-cpp-development"
nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp-development"
- !!merge <<: *mosstranscribecpp
name: "nvidia-l4t-arm64-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-arm64-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-nvidia-l4t-arm64-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "nvidia-l4t-arm64-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-arm64-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-nvidia-l4t-arm64-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cuda13-nvidia-l4t-arm64-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cpu-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-cpu-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cpu-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-cpu-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-cpu-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "metal-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-metal-darwin-arm64-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "metal-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-metal-darwin-arm64-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cuda12-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-gpu-nvidia-cuda-12-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cuda12-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-gpu-nvidia-cuda-12-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "rocm-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-gpu-rocm-hipblas-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "rocm-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-gpu-rocm-hipblas-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "intel-sycl-f32-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-gpu-intel-sycl-f32-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "intel-sycl-f32-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-gpu-intel-sycl-f32-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "intel-sycl-f16-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f16-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-gpu-intel-sycl-f16-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "intel-sycl-f16-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f16-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-gpu-intel-sycl-f16-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "vulkan-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-gpu-vulkan-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "vulkan-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-gpu-vulkan-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cuda13-moss-transcribe-cpp"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:latest-gpu-nvidia-cuda-13-moss-transcribe-cpp
- !!merge <<: *mosstranscribecpp
name: "cuda13-moss-transcribe-cpp-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-moss-transcribe-cpp"
mirrors:
- localai/localai-backends:master-gpu-nvidia-cuda-13-moss-transcribe-cpp
## ced
- !!merge <<: *ced
name: "ced-development"

View File

@@ -119,6 +119,10 @@ var defaultImporters = []Importer{
// kept ahead of LlamaCPPImporter so its .gguf bundles aren't claimed by the
// generic GGUF importer.
&ParakeetCppImporter{},
// MossTranscribeCppImporter matches only moss-transcribe GGUFs
// (moss-transcribe[-<quant>].gguf); like parakeet-cpp it stays ahead of
// LlamaCPPImporter so its .gguf bundles aren't claimed by the generic GGUF importer.
&MossTranscribeCppImporter{},
// TTS (Batch 2)
&PiperImporter{},
&BarkImporter{},

View File

@@ -0,0 +1,178 @@
package importers
import (
"encoding/json"
"path/filepath"
"strings"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/schema"
"github.com/mudler/LocalAI/pkg/downloader"
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
"go.yaml.in/yaml/v2"
)
var _ Importer = &MossTranscribeCppImporter{}
// MossTranscribeCppImporter recognises moss-transcribe.cpp GGUF weights, the
// C++/ggml port of OpenMOSS MOSS-Transcribe-Diarize. The signal is narrow on
// purpose: moss-transcribe.cpp names its weights "moss-transcribe[-<quant>].gguf"
// (e.g. moss-transcribe-q5_k.gguf, moss-transcribe-q8_0.gguf), so we only match a
// .gguf whose name carries the "moss-transcribe" token. That keeps us from
// claiming arbitrary llama-style GGUFs (the importer is registered before
// llama-cpp). preferences.backend="moss-transcribe-cpp" forces the importer
// regardless.
type MossTranscribeCppImporter struct{}
func (i *MossTranscribeCppImporter) Name() string { return "moss-transcribe-cpp" }
func (i *MossTranscribeCppImporter) Modality() string { return "asr" }
func (i *MossTranscribeCppImporter) AutoDetects() bool { return true }
func (i *MossTranscribeCppImporter) Match(details Details) bool {
preferences, err := details.Preferences.MarshalJSON()
if err != nil {
return false
}
preferencesMap := make(map[string]any)
if len(preferences) > 0 {
if err := json.Unmarshal(preferences, &preferencesMap); err != nil {
return false
}
}
if b, ok := preferencesMap["backend"].(string); ok && b == "moss-transcribe-cpp" {
return true
}
// Direct URL or path to a moss-transcribe GGUF.
if isMossTranscribeGGUF(filepath.Base(details.URI)) {
return true
}
// HF repo shipping at least one moss-transcribe GGUF.
if details.HuggingFace != nil {
for _, f := range details.HuggingFace.Files {
if isMossTranscribeGGUF(filepath.Base(f.Path)) {
return true
}
}
}
return false
}
func (i *MossTranscribeCppImporter) Import(details Details) (gallery.ModelConfig, error) {
preferences, err := details.Preferences.MarshalJSON()
if err != nil {
return gallery.ModelConfig{}, err
}
preferencesMap := make(map[string]any)
if len(preferences) > 0 {
if err := json.Unmarshal(preferences, &preferencesMap); err != nil {
return gallery.ModelConfig{}, err
}
}
name, ok := preferencesMap["name"].(string)
if !ok {
name = filepath.Base(details.URI)
}
description, ok := preferencesMap["description"].(string)
if !ok {
description = "Imported from " + details.URI
}
// MOSS quants are byte-exact against the reference even at Q5_K (and ~1/6 the
// size), so default to q5_k, then fall back up the size ladder; the last file
// wins if none match (mirrors parakeet-cpp / whisper).
preferredQuants, _ := preferencesMap["quantizations"].(string)
quants := []string{"q5_k", "q4_k", "q6_k", "q8_0", "f16"}
if preferredQuants != "" {
quants = strings.Split(preferredQuants, ",")
}
cfg := gallery.ModelConfig{
Name: name,
Description: description,
}
modelConfig := config.ModelConfig{
Name: name,
Description: description,
Backend: "moss-transcribe-cpp",
KnownUsecaseStrings: []string{"transcript"},
}
uri := downloader.URI(details.URI)
directGGUF := isMossTranscribeGGUF(filepath.Base(details.URI))
switch {
case uri.LooksLikeURL() && directGGUF:
// Direct file URL (e.g. .../resolve/main/moss-transcribe-q5_k.gguf). The
// exact file is known, no quant pick.
fileName, err := uri.FilenameFromUrl()
if err != nil {
return gallery.ModelConfig{}, err
}
target := filepath.Join("moss-transcribe-cpp", "models", name, fileName)
cfg.Files = append(cfg.Files, gallery.File{
URI: details.URI,
Filename: target,
})
modelConfig.PredictionOptions = schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{Model: target},
}
case details.HuggingFace != nil:
// HF repo: collect every moss-transcribe GGUF, pick the preferred quant,
// and nest under moss-transcribe-cpp/models/<name>/ so a multi-quant repo
// doesn't collide on disk.
var ggufFiles []hfapi.ModelFile
for _, f := range details.HuggingFace.Files {
if isMossTranscribeGGUF(filepath.Base(f.Path)) {
ggufFiles = append(ggufFiles, f)
}
}
if chosen, ok := pickPreferredGGMLFile(ggufFiles, quants); ok {
target := filepath.Join("moss-transcribe-cpp", "models", name, filepath.Base(chosen.Path))
cfg.Files = append(cfg.Files, gallery.File{
URI: chosen.URL,
Filename: target,
SHA256: chosen.SHA256,
})
modelConfig.PredictionOptions = schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{Model: target},
}
}
default:
// Bare URI with no HF metadata (pref-only path): point at the basename so
// users can tweak the YAML after import.
modelConfig.PredictionOptions = schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{Model: filepath.Base(details.URI)},
}
}
data, err := yaml.Marshal(modelConfig)
if err != nil {
return gallery.ModelConfig{}, err
}
cfg.ConfigFile = string(data)
return cfg, nil
}
// isMossTranscribeGGUF reports whether name is a moss-transcribe.cpp GGUF: a
// .gguf file whose name carries the "moss-transcribe" token (hyphen or
// underscore form). The .gguf check is case-insensitive.
func isMossTranscribeGGUF(name string) bool {
lower := strings.ToLower(name)
if !strings.HasSuffix(lower, ".gguf") {
return false
}
for _, tok := range []string{"moss-transcribe", "moss_transcribe"} {
if strings.Contains(lower, tok) {
return true
}
}
return false
}

View File

@@ -0,0 +1,96 @@
package importers_test
import (
"encoding/json"
"fmt"
"github.com/mudler/LocalAI/core/gallery/importers"
hfapi "github.com/mudler/LocalAI/pkg/huggingface-api"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// mossDetails builds Details carrying a synthetic HF file list so detection can
// be exercised without hitting the network.
func mossDetails(uri string, prefs string, files ...hfapi.ModelFile) importers.Details {
return importers.Details{
URI: uri,
Preferences: json.RawMessage(prefs),
HuggingFace: &hfapi.ModelDetails{Files: files},
}
}
var _ = Describe("MossTranscribeCppImporter", func() {
imp := &importers.MossTranscribeCppImporter{}
Context("Importer interface metadata", func() {
It("exposes name/modality/autodetect", func() {
Expect(imp.Name()).To(Equal("moss-transcribe-cpp"))
Expect(imp.Modality()).To(Equal("asr"))
Expect(imp.AutoDetects()).To(BeTrue())
})
})
Context("detection (Match)", func() {
It("matches an HF repo shipping a moss-transcribe GGUF", func() {
d := mossDetails("huggingface://mudler/moss-transcribe.cpp-gguf", `{}`,
hfapi.ModelFile{Path: "moss-transcribe-q5_k.gguf"},
hfapi.ModelFile{Path: "README.md"},
)
Expect(imp.Match(d)).To(BeTrue())
})
It("matches a direct URL to a moss-transcribe GGUF", func() {
d := mossDetails("https://huggingface.co/mudler/moss-transcribe.cpp-gguf/resolve/main/moss-transcribe-q8_0.gguf", `{}`)
Expect(imp.Match(d)).To(BeTrue())
})
It("honours preferences.backend=moss-transcribe-cpp for arbitrary URIs", func() {
d := mossDetails("https://example.com/whatever", `{"backend": "moss-transcribe-cpp"}`)
Expect(imp.Match(d)).To(BeTrue())
})
It("does NOT claim a generic llama-style GGUF", func() {
d := mossDetails("huggingface://someorg/some-llm-gguf", `{}`,
hfapi.ModelFile{Path: "llama-3-8b-instruct-q4_k_m.gguf"},
)
Expect(imp.Match(d)).To(BeFalse())
})
})
Context("import (Import)", func() {
It("picks the default quant (q5_k) from a multi-quant HF repo", func() {
d := mossDetails("huggingface://mudler/moss-transcribe.cpp-gguf", `{"name":"moss-transcribe"}`,
hfapi.ModelFile{Path: "moss-transcribe-f16.gguf", URL: "https://hf/f16", SHA256: "aaa"},
hfapi.ModelFile{Path: "moss-transcribe-q5_k.gguf", URL: "https://hf/q5k", SHA256: "bbb"},
hfapi.ModelFile{Path: "moss-transcribe-q8_0.gguf", URL: "https://hf/q8", SHA256: "ccc"},
)
cfg, err := imp.Import(d)
Expect(err).ToNot(HaveOccurred())
Expect(cfg.ConfigFile).To(ContainSubstring("backend: moss-transcribe-cpp"), fmt.Sprintf("%+v", cfg))
Expect(cfg.ConfigFile).To(ContainSubstring("transcript"))
Expect(cfg.Files).To(HaveLen(1))
Expect(cfg.Files[0].URI).To(Equal("https://hf/q5k"), "default quant should be q5_k")
Expect(cfg.Files[0].Filename).To(ContainSubstring("moss-transcribe-cpp/models/moss-transcribe/moss-transcribe-q5_k.gguf"))
})
It("honours a preferred quantization override", func() {
d := mossDetails("huggingface://mudler/moss-transcribe.cpp-gguf", `{"name":"m","quantizations":"q8_0"}`,
hfapi.ModelFile{Path: "moss-transcribe-q5_k.gguf", URL: "https://hf/q5k"},
hfapi.ModelFile{Path: "moss-transcribe-q8_0.gguf", URL: "https://hf/q8"},
)
cfg, err := imp.Import(d)
Expect(err).ToNot(HaveOccurred())
Expect(cfg.Files).To(HaveLen(1))
Expect(cfg.Files[0].URI).To(Equal("https://hf/q8"))
})
It("uses the exact file for a direct GGUF URL", func() {
d := mossDetails("https://huggingface.co/mudler/moss-transcribe.cpp-gguf/resolve/main/moss-transcribe-q5_k.gguf", `{"name":"moss"}`)
cfg, err := imp.Import(d)
Expect(err).ToNot(HaveOccurred())
Expect(cfg.Files).To(HaveLen(1))
Expect(cfg.Files[0].Filename).To(ContainSubstring("moss-transcribe-cpp/models/moss/moss-transcribe-q5_k.gguf"))
})
})
})

View File

@@ -125,7 +125,7 @@ For getting started, see the available backends in LocalAI here: https://github.
LocalAI supports various types of backends:
- **LLM Backends**: For running language models (e.g., llama.cpp, vLLM, SGLang, transformers, MLX)
- **Speech-to-Text Backends**: For transcription (e.g., whisper.cpp, parakeet.cpp, faster-whisper, NeMo)
- **Speech-to-Text Backends**: For transcription and speaker diarization (e.g., whisper.cpp, parakeet.cpp, moss-transcribe.cpp, faster-whisper, NeMo)
- **Text-to-Speech Backends**: For speech synthesis (e.g., piper, Kokoro, VibeVoice, Qwen3-TTS)
- **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step)
- **Sound Classification Backends**: For sound-event classification / audio tagging - identifying everyday sounds like baby cry, glass breaking, alarms (e.g., ced.cpp)

View File

@@ -36199,6 +36199,37 @@
- filename: parakeet-cpp/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf
uri: huggingface://mudler/parakeet-cpp-gguf/nemotron-3.5-asr-streaming-0.6b-q8_0.gguf
sha256: ba2f13eccd4a5245be728f77e6149bd6a4fdcdd133ff2e08ac6005bcef7a99f1
- name: moss-transcribe-cpp-0.9b
url: github:mudler/LocalAI/gallery/virtual.yaml@master
urls:
- https://huggingface.co/mudler/moss-transcribe.cpp-gguf
- https://github.com/mudler/moss-transcribe.cpp
description: |
MOSS-Transcribe-Diarize 0.9B, an end-to-end audio understanding model for joint multi-speaker transcription,
speaker diarization and timestamping in a single pass. Q5_K GGUF for the moss-transcribe-cpp backend (C++/ggml
port of OpenMOSS MOSS-Transcribe-Diarize), byte-identical to the reference at ~1/6 the size. Faster than the
reference PyTorch runtime on CPU (1.6 to 2.2x).
license: apache-2.0
tags:
- moss
- moss-transcribe-cpp
- asr
- diarization
- speech-recognition
- stt
- gguf
- ggml
overrides:
backend: moss-transcribe-cpp
known_usecases:
- transcript
name: moss-transcribe-cpp-0.9b
parameters:
model: moss-transcribe-cpp/moss-transcribe-q5_k.gguf
files:
- filename: moss-transcribe-cpp/moss-transcribe-q5_k.gguf
uri: huggingface://mudler/moss-transcribe.cpp-gguf/moss-transcribe-q5_k.gguf
sha256: 6fe985fd9a9cd043728f7be49659d10de6901872dad73b49e3c30e1340582e79
- name: parakeet-crispasr
url: github:mudler/LocalAI/gallery/virtual.yaml@master
urls:

View File

@@ -33,6 +33,14 @@ function inferBackendPath(item) {
if (item.backend === "ced") {
return `backend/go/ced/`;
}
// moss-transcribe-cpp is a Go backend (Dockerfile.golang) wrapping the
// moss-transcribe.cpp ggml port via purego, living in
// backend/go/moss-transcribe-cpp/. Same explicit-branch rationale as
// parakeet-cpp / ced: the generic golang fallthrough would also resolve it,
// but this documents the mapping and guards a future dockerfile-suffix change.
if (item.backend === "moss-transcribe-cpp") {
return `backend/go/moss-transcribe-cpp/`;
}
if (item.dockerfile.endsWith("golang")) {
return `backend/go/${item.backend}/`;
}