From a7bce6a128d968971acb8f5489694a38357058e6 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 18 Aug 2026 09:03:16 +0200 Subject: [PATCH] fix(audio): reject incompatible transform streams (#11565) The transform WebSocket accepted any model and opened its frame-based RPC. Any-to-any models use a different stream contract, so liquid-audio failed with an unimplemented RPC after the handshake. Reject incompatible model use cases before loading the backend. Direct realtime-audio callers to the OpenAI Realtime API. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- .../http/endpoints/localai/audio_transform.go | 20 ++++++++++++++++++- .../localai/audio_transform_internal_test.go | 16 +++++++++++++++ docs/content/features/audio-transform.md | 5 +++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/core/http/endpoints/localai/audio_transform.go b/core/http/endpoints/localai/audio_transform.go index 5b3be15b5..db74c43f0 100644 --- a/core/http/endpoints/localai/audio_transform.go +++ b/core/http/endpoints/localai/audio_transform.go @@ -99,6 +99,20 @@ func validateAudioTransformSampleRate(sampleRate int) error { return nil } +func validateAudioTransformStreamModel(cfg *config.ModelConfig) error { + if cfg != nil && cfg.HasUsecases(config.FLAG_AUDIO_TRANSFORM) { + return nil + } + if cfg != nil && cfg.HasUsecases(config.FLAG_REALTIME_AUDIO) { + return fmt.Errorf("model %q is an any-to-any model; use the OpenAI Realtime API instead", cfg.Name) + } + modelName := "" + if cfg != nil { + modelName = cfg.Name + } + return fmt.Errorf("model %q does not support streaming audio transforms", modelName) +} + // AudioTransformEndpoint implements the batch audio-transform API. Accepts a // multipart/form-data request with `audio` (required) and an optional // `reference` file. Backend-specific tuning is forwarded via repeated @@ -239,7 +253,7 @@ func AudioTransformEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, // and on schema.AudioTransformStreamControl. // // @Summary Bidirectional realtime audio transform over WebSocket. -// @Description Streams binary PCM frames in (interleaved stereo: ch0=audio, ch1=reference) and out (mono). The first message must be a JSON `session.update` envelope describing model + sample format + frame size + backend params. Server emits binary PCM on the same cadence. +// @Description Streams binary PCM frames in (interleaved stereo: ch0=audio, ch1=reference) and out (mono). The model must support the audio_transform use case. Any-to-any models such as liquid-audio use the OpenAI Realtime API instead. The first message must be a JSON `session.update` envelope describing model + sample format + frame size + backend params. Server emits binary PCM on the same cadence. // @Tags audio // @Router /audio/transformations/stream [get] func AudioTransformStreamEndpoint(app *application.Application) echo.HandlerFunc { @@ -280,6 +294,10 @@ func AudioTransformStreamEndpoint(app *application.Application) echo.HandlerFunc sendWSError(ws, fmt.Sprintf("failed to load model config: %v", err)) return nil } + if err := validateAudioTransformStreamModel(cfg); err != nil { + sendWSError(ws, err.Error()) + return nil + } ctx, cancel := context.WithCancel(c.Request().Context()) defer cancel() diff --git a/core/http/endpoints/localai/audio_transform_internal_test.go b/core/http/endpoints/localai/audio_transform_internal_test.go index 64a6becb3..0f831883b 100644 --- a/core/http/endpoints/localai/audio_transform_internal_test.go +++ b/core/http/endpoints/localai/audio_transform_internal_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" laudio "github.com/mudler/LocalAI/pkg/audio" . "github.com/onsi/ginkgo/v2" @@ -119,6 +120,21 @@ var _ = Describe("audio transform sample_rate bounds", func() { }) }) +var _ = Describe("audio transform stream model validation", func() { + It("accepts models that advertise audio transforms", func() { + usecases := config.FLAG_AUDIO_TRANSFORM + cfg := &config.ModelConfig{KnownUsecases: &usecases} + Expect(validateAudioTransformStreamModel(cfg)).To(Succeed()) + }) + + It("rejects realtime any-to-any models", func() { + usecases := config.FLAG_REALTIME_AUDIO + cfg := &config.ModelConfig{KnownUsecases: &usecases, Backend: "liquid-audio"} + err := validateAudioTransformStreamModel(cfg) + Expect(err).To(MatchError(ContainSubstring("OpenAI Realtime API"))) + }) +}) + // Both uploads land in the SAME temp dir, and a client is free to send the same // BASENAME on both parts (`-F audio=@mic/clip.wav -F reference=@loopback/clip.wav` // is an ordinary request, not an attack). The raw copy is therefore named after diff --git a/docs/content/features/audio-transform.md b/docs/content/features/audio-transform.md index 8b37f42cd..58733f51f 100644 --- a/docs/content/features/audio-transform.md +++ b/docs/content/features/audio-transform.md @@ -172,6 +172,11 @@ noise-suppression request is unaffected. client message is a JSON envelope; subsequent client messages are binary PCM frames; server emits binary PCM frames at the same cadence. +This endpoint accepts models with the `audio_transform` use case. It does not +accept any-to-any models with the `realtime_audio` use case. For example, +`liquid-audio` uses the [OpenAI Realtime API]({{% relref "features/openai-realtime" %}}) +instead. + ### Wire format **Client → server** (text frame, first):