fix(audio): reject incompatible transform streams (#11565)

The transform WebSocket accepted any model and opened its frame-based RPC. Any-to-any models use a different stream contract, so liquid-audio failed with an unimplemented RPC after the handshake.

Reject incompatible model use cases before loading the backend. Direct realtime-audio callers to the OpenAI Realtime API.

Assisted-by: Codex:gpt-5

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

core/http/endpoints/localai/audio_transform.go

@@ -99,6 +99,20 @@ func validateAudioTransformSampleRate(sampleRate int) error {
 	return nil
 }
 
+func validateAudioTransformStreamModel(cfg *config.ModelConfig) error {
+	if cfg != nil && cfg.HasUsecases(config.FLAG_AUDIO_TRANSFORM) {
+		return nil
+	}
+	if cfg != nil && cfg.HasUsecases(config.FLAG_REALTIME_AUDIO) {
+		return fmt.Errorf("model %q is an any-to-any model; use the OpenAI Realtime API instead", cfg.Name)
+	}
+	modelName := ""
+	if cfg != nil {
+		modelName = cfg.Name
+	}
+	return fmt.Errorf("model %q does not support streaming audio transforms", modelName)
+}
+
 // AudioTransformEndpoint implements the batch audio-transform API. Accepts a
 // multipart/form-data request with `audio` (required) and an optional
 // `reference` file. Backend-specific tuning is forwarded via repeated
@@ -239,7 +253,7 @@ func AudioTransformEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
 // and on schema.AudioTransformStreamControl.
 //
 // @Summary Bidirectional realtime audio transform over WebSocket.
-// @Description Streams binary PCM frames in (interleaved stereo: ch0=audio, ch1=reference) and out (mono). The first message must be a JSON `session.update` envelope describing model + sample format + frame size + backend params. Server emits binary PCM on the same cadence.
+// @Description Streams binary PCM frames in (interleaved stereo: ch0=audio, ch1=reference) and out (mono). The model must support the audio_transform use case. Any-to-any models such as liquid-audio use the OpenAI Realtime API instead. The first message must be a JSON `session.update` envelope describing model + sample format + frame size + backend params. Server emits binary PCM on the same cadence.
 // @Tags audio
 // @Router /audio/transformations/stream [get]
 func AudioTransformStreamEndpoint(app *application.Application) echo.HandlerFunc {
@@ -280,6 +294,10 @@ func AudioTransformStreamEndpoint(app *application.Application) echo.HandlerFunc
 			sendWSError(ws, fmt.Sprintf("failed to load model config: %v", err))
 			return nil
 		}
+		if err := validateAudioTransformStreamModel(cfg); err != nil {
+			sendWSError(ws, err.Error())
+			return nil
+		}
 
 		ctx, cancel := context.WithCancel(c.Request().Context())
 		defer cancel()

core/http/endpoints/localai/audio_transform_internal_test.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 
 	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
 	laudio "github.com/mudler/LocalAI/pkg/audio"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -119,6 +120,21 @@ var _ = Describe("audio transform sample_rate bounds", func() {
 	})
 })
 
+var _ = Describe("audio transform stream model validation", func() {
+	It("accepts models that advertise audio transforms", func() {
+		usecases := config.FLAG_AUDIO_TRANSFORM
+		cfg := &config.ModelConfig{KnownUsecases: &usecases}
+		Expect(validateAudioTransformStreamModel(cfg)).To(Succeed())
+	})
+
+	It("rejects realtime any-to-any models", func() {
+		usecases := config.FLAG_REALTIME_AUDIO
+		cfg := &config.ModelConfig{KnownUsecases: &usecases, Backend: "liquid-audio"}
+		err := validateAudioTransformStreamModel(cfg)
+		Expect(err).To(MatchError(ContainSubstring("OpenAI Realtime API")))
+	})
+})
+
 // Both uploads land in the SAME temp dir, and a client is free to send the same
 // BASENAME on both parts (`-F audio=@mic/clip.wav -F reference=@loopback/clip.wav`
 // is an ordinary request, not an attack). The raw copy is therefore named after

docs/content/features/audio-transform.md

@@ -172,6 +172,11 @@ noise-suppression request is unaffected.
 client message is a JSON envelope; subsequent client messages are binary
 PCM frames; server emits binary PCM frames at the same cadence.
 
+This endpoint accepts models with the `audio_transform` use case. It does not
+accept any-to-any models with the `realtime_audio` use case. For example,
+`liquid-audio` uses the [OpenAI Realtime API]({{% relref "features/openai-realtime" %}})
+instead.
+
 ### Wire format
 
 **Client → server** (text frame, first):