From e1c06465bd787516d5a7a90ee03e7e1a50639167 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 12 Jun 2026 22:13:48 +0000 Subject: [PATCH] fix(realtime): resolve platform opus codec backend by alias (#9813) The realtime WebRTC transport hardcoded loading the literal "opus" backend. On darwin/arm64 the only installable opus codec is "metal-opus" (it shares the gallery alias "opus"). Backend names resolve through the model loader's external-backends map, and the "opus" alias key is only registered when a user-path variant's alias was collected; a system-path metal-opus registers only under its concrete name. As a result, with metal-opus installed the realtime path still failed with "opus backend not available". Resolve the opus codec from the set of currently loadable backends instead of the hardcoded literal: an exact "opus" match wins (covers the plain backend and the alias key), otherwise fall back to a platform-appropriate "*opus*" codec, preferring the metal build on darwin/arm64. Behavior is unchanged when a plain "opus" backend is present, and the same error is surfaced when no opus codec is installed at all. The selection logic is extracted into resolveOpusBackend and unit-tested. Assisted-by: claude:claude-opus-4-8 [Claude Code] Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/openai/realtime_webrtc.go | 68 +++++++++++++++++-- .../openai/realtime_webrtc_opus_test.go | 32 +++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 core/http/endpoints/openai/realtime_webrtc_opus_test.go diff --git a/core/http/endpoints/openai/realtime_webrtc.go b/core/http/endpoints/openai/realtime_webrtc.go index 0ac982c19..066200088 100644 --- a/core/http/endpoints/openai/realtime_webrtc.go +++ b/core/http/endpoints/openai/realtime_webrtc.go @@ -2,6 +2,8 @@ package openai import ( "net/http" + "runtime" + "strings" "time" "github.com/labstack/echo/v4" @@ -11,6 +13,52 @@ import ( "github.com/pion/webrtc/v4" ) +// opusBackendName is the canonical gallery name/alias of the opus audio codec +// backend that the realtime WebRTC transport needs. +const opusBackendName = "opus" + +// resolveOpusBackend picks which installed opus-codec backend the realtime +// WebRTC transport should load. The transport historically hardcoded the +// literal "opus" backend name, but on darwin/arm64 the only installable opus +// codec is "metal-opus" (it shares the gallery alias "opus"). When that +// platform-specific variant is registered under its concrete directory name +// rather than the "opus" alias key, loading the literal "opus" fails with +// "opus backend not available" (issue #9813). Given the set of currently +// loadable backend names, this returns the best opus codec to load for the +// running platform, falling back to the literal name so the caller surfaces +// the same error as before when no opus codec is installed at all. +func resolveOpusBackend(installed []string, goos, goarch string) string { + // An exact match wins: this covers the plain "opus" backend as well as the + // "opus" alias key registered by gallery alias resolution for a + // user-installed platform variant. + for _, b := range installed { + if b == opusBackendName { + return opusBackendName + } + } + + // No "opus" key is registered (e.g. a system-path metal-opus whose alias + // was never collected). Fall back to a platform-appropriate "*opus*" codec + // backend; on darwin/arm64 prefer the metal build. + var fallback string + for _, b := range installed { + if !strings.Contains(strings.ToLower(b), opusBackendName) { + continue + } + if goos == "darwin" && goarch == "arm64" && strings.Contains(strings.ToLower(b), "metal") { + return b + } + if fallback == "" { + fallback = b + } + } + if fallback != "" { + return fallback + } + + return opusBackendName +} + // RealtimeCallRequest is the JSON body for POST /v1/realtime/calls. type RealtimeCallRequest struct { SDP string `json:"sdp"` @@ -94,15 +142,25 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { } }() - // Load the Opus backend - opusBackend, err := application.ModelLoader().Load( - model.WithBackendString("opus"), + // Load the Opus backend. The opus codec ships under different backend + // names per platform (e.g. "metal-opus" on darwin/arm64), so resolve the + // platform-appropriate variant from the installed backends instead of + // hardcoding the literal "opus" name (issue #9813). + ml := application.ModelLoader() + installed := make([]string, 0) + for name := range ml.GetAllExternalBackends(nil) { + installed = append(installed, name) + } + opusName := resolveOpusBackend(installed, runtime.GOOS, runtime.GOARCH) + + opusBackend, err := ml.Load( + model.WithBackendString(opusName), model.WithModelID("__opus_codec__"), - model.WithModel("opus"), + model.WithModel(opusName), ) if err != nil { pc.Close() - xlog.Error("failed to load opus backend", "error", err) + xlog.Error("failed to load opus backend", "error", err, "backend", opusName) return c.JSON(http.StatusInternalServerError, map[string]string{"error": "opus backend not available"}) } diff --git a/core/http/endpoints/openai/realtime_webrtc_opus_test.go b/core/http/endpoints/openai/realtime_webrtc_opus_test.go new file mode 100644 index 000000000..cc309c432 --- /dev/null +++ b/core/http/endpoints/openai/realtime_webrtc_opus_test.go @@ -0,0 +1,32 @@ +package openai + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("resolveOpusBackend", func() { + It("prefers the exact opus backend when it is installed", func() { + Expect(resolveOpusBackend([]string{"opus", "metal-opus"}, "linux", "amd64")).To(Equal("opus")) + }) + + It("resolves to the opus alias key on linux", func() { + Expect(resolveOpusBackend([]string{"opus"}, "linux", "amd64")).To(Equal("opus")) + }) + + It("selects metal-opus on darwin/arm64 when no plain opus is installed", func() { + Expect(resolveOpusBackend([]string{"metal-opus"}, "darwin", "arm64")).To(Equal("metal-opus")) + }) + + It("selects metal-opus on darwin/arm64 even when other backends are present", func() { + Expect(resolveOpusBackend([]string{"silero-vad", "metal-opus", "whisper"}, "darwin", "arm64")).To(Equal("metal-opus")) + }) + + It("falls back to any opus codec backend when there is no exact match (non-darwin)", func() { + Expect(resolveOpusBackend([]string{"metal-opus"}, "linux", "amd64")).To(Equal("metal-opus")) + }) + + It("returns the literal opus name when no opus codec is installed", func() { + Expect(resolveOpusBackend([]string{"silero-vad", "whisper"}, "darwin", "arm64")).To(Equal("opus")) + }) +})