diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 969962a76..724996974 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -236,10 +236,19 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip return pb.TranscriptResult{}, errors.New("parakeet-cpp: TranscriptRequest.dst (audio path) is required") } - // Fallback when the batched C-API is unavailable: transcribe directly from - // the file path (original behavior, no batching). + // Fallback when the batched C-API is unavailable: transcribe from a file + // path (original behavior, no batching). The C library's audio loader only + // understands 16 kHz mono WAV/PCM, so convert the input first - otherwise + // any non-WAV upload (MP3, etc.) fails with "failed to load audio". This + // mirrors what every other audio backend (whisper, crispasr) does via + // utils.AudioToWav before handing the file to the engine. if p.bat == nil { - cstr := CppTranscribePathJSON(p.ctxPtr, opts.Dst, 0) + converted, cleanup, err := convertToWavMono16k(opts.Dst) + if err != nil { + return pb.TranscriptResult{}, err + } + defer cleanup() + cstr := CppTranscribePathJSON(p.ctxPtr, converted, 0) if cstr == 0 { return pb.TranscriptResult{}, fmt.Errorf("parakeet-cpp: transcribe_path_json failed: %s", CppLastError(p.ctxPtr)) } @@ -460,17 +469,33 @@ func (p *ParakeetCpp) AudioTranscriptionStream(ctx context.Context, opts *pb.Tra // float samples plus the clip duration in seconds. Mirrors the whisper // backend: utils.AudioToWav (ffmpeg) normalises rate/channels, go-audio // decodes the PCM. -func decodeWavMono16k(path string) ([]float32, float32, error) { +// convertToWavMono16k converts an arbitrary audio file to a 16 kHz mono WAV in +// a fresh temp dir and returns the path together with a cleanup func the caller +// must defer. WAV inputs already at 16 kHz/mono/16-bit are passed through by +// utils.AudioToWav (hardlink/copy), everything else is transcoded via ffmpeg. +// Used by the direct (non-batched) transcription path, which hands a file path +// to the C library's WAV-only audio loader. +func convertToWavMono16k(path string) (string, func(), error) { dir, err := os.MkdirTemp("", "parakeet") if err != nil { - return nil, 0, err + return "", func() {}, err } - defer func() { _ = os.RemoveAll(dir) }() + cleanup := func() { _ = os.RemoveAll(dir) } converted := filepath.Join(dir, "converted.wav") if err := utils.AudioToWav(path, converted); err != nil { + cleanup() + return "", func() {}, err + } + return converted, cleanup, nil +} + +func decodeWavMono16k(path string) ([]float32, float32, error) { + converted, cleanup, err := convertToWavMono16k(path) + if err != nil { return nil, 0, err } + defer cleanup() fh, err := os.Open(converted) if err != nil { diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index c059eb4bf..d72e10d05 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -3,11 +3,14 @@ package main import ( "context" "os" + "path/filepath" "strings" "sync" "testing" "github.com/ebitengine/purego" + "github.com/go-audio/audio" + "github.com/go-audio/wav" pb "github.com/mudler/LocalAI/pkg/grpc/proto" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -70,6 +73,24 @@ func fixturesOrSkip() (string, string) { return modelPath, audioPath } +// writeMono16kWav writes `samples` frames of 16 kHz mono 16-bit silence to +// path. The result is already in AudioToWav's target format, so the conversion +// helper copies it through without invoking ffmpeg. +func writeMono16kWav(path string, samples int) { + GinkgoHelper() + f, err := os.Create(path) + Expect(err).ToNot(HaveOccurred()) + enc := wav.NewEncoder(f, 16000, 16, 1, 1) + buf := &audio.IntBuffer{ + Format: &audio.Format{NumChannels: 1, SampleRate: 16000}, + SourceBitDepth: 16, + Data: make([]int, samples), + } + Expect(enc.Write(buf)).To(Succeed()) + Expect(enc.Close()).To(Succeed()) + Expect(f.Close()).To(Succeed()) +} + var _ = Describe("ParakeetCpp", func() { Context("AudioTranscription", func() { It("transcribes a WAV via the parakeet C-API", func() { @@ -120,6 +141,39 @@ var _ = Describe("ParakeetCpp", func() { }) }) + Context("convertToWavMono16k", func() { + // The non-batched transcription path hands a file path to the C + // library's WAV-only audio loader, so it must convert first. + // utils.AudioToWav passes an already-16kHz/mono/16-bit WAV through + // without ffmpeg, which lets us exercise the helper (and the + // regression: the direct path used to skip conversion entirely) + // without a model, the C library, or ffmpeg. + It("returns a decodable 16kHz mono WAV copy and cleans it up", func() { + dir := GinkgoT().TempDir() + src := filepath.Join(dir, "input.wav") + writeMono16kWav(src, 16000) // 1s of silence at 16 kHz + + converted, cleanup, err := convertToWavMono16k(src) + Expect(err).ToNot(HaveOccurred()) + + // It must produce a fresh temp file, not return the original path. + Expect(converted).ToNot(Equal(src)) + Expect(converted).To(BeAnExistingFile()) + + pcm, _, err := decodeWavMono16k(converted) + Expect(err).ToNot(HaveOccurred()) + Expect(pcm).To(HaveLen(16000), "round-trips the sample count") + + cleanup() + Expect(converted).ToNot(BeAnExistingFile(), "cleanup removes the temp dir") + }) + + It("errors on a non-existent input rather than passing the path through", func() { + _, _, err := convertToWavMono16k(filepath.Join(GinkgoT().TempDir(), "missing.mp3")) + Expect(err).To(HaveOccurred()) + }) + }) + Context("AudioTranscriptionStream", func() { It("streams deltas and a closing FinalResult from a cache-aware model", func() { // Streaming needs a cache-aware streaming model (e.g.