From d7020708f2d9d14c5639b2bc6db33ca64ce57cd7 Mon Sep 17 00:00:00 2001 From: Tai An Date: Wed, 22 Jul 2026 00:29:11 -0700 Subject: [PATCH] fix(completions): reject empty PromptStrings in streaming to avoid index-out-of-range panic (#11028) * fix(completions): reject empty PromptStrings in streaming to avoid index-out-of-range panic The streaming branch of CompletionEndpoint only guarded len(config.PromptStrings) > 1 before unconditionally reading config.PromptStrings[0]. A completion request whose prompt field is an empty array, an array of non-strings, or omitted leaves PromptStrings with length 0, so PromptStrings[0] panics with index out of range and crashes the handler goroutine. Guard for exactly one prompt string instead, returning a clean error for the 0-length case as well as the pre-existing multi-prompt case. Signed-off-by: Tai An * fix(completions): return 400 for malformed streaming prompt Reject streaming completion requests whose prompt does not resolve to exactly one string (omitted prompt, empty array, or a multi-element array) with an HTTP 400 before writing any SSE headers, instead of returning a plain error that Echo surfaces as a 500. Extract the guard into validateStreamingPromptStrings and cover the three reported payloads with a regression test. Fixes #11021 Signed-off-by: Tai An --------- Signed-off-by: Tai An --- core/http/endpoints/openai/completion.go | 26 +++++++++--- .../completion_stream_validation_test.go | 40 +++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 core/http/endpoints/openai/completion_stream_validation_test.go diff --git a/core/http/endpoints/openai/completion.go b/core/http/endpoints/openai/completion.go index e771fed46..a598af9cd 100644 --- a/core/http/endpoints/openai/completion.go +++ b/core/http/endpoints/openai/completion.go @@ -2,8 +2,8 @@ package openai import ( "encoding/json" - "errors" "fmt" + "net/http" "time" "github.com/labstack/echo/v4" @@ -19,6 +19,18 @@ import ( "github.com/mudler/xlog" ) +// validateStreamingPromptStrings enforces that a streaming completion request +// resolves to exactly one prompt string. Malformed inputs — an omitted prompt, +// an empty array, or an array whose elements do not reduce to a single string — +// return a 400 error instead of panicking on PromptStrings[0] or opening a +// half-written event stream (which Echo surfaces as a 500). See issue #11021. +func validateStreamingPromptStrings(cfg *config.ModelConfig) error { + if len(cfg.PromptStrings) != 1 { + return echo.NewHTTPError(http.StatusBadRequest, "streaming completions require exactly one prompt string") + } + return nil +} + // CompletionEndpoint is the OpenAI Completion API endpoint https://platform.openai.com/docs/api-reference/completions // @Summary Generate completions for a given prompt and model. // @Tags inference @@ -102,14 +114,18 @@ func CompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eva if input.Stream { xlog.Debug("Stream request received") + + // Validate before writing any SSE headers so a malformed request + // yields a 400 JSON error instead of a half-opened event stream + // (which Echo would otherwise surface as a 500). See issue #11021. + if err := validateStreamingPromptStrings(config); err != nil { + return err + } + c.Response().Header().Set("Content-Type", "text/event-stream") c.Response().Header().Set("Cache-Control", "no-cache") c.Response().Header().Set("Connection", "keep-alive") - if len(config.PromptStrings) > 1 { - return errors.New("cannot handle more than 1 `PromptStrings` when Streaming") - } - // Response/output PII redaction is out of scope for now — // redaction runs request-side via the NER middleware only. predInput := config.PromptStrings[0] diff --git a/core/http/endpoints/openai/completion_stream_validation_test.go b/core/http/endpoints/openai/completion_stream_validation_test.go new file mode 100644 index 000000000..eb504a675 --- /dev/null +++ b/core/http/endpoints/openai/completion_stream_validation_test.go @@ -0,0 +1,40 @@ +package openai + +import ( + "net/http" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// These tests pin the streaming completion request validation added for issue +// #11021: a streaming request whose prompt does not resolve to exactly one +// string must be rejected with an HTTP 400 before any SSE headers are written, +// rather than panicking on PromptStrings[0] or returning a 500 on a +// half-opened event stream. +var _ = Describe("streaming completion prompt validation (issue #11021)", func() { + DescribeTable("rejects malformed prompts with HTTP 400", + func(prompts []string) { + cfg := &config.ModelConfig{} + cfg.PromptStrings = prompts + + err := validateStreamingPromptStrings(cfg) + Expect(err).To(HaveOccurred()) + + he, ok := err.(*echo.HTTPError) + Expect(ok).To(BeTrue(), "expected an *echo.HTTPError, got %T", err) + Expect(he.Code).To(Equal(http.StatusBadRequest)) + }, + Entry("omitted prompt (nil slice)", []string(nil)), + Entry("empty array", []string{}), + Entry("multiple prompt strings", []string{"a", "b", "c"}), + ) + + It("accepts exactly one prompt string", func() { + cfg := &config.ModelConfig{} + cfg.PromptStrings = []string{"hello"} + Expect(validateStreamingPromptStrings(cfg)).To(Succeed()) + }) +})