diff --git a/core/http/auth/features.go b/core/http/auth/features.go index 4c1f53ec2..ac7cf4106 100644 --- a/core/http/auth/features.go +++ b/core/http/auth/features.go @@ -71,7 +71,8 @@ var RouteFeatureRegistry = []RouteFeature{ // Detection {"POST", "/v1/detection", FeatureDetection}, - // Decisions API (SystemOne wire contract) + // Decisions API (OpenAI and SystemOne wire contracts) + {"POST", "/v1/decisions", FeatureDecisions}, {"POST", "/v1/systemone", FeatureDecisions}, {"POST", "/v1/systemone/permute", FeatureDecisions}, {"POST", "/v1/systemone/separate", FeatureDecisions}, diff --git a/core/http/auth/features_decisions_test.go b/core/http/auth/features_decisions_test.go index 4c5528618..101272cdd 100644 --- a/core/http/auth/features_decisions_test.go +++ b/core/http/auth/features_decisions_test.go @@ -7,7 +7,7 @@ import ( ) var _ = Describe("Decisions feature registration", func() { - It("gates the three decision routes behind one default-on API feature", func() { + It("gates the four decision routes behind one default-on API feature", func() { Expect(APIFeatures).To(ContainElement(FeatureDecisions)) patterns := []string{} @@ -17,7 +17,7 @@ var _ = Describe("Decisions feature registration", func() { patterns = append(patterns, route.Pattern) } } - Expect(patterns).To(ConsistOf("/v1/systemone", "/v1/systemone/permute", "/v1/systemone/separate")) + Expect(patterns).To(ConsistOf("/v1/decisions", "/v1/systemone", "/v1/systemone/permute", "/v1/systemone/separate")) Expect(APIFeatureMetas()).To(ContainElement(FeatureMeta{Key: FeatureDecisions, Label: "Decisions", DefaultValue: true})) }) diff --git a/core/http/endpoints/localai/api_instructions.go b/core/http/endpoints/localai/api_instructions.go index 06295b0ef..2e849bf6a 100644 --- a/core/http/endpoints/localai/api_instructions.go +++ b/core/http/endpoints/localai/api_instructions.go @@ -108,9 +108,9 @@ var instructionDefs = []instructionDef{ }, { Name: "decisions", - Description: "Typed decisions (choice, noul, score) over a state text with calibrated confidence", + Description: "Typed local decisions through OpenAI-compatible /v1/decisions and SystemOne APIs", Tags: []string{"systemone"}, - Intro: "POST /v1/systemone answers every question in one pass; /v1/systemone/permute re-runs one choice question under n_perm option orders; /v1/systemone/separate answers each question in its own pass. Request: { model, state, questions: { : { type: choice|noul|score, instructions, criteria } } }. A decision model declares known_usecases: [decisions] and serves only /v1/systemone; a zero-shot NER model declares token_classify and serves all three routes (through the NER path); /permute and /separate return 400 for decision models. A vllm-cpp config that declares no usecases is treated as a decision model. Responses carry per-question answers with confidence and probabilities plus token usage. Field names and question types follow Ollama's /v1/systemone, with differences in confidence, error shape and keep_alive (see the Decisions API docs). A request over 64 KiB, with more than 64 questions, or with a malformed question is refused.", + Intro: "POST /v1/decisions accepts { model, input, questions: [{ type: predicate|choice|score, name, instructions, choices, levels }] } and returns ordered answers with typed string/boolean choices. It shares SystemOne execution and backend limits. POST /v1/systemone accepts { model, state, questions: { : { type: choice|noul|score, instructions, criteria } } }. Both answer all questions in one pass. /v1/systemone/permute re-runs one choice question under n_perm option orders; /v1/systemone/separate answers each question in its own pass. Decision models declare known_usecases: [decisions]; /permute and /separate return 400 for these models. Zero-shot NER models declare token_classify and serve all four routes through the NER path. A vllm-cpp config with no declared usecases is treated as a decision model. Confidence comes from the local pipeline, without an OpenAI calibration guarantee. Usage reflects backend counts; unavailable detail breakdowns are empty objects. /v1/decisions accepts user messages with input_text and inline PNG/JPEG input_image parts; only default auto detail is supported. Text and images are flattened into SystemOne state and an image list. External URLs are never fetched. safety_identifier is compatibility metadata, not a safety service or authentication identity. Text-only bodies are limited to 64 KiB, image-bearing bodies to 16 MiB, and requests to 64 questions. Shared image limits and backend capability checks apply; see the Decisions API docs.", }, { Name: "branding", diff --git a/core/http/endpoints/localai/decisions.go b/core/http/endpoints/localai/decisions.go new file mode 100644 index 000000000..bf7428cb3 --- /dev/null +++ b/core/http/endpoints/localai/decisions.go @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: MIT +package localai + +import ( + "encoding/json" + "fmt" + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/http/middleware" + "io" + "math" + "net/http" + "strconv" + "strings" + + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/systemone" +) + +// All identifiers are adapter-owned; public names never enter the internal namespace. +func decisionQuestionID(i int) string { return fmt.Sprintf("q%06d", i) } +func decisionChoiceID(i int) string { return fmt.Sprintf("c%06d", i) } + +func convertDecisionsRequest(req *schema.DecisionsRequest) (*schema.SystemOneRequest, error) { + out := &schema.SystemOneRequest{Model: req.Model, Questions: map[string]schema.SystemOneQuestion{}} + if strings.TrimSpace(req.Model) == "" { + return nil, fmt.Errorf("model is required") + } + var texts, images []string + var text string + if len(req.Input) > 0 && req.Input[0] == '"' { + if err := json.Unmarshal(req.Input, &text); err != nil { + return nil, err + } + texts = append(texts, text) + } else { + var messages []schema.DecisionInputMessage + if err := json.Unmarshal(req.Input, &messages); err != nil || len(messages) == 0 { + return nil, fmt.Errorf("input must be text or user messages") + } + for _, message := range messages { + if message.Role != "user" || (message.Type != "" && message.Type != "message") { + return nil, fmt.Errorf("input supports only user messages") + } + if len(message.Content) > 0 && message.Content[0] == '"' { + if err := json.Unmarshal(message.Content, &text); err != nil { + return nil, err + } + texts = append(texts, text) + continue + } + var parts []schema.DecisionInputPart + if err := json.Unmarshal(message.Content, &parts); err != nil || len(parts) == 0 { + return nil, fmt.Errorf("message content must be text or input parts") + } + for _, part := range parts { + switch part.Type { + case "input_text": + if part.Text == nil { + return nil, fmt.Errorf("input_text requires text") + } + texts = append(texts, *part.Text) + case "input_image": + if part.Detail != nil && *part.Detail != "auto" { + return nil, fmt.Errorf("only auto image detail is supported") + } + images = append(images, part.ImageURL) + default: + return nil, fmt.Errorf("unsupported input part %q", part.Type) + } + } + } + } + out.State, _ = json.Marshal(strings.Join(texts, "\n")) + if len(images) > 0 { + out.Images, _ = json.Marshal(images) + // Image-only evidence must not become an invalid blank text state. + if strings.TrimSpace(strings.Join(texts, "\n")) == "" { + out.State = json.RawMessage(`{}`) + } + } + names := map[string]bool{} + for i, q := range req.Questions { + if q.Name != nil { + if names[*q.Name] { + return nil, fmt.Errorf("duplicate question name %q", *q.Name) + } + names[*q.Name] = true + } + if q.Instructions == nil { + return nil, fmt.Errorf("question instructions are required") + } + internal := schema.SystemOneQuestion{Type: q.Type} + internal.Instructions, _ = json.Marshal(*q.Instructions) + switch q.Type { + case "predicate": + if q.Choices != nil || q.Levels != nil { + return nil, fmt.Errorf("predicate does not support choices or levels") + } + internal.Type = "noul" + case "choice": + if q.Levels != nil { + return nil, fmt.Errorf("choice does not support levels") + } + criteria := map[string]string{} + values := map[string]bool{} + for j, choice := range q.Choices { + var v any + if err := json.Unmarshal(choice.Value, &v); err != nil { + return nil, fmt.Errorf("choice value must be string or boolean") + } + switch v.(type) { + case string, bool: + default: + return nil, fmt.Errorf("choice value must be string or boolean") + } + canonical, _ := json.Marshal(v) + if values[string(canonical)] { + return nil, fmt.Errorf("duplicate choice value") + } + values[string(canonical)] = true + label := string(canonical) + if choice.Description != "" { + label += ": " + choice.Description + } + criteria[decisionChoiceID(j)] = label + } + internal.Criteria, _ = json.Marshal(criteria) + case "score": + if q.Choices != nil { + return nil, fmt.Errorf("score does not support choices") + } + levels := []string{} + for _, level := range q.Levels { + if level.Label == nil { + return nil, fmt.Errorf("score level requires label") + } + label := *level.Label + if level.Description != "" { + label += ": " + level.Description + } + levels = append(levels, label) + } + internal.Criteria, _ = json.Marshal(levels) + default: + return nil, fmt.Errorf("unsupported question type %q", q.Type) + } + out.Questions[decisionQuestionID(i)] = internal + } + if err := systemone.ValidateRequestStructure(out); err != nil { + return nil, err + } + return out, nil +} + +func decisionUnit(value *float64) bool { + return value != nil && !math.IsNaN(*value) && !math.IsInf(*value, 0) && *value >= 0 && *value <= 1 +} + +func convertDecisionsResponse(req *schema.DecisionsRequest, raw []byte) (*schema.DecisionsResponse, error) { + var source struct { + Model string `json:"model"` + Answers map[string]struct { + schema.SystemOneAnswer + // Pointers distinguish an explicit JSON null from a valid zero. + Probabilities map[string]*float64 `json:"probabilities"` + } `json:"answers"` + Usage *struct { + Input *int `json:"input_tokens"` + Output *int `json:"output_tokens"` + InputDetails map[string]int `json:"input_tokens_details"` + OutputDetails map[string]int `json:"output_tokens_details"` + } `json:"usage"` + } + invalid := fmt.Errorf("invalid decision backend response") + if len(raw) > systemone.MaxResponseBytes { + return nil, invalid + } + if err := json.Unmarshal(raw, &source); err != nil { + return nil, invalid + } + if source.Model == "" || len(source.Answers) != len(req.Questions) || source.Usage == nil || source.Usage.Input == nil || source.Usage.Output == nil { + return nil, invalid + } + u := source.Usage + if *u.Input < 0 || *u.Output < 0 || *u.Input > int(^uint(0)>>1)-*u.Output { + return nil, invalid + } + if u.InputDetails == nil { + u.InputDetails = map[string]int{} + } + if u.OutputDetails == nil { + u.OutputDetails = map[string]int{} + } + for _, details := range []map[string]int{u.InputDetails, u.OutputDetails} { + for _, n := range details { + if n < 0 { + return nil, invalid + } + } + } + out := &schema.DecisionsResponse{Model: source.Model, Answers: make([]schema.DecisionAnswer, 0, len(req.Questions)), Usage: schema.DecisionUsage{InputTokens: *u.Input, OutputTokens: *u.Output, TotalTokens: *u.Input + *u.Output, InputTokensDetails: u.InputDetails, OutputTokensDetails: u.OutputDetails}} + for i, q := range req.Questions { + a, ok := source.Answers[decisionQuestionID(i)] + if !ok { + return nil, invalid + } + answer := schema.DecisionAnswer{Type: q.Type, Name: q.Name} + if a.Type == "refusal" { + answer.Type = "refusal" + out.Answers = append(out.Answers, answer) + continue + } + switch q.Type { + case "predicate": + if a.Type != "noul" || !decisionUnit(a.Noul) { + return nil, invalid + } + answer.Probability = a.Noul + case "choice", "score": + if a.Type != q.Type || !decisionUnit(a.Confidence) { + return nil, invalid + } + answer.Confidence = a.Confidence + count := len(q.Choices) + if q.Type == "score" { + count = len(q.Levels) + } + if len(a.Probabilities) != count { + return nil, invalid + } + total := 0.0 + chosen := false + for j := 0; j < count; j++ { + key := decisionChoiceID(j) + if q.Type == "score" { + key = strconv.Itoa(j) + } + p, ok := a.Probabilities[key] + if !ok || !decisionUnit(p) { + return nil, invalid + } + total += *p + probability := schema.DecisionProbability{Probability: *p} + if q.Type == "choice" { + probability.Value = q.Choices[j].Value + if a.Choice != nil && *a.Choice == key { + answer.Choice = q.Choices[j].Value + chosen = true + } + } else { + probability.Value = json.RawMessage(strconv.Itoa(j)) + probability.Label = q.Levels[j].Label + } + answer.Probabilities = append(answer.Probabilities, probability) + } + // SystemOne rounds probabilities to two decimals independently. + if math.Abs(total-1) > 0.005*float64(count)+1e-9 { + return nil, invalid + } + if q.Type == "choice" && !chosen { + return nil, invalid + } + if q.Type == "score" { + if a.Score == nil || math.IsNaN(*a.Score) || math.IsInf(*a.Score, 0) || *a.Score < 0 || *a.Score > float64(count-1) { + return nil, invalid + } + // Keep the score consistent with the rounded distribution we emit. + score := 0.0 + for j, probability := range answer.Probabilities { + score += float64(j) * probability.Probability + } + answer.Score = &score + } + default: + return nil, invalid + } + out.Answers = append(out.Answers, answer) + } + return out, nil +} + +// DecisionsEndpoint handles the OpenAI Decisions wire contract. +// @Summary Answer ordered predicate, choice and score questions. +// @Description Uses the same local decision and NER execution as SystemOne. Supports inline images on capable backends; confidence is not OpenAI calibrated. +// @Tags systemone +// @Accept json +// @Produce json +// @Param request body schema.DecisionsRequest true "input + questions + model" +// @Success 200 {object} schema.DecisionsResponse +// @Router /v1/decisions [post] +func DecisionsEndpoint(app *application.Application) echo.HandlerFunc { + return decisionsEndpoint(func(c echo.Context, req *schema.SystemOneRequest, respond func(string) error) error { + return executeSystemOne(c, app, req, respond) + }) +} + +// The seam keeps handler tests independent of model downloads, not of HTTP admission. +func decisionsEndpoint(execute func(echo.Context, *schema.SystemOneRequest, func(string) error) error) echo.HandlerFunc { + return func(c echo.Context) error { + release, err := systemone.AcquireAdmission(c.Request().Context()) + if err != nil { + return systemOneError(c, http.StatusServiceUnavailable, err.Error()) + } + defer release() + raw, err := io.ReadAll(http.MaxBytesReader(c.Response(), c.Request().Body, systemone.MaxImageBodyBytes)) + if err != nil { + return systemOneError(c, systemOneBindStatus(err), systemOneBindMessage(err)) + } + var req schema.DecisionsRequest + err = json.Unmarshal(raw, &req) + var internal *schema.SystemOneRequest + if err == nil { + internal, err = convertDecisionsRequest(&req) + } + limit := systemone.MaxBodyBytes + if err == nil { + limit, err = systemone.RequestBodyLimit(internal) + } + if len(raw) > limit { + return systemOneError(c, http.StatusRequestEntityTooLarge, "decision request body exceeds limit") + } + if err != nil { + return systemOneError(c, systemOneInputStatus(err), err.Error()) + } + return execute(c, internal, func(raw string) error { + response, err := convertDecisionsResponse(&req, []byte(raw)) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + middleware.StampUsage(c, response.Model, response.Usage.InputTokens, response.Usage.OutputTokens) + return c.JSON(http.StatusOK, response) + }) + } +} diff --git a/core/http/endpoints/localai/decisions_test.go b/core/http/endpoints/localai/decisions_test.go new file mode 100644 index 000000000..455167189 --- /dev/null +++ b/core/http/endpoints/localai/decisions_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: MIT +package localai + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "image" + "image/png" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Decisions conversion", func() { + It("preserves question order, unnamed questions and typed choices", func() { + var req schema.DecisionsRequest + Expect(json.Unmarshal([]byte(`{"model":"m","input":"evidence","questions":[{"type":"predicate","instructions":"yes?"},{"type":"choice","name":"q000000","instructions":"pick","choices":[{"value":true},{"value":"true"}]},{"type":"score","instructions":"rate","levels":[{"label":"low"},{"label":"high"}]}]}`), &req)).To(Succeed()) + translated, err := convertDecisionsRequest(&req) + Expect(err).NotTo(HaveOccurred()) + Expect(translated.Questions).To(HaveLen(3)) + Expect(translated.Questions[decisionQuestionID(0)].Type).To(Equal("noul")) + response, err := convertDecisionsResponse(&req, []byte(`{"model":"m","answers":{"q000000":{"type":"noul","noul":0.7},"q000001":{"type":"choice","choice":"c000000","confidence":0.8,"probabilities":{"c000000":0.9,"c000001":0.1}},"q000002":{"type":"score","score":0.4,"confidence":0.6,"probabilities":{"0":0.6,"1":0.4}}},"usage":{"input_tokens":7,"output_tokens":2}}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(response.Answers[0].Probability).To(HaveValue(Equal(0.7))) + Expect(response.Answers[1].Choice).To(Equal(json.RawMessage(`true`))) + Expect(response.Answers[1].Probabilities[1].Value).To(Equal(json.RawMessage(`"true"`))) + Expect(response.Answers[2].Probabilities[1].Label).To(HaveValue(Equal("high"))) + Expect(response.Usage.TotalTokens).To(Equal(9)) + }) + DescribeTable("keeps typed choice values distinct in model-visible criteria", + func(description string) { + var req schema.DecisionsRequest + Expect(json.Unmarshal([]byte(`{"model":"m","input":"evidence","questions":[{"type":"choice","instructions":"pick","choices":[{"value":true},{"value":"true"},{"value":false},{"value":"false"}]}]}`), &req)).To(Succeed()) + for i := range req.Questions[0].Choices { + req.Questions[0].Choices[i].Description = description + } + translated, err := convertDecisionsRequest(&req) + Expect(err).NotTo(HaveOccurred()) + var criteria map[string]string + Expect(json.Unmarshal(translated.Questions[decisionQuestionID(0)].Criteria, &criteria)).To(Succeed()) + suffix := "" + if description != "" { + suffix = ": " + description + } + Expect(criteria).To(Equal(map[string]string{ + decisionChoiceID(0): `true` + suffix, + decisionChoiceID(1): `"true"` + suffix, + decisionChoiceID(2): `false` + suffix, + decisionChoiceID(3): `"false"` + suffix, + })) + }, + Entry("without descriptions", ""), + Entry("with identical descriptions", "same description"), + ) + It("rejects duplicate names and typed values", func() { + for _, questions := range []string{`[{"type":"predicate","name":"a","instructions":"x"},{"type":"predicate","name":"a","instructions":"x"}]`, `[{"type":"choice","instructions":"x","choices":[{"value":true},{"value":true}]}]`} { + var req schema.DecisionsRequest + Expect(json.Unmarshal([]byte(`{"model":"m","input":"x","questions":`+questions+`}`), &req)).To(Succeed()) + _, err := convertDecisionsRequest(&req) + Expect(err).To(HaveOccurred()) + } + }) +}) + +var _ = Describe("Decisions runtime contract", func() { + const request = `{"model":"m","input":"evidence","questions":[{"type":"predicate","instructions":"yes?"},{"type":"choice","instructions":"pick","choices":[{"value":false},{"value":"false"}]},{"type":"score","instructions":"rate","levels":[{"label":"low"},{"label":"middle"},{"label":"high"}]}]}` + const result = `{"model":"resolved","answers":{"q000002":{"type":"score","score":0.99,"confidence":0,"probabilities":{"0":0.33,"1":0.33,"2":0.34}},"q000001":{"type":"choice","choice":"c000000","confidence":0,"probabilities":{"c000000":1,"c000001":0}},"q000000":{"type":"noul","noul":0}},"usage":{"input_tokens":7,"output_tokens":0,"input_tokens_details":{"cached_tokens":2}}}` + parse := func() *schema.DecisionsRequest { + var req schema.DecisionsRequest + Expect(json.Unmarshal([]byte(request), &req)).To(Succeed()) + return &req + } + It("derives score from the emitted ordered distribution and retains false and zero", func() { + out, err := convertDecisionsResponse(parse(), []byte(result)) + Expect(err).NotTo(HaveOccurred()) + Expect(out.Answers[2].Score).To(HaveValue(BeNumerically("~", 1.01, 1e-12))) + wire, err := json.Marshal(out) + Expect(err).NotTo(HaveOccurred()) + Expect(string(wire)).To(ContainSubstring(`"choice":false`)) + Expect(string(wire)).To(ContainSubstring(`"probability":0`)) + Expect(string(wire)).To(ContainSubstring(`"confidence":0`)) + for _, a := range out.Answers { + Expect(a.Name).To(BeNil()) + } + out, err = convertDecisionsResponse(parse(), []byte(strings.ReplaceAll(strings.ReplaceAll(result, `"score":0.99`, `"score":0`), `"0":0.33,"1":0.33,"2":0.34`, `"0":1,"1":0,"2":0`))) + Expect(err).NotTo(HaveOccurred()) + wire, err = json.Marshal(out) + Expect(err).NotTo(HaveOccurred()) + Expect(string(wire)).To(ContainSubstring(`"score":0`)) + }) + It("serializes accepted empty score labels without labeling choice probabilities", func() { + req := parse() + empty := "" + req.Questions[2].Levels[0].Label = &empty + _, err := convertDecisionsRequest(req) + Expect(err).NotTo(HaveOccurred()) + out, err := convertDecisionsResponse(req, []byte(result)) + Expect(err).NotTo(HaveOccurred()) + wire, err := json.Marshal(out) + Expect(err).NotTo(HaveOccurred()) + var decoded struct { + Answers []struct { + Probabilities []map[string]any `json:"probabilities"` + } `json:"answers"` + } + Expect(json.Unmarshal(wire, &decoded)).To(Succeed()) + Expect(decoded.Answers[2].Probabilities[0]).To(HaveKeyWithValue("label", "")) + for _, probability := range decoded.Answers[1].Probabilities { + Expect(probability).NotTo(HaveKey("label")) + } + }) + It("rejects missing answers, malformed distributions and invalid accounting", func() { + for _, mutation := range [][2]string{ + {`"q000000"`, `"unexpected"`}, + {`"noul":0`, `"noul":null`}, + {`"noul":0`, `"noul":1.1`}, + {`"c000000":1`, `"c000000":0.2`}, + {`"c000001":0`, `"c000001":-0.1`}, + {`"c000001":0`, `"c000001":null`}, + {`"0":0.33`, `"0":null`}, + {`"c000001":0`, `"wrong":0`}, + {`"choice":"c000000"`, `"choice":"missing"`}, + {`"confidence":0`, `"confidence":null`}, + {`"score":0.99`, `"score":null`}, + {`"input_tokens":7`, `"input_tokens":-1`}, + {`"output_tokens":0`, `"output_tokens":null`}, + {`"cached_tokens":2`, `"cached_tokens":-2`}, + } { + _, err := convertDecisionsResponse(parse(), []byte(strings.ReplaceAll(result, mutation[0], mutation[1]))) + Expect(err).To(HaveOccurred(), mutation[0]) + } + }) + It("passes refusal through without inventing values", func() { + out, err := convertDecisionsResponse(parse(), []byte(strings.Replace(result, `{"type":"noul","noul":0}`, `{"type":"refusal"}`, 1))) + Expect(err).NotTo(HaveOccurred()) + Expect(out.Answers[0].Type).To(Equal("refusal")) + Expect(out.Answers[0].Probability).To(BeNil()) + }) + It("preserves user text and inline images and rejects unsupported input", func() { + req := parse() + var pixel bytes.Buffer + Expect(png.Encode(&pixel, image.NewRGBA(image.Rect(0, 0, 1, 1)))).To(Succeed()) + image := "data:image/png;base64," + base64.StdEncoding.EncodeToString(pixel.Bytes()) + req.Input = json.RawMessage(`[{"role":"user","content":"first"},{"type":"message","role":"user","content":[{"type":"input_text","text":"second"},{"type":"input_image","image_url":"` + image + `","detail":"auto"}]}]`) + internal, err := convertDecisionsRequest(req) + Expect(err).NotTo(HaveOccurred()) + Expect(string(internal.State)).To(Equal(`"first\nsecond"`)) + Expect(string(internal.Images)).To(Equal(`["` + image + `"]`)) + for _, input := range []string{ + `[{"role":"system","content":"x"}]`, + `[{"role":"user","content":[{"type":"input_audio"}]}]`, + `[{"role":"user","content":[{"type":"input_text"}]}]`, + `[{"role":"user","content":[{"type":"input_image","image_url":"` + image + `","detail":"high"}]}]`, + `[{"role":"user","content":[{"type":"input_image","image_url":"https://example.org/a.png"}]}]`, + } { + req.Input = json.RawMessage(input) + _, err := convertDecisionsRequest(req) + Expect(err).To(HaveOccurred(), input) + } + }) + It("executes in process and stamps actual usage", func() { + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/decisions", strings.NewReader(request)), rec) + calls := 0 + err := decisionsEndpoint(func(got echo.Context, req *schema.SystemOneRequest, respond func(string) error) error { + calls++ + Expect(got).To(Equal(c)) + Expect(req.Questions).To(HaveLen(3)) + return respond(result) + })(c) + Expect(err).NotTo(HaveOccurred()) + Expect(calls).To(Equal(1)) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(c.Get(middleware.ContextKeyResponseModel)).To(Equal("resolved")) + Expect(c.Get(middleware.ContextKeyTotalTokens)).To(Equal(int64(7))) + }) + It("does not execute invalid requests or publish invalid backend answers", func() { + for _, body := range []string{`{`, `{"model":"m","input":"x","questions":[]}`, request} { + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/decisions", strings.NewReader(body)), rec) + calls := 0 + Expect(decisionsEndpoint(func(_ echo.Context, _ *schema.SystemOneRequest, respond func(string) error) error { + calls++ + return respond(`{}`) + })(c)).To(Succeed()) + if body == request { + Expect(calls).To(Equal(1)) + Expect(rec.Code).To(Equal(500)) + } else { + Expect(calls).To(BeZero()) + Expect(rec.Code).To(Equal(400)) + } + Expect(c.Get(middleware.ContextKeyTotalTokens)).To(BeNil()) + Expect(rec.Body.String()).NotTo(ContainSubstring(`"refusal"`)) + } + }) + It("preserves execution errors and request cancellation", func() { + for _, failure := range []error{errors.New("backend unavailable"), context.Canceled} { + ctx, cancel := context.WithCancel(context.Background()) + e := echo.New() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/decisions", strings.NewReader(request)).WithContext(ctx), httptest.NewRecorder()) + err := decisionsEndpoint(func(got echo.Context, _ *schema.SystemOneRequest, _ func(string) error) error { + cancel() + Expect(got.Request().Context().Err()).To(MatchError(context.Canceled)) + return failure + })(c) + cancel() + Expect(err).To(MatchError(failure)) + Expect(c.Response().Committed).To(BeFalse()) + Expect(c.Get(middleware.ContextKeyTotalTokens)).To(BeNil()) + } + }) +}) diff --git a/core/http/endpoints/localai/systemone.go b/core/http/endpoints/localai/systemone.go index de2e83f98..d81a7f843 100644 --- a/core/http/endpoints/localai/systemone.go +++ b/core/http/endpoints/localai/systemone.go @@ -628,60 +628,76 @@ func SystemOneEndpoint(app *application.Application) echo.HandlerFunc { if err := systemOneBind(c, &req); err != nil { return systemOneError(c, systemOneBindStatus(err), systemOneBindMessage(err)) } - if req.Model == "" { - return systemOneError(c, http.StatusBadRequest, "model is required") - } - if err := checkSystemOneModel(app, req.Model); err != nil { - return systemOneError(c, http.StatusBadRequest, err.Error()) - } - if err := validateSystemOneRequest(&req); err != nil { - return systemOneError(c, systemOneInputStatus(err), err.Error()) - } - // vllm-cpp models (kev/laya) implement the decision pipeline natively - // via the vllm_decide C ABI. Forward the raw request JSON through the - // Score RPC and return the backend's response as-is. - cl := app.ModelConfigLoader() - if cl != nil { - if cfg, ok := cl.GetModelConfig(req.Model); ok && systemOneUsesDecisionPipeline(cfg) { - reqJSON, err := json.Marshal(req) - if err != nil { - return systemOneError(c, http.StatusInternalServerError, "failed to marshal request: "+err.Error()) - } - fn, err := backend.ModelSystemOne(string(reqJSON), app.ModelLoader(), cfg, app.ApplicationConfig()) - if err != nil { - return systemOneError(c, http.StatusInternalServerError, err.Error()) - } - return respondSystemOne(c, req.Model, fn) - } - } - // NER-based path (GLiNER2.5 zero-shot NER). - parsed, err := parseSystemOneRequest(&req) - if err != nil { - return systemOneError(c, systemOneInputStatus(err), err.Error()) - } - classifier, err := resolveClassifier(app, req.Model, parsed.threshold) - if err != nil { - return systemOneError(c, http.StatusNotFound, err.Error()) - } - start := time.Now() - entities, err := classifier.TokenClassifyWithLabels(c.Request().Context(), parsed.text, parsed.allLabels) - if err != nil { - return systemOneError(c, http.StatusInternalServerError, err.Error()) - } - latencyMs := float64(time.Since(start).Microseconds()) / 1000.0 - answers := make(map[string]schema.SystemOneAnswer, len(parsed.questions)) - for i := range parsed.questions { - answers[parsed.questions[i].id] = buildSystemOneAnswer(&parsed.questions[i], entities) - } - return c.JSON(http.StatusOK, schema.SystemOneResponse{ - Model: req.Model, - Answers: answers, - Usage: schema.SystemOneUsage{InputTokens: 0, OutputTokens: 0}, - LatencyMs: r2(latencyMs), + return executeSystemOne(c, app, &req, func(response string) error { + return respondSystemOne(c, req.Model, func(context.Context) (string, error) { return response, nil }) }) } } +// executeSystemOne shares native and NER execution without routing through HTTP. +// Admission and wire binding belong to each public adapter. +func executeSystemOne(c echo.Context, app *application.Application, req *schema.SystemOneRequest, respond func(string) error) error { + if req.Model == "" { + return systemOneError(c, http.StatusBadRequest, "model is required") + } + if err := checkSystemOneModel(app, req.Model); err != nil { + return systemOneError(c, http.StatusBadRequest, err.Error()) + } + if err := validateSystemOneRequest(req); err != nil { + return systemOneError(c, systemOneInputStatus(err), err.Error()) + } + // vllm-cpp models (kev/laya) implement the decision pipeline natively + // via the vllm_decide C ABI. Forward the raw request JSON through the + // Score RPC and return the backend's response as-is. + cl := app.ModelConfigLoader() + if cl != nil { + if cfg, ok := cl.GetModelConfig(req.Model); ok && systemOneUsesDecisionPipeline(cfg) { + reqJSON, err := json.Marshal(req) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, "failed to marshal request: "+err.Error()) + } + fn, err := backend.ModelSystemOne(string(reqJSON), app.ModelLoader(), cfg, app.ApplicationConfig()) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + response, err := fn(c.Request().Context()) + if err != nil { + return systemOneError(c, systemOneBackendStatus(err), err.Error()) + } + return respond(response) + } + } + // NER-based path (GLiNER2.5 zero-shot NER). + parsed, err := parseSystemOneRequest(req) + if err != nil { + return systemOneError(c, systemOneInputStatus(err), err.Error()) + } + classifier, err := resolveClassifier(app, req.Model, parsed.threshold) + if err != nil { + return systemOneError(c, http.StatusNotFound, err.Error()) + } + start := time.Now() + entities, err := classifier.TokenClassifyWithLabels(c.Request().Context(), parsed.text, parsed.allLabels) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + latencyMs := float64(time.Since(start).Microseconds()) / 1000.0 + answers := make(map[string]schema.SystemOneAnswer, len(parsed.questions)) + for i := range parsed.questions { + answers[parsed.questions[i].id] = buildSystemOneAnswer(&parsed.questions[i], entities) + } + response, err := json.Marshal(schema.SystemOneResponse{ + Model: req.Model, + Answers: answers, + Usage: schema.SystemOneUsage{InputTokens: 0, OutputTokens: 0}, + LatencyMs: r2(latencyMs), + }) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + return respond(string(response)) +} + // SystemOnePermuteEndpoint handles POST /v1/systemone/permute. // Re-runs one choice question under n_perm option orders with a seeded RNG. // @Summary Re-run a choice question under multiple option orders. diff --git a/core/http/routes/decisions_auth_test.go b/core/http/routes/decisions_auth_test.go new file mode 100644 index 000000000..04f3d9094 --- /dev/null +++ b/core/http/routes/decisions_auth_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: MIT +//go:build auth + +package routes_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/routing/billing" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("registered Decisions auth", func() { + DescribeTable("enforces the shared feature before inference", func(role string, authenticated bool, permission *bool, code int) { + root := GinkgoT().TempDir() + app, err := application.New(config.WithDataPath(root), config.WithDisableLocalAIAssistant(true), config.WithDisableCSRF(true), config.WithSystemState(&system.SystemState{Model: system.Model{ModelsPath: root}, Backend: system.Backend{BackendsPath: root}})) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(app.Shutdown()).To(Succeed()) }) + cfg := config.ModelConfig{Name: "decision", Backend: "llama-cpp", KnownUsecases: config.GetUsecasesFromYAML([]string{"decisions"})} + cfg.SetDefaults() + cfg.Model = "fixture.gguf" + app.ModelConfigLoader().ReplaceModelConfigs([]config.ModelConfig{cfg}) + calls := 0 + fixture := &nativeDecisionFixture{body: `{"model":"decision","answers":{"q000000":{"type":"noul","noul":0.5}},"usage":{"input_tokens":12,"output_tokens":0}}`} + app.ModelLoader().SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) { + calls++ + return model.NewModelWithClient(id, "test://decision", fixture), nil + }) + db, err := auth.InitDB(":memory:") + Expect(err).NotTo(HaveOccurred()) + sqlDB, err := db.DB() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(sqlDB.Close) + user := &auth.User{ID: "decision-user", Email: "decision@example.test", Name: "Decision user", Role: role, Status: auth.StatusActive, Provider: auth.ProviderLocal} + Expect(db.Create(user).Error).To(Succeed()) + if permission != nil { + Expect(auth.UpdateUserPermissions(db, user.ID, auth.PermissionMap{auth.FeatureDecisions: *permission})).To(Succeed()) + } + session, err := auth.CreateSession(db, user.ID, "") + Expect(err).NotTo(HaveOccurred()) + e := echo.New() + e.Use(auth.Middleware(db, app.ApplicationConfig())) + e.Use(auth.RequireRouteFeature(db)) + routes.RegisterSystemOneRoutes(e, app) + req := httptest.NewRequest(http.MethodPost, "/v1/decisions", strings.NewReader(`{"model":"decision","input":"x","questions":[{"type":"predicate","instructions":"yes?"}]}`)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + if authenticated { + req.AddCookie(&http.Cookie{Name: "session", Value: session}) + } + w := httptest.NewRecorder() + e.ServeHTTP(w, req) + Expect(w.Code).To(Equal(code), w.Body.String()) + buckets, err := app.StatsRecorder().Aggregate(context.Background(), billing.AggregateQuery{}) + Expect(err).NotTo(HaveOccurred()) + var count int64 + for _, bucket := range buckets { + count += bucket.RequestCount + } + if code == http.StatusOK { + Expect(calls).To(Equal(1)) + Expect(count).To(Equal(int64(1))) + } else { + Expect(calls).To(BeZero()) + Expect(count).To(BeZero()) + } + }, + Entry("missing credentials", auth.RoleUser, false, (*bool)(nil), http.StatusUnauthorized), + Entry("default-on feature", auth.RoleUser, true, (*bool)(nil), http.StatusOK), + Entry("explicitly enabled", auth.RoleUser, true, boolPointer(true), http.StatusOK), + Entry("explicitly disabled", auth.RoleUser, true, boolPointer(false), http.StatusForbidden), + Entry("admin bypass", auth.RoleAdmin, true, boolPointer(false), http.StatusOK), + ) +}) + +func boolPointer(value bool) *bool { return &value } diff --git a/core/http/routes/decisions_billing_test.go b/core/http/routes/decisions_billing_test.go new file mode 100644 index 000000000..279c0f9f6 --- /dev/null +++ b/core/http/routes/decisions_billing_test.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MIT +package routes_test + +import ( + "context" + "encoding/json" + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/routing/billing" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" + "strings" +) + +var _ = Describe("registered Decisions billing", func() { + DescribeTable("serves the adapter without auth and records validated usage once", func(body string, code int, count, tokens int64) { + root := GinkgoT().TempDir() + app, err := application.New(config.WithDataPath(root), config.WithDisableLocalAIAssistant(true), config.WithDisableCSRF(true), config.WithSystemState(&system.SystemState{Model: system.Model{ModelsPath: root}, Backend: system.Backend{BackendsPath: root}})) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(app.Shutdown()).To(Succeed()) }) + cfg := config.ModelConfig{Name: "decision", Backend: "llama-cpp", KnownUsecases: config.GetUsecasesFromYAML([]string{"decisions"})} + cfg.SetDefaults() + cfg.Model = "fixture.gguf" + app.ModelConfigLoader().ReplaceModelConfigs([]config.ModelConfig{cfg}) + fixture := &nativeDecisionFixture{body: body} + app.ModelLoader().SetModelRouter(func(_ context.Context, id string, _, _, _, _ string, _ *pb.ModelOptions, _ bool) (*model.Model, error) { + return model.NewModelWithClient(id, "test://decision", fixture), nil + }) + e := echo.New() + e.Use(auth.Middleware(nil, app.ApplicationConfig())) + e.Use(auth.RequireRouteFeature(nil)) + routes.RegisterSystemOneRoutes(e, app) + req := httptest.NewRequest(http.MethodPost, "/v1/decisions", strings.NewReader(`{"model":"decision","input":"x","questions":[{"type":"predicate","name":"refund","instructions":"yes?"}]}`)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + w := httptest.NewRecorder() + e.ServeHTTP(w, req) + Expect(w.Code).To(Equal(code), w.Body.String()) + if code == http.StatusOK { + var response schema.DecisionsResponse + Expect(json.Unmarshal(w.Body.Bytes(), &response)).To(Succeed()) + Expect(response.Answers).To(HaveLen(1)) + Expect(response.Answers[0].Type).To(Equal("predicate")) + Expect(*response.Answers[0].Name).To(Equal("refund")) + Expect(*response.Answers[0].Probability).To(Equal(0.5)) + Expect(response.Usage.InputTokens).To(Equal(int(tokens))) + } + buckets, err := app.StatsRecorder().Aggregate(context.Background(), billing.AggregateQuery{}) + Expect(err).NotTo(HaveOccurred()) + var actual, prompt int64 + for _, b := range buckets { + actual += b.RequestCount + prompt += b.PromptTokens + Expect(b.CompletionTokens).To(Equal(int64(0))) + } + Expect(actual).To(Equal(count)) + Expect(prompt).To(Equal(tokens)) + }, + Entry("positive input", `{"model":"decision","answers":{"q000000":{"type":"noul","noul":0.5}},"usage":{"input_tokens":12,"output_tokens":0}}`, 200, int64(1), int64(12)), + Entry("explicit zero usage", `{"model":"decision","answers":{"q000000":{"type":"noul","noul":0.5}},"usage":{"input_tokens":0,"output_tokens":0}}`, 200, int64(1), int64(0)), + Entry("missing usage", `{"model":"decision","answers":{"q000000":{"type":"noul","noul":0.5}}}`, 500, int64(0), int64(0)), + Entry("invalid adapter answer", `{"model":"decision","answers":{"q000000":{"type":"noul","noul":2}},"usage":{"input_tokens":12,"output_tokens":0}}`, 500, int64(0), int64(0)), + ) +}) diff --git a/core/http/routes/systemone.go b/core/http/routes/systemone.go index 55e6c9561..494306f01 100644 --- a/core/http/routes/systemone.go +++ b/core/http/routes/systemone.go @@ -14,7 +14,9 @@ import ( // one NER pass; POST /v1/systemone/permute re-runs one choice question // under n_perm option orders; POST /v1/systemone/separate answers each // question in its own NER pass. +// POST /v1/decisions adapts the OpenAI contract to the same execution path. func RegisterSystemOneRoutes(e *echo.Echo, app *application.Application) { + e.POST("/v1/decisions", localai.DecisionsEndpoint(app), middleware.UsageMiddleware(app.StatsRecorder(), app.FallbackUser())) e.POST("/v1/systemone", localai.SystemOneEndpoint(app), middleware.UsageMiddleware(app.StatsRecorder(), app.FallbackUser())) e.POST("/v1/systemone/permute", localai.SystemOnePermuteEndpoint(app)) e.POST("/v1/systemone/separate", localai.SystemOneSeparateEndpoint(app)) diff --git a/core/schema/decisions.go b/core/schema/decisions.go new file mode 100644 index 000000000..1ea0836ad --- /dev/null +++ b/core/schema/decisions.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MIT +package schema + +import "encoding/json" + +// DecisionsRequest is the OpenAI Decisions wire contract, distinct from SystemOne. +type DecisionsRequest struct { + Model string `json:"model"` + // Input is a text string or an array of user messages. Swagger 2 uses a loose object placeholder for this union. + Input json.RawMessage `json:"input" swaggertype:"object"` + Questions []DecisionQuestion `json:"questions"` + SafetyIdentifier *string `json:"safety_identifier,omitempty"` +} + +type DecisionQuestion struct { + Type string `json:"type"` + Name *string `json:"name,omitempty"` + Instructions *string `json:"instructions"` + Choices []DecisionChoice `json:"choices,omitempty"` + Levels []DecisionLevel `json:"levels,omitempty"` +} +type DecisionChoice struct { + // Value is a string or boolean choice. Swagger 2 uses a loose object placeholder for this union. + Value json.RawMessage `json:"value" swaggertype:"object"` + Description string `json:"description,omitempty"` +} +type DecisionLevel struct { + Label *string `json:"label"` + Description string `json:"description,omitempty"` +} +type DecisionInputMessage struct { + Type string `json:"type,omitempty"` + Role string `json:"role"` + // Content is a text string or an array of input_text/input_image parts. Swagger 2 uses a loose object placeholder for this union. + Content json.RawMessage `json:"content" swaggertype:"object"` +} +type DecisionInputPart struct { + Type string `json:"type"` + Text *string `json:"text,omitempty"` + ImageURL string `json:"image_url,omitempty"` + Detail *string `json:"detail,omitempty"` +} +type DecisionsResponse struct { + Model string `json:"model"` + Answers []DecisionAnswer `json:"answers"` + Usage DecisionUsage `json:"usage"` +} +type DecisionAnswer struct { + Type string `json:"type"` + Name *string `json:"name,omitempty"` + Probability *float64 `json:"probability,omitempty"` + // Choice preserves the selected string or boolean value. Swagger 2 uses a loose object placeholder for this union. + Choice json.RawMessage `json:"choice,omitempty" swaggertype:"object"` + Score *float64 `json:"score,omitempty"` + Confidence *float64 `json:"confidence,omitempty"` + Probabilities []DecisionProbability `json:"probabilities,omitempty"` +} +type DecisionProbability struct { + // Value is a string or boolean for choices, or a zero-based integer for score levels. Swagger 2 uses a loose object placeholder for this union. + Value json.RawMessage `json:"value" swaggertype:"object"` + Label *string `json:"label,omitempty"` + Probability float64 `json:"probability"` +} + +// Unknown accounting details are omitted, not invented as zero counts. +type DecisionUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + TotalTokens int `json:"total_tokens"` + InputTokensDetails map[string]int `json:"input_tokens_details"` + OutputTokensDetails map[string]int `json:"output_tokens_details"` +} diff --git a/core/schema/decisions_test.go b/core/schema/decisions_test.go new file mode 100644 index 000000000..ee571dd9d --- /dev/null +++ b/core/schema/decisions_test.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT +package schema_test + +import ( + "encoding/json" + + "github.com/mudler/LocalAI/core/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Decisions probability serialization", func() { + DescribeTable("preserves label presence", + func(wire string) { + var probability schema.DecisionProbability + Expect(json.Unmarshal([]byte(wire), &probability)).To(Succeed()) + encoded, err := json.Marshal(probability) + Expect(err).NotTo(HaveOccurred()) + Expect(encoded).To(MatchJSON(wire)) + }, + Entry("empty score label", `{"value":0,"label":"","probability":0.5}`), + Entry("nonempty score label", `{"value":1,"label":"high","probability":0.5}`), + Entry("absent choice label", `{"value":true,"probability":0.5}`), + ) +}) diff --git a/docs/content/features/decisions.md b/docs/content/features/decisions.md index 7dd000dfc..94d8096bf 100644 --- a/docs/content/features/decisions.md +++ b/docs/content/features/decisions.md @@ -17,14 +17,122 @@ endpoint (Ollama 0.35 and later). The wire contract is called SystemOne; the capability a model declares is called `decisions`. See [Compatibility with Ollama](#compatibility-with-ollama) for what differs. -OpenAI announced its own Decisions API in limited preview on 2026-09-29. It has no -public request or response schema yet, so LocalAI does not serve a `/v1/decisions` -route. +LocalAI also serves the OpenAI-compatible `/v1/decisions` contract with `input` +and an ordered `questions` array. Both APIs share SystemOne execution, model +selection, validation, admission limits, and backend support. They differ in +request and response shape, not in the underlying model. + +## OpenAI-compatible decisions + +Use an installed local decision model, such as `julia-1-llama-cpp`: + +```bash +curl http://localhost:8080/v1/decisions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "julia-1-llama-cpp", + "input": "My order arrived broken and I want my money back. This is the second time.", + "questions": [ + { + "type": "predicate", + "name": "refund_requested", + "instructions": "The customer explicitly asks for a refund" + }, + { + "type": "choice", + "name": "team", + "instructions": "Which team should handle this ticket?", + "choices": [ + {"value": "billing", "description": "Payments, invoices, and refunds"}, + {"value": "shipping", "description": "Delivery and damaged goods"}, + {"value": "product", "description": "Questions about how the product works"} + ] + }, + { + "type": "score", + "name": "urgency", + "instructions": "How urgent is this ticket?", + "levels": [ + {"label": "not urgent"}, + {"label": "somewhat urgent"}, + {"label": "urgent"}, + {"label": "critical"} + ] + } + ] + }' +``` + +The response contains `model`, an ordered `answers` array, and `usage`. +Answers preserve question order and optional names. Named questions must have +unique names. Each question requires `instructions`. + +| Question type | Input | Answer | +|---|---|---| +| `predicate` | A statement in `instructions` | `probability` from 0 to 1; maps to SystemOne `noul` | +| `choice` | `choices` with unique string or boolean `value` fields and optional descriptions | Typed `choice`, `confidence`, and a `probabilities` array | +| `score` | Ordered `levels` with required `label` fields and optional descriptions | `score`, `confidence`, and a `probabilities` array with zero-based numeric values and labels | + +Choice values retain their JSON types: `true` and `"true"` are different choices. +Numbers, objects, arrays, and null are not supported choice values. +Scores are probability-weighted sums of zero-based level indices, not necessarily integers. +Backend refusals appear as answers with `type: "refusal"`. + +`usage` contains `input_tokens`, `output_tokens`, their sum as `total_tokens`, +`input_tokens_details`, and `output_tokens_details`. Counts reflect available +backend accounting, including explicit zero output tokens. The NER path reports +zero token counts. Detail objects contain only backend-supplied counts and are +empty objects when unavailable. LocalAI does not invent cached, reasoning, text, +or image token breakdowns. + +### Input and compatibility limits + +`input` accepts a text string or an array of user messages. Each message has +`role: "user"`, optional `type: "message"`, and `content`. Content accepts a +string or an array of `input_text` and `input_image` parts: + +```json +{ + "input": [{ + "role": "user", + "content": [ + {"type": "input_text", "text": "Does this package look damaged?"}, + {"type": "input_image", "image_url": "data:image/png;base64,...", "detail": "auto"} + ] + }] +} +``` + +This fragment requires a complete PNG or JPEG data URL and an image-capable +model, such as OpenJev with its projector. Image-only input is supported. +Only omitted or default (`"auto"`) image `detail` is supported; other values +return an error. LocalAI never fetches external image URLs or file paths. + +The adapter joins text parts with newlines and collects images in encounter order. +It flattens message boundaries and text/image interleaving into SystemOne text +and an image list. It does not preserve a conversation layout for inference. + +Requests support at most 64 questions. Text-only bodies have a 64 KiB limit; +image-bearing bodies have a 16 MiB limit. The shared image limits are 8 images, +12 MiB aggregate encoded data-URL bytes, 8 MiB aggregate decoded bytes, +4096 pixels per dimension, and 16 million aggregate pixels. +See [Bounded image input](#bounded-image-input) for validation and admission behavior. +Text-only models and the NER path reject images instead of dropping them. +Backend-specific question limits still apply, including llama.cpp's 2–10 score levels. + +`safety_identifier` is accepted as compatibility metadata. It does not establish +an authentication identity or enable a safety service. Normal LocalAI +authentication and the `decisions` permission still apply. + +Probabilities and confidence come from the local model and its pipeline. +OpenAI-compatible fields do not guarantee OpenAI calibration, model quality, +or safety behavior. Evaluate the installed model for your task. ## Endpoints | Endpoint | Method | Description | |---|---|---| +| `/v1/decisions` | POST | Answer ordered predicate, choice, and score questions using the OpenAI-compatible contract | | `/v1/systemone` | POST | Answer all questions in one pass | | `/v1/systemone/permute` | POST | Re-run one choice question under `n_perm` option orders | | `/v1/systemone/separate` | POST | Answer each question in its own pass | @@ -167,7 +275,7 @@ common case. These behaviors differ: ## Access control -When authentication is on, the three routes need the `decisions` feature. It is +When authentication is on, all four routes need the `decisions` feature. It is on by default for every user, like the other API features, and an administrator can turn it off per user. diff --git a/swagger/docs.go b/swagger/docs.go index 063447074..f8e1f0756 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -3294,6 +3294,40 @@ const docTemplate = `{ } } }, + "/v1/decisions": { + "post": { + "description": "Uses the same local decision and NER execution as SystemOne. Supports inline images on capable backends; confidence is not OpenAI calibrated.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "systemone" + ], + "summary": "Answer ordered predicate, choice and score questions.", + "parameters": [ + { + "description": "input + questions + model", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.DecisionsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/schema.DecisionsResponse" + } + } + } + } + }, "/v1/depth": { "post": { "tags": [ @@ -5811,6 +5845,164 @@ const docTemplate = `{ } } }, + "schema.DecisionAnswer": { + "type": "object", + "properties": { + "choice": { + "description": "Choice preserves the selected string or boolean value. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + }, + "confidence": { + "type": "number" + }, + "name": { + "type": "string" + }, + "probabilities": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionProbability" + } + }, + "probability": { + "type": "number" + }, + "score": { + "type": "number" + }, + "type": { + "type": "string" + } + } + }, + "schema.DecisionChoice": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "value": { + "description": "Value is a string or boolean choice. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + } + } + }, + "schema.DecisionLevel": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "schema.DecisionProbability": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "probability": { + "type": "number" + }, + "value": { + "description": "Value is a string or boolean for choices, or a zero-based integer for score levels. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + } + } + }, + "schema.DecisionQuestion": { + "type": "object", + "properties": { + "choices": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionChoice" + } + }, + "instructions": { + "type": "string" + }, + "levels": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionLevel" + } + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "schema.DecisionUsage": { + "type": "object", + "properties": { + "input_tokens": { + "type": "integer" + }, + "input_tokens_details": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "output_tokens": { + "type": "integer" + }, + "output_tokens_details": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "total_tokens": { + "type": "integer" + } + } + }, + "schema.DecisionsRequest": { + "type": "object", + "properties": { + "input": { + "description": "Input is a text string or an array of user messages. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + }, + "model": { + "type": "string" + }, + "questions": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionQuestion" + } + }, + "safety_identifier": { + "type": "string" + } + } + }, + "schema.DecisionsResponse": { + "type": "object", + "properties": { + "answers": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionAnswer" + } + }, + "model": { + "type": "string" + }, + "usage": { + "$ref": "#/definitions/schema.DecisionUsage" + } + } + }, "schema.DepthRequest": { "type": "object", "properties": { diff --git a/swagger/swagger.json b/swagger/swagger.json index 7ccfd6984..1f98a5070 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -3291,6 +3291,40 @@ } } }, + "/v1/decisions": { + "post": { + "description": "Uses the same local decision and NER execution as SystemOne. Supports inline images on capable backends; confidence is not OpenAI calibrated.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "systemone" + ], + "summary": "Answer ordered predicate, choice and score questions.", + "parameters": [ + { + "description": "input + questions + model", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.DecisionsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/schema.DecisionsResponse" + } + } + } + } + }, "/v1/depth": { "post": { "tags": [ @@ -5808,6 +5842,164 @@ } } }, + "schema.DecisionAnswer": { + "type": "object", + "properties": { + "choice": { + "description": "Choice preserves the selected string or boolean value. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + }, + "confidence": { + "type": "number" + }, + "name": { + "type": "string" + }, + "probabilities": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionProbability" + } + }, + "probability": { + "type": "number" + }, + "score": { + "type": "number" + }, + "type": { + "type": "string" + } + } + }, + "schema.DecisionChoice": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "value": { + "description": "Value is a string or boolean choice. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + } + } + }, + "schema.DecisionLevel": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "schema.DecisionProbability": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "probability": { + "type": "number" + }, + "value": { + "description": "Value is a string or boolean for choices, or a zero-based integer for score levels. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + } + } + }, + "schema.DecisionQuestion": { + "type": "object", + "properties": { + "choices": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionChoice" + } + }, + "instructions": { + "type": "string" + }, + "levels": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionLevel" + } + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "schema.DecisionUsage": { + "type": "object", + "properties": { + "input_tokens": { + "type": "integer" + }, + "input_tokens_details": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "output_tokens": { + "type": "integer" + }, + "output_tokens_details": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "total_tokens": { + "type": "integer" + } + } + }, + "schema.DecisionsRequest": { + "type": "object", + "properties": { + "input": { + "description": "Input is a text string or an array of user messages. Swagger 2 uses a loose object placeholder for this union.", + "type": "object" + }, + "model": { + "type": "string" + }, + "questions": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionQuestion" + } + }, + "safety_identifier": { + "type": "string" + } + } + }, + "schema.DecisionsResponse": { + "type": "object", + "properties": { + "answers": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.DecisionAnswer" + } + }, + "model": { + "type": "string" + }, + "usage": { + "$ref": "#/definitions/schema.DecisionUsage" + } + } + }, "schema.DepthRequest": { "type": "object", "properties": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 14224a6b4..d663d42ce 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -967,6 +967,114 @@ definitions: summary_tokens: type: integer type: object + schema.DecisionAnswer: + properties: + choice: + description: Choice preserves the selected string or boolean value. Swagger + 2 uses a loose object placeholder for this union. + type: object + confidence: + type: number + name: + type: string + probabilities: + items: + $ref: '#/definitions/schema.DecisionProbability' + type: array + probability: + type: number + score: + type: number + type: + type: string + type: object + schema.DecisionChoice: + properties: + description: + type: string + value: + description: Value is a string or boolean choice. Swagger 2 uses a loose object + placeholder for this union. + type: object + type: object + schema.DecisionLevel: + properties: + description: + type: string + label: + type: string + type: object + schema.DecisionProbability: + properties: + label: + type: string + probability: + type: number + value: + description: Value is a string or boolean for choices, or a zero-based integer + for score levels. Swagger 2 uses a loose object placeholder for this union. + type: object + type: object + schema.DecisionQuestion: + properties: + choices: + items: + $ref: '#/definitions/schema.DecisionChoice' + type: array + instructions: + type: string + levels: + items: + $ref: '#/definitions/schema.DecisionLevel' + type: array + name: + type: string + type: + type: string + type: object + schema.DecisionUsage: + properties: + input_tokens: + type: integer + input_tokens_details: + additionalProperties: + type: integer + type: object + output_tokens: + type: integer + output_tokens_details: + additionalProperties: + type: integer + type: object + total_tokens: + type: integer + type: object + schema.DecisionsRequest: + properties: + input: + description: Input is a text string or an array of user messages. Swagger + 2 uses a loose object placeholder for this union. + type: object + model: + type: string + questions: + items: + $ref: '#/definitions/schema.DecisionQuestion' + type: array + safety_identifier: + type: string + type: object + schema.DecisionsResponse: + properties: + answers: + items: + $ref: '#/definitions/schema.DecisionAnswer' + type: array + model: + type: string + usage: + $ref: '#/definitions/schema.DecisionUsage' + type: object schema.DepthRequest: properties: dst: @@ -5795,6 +5903,29 @@ paths: summary: Generate completions for a given prompt and model. tags: - inference + /v1/decisions: + post: + consumes: + - application/json + description: Uses the same local decision and NER execution as SystemOne. Supports + inline images on capable backends; confidence is not OpenAI calibrated. + parameters: + - description: input + questions + model + in: body + name: request + required: true + schema: + $ref: '#/definitions/schema.DecisionsRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/schema.DecisionsResponse' + summary: Answer ordered predicate, choice and score questions. + tags: + - systemone /v1/depth: post: parameters: