From a2e5c90ff0b9fb28809613881e5bdd41cd419a96 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 19 Sep 2026 17:50:07 +0000 Subject: [PATCH] feat(vllm-cpp): add kev-compatible SystemOne API endpoints Add POST /v1/systemone, /v1/systemone/permute, and /v1/systemone/separate to LocalAI, mirroring the kev project's structured-extraction API. Each endpoint runs zero-shot NER over the rendered state text and builds kev-compatible answers for three question types: noul (binary entity presence), choice (pick one option), and score (pick one level). The TokenClassifyRequest proto gains a `repeated string labels` field so each question can supply its own labels at inference time, and TokenClassifier gains TokenClassifyWithLabels for per-call label selection. The vllm-cpp backend uses request labels when non-empty, falling back to configured ner_labels then the built-in defaults. Helpers (renderState, softmax, choiceConfidence, scoreConfidence, r2) are ported from kev/api.py and mirrored in vllm.cpp's api_server.cpp so both servers produce the same answer shape. Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:regolo/glm5.2 [maki] --- backend/backend.proto | 6 + backend/go/vllm-cpp/backend.go | 3 + core/backend/token_classify.go | 22 + core/http/app.go | 1 + core/http/endpoints/localai/systemone.go | 561 +++++++++++++++++++++++ core/http/routes/systemone.go | 20 + core/schema/systemone.go | 92 ++++ docs/content/features/vllm-cpp.md | 17 + 8 files changed, 722 insertions(+) create mode 100644 core/http/endpoints/localai/systemone.go create mode 100644 core/http/routes/systemone.go create mode 100644 core/schema/systemone.go diff --git a/backend/backend.proto b/backend/backend.proto index 7fcfe183a..f2c0eb65b 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -144,6 +144,12 @@ message TokenClassifyRequest { // PredictOptions.ModelIdentity for the full rationale. Empty means "no // identity supplied" and backends MUST skip the check. string ModelIdentity = 3; + // Labels overrides the backend's configured entity labels for this + // request. Empty means "use the model's configured labels" (the + // default for PII detection, where labels are fixed at load time). + // Non-empty enables zero-shot per-request label selection (kev / + // SystemOne: each question type supplies its own labels). + repeated string labels = 4; } // TokenClassifyEntity is one detected entity span. Byte offsets are diff --git a/backend/go/vllm-cpp/backend.go b/backend/go/vllm-cpp/backend.go index 21ad78940..c99399a13 100644 --- a/backend/go/vllm-cpp/backend.go +++ b/backend/go/vllm-cpp/backend.go @@ -293,6 +293,9 @@ func (v *VllmCpp) TokenClassify(_ context.Context, in *pb.TokenClassifyRequest) return nil, fmt.Errorf("vllm-cpp: model not loaded") } labels := v.opts.nerLabels + if len(in.Labels) > 0 { + labels = in.Labels + } if len(labels) == 0 { labels = defaultNerLabels } diff --git a/core/backend/token_classify.go b/core/backend/token_classify.go index e316f2253..588d4696e 100644 --- a/core/backend/token_classify.go +++ b/core/backend/token_classify.go @@ -30,6 +30,11 @@ type TokenClassifyOptions struct { // callers (e.g. the PII redactor's MinScore) can still filter // further once they know the per-request policy. Threshold float32 + // Labels overrides the backend's configured entity labels for this + // request. Empty means "use the model's configured labels" (the PII + // default). Non-empty enables zero-shot per-request label selection + // (kev / SystemOne questions). + Labels []string } // TokenClassifier runs a token-classification model over text and @@ -39,6 +44,9 @@ type TokenClassifyOptions struct { // core/services/routing/piidetector). type TokenClassifier interface { TokenClassify(ctx context.Context, text string) ([]TokenEntity, error) + // TokenClassifyWithLabels runs NER with the given labels, overriding + // the model's configured labels for this call. + TokenClassifyWithLabels(ctx context.Context, text string, labels []string) ([]TokenEntity, error) } // NewTokenClassifier binds (loader, modelConfig, appConfig) into a @@ -63,6 +71,19 @@ func (m *modelTokenClassifier) TokenClassify(ctx context.Context, text string) ( return fn(ctx) } +// TokenClassifyWithLabels runs NER with the given labels, overriding the +// model's configured labels for this call. Used by the SystemOne endpoints +// where each question supplies its own labels. +func (m *modelTokenClassifier) TokenClassifyWithLabels(ctx context.Context, text string, labels []string) ([]TokenEntity, error) { + opts := m.opts + opts.Labels = labels + fn, err := ModelTokenClassify(text, opts, m.loader, m.modelConfig, m.appConfig) + if err != nil { + return nil, err + } + return fn(ctx) +} + // ModelTokenClassify loads the backend for modelConfig and returns a // closure that classifies `text`. Mirrors ModelScore: the closure is // bound to the loaded model so a caller can reuse it within a request @@ -98,6 +119,7 @@ func ModelTokenClassify(text string, opts TokenClassifyOptions, loader *model.Mo ModelIdentity: modelConfig.Model, Text: text, Threshold: opts.Threshold, + Labels: opts.Labels, }) entities := tokenClassifyResponseToEntities(resp) if appConfig.EnableTracing { diff --git a/core/http/app.go b/core/http/app.go index 2e1453ac0..a2a46cb35 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -491,6 +491,7 @@ func API(application *application.Application) (*echo.Echo, error) { // mode by attributing requests to the synthetic "local" user. routes.RegisterUsageRoutes(e, application) routes.RegisterPIIRoutes(e, application) + routes.RegisterSystemOneRoutes(e, application) routes.RegisterMiddlewareRoutes(e, application) routes.RegisterElevenLabsRoutes(e, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()) diff --git a/core/http/endpoints/localai/systemone.go b/core/http/endpoints/localai/systemone.go new file mode 100644 index 000000000..c4e249b63 --- /dev/null +++ b/core/http/endpoints/localai/systemone.go @@ -0,0 +1,561 @@ +package localai + +import ( + "encoding/json" + "fmt" + "math" + "math/rand" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/schema" +) + +// --------------------------------------------------------------------------- +// Helpers — ported from kev/api.py (render, r2, choice_confidence, +// score_confidence, softmax) and mirrored in vllm.cpp api_server.cpp. +// --------------------------------------------------------------------------- + +// renderState flattens a JSON value into text, mirroring kev's render(). +// Field names are kept as labels; arrays become "- item" bullets; objects +// become "key: value" lines. +func renderState(v interface{}, indent int) string { + switch val := v.(type) { + case nil: + return "" + case string: + return val + case bool: + if val { + return "true" + } + return "false" + case float64: + b, _ := json.Marshal(val) + return string(b) + case []interface{}: + pad := strings.Repeat(" ", indent) + var parts []string + for _, item := range val { + rendered := renderState(item, indent+1) + rendered = strings.TrimLeft(rendered, " \t\n") + parts = append(parts, pad+"- "+rendered) + } + return strings.Join(parts, "\n") + case map[string]interface{}: + pad := strings.Repeat(" ", indent) + keys := make([]string, 0, len(val)) + for k := range val { + keys = append(keys, k) + } + sort.Strings(keys) + var parts []string + for i, k := range keys { + if i > 0 { + parts = append(parts, "\n") + } + switch vv := val[k].(type) { + case map[string]interface{}, []interface{}: + parts = append(parts, pad+k+":\n"+renderState(vv, indent+1)) + default: + parts = append(parts, pad+k+": "+renderState(vv, indent)) + } + } + return strings.Join(parts, "") + default: + b, _ := json.Marshal(v) + return string(b) + } +} + +// r2 rounds to 2 decimal places. +func r2(x float64) float64 { + return math.Round(x*100) / 100 +} + +// choiceConfidence is the normalized margin (kev/api.py:choice_confidence). +func choiceConfidence(p []float64) float64 { + k := len(p) + if k <= 1 { + return 1.0 + } + mx := p[0] + for _, v := range p[1:] { + if v > mx { + mx = v + } + } + return (mx - 1.0/float64(k)) / (1.0 - 1.0/float64(k)) +} + +// scoreConfidence is 1 - E|level - mode| / (L - 1) +// (kev/api.py:score_confidence). +func scoreConfidence(p []float64) float64 { + l := len(p) + if l <= 1 { + return 1.0 + } + mode := 0 + maxP := p[0] + for i := 1; i < l; i++ { + if p[i] > maxP { + maxP = p[i] + mode = i + } + } + s := 0.0 + for i := 0; i < l; i++ { + s += p[i] * math.Abs(float64(i)-float64(mode)) + } + return 1.0 - s/float64(l-1) +} + +// softmax is a numerically stable softmax. +func softmax(scores []float64) []float64 { + if len(scores) == 0 { + return nil + } + mx := scores[0] + for _, s := range scores[1:] { + if s > mx { + mx = s + } + } + exps := make([]float64, len(scores)) + sum := 0.0 + for i, s := range scores { + exps[i] = math.Exp(s - mx) + sum += exps[i] + } + if sum <= 0 { + inv := 1.0 / float64(len(scores)) + for i := range exps { + exps[i] = inv + } + return exps + } + for i := range exps { + exps[i] /= sum + } + return exps +} + +// --------------------------------------------------------------------------- +// Parsed question (internal). +// --------------------------------------------------------------------------- + +type parsedQuestion struct { + id string + qtype string // "noul", "choice", "score" + keys []string + labels []string +} + +type parsedSystemOne struct { + text string + model string + threshold float32 + questions []parsedQuestion + allLabels []string +} + +func parseSystemOneRequest(req *schema.SystemOneRequest) (*parsedSystemOne, error) { + p := &parsedSystemOne{ + model: req.Model, + threshold: 0.5, + } + if req.Threshold != nil { + p.threshold = *req.Threshold + } + + var stateVal interface{} + if err := json.Unmarshal(req.State, &stateVal); err != nil { + return nil, fmt.Errorf("state is not valid JSON: %w", err) + } + p.text = renderState(stateVal, 0) + + if len(req.Questions) == 0 { + return nil, fmt.Errorf("questions is required and must contain at least one question") + } + + qids := make([]string, 0, len(req.Questions)) + for k := range req.Questions { + qids = append(qids, k) + } + sort.Strings(qids) + + for _, qid := range qids { + q := req.Questions[qid] + pq := parsedQuestion{id: qid, qtype: q.Type} + switch q.Type { + case "noul": + pq.labels = []string{qid} + pq.keys = []string{"no", "yes"} + case "choice": + var criteria map[string]json.RawMessage + if err := json.Unmarshal(q.Criteria, &criteria); err != nil || len(criteria) == 0 { + return nil, fmt.Errorf("question %q (choice) requires a non-empty criteria object", qid) + } + ckeys := make([]string, 0, len(criteria)) + for k := range criteria { + ckeys = append(ckeys, k) + } + sort.Strings(ckeys) + for _, ck := range ckeys { + pq.keys = append(pq.keys, ck) + pq.labels = append(pq.labels, ck) + } + case "score": + var criteria []json.RawMessage + if err := json.Unmarshal(q.Criteria, &criteria); err != nil || len(criteria) < 2 { + return nil, fmt.Errorf("question %q (score) requires a criteria array with >= 2 levels", qid) + } + for _, level := range criteria { + var lv interface{} + _ = json.Unmarshal(level, &lv) + rendered := renderState(lv, 0) + pq.keys = append(pq.keys, rendered) + pq.labels = append(pq.labels, rendered) + } + default: + return nil, fmt.Errorf("question %q has unknown type: %s", qid, q.Type) + } + p.questions = append(p.questions, pq) + p.allLabels = append(p.allLabels, pq.labels...) + } + return p, nil +} + +// buildSystemOneAnswer produces one kev answer from NER entities. +func buildSystemOneAnswer(q *parsedQuestion, entities []backend.TokenEntity) schema.SystemOneAnswer { + scores := make([]float64, len(q.labels)) + for i, label := range q.labels { + var maxConf float32 + for _, e := range entities { + if e.Group == label && e.Score > maxConf { + maxConf = e.Score + } + } + scores[i] = float64(maxConf) + } + + switch q.qtype { + case "noul": + probs := []float64{1.0 - scores[0], scores[0]} + var ents []schema.SystemOneEntity + for _, e := range entities { + if e.Group == q.labels[0] { + ents = append(ents, schema.SystemOneEntity{ + Text: e.Text, + Start: e.Start, + End: e.End, + Confidence: e.Score, + }) + } + } + noul := r2(probs[1]) + return schema.SystemOneAnswer{ + Type: "noul", + Noul: &noul, + Entities: ents, + } + + case "choice": + probs := softmax(scores) + argmax := 0 + for i := 1; i < len(probs); i++ { + if probs[i] > probs[argmax] { + argmax = i + } + } + dist := make(map[string]float64, len(q.keys)) + for i, k := range q.keys { + dist[k] = r2(probs[i]) + } + choice := q.keys[argmax] + conf := r2(choiceConfidence(probs)) + return schema.SystemOneAnswer{ + Type: "choice", + Choice: &choice, + Confidence: &conf, + Probabilities: dist, + } + + default: // score + probs := softmax(scores) + var score float64 + for i, pr := range probs { + score += float64(i) * pr + } + legend := make(map[string]string, len(q.keys)) + dist := make(map[string]float64, len(q.keys)) + for i, k := range q.keys { + legend[strconv.Itoa(i)] = k + dist[strconv.Itoa(i)] = r2(probs[i]) + } + sc := r2(score) + conf := r2(scoreConfidence(probs)) + return schema.SystemOneAnswer{ + Type: "score", + Score: &sc, + Legend: legend, + Probabilities: dist, + Confidence: &conf, + } + } +} + +// --------------------------------------------------------------------------- +// Model resolution. +// --------------------------------------------------------------------------- + +func resolveClassifier(app *application.Application, modelName string, threshold float32) (backend.TokenClassifier, error) { + cl := app.ModelConfigLoader() + if cl == nil { + return nil, fmt.Errorf("model config loader unavailable") + } + cfg, ok := cl.GetModelConfig(modelName) + if !ok { + return nil, fmt.Errorf("model %q not found", modelName) + } + opts := backend.TokenClassifyOptions{ + Threshold: threshold, + } + return backend.NewTokenClassifier(app.ModelLoader(), cfg, app.ApplicationConfig(), opts), nil +} + +func systemOneError(c echo.Context, status int, msg string) error { + return c.JSON(status, map[string]any{ + "error": map[string]string{ + "message": msg, + "type": "invalid_request", + }, + }) +} + +// --------------------------------------------------------------------------- +// Endpoints. +// --------------------------------------------------------------------------- + +// SystemOneEndpoint handles POST /v1/systemone. +// Runs one NER pass over the rendered state with all question labels, then +// builds a kev-compatible answer for each question. +// @Summary Answer structured-extraction questions over state text. +// @Description Runs zero-shot NER over the supplied state and answers each question. Question types: noul (binary entity presence), choice (pick one option), score (pick one level). +// @Tags systemone +// @Param request body schema.SystemOneRequest true "state + questions" +// @Success 200 {object} schema.SystemOneResponse +// @Router /v1/systemone [post] +func SystemOneEndpoint(app *application.Application) echo.HandlerFunc { + return func(c echo.Context) error { + var req schema.SystemOneRequest + if err := c.Bind(&req); err != nil { + return systemOneError(c, http.StatusBadRequest, "invalid request body") + } + if req.Model == "" { + return systemOneError(c, http.StatusBadRequest, "model is required") + } + parsed, err := parseSystemOneRequest(&req) + if err != nil { + return systemOneError(c, http.StatusBadRequest, err.Error()) + } + classifier, err := resolveClassifier(app, req.Model, parsed.threshold) + if err != nil { + return systemOneError(c, http.StatusNotFound, err.Error()) + } + start := time.Now() + entities, err := classifier.TokenClassifyWithLabels(c.Request().Context(), parsed.text, parsed.allLabels) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + latencyMs := float64(time.Since(start).Microseconds()) / 1000.0 + answers := make(map[string]schema.SystemOneAnswer, len(parsed.questions)) + for i := range parsed.questions { + answers[parsed.questions[i].id] = buildSystemOneAnswer(&parsed.questions[i], entities) + } + return c.JSON(http.StatusOK, schema.SystemOneResponse{ + Model: req.Model, + Answers: answers, + Usage: schema.SystemOneUsage{InputTokens: 0, OutputTokens: 0}, + LatencyMs: r2(latencyMs), + }) + } +} + +// SystemOnePermuteEndpoint handles POST /v1/systemone/permute. +// Re-runs one choice question under n_perm option orders with a seeded RNG. +// @Summary Re-run a choice question under multiple option orders. +// @Description Re-runs one choice question under n_perm option orders. Reports per-order probabilities, argmax stability, and spread. +// @Tags systemone +// @Param request body schema.SystemOnePermuteRequest true "request + question + n_perm + seed" +// @Success 200 {object} schema.SystemOnePermuteResponse +// @Router /v1/systemone/permute [post] +func SystemOnePermuteEndpoint(app *application.Application) echo.HandlerFunc { + return func(c echo.Context) error { + var req schema.SystemOnePermuteRequest + if err := c.Bind(&req); err != nil { + return systemOneError(c, http.StatusBadRequest, "invalid request body") + } + if req.Request.Model == "" { + return systemOneError(c, http.StatusBadRequest, "model is required") + } + if req.Question == "" { + return systemOneError(c, http.StatusBadRequest, "question is required") + } + parsed, err := parseSystemOneRequest(&req.Request) + if err != nil { + return systemOneError(c, http.StatusBadRequest, err.Error()) + } + var target *parsedQuestion + for i := range parsed.questions { + if parsed.questions[i].id == req.Question { + target = &parsed.questions[i] + break + } + } + if target == nil { + return systemOneError(c, http.StatusBadRequest, fmt.Sprintf("question %q not found", req.Question)) + } + if target.qtype != "choice" { + return systemOneError(c, http.StatusBadRequest, "question must be a choice question") + } + classifier, err := resolveClassifier(app, req.Request.Model, parsed.threshold) + if err != nil { + return systemOneError(c, http.StatusNotFound, err.Error()) + } + nPerm := req.NPerm + if nPerm <= 0 { + nPerm = 6 + } + rng := rand.New(rand.NewSource(req.Seed)) + runs := make([]schema.SystemOnePermuteRun, 0, nPerm) + minProb := make([]float64, len(target.keys)) + maxProb := make([]float64, len(target.keys)) + for i := range minProb { + minProb[i] = 1.0 + maxProb[i] = 0.0 + } + firstChoice := "" + argmaxStable := true + + for i := 0; i < nPerm; i++ { + order := make([]string, len(target.keys)) + copy(order, target.keys) + if i > 0 { + rng.Shuffle(len(order), func(a, b int) { order[a], order[b] = order[b], order[a] }) + } + start := time.Now() + entities, err := classifier.TokenClassifyWithLabels(c.Request().Context(), parsed.text, order) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + latencyMs := float64(time.Since(start).Microseconds()) / 1000.0 + + scores := make([]float64, len(order)) + for j, label := range order { + var maxConf float32 + for _, e := range entities { + if e.Group == label && e.Score > maxConf { + maxConf = e.Score + } + } + scores[j] = float64(maxConf) + } + probs := softmax(scores) + argmax := 0 + for j := 1; j < len(probs); j++ { + if probs[j] > probs[argmax] { + argmax = j + } + } + probDist := make(map[string]float64, len(order)) + for j, label := range order { + probDist[label] = r2(probs[j]) + for k, key := range target.keys { + if label == key { + if probs[j] < minProb[k] { + minProb[k] = probs[j] + } + if probs[j] > maxProb[k] { + maxProb[k] = probs[j] + } + break + } + } + } + choice := order[argmax] + if i == 0 { + firstChoice = choice + } else if choice != firstChoice { + argmaxStable = false + } + runs = append(runs, schema.SystemOnePermuteRun{ + Order: order, + Probabilities: probDist, + Choice: choice, + LatencyMs: r2(latencyMs), + }) + } + + spread := make(map[string]float64, len(target.keys)) + for k, key := range target.keys { + spread[key] = r2(maxProb[k] - minProb[k]) + } + return c.JSON(http.StatusOK, schema.SystemOnePermuteResponse{ + Runs: runs, + ArgmaxStable: argmaxStable, + Spread: spread, + }) + } +} + +// SystemOneSeparateEndpoint handles POST /v1/systemone/separate. +// Answers each question in its own NER call (N passes). Response shape +// matches /v1/systemone. +// @Summary Answer each question in a separate NER pass. +// @Description Runs N independent NER passes, one per question, against the same state. Response shape matches /v1/systemone. +// @Tags systemone +// @Param request body schema.SystemOneRequest true "state + questions" +// @Success 200 {object} schema.SystemOneResponse +// @Router /v1/systemone/separate [post] +func SystemOneSeparateEndpoint(app *application.Application) echo.HandlerFunc { + return func(c echo.Context) error { + var req schema.SystemOneRequest + if err := c.Bind(&req); err != nil { + return systemOneError(c, http.StatusBadRequest, "invalid request body") + } + if req.Model == "" { + return systemOneError(c, http.StatusBadRequest, "model is required") + } + parsed, err := parseSystemOneRequest(&req) + if err != nil { + return systemOneError(c, http.StatusBadRequest, err.Error()) + } + classifier, err := resolveClassifier(app, req.Model, parsed.threshold) + if err != nil { + return systemOneError(c, http.StatusNotFound, err.Error()) + } + start := time.Now() + answers := make(map[string]schema.SystemOneAnswer, len(parsed.questions)) + for i := range parsed.questions { + entities, err := classifier.TokenClassifyWithLabels(c.Request().Context(), parsed.text, parsed.questions[i].labels) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + answers[parsed.questions[i].id] = buildSystemOneAnswer(&parsed.questions[i], entities) + } + latencyMs := float64(time.Since(start).Microseconds()) / 1000.0 + return c.JSON(http.StatusOK, schema.SystemOneResponse{ + Model: req.Model, + Answers: answers, + Usage: schema.SystemOneUsage{InputTokens: 0, OutputTokens: 0}, + LatencyMs: r2(latencyMs), + }) + } +} diff --git a/core/http/routes/systemone.go b/core/http/routes/systemone.go new file mode 100644 index 000000000..d6a70e5fc --- /dev/null +++ b/core/http/routes/systemone.go @@ -0,0 +1,20 @@ +package routes + +import ( + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/http/endpoints/localai" +) + +// RegisterSystemOneRoutes wires the kev-compatible SystemOne endpoints. +// These provide zero-shot structured extraction over arbitrary state text +// using a GLiNER2-backed NER model. The API mirrors the kev project +// (jaredpalmer/kev serve.py): POST /v1/systemone answers all questions in +// one NER pass; POST /v1/systemone/permute re-runs one choice question +// under n_perm option orders; POST /v1/systemone/separate answers each +// question in its own NER pass. +func RegisterSystemOneRoutes(e *echo.Echo, app *application.Application) { + e.POST("/v1/systemone", localai.SystemOneEndpoint(app)) + e.POST("/v1/systemone/permute", localai.SystemOnePermuteEndpoint(app)) + e.POST("/v1/systemone/separate", localai.SystemOneSeparateEndpoint(app)) +} diff --git a/core/schema/systemone.go b/core/schema/systemone.go new file mode 100644 index 000000000..b7282f39d --- /dev/null +++ b/core/schema/systemone.go @@ -0,0 +1,92 @@ +package schema + +import "encoding/json" + +// SystemOneRequest is the body for POST /v1/systemone, +// /v1/systemone/separate, and the inner `request` of /v1/systemone/permute. +// Mirrors the kev project's SystemOneRequest (jaredpalmer/kev serve.py). +type SystemOneRequest struct { + // State is the text (or any JSON value) to extract from. A non-string + // value is rendered to its JSON representation before NER. + State json.RawMessage `json:"state"` + // Questions maps question IDs to their definitions. + Questions map[string]SystemOneQuestion `json:"questions"` + // Model names the NER model to use. Optional. + Model string `json:"model,omitempty"` + // Threshold is the minimum entity confidence (0–1). Default 0.5. + Threshold *float32 `json:"threshold,omitempty"` + // MaxWidth is the maximum span width in tokens. Default 12. + MaxWidth *int `json:"max_width,omitempty"` +} + +// SystemOneQuestion defines one question. Type is "noul", "choice", or +// "score". Instr is optional human-readable instruction text. Criteria +// is: +// - noul: omitted +// - choice: a map of option_name → description (each key is a NER label) +// - score: an array of level descriptions (each is a NER label) +type SystemOneQuestion struct { + Type string `json:"type"` + Instr string `json:"instr,omitempty"` + Criteria json.RawMessage `json:"criteria,omitempty"` +} + +// SystemOneResponse is the shared response shape for /v1/systemone and +// /v1/systemone/separate. +type SystemOneResponse struct { + Model string `json:"model"` + Answers map[string]SystemOneAnswer `json:"answers"` + Usage SystemOneUsage `json:"usage"` + LatencyMs float64 `json:"latency_ms"` +} + +type SystemOneUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +// SystemOneAnswer is one question's answer. The fields populated depend on +// the question type: +// - noul: Noul (float 0–1), Entities +// - choice: Choice (string), Confidence, Probabilities (map) +// - score: Score (float), Legend (map), Probabilities (map), Confidence +type SystemOneAnswer struct { + Type string `json:"type"` + Noul *float64 `json:"noul,omitempty"` + Entities []SystemOneEntity `json:"entities,omitempty"` + Choice *string `json:"choice,omitempty"` + Confidence *float64 `json:"confidence,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + Score *float64 `json:"score,omitempty"` + Legend map[string]string `json:"legend,omitempty"` +} + +type SystemOneEntity struct { + Text string `json:"text"` + Start int `json:"start"` + End int `json:"end"` + Confidence float32 `json:"confidence"` +} + +// SystemOnePermuteRequest is the body for POST /v1/systemone/permute. +type SystemOnePermuteRequest struct { + Request SystemOneRequest `json:"request"` + Question string `json:"question"` + NPerm int `json:"n_perm,omitempty"` + Seed int64 `json:"seed,omitempty"` +} + +// SystemOnePermuteRun is one permutation's result. +type SystemOnePermuteRun struct { + Order []string `json:"order"` + Probabilities map[string]float64 `json:"probabilities"` + Choice string `json:"choice"` + LatencyMs float64 `json:"latency_ms"` +} + +// SystemOnePermuteResponse is the response for POST /v1/systemone/permute. +type SystemOnePermuteResponse struct { + Runs []SystemOnePermuteRun `json:"runs"` + ArgmaxStable bool `json:"argmax_stable"` + Spread map[string]float64 `json:"spread"` +} diff --git a/docs/content/features/vllm-cpp.md b/docs/content/features/vllm-cpp.md index 49cb5f872..74e3c0d38 100644 --- a/docs/content/features/vllm-cpp.md +++ b/docs/content/features/vllm-cpp.md @@ -160,6 +160,23 @@ forward, which is the required contract for pooling models in vllm.cpp. A device-resident forward is tracked as a performance optimization, not a correctness gap. +### SystemOne structured-extraction API + +The `vllm-cpp` backend also exposes kev-compatible SystemOne endpoints that +turn zero-shot NER into structured question answering. These mirror the API +from the [kev](https://github.com/jaredpalmer/kev) project: + +| Endpoint | Method | Description | +|---|---|---| +| `/v1/systemone` | POST | Answer all questions in one NER pass | +| `/v1/systemone/permute` | POST | Re-run one choice question under n_perm option orders | +| `/v1/systemone/separate` | POST | Answer each question in its own NER pass (N passes) | + +Each question has a `type` of `noul` (binary entity presence), `choice` (pick +one option), or `score` (pick one level). The `model` field in the request body +selects the NER model. Labels are derived from the question definition, so no +`ner_labels` configuration is needed for these endpoints. + ## Beyond text generation The `vllm-cpp` backend also serves MiniMax-H3, which generates video and audio