diff --git a/backend/backend.proto b/backend/backend.proto index 4f7c60fe9..54255528e 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -198,6 +198,12 @@ message ScoreRequest { // use it to place a reuse point exactly at the boundary, so the next // call re-processes only the tokens after it. 0 means unknown. int32 stable_prefix_len = 6; + // question_type signals a decision-pipeline request (kev/laya) + // rather than plain candidate scoring. When set to "systemone", the + // backend treats prompt as the raw /v1/systemone request JSON and + // returns the full response in ScoreResponse.response_json. Empty + // means plain candidate scoring. + string question_type = 7; } // CandidateScore is one row in the ScoreResponse, matching by index @@ -223,6 +229,9 @@ message TokenLogProb { message ScoreResponse { repeated CandidateScore candidates = 1; + // response_json carries the full decision-pipeline JSON response when + // question_type is set on the request. Empty for plain scoring. + string response_json = 2; } message RerankRequest { diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 0b18c01e4..05ffd4628 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=b24f8094cba9b4f02df71bcff8d41ddc7e88b4ef +VLLM_CPP_VERSION?=bc5dbc367 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/go/vllm-cpp/backend.go b/backend/go/vllm-cpp/backend.go index c99399a13..40e06fe8c 100644 --- a/backend/go/vllm-cpp/backend.go +++ b/backend/go/vllm-cpp/backend.go @@ -11,7 +11,9 @@ package main import ( "context" + "encoding/json" "fmt" + "math" "os" "path/filepath" "runtime" @@ -90,10 +92,16 @@ func validModelPath(model string) error { return fmt.Errorf("vllm-cpp: model path %q not found: %w", model, err) } if info.IsDir() { - if _, err := os.Stat(filepath.Join(model, "config.json")); err != nil { - return fmt.Errorf("vllm-cpp: model dir %q has no config.json", model) + // vllm.cpp accepts three config filenames: config.json (standard), + // cua-s1-forms.json (cua-s1-forms scoring model), and + // rl_agent_config.json (laya decision model). The engine's + // model_loader.cpp checks them in that order. + for _, cfg := range []string{"config.json", "cua-s1-forms.json", "rl_agent_config.json"} { + if _, err := os.Stat(filepath.Join(model, cfg)); err == nil { + return nil + } } - return nil + return fmt.Errorf("vllm-cpp: model dir %q has no config.json, cua-s1-forms.json, or rl_agent_config.json", model) } if strings.EqualFold(filepath.Ext(model), ".gguf") { return nil @@ -360,3 +368,68 @@ func (v *VllmCpp) PredictStream(opts *pb.PredictOptions, results chan string) er }() return nil } + +// Score runs the unified decision pipeline via the vllm_decide C ABI +// (ABI v29). When question_type is "systemone", the prompt carries the +// raw /v1/systemone request JSON and the response is returned in +// response_json. When question_type is empty, the prompt and candidates +// are scored as candidate continuations. +func (v *VllmCpp) Score(_ context.Context, in *pb.ScoreRequest) (*pb.ScoreResponse, error) { + if v.engine == 0 { + return nil, fmt.Errorf("vllm-cpp: model not loaded") + } + if in.QuestionType == "systemone" { + // Decision pipeline (kev/laya): forward the raw request JSON. + var out uintptr + rc := vllmDecide(v.engine, in.Prompt, unsafe.Pointer(&out)) // #nosec G103 -- char** out-param + if rc != vllmOK { + return nil, fmt.Errorf("vllm-cpp: decide failed: %s", vllmLastError()) + } + payload := goString(out) + vllmDecideFree(out) + return &pb.ScoreResponse{ResponseJson: payload}, nil + } + // Candidate scoring (cua-s1-forms): build a scoring request JSON. + if len(in.Candidates) == 0 { + return nil, fmt.Errorf("vllm-cpp: score requires at least one candidate") + } + reqJSON, err := json.Marshal(map[string]any{ + "context": in.Prompt, + "options": in.Candidates, + }) + if err != nil { + return nil, fmt.Errorf("vllm-cpp: score request encode: %w", err) + } + var out uintptr + rc := vllmDecide(v.engine, string(reqJSON), unsafe.Pointer(&out)) // #nosec G103 -- char** out-param + if rc != vllmOK { + return nil, fmt.Errorf("vllm-cpp: decide failed: %s", vllmLastError()) + } + payload := goString(out) + vllmDecideFree(out) + + var resp struct { + Probabilities []float64 `json:"probabilities"` + } + if err := json.Unmarshal([]byte(payload), &resp); err != nil { + return nil, fmt.Errorf("vllm-cpp: unparseable score response: %w", err) + } + candidates := make([]*pb.CandidateScore, len(in.Candidates)) + for i, c := range in.Candidates { + var p float64 + if i < len(resp.Probabilities) { + p = resp.Probabilities[i] + } + lp := math.Log(p) + if p <= 0 { + lp = -999.0 // JSON cannot encode -Inf; use a large negative sentinel + } + nTok := max((len(c)+3)/4, 1) + candidates[i] = &pb.CandidateScore{ + LogProb: lp, + NumTokens: int32(nTok), + LengthNormalizedLogProb: lp / float64(nTok), + } + } + return &pb.ScoreResponse{Candidates: candidates}, nil +} diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index 1f432ea39..f3723d096 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 27 +const abiVersion = 29 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -256,6 +256,11 @@ var ( // Zero-shot NER (ABI v27, GLiNER2.5). vllmGlinerNer func(engine uintptr, text string, labels uintptr, nLabels int32, threshold float32, maxWidth int32, out unsafe.Pointer) int32 vllmNerResultFree func(out unsafe.Pointer) + + // Decide: unified decision pipeline (ABI v29, MODEL-KEV / MODEL-LAYA / + // MODEL-CUA-S1-FORMS). Replaces vllm_systemone + vllm_score from v28. + vllmDecide func(engine uintptr, requestJSON string, out unsafe.Pointer) int32 + vllmDecideFree func(json uintptr) ) // cNerEntity mirrors vllm_ner_entity. Layout matches the C struct on LP64: @@ -309,6 +314,8 @@ func registerLib(libName string) error { {&vllmVideoMuxArgvFre, "vllm_video_mux_argv_free"}, {&vllmGlinerNer, "vllm_gliner_ner"}, {&vllmNerResultFree, "vllm_ner_result_free"}, + {&vllmDecide, "vllm_decide"}, + {&vllmDecideFree, "vllm_decide_free"}, } { purego.RegisterLibFunc(lf.ptr, lib, lf.name) } diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index a4de363d0..51ca81ace 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v27) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v29) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(27)) + Expect(abiVersion).To(Equal(29)) }) It("cModelParams matches vllm_model_params", func() { diff --git a/core/backend/systemone.go b/core/backend/systemone.go new file mode 100644 index 000000000..8b1077672 --- /dev/null +++ b/core/backend/systemone.go @@ -0,0 +1,76 @@ +package backend + +import ( + "context" + "fmt" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/trace" + "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + model "github.com/mudler/LocalAI/pkg/model" +) + +// ModelSystemOne loads the backend for modelConfig and returns a closure +// that runs the kev/laya decision pipeline via the Score gRPC RPC with +// question_type set to "systemone". requestJSON is the raw /v1/systemone +// POST body; the closure returns the full response JSON from the backend. +func ModelSystemOne(requestJSON string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func(ctx context.Context) (string, error), error) { + modelOpts := ModelOptions(modelConfig, appConfig) + inferenceModel, err := loader.Load(modelOpts...) + if err != nil { + recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) + return nil, err + } + b, ok := inferenceModel.(grpc.Backend) + if !ok { + return nil, fmt.Errorf("systemone not supported by backend %q", modelConfig.Backend) + } + return func(ctx context.Context) (string, error) { + release, err := AcquireGlobalBackendSlot() + if err != nil { + return "", err + } + defer release() + var startTime time.Time + var traceID string + if appConfig.EnableTracing { + trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) + startTime = time.Now() + traceID = trace.BeginBackendTrace(trace.BackendTrace{ + Timestamp: startTime, + Type: trace.BackendTraceScore, + ModelName: modelConfig.Name, + Backend: modelConfig.Backend, + Summary: trace.TruncateString(requestJSON, 200), + }) + } + defer trace.CancelBackendTrace(traceID) + resp, err := b.Score(ctx, &pb.ScoreRequest{ + Prompt: requestJSON, + QuestionType: "systemone", + ModelIdentity: modelConfig.Model, + }) + if appConfig.EnableTracing { + errStr := "" + if err != nil { + errStr = err.Error() + } + trace.RecordBackendTrace(trace.BackendTrace{ + ID: traceID, + Timestamp: startTime, + Duration: time.Since(startTime), + Type: trace.BackendTraceScore, + ModelName: modelConfig.Name, + Backend: modelConfig.Backend, + Summary: trace.TruncateString(requestJSON, 200), + Error: errStr, + }) + } + if err != nil { + return "", err + } + return resp.GetResponseJson(), nil + }, nil +} diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index 19c628535..886ddd22a 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -348,11 +348,11 @@ var BackendCapabilities = map[string]BackendCapability{ // engine refuses non-BoundaryExtractor architectures, so a chat or embedding // model returns an error rather than silent garbage. "vllm-cpp": { - GRPCMethods: []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo, MethodTokenClassify}, - PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo, UsecaseTokenClassify}, + GRPCMethods: []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo, MethodTokenClassify, MethodScore}, + PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo, UsecaseTokenClassify, UsecaseScore}, DefaultUsecases: []string{UsecaseChat}, AcceptsImages: true, - Description: "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation, MiniMax-H3 video+audio generation, and GLiNER2.5 zero-shot NER", + Description: "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation, MiniMax-H3 video+audio generation, GLiNER2.5 zero-shot NER, cua-s1-forms scoring, and kev/laya decision pipelines", }, "vllm-omni": { GRPCMethods: []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateImage, MethodGenerateVideo, MethodTTS}, diff --git a/core/http/endpoints/localai/systemone.go b/core/http/endpoints/localai/systemone.go index 9331dbf73..e272e1b8b 100644 --- a/core/http/endpoints/localai/systemone.go +++ b/core/http/endpoints/localai/systemone.go @@ -6,6 +6,7 @@ import ( "math" "math/rand" "net/http" + "slices" "sort" "strconv" "strings" @@ -14,6 +15,7 @@ import ( "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/application" "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/schema" ) @@ -369,13 +371,28 @@ func systemOneError(c echo.Context, status int, msg string) error { }) } +// backendSupportsScore reports whether the named backend implements the +// Score gRPC RPC. vllm-cpp does (kev/laya decision pipeline and cua-s1-forms +// scoring via the unified vllm_decide C ABI); other backends fall through to +// the NER-based path. +func backendSupportsScore(backendName string) bool { + cap := config.GetBackendCapability(backendName) + if cap == nil { + return false + } + return slices.Contains(cap.GRPCMethods, config.MethodScore) +} + // --------------------------------------------------------------------------- // Endpoints. // --------------------------------------------------------------------------- // SystemOneEndpoint handles POST /v1/systemone. -// Runs one NER pass over the rendered state with all question labels, then -// builds a kev-compatible answer for each question. +// For vllm-cpp models (kev/laya), forwards the raw request to the backend's +// Score gRPC RPC with question_type set to "systemone" and returns the +// response JSON as-is. For other backends, runs one NER pass over the +// rendered state with all question labels, then builds a kev-compatible +// answer for each question. // @Summary Answer structured-extraction questions over state text. // @Description Runs zero-shot NER over the supplied state and answers each question. Question types: noul (binary entity presence), choice (pick one option), score (pick one level). // @Tags systemone @@ -391,6 +408,28 @@ func SystemOneEndpoint(app *application.Application) echo.HandlerFunc { if req.Model == "" { return systemOneError(c, http.StatusBadRequest, "model is required") } + // vllm-cpp models (kev/laya) implement the decision pipeline natively + // via the vllm_decide C ABI. Forward the raw request JSON through the + // Score RPC and return the backend's response as-is. + cl := app.ModelConfigLoader() + if cl != nil { + if cfg, ok := cl.GetModelConfig(req.Model); ok && backendSupportsScore(cfg.Backend) { + reqJSON, err := json.Marshal(req) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, "failed to marshal request: "+err.Error()) + } + fn, err := backend.ModelSystemOne(string(reqJSON), app.ModelLoader(), cfg, app.ApplicationConfig()) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + respJSON, err := fn(c.Request().Context()) + if err != nil { + return systemOneError(c, http.StatusInternalServerError, err.Error()) + } + return c.JSON(http.StatusOK, json.RawMessage(respJSON)) + } + } + // NER-based path (GLiNER2.5 zero-shot NER). parsed, err := parseSystemOneRequest(&req) if err != nil { return systemOneError(c, http.StatusBadRequest, err.Error()) diff --git a/pkg/grpc/interface.go b/pkg/grpc/interface.go index 998eb0745..3da403552 100644 --- a/pkg/grpc/interface.go +++ b/pkg/grpc/interface.go @@ -113,3 +113,13 @@ type AIModelRich interface { type ClassifyModel interface { TokenClassify(context.Context, *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error) } + +// ScoreModel is an optional extension to AIModel for backends that +// implement the Score RPC (candidate scoring and decision pipelines). +// The gRPC server type-asserts to this interface; backends that do not +// implement it fall through to the UnimplementedBackendServer default. +// This mirrors the ClassifyModel pattern: adding a method to AIModel +// itself would break every backend, so the capability is opt-in. +type ScoreModel interface { + Score(context.Context, *pb.ScoreRequest) (*pb.ScoreResponse, error) +} diff --git a/pkg/grpc/server.go b/pkg/grpc/server.go index 7841d1c40..40eec06fe 100644 --- a/pkg/grpc/server.go +++ b/pkg/grpc/server.go @@ -117,6 +117,21 @@ func (s *server) TokenClassify(ctx context.Context, in *pb.TokenClassifyRequest) return cm.TokenClassify(ctx, in) } +func (s *server) Score(ctx context.Context, in *pb.ScoreRequest) (*pb.ScoreResponse, error) { + if err := s.checkModelIdentity(in); err != nil { + return nil, err + } + sm, ok := s.llm.(ScoreModel) + if !ok { + return nil, status.Errorf(codes.Unimplemented, "method Score not implemented") + } + if s.llm.Locking() { + s.llm.Lock() + defer s.llm.Unlock() + } + return sm.Score(ctx, in) +} + func (s *server) LoadModel(ctx context.Context, in *pb.ModelOptions) (*pb.Result, error) { if s.llm.Locking() { s.llm.Lock()