From 49efeff7292afccffe40ca1f74f4e116e89fba76 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 19 Sep 2026 07:46:07 +0000 Subject: [PATCH] feat(vllm-cpp): add GLiNER2.5 NER via TokenClassify Wire the vllm-cpp backend to the C ABI NER surface (vllm_gliner_ner, ABI v27) so LocalAI can serve zero-shot named entity recognition through the existing TokenClassify gRPC method. backend.go: TokenClassify method on *VllmCpp calls vllm_gliner_ner with the text and labels, copies the C-owned entity array into protobuf TokenClassifyEntity messages, and frees the result. govllmcpp.go: cNerEntity and cNerResult Go POD mirrors matching the C structs; vllmGlinerNer and vllmNerResultFree purego bindings; abiVersion bumped 26 -> 27. options.go: ner_labels, ner_threshold, ner_max_width parsed from engine_args. pkg/grpc: ClassifyModel interface and TokenClassify server handler (follows the Embedding locking pattern). core/config: vllm-cpp backend declares MethodTokenClassify and UsecaseTokenClassify. docs/content/features/vllm-cpp.md: NER section documenting the engine_args keys and the host-forward contract. Assisted-by: MAKI:regolo/glm5.2 [maki] Signed-off-by: Ettore Di Giacinto --- backend/go/vllm-cpp/backend.go | 59 +++++++++++++++++++++++++++++ backend/go/vllm-cpp/govllmcpp.go | 26 ++++++++++++- backend/go/vllm-cpp/options.go | 25 ++++++++++++ backend/go/vllm-cpp/vllmcpp_test.go | 23 ++++++++++- core/config/backend_capabilities.go | 11 ++++-- docs/content/features/vllm-cpp.md | 27 +++++++++++++ pkg/grpc/interface.go | 10 +++++ pkg/grpc/server.go | 15 ++++++++ 8 files changed, 190 insertions(+), 6 deletions(-) diff --git a/backend/go/vllm-cpp/backend.go b/backend/go/vllm-cpp/backend.go index 5292c42ea..30204890a 100644 --- a/backend/go/vllm-cpp/backend.go +++ b/backend/go/vllm-cpp/backend.go @@ -10,6 +10,7 @@ package main // backend embeds base.Base and not base.SingleThread). import ( + "context" "fmt" "os" "path/filepath" @@ -275,6 +276,64 @@ func (v *VllmCpp) Predict(opts *pb.PredictOptions) (string, error) { return text, nil } +// defaultNerLabels is the general-purpose entity type set used when the model +// config does not supply ner_labels. These cover the most common NER use cases +// and match the categories the GLiNER2.5 model card demonstrates. +var defaultNerLabels = []string{ + "person", "organization", "location", + "date", "time", "money", "quantity", +} + +// TokenClassify runs zero-shot NER on the loaded GLiNER2.5 engine via the +// vllm_gliner_ner C ABI (ABI v27). The engine refuses non-BoundaryExtractor +// architectures, so a model loaded for chat or embeddings returns an error +// here rather than silent garbage. +func (v *VllmCpp) TokenClassify(_ context.Context, in *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error) { + if v.engine == 0 { + return nil, fmt.Errorf("vllm-cpp: model not loaded") + } + labels := v.opts.nerLabels + if len(labels) == 0 { + labels = defaultNerLabels + } + threshold := v.opts.nerThreshold + if in.Threshold > 0 { + threshold = in.Threshold + } + maxWidth := v.opts.nerMaxWidth + + labelPtrs, labelBacking := cStringArray(labels) + if len(labelPtrs) == 0 { + return nil, fmt.Errorf("vllm-cpp: no NER labels configured") + } + labelsPtr := uintptr(unsafe.Pointer(&labelPtrs[0])) // #nosec G103 -- borrowed by C for the call only + + var out cNerResult + rc := vllmGlinerNer(v.engine, in.Text, labelsPtr, int32(len(labelPtrs)), threshold, maxWidth, unsafe.Pointer(&out)) // #nosec G103 -- POD in/out params + runtime.KeepAlive(labelBacking) + if rc != vllmOK { + return nil, fmt.Errorf("vllm-cpp: NER failed: %s", vllmLastError()) + } + defer vllmNerResultFree(unsafe.Pointer(&out)) // #nosec G103 -- frees C-owned members + + entities := make([]*pb.TokenClassifyEntity, 0, out.nEntities) + if out.nEntities > 0 && out.entities != 0 { + // #nosec:govet // C-owned array, valid for this call before vllmNerResultFree + cents := unsafe.Slice((*cNerEntity)(unsafe.Pointer(out.entities)), int(out.nEntities)) // #nosec G103 -- C-owned, copied out immediately + for i := range cents { + e := ¢s[i] + entities = append(entities, &pb.TokenClassifyEntity{ + EntityGroup: goString(e.label), + Start: e.charStart, + End: e.charEnd, + Score: e.confidence, + Text: goString(e.text), + }) + } + } + return &pb.TokenClassifyResponse{Entities: entities}, nil +} + func (v *VllmCpp) PredictStream(opts *pb.PredictOptions, results chan string) error { if v.engine == 0 { close(results) diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index db7555cd2..a6f1935d3 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 26 +const abiVersion = 27 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -252,8 +252,30 @@ var ( vllmVideoResultFree func(out unsafe.Pointer) vllmVideoMuxArgv func(params, outArgv, outArgc unsafe.Pointer) int32 vllmVideoMuxArgvFre func(argv uintptr, argc int32) + + // Zero-shot NER (ABI v27, GLiNER2.5). + vllmGlinerNer func(engine uintptr, text string, labels uintptr, nLabels int32, threshold float32, maxWidth int32, out unsafe.Pointer) int32 + vllmNerResultFree func(out unsafe.Pointer) ) +// cNerEntity mirrors vllm_ner_entity. Layout matches the C struct on LP64: +// two pointer-width fields, four int32, one float, padded to 40 bytes. +type cNerEntity struct { + label uintptr // char* + text uintptr // char* + charStart int32 + charEnd int32 + tokenStart int32 + tokenEnd int32 + confidence float32 +} + +// cNerResult mirrors vllm_ner_result. +type cNerResult struct { + entities uintptr // vllm_ner_entity* + nEntities int32 +} + type libFunc struct { ptr any name string @@ -285,6 +307,8 @@ func registerLib(libName string) error { {&vllmVideoResultFree, "vllm_video_result_free"}, {&vllmVideoMuxArgv, "vllm_video_mux_argv"}, {&vllmVideoMuxArgvFre, "vllm_video_mux_argv_free"}, + {&vllmGlinerNer, "vllm_gliner_ner"}, + {&vllmNerResultFree, "vllm_ner_result_free"}, } { purego.RegisterLibFunc(lf.ptr, lib, lf.name) } diff --git a/backend/go/vllm-cpp/options.go b/backend/go/vllm-cpp/options.go index 7e3edf4d3..1a36e62b3 100644 --- a/backend/go/vllm-cpp/options.go +++ b/backend/go/vllm-cpp/options.go @@ -65,6 +65,15 @@ type loadOptions struct { // MiniMax-H3 video+audio generation (ABI v12). Present only when the config // carries at least one of its keys; see videoOptions.engaged. video videoOptions + // Zero-shot NER labels (ABI v27, GLiNER2.5). GLiNER2.5 is truly zero-shot: + // the model ships no default labels, so the entity types to extract are + // supplied here from engine_args.ner_labels. When empty, a general-purpose + // default set is used. + nerLabels []string + // nerThreshold is the default sigmoid floor (0 = model default 0.5). + nerThreshold float32 + // nerMaxWidth is the maximum span width in tokens (0 = engine default 12). + nerMaxWidth int32 } // videoOptions is the MiniMax-H3 checkpoint SET plus its generation defaults. @@ -342,6 +351,22 @@ func applyEngineArgs(lo *loadOptions, engineArgs string) { if b, ok := v.(bool); ok { lo.enableJumpForward = boolTriState(b) } + case "ner_labels": + if arr, ok := v.([]any); ok { + for _, e := range arr { + if s, ok := e.(string); ok && s != "" { + lo.nerLabels = append(lo.nerLabels, s) + } + } + } + case "ner_threshold": + if f, ok := v.(float64); ok { + lo.nerThreshold = float32(f) + } + case "ner_max_width": + if f, ok := v.(float64); ok { + lo.nerMaxWidth = int32(f) + } default: if s, ok := videoScalarString(v); ok && applyVideoOption(&lo.video, k, s) { continue diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 201fd1692..a4de363d0 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v26) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v27) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(26)) + Expect(abiVersion).To(Equal(27)) }) It("cModelParams matches vllm_model_params", func() { @@ -92,6 +92,25 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(c.CompletionTokens)).To(Equal(uintptr(20))) Expect(unsafe.Sizeof(c)).To(Equal(uintptr(24))) }) + + It("cNerEntity matches vllm_ner_entity (ABI v27)", func() { + var e cNerEntity + Expect(unsafe.Offsetof(e.label)).To(Equal(uintptr(0))) + Expect(unsafe.Offsetof(e.text)).To(Equal(uintptr(8))) + Expect(unsafe.Offsetof(e.charStart)).To(Equal(uintptr(16))) + Expect(unsafe.Offsetof(e.charEnd)).To(Equal(uintptr(20))) + Expect(unsafe.Offsetof(e.tokenStart)).To(Equal(uintptr(24))) + Expect(unsafe.Offsetof(e.tokenEnd)).To(Equal(uintptr(28))) + Expect(unsafe.Offsetof(e.confidence)).To(Equal(uintptr(32))) + Expect(unsafe.Sizeof(e)).To(Equal(uintptr(40))) + }) + + It("cNerResult matches vllm_ner_result (ABI v27)", func() { + var r cNerResult + Expect(unsafe.Offsetof(r.entities)).To(Equal(uintptr(0))) + Expect(unsafe.Offsetof(r.nEntities)).To(Equal(uintptr(8))) + Expect(unsafe.Sizeof(r)).To(Equal(uintptr(16))) + }) }) // Pin/mirror skew is the failure mode this backend is most exposed to: the Go diff --git a/core/config/backend_capabilities.go b/core/config/backend_capabilities.go index e97390645..19c628535 100644 --- a/core/config/backend_capabilities.go +++ b/core/config/backend_capabilities.go @@ -342,12 +342,17 @@ var BackendCapabilities = map[string]BackendCapability{ // // AcceptsImages is the fl2va keyframe (start_image/end_image), the same // reason longcat-video declares it; the text path takes no image input. + // + // TokenClassify is possible (GLiNER2.5 zero-shot NER via vllm_gliner_ner, + // ABI v27), declared explicitly via known_usecases: [token_classify]. The + // engine refuses non-BoundaryExtractor architectures, so a chat or embedding + // model returns an error rather than silent garbage. "vllm-cpp": { - GRPCMethods: []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo}, - PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo}, + GRPCMethods: []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo, MethodTokenClassify}, + PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo, UsecaseTokenClassify}, DefaultUsecases: []string{UsecaseChat}, AcceptsImages: true, - Description: "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation plus MiniMax-H3 video+audio generation", + Description: "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation, MiniMax-H3 video+audio generation, and GLiNER2.5 zero-shot NER", }, "vllm-omni": { GRPCMethods: []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateImage, MethodGenerateVideo, MethodTTS}, diff --git a/docs/content/features/vllm-cpp.md b/docs/content/features/vllm-cpp.md index d4bd2561d..49cb5f872 100644 --- a/docs/content/features/vllm-cpp.md +++ b/docs/content/features/vllm-cpp.md @@ -133,6 +133,33 @@ engine_args: tool_parser: qwen3_coder ``` +## Named entity recognition (GLiNER2.5) + +The `vllm-cpp` backend serves [GLiNER2.5](https://huggingface.co/fastino/gliner2.5-multi-v1), +a zero-shot NER and structured-extraction model. Point the backend at the +safetensors directory and the backend exposes the `TokenClassify` gRPC method, +which LocalAI maps to its standard NER API surface. + +Labels are supplied at inference time, not baked into the model config. Set +them in `engine_args`: + +```yaml +engine_args: + ner_labels: "person,organization,location,date,time,money,quantity" + ner_threshold: 0.5 + ner_max_width: 12 +``` + +`ner_labels` is a comma-separated list. When omitted, the backend falls back to +a built-in default set (`person`, `organization`, `location`, `date`, `time`, +`money`, `quantity`). `ner_threshold` is the sigmoid cutoff (default 0.5); +`ner_max_width` is the maximum span length in tokens (default 12). + +The model runs the DeBERTa v2 encoder with disentangled attention on the host +forward, which is the required contract for pooling models in vllm.cpp. A +device-resident forward is tracked as a performance optimization, not a +correctness gap. + ## Beyond text generation The `vllm-cpp` backend also serves MiniMax-H3, which generates video and audio diff --git a/pkg/grpc/interface.go b/pkg/grpc/interface.go index 8370602a3..4492763c2 100644 --- a/pkg/grpc/interface.go +++ b/pkg/grpc/interface.go @@ -97,3 +97,13 @@ type AIModelRich interface { PredictRich(*pb.PredictOptions) (*pb.Reply, error) PredictStreamRich(*pb.PredictOptions, chan<- *pb.Reply) error } + +// ClassifyModel is an optional extension to AIModel for backends that +// implement the TokenClassify RPC (zero-shot NER). The gRPC server +// type-asserts to this interface; backends that do not implement it +// fall through to the UnimplementedBackendServer default. This mirrors +// the AIModelRich pattern: adding a method to AIModel itself would +// break every backend, so the capability is opt-in. +type ClassifyModel interface { + TokenClassify(context.Context, *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error) +} diff --git a/pkg/grpc/server.go b/pkg/grpc/server.go index 092e68b65..d4757cbb8 100644 --- a/pkg/grpc/server.go +++ b/pkg/grpc/server.go @@ -102,6 +102,21 @@ func (s *server) Embedding(ctx context.Context, in *pb.PredictOptions) (*pb.Embe }, nil } +func (s *server) TokenClassify(ctx context.Context, in *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error) { + if err := s.checkModelIdentity(in); err != nil { + return nil, err + } + cm, ok := s.llm.(ClassifyModel) + if !ok { + return nil, status.Errorf(codes.Unimplemented, "method TokenClassify not implemented") + } + if s.llm.Locking() { + s.llm.Lock() + defer s.llm.Unlock() + } + return cm.TokenClassify(ctx, in) +} + func (s *server) LoadModel(ctx context.Context, in *pb.ModelOptions) (*pb.Result, error) { if s.llm.Locking() { s.llm.Lock()