feat(vllm-cpp): add GLiNER2.5 NER via TokenClassify

Wire the vllm-cpp backend to the C ABI NER surface (vllm_gliner_ner,
ABI v27) so LocalAI can serve zero-shot named entity recognition through
the existing TokenClassify gRPC method.

backend.go: TokenClassify method on *VllmCpp calls vllm_gliner_ner with
the text and labels, copies the C-owned entity array into protobuf
TokenClassifyEntity messages, and frees the result.

govllmcpp.go: cNerEntity and cNerResult Go POD mirrors matching the C
structs; vllmGlinerNer and vllmNerResultFree purego bindings; abiVersion
bumped 26 -> 27.

options.go: ner_labels, ner_threshold, ner_max_width parsed from
engine_args.

pkg/grpc: ClassifyModel interface and TokenClassify server handler
(follows the Embedding locking pattern).

core/config: vllm-cpp backend declares MethodTokenClassify and
UsecaseTokenClassify.

docs/content/features/vllm-cpp.md: NER section documenting the
engine_args keys and the host-forward contract.

Assisted-by: MAKI:regolo/glm5.2 [maki]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

backend/go/vllm-cpp/backend.go

@@ -10,6 +10,7 @@ package main
 // backend embeds base.Base and not base.SingleThread).
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -275,6 +276,64 @@ func (v *VllmCpp) Predict(opts *pb.PredictOptions) (string, error) {
 	return text, nil
 }
 
+// defaultNerLabels is the general-purpose entity type set used when the model
+// config does not supply ner_labels. These cover the most common NER use cases
+// and match the categories the GLiNER2.5 model card demonstrates.
+var defaultNerLabels = []string{
+	"person", "organization", "location",
+	"date", "time", "money", "quantity",
+}
+
+// TokenClassify runs zero-shot NER on the loaded GLiNER2.5 engine via the
+// vllm_gliner_ner C ABI (ABI v27). The engine refuses non-BoundaryExtractor
+// architectures, so a model loaded for chat or embeddings returns an error
+// here rather than silent garbage.
+func (v *VllmCpp) TokenClassify(_ context.Context, in *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error) {
+	if v.engine == 0 {
+		return nil, fmt.Errorf("vllm-cpp: model not loaded")
+	}
+	labels := v.opts.nerLabels
+	if len(labels) == 0 {
+		labels = defaultNerLabels
+	}
+	threshold := v.opts.nerThreshold
+	if in.Threshold > 0 {
+		threshold = in.Threshold
+	}
+	maxWidth := v.opts.nerMaxWidth
+
+	labelPtrs, labelBacking := cStringArray(labels)
+	if len(labelPtrs) == 0 {
+		return nil, fmt.Errorf("vllm-cpp: no NER labels configured")
+	}
+	labelsPtr := uintptr(unsafe.Pointer(&labelPtrs[0])) // #nosec G103 -- borrowed by C for the call only
+
+	var out cNerResult
+	rc := vllmGlinerNer(v.engine, in.Text, labelsPtr, int32(len(labelPtrs)), threshold, maxWidth, unsafe.Pointer(&out)) // #nosec G103 -- POD in/out params
+	runtime.KeepAlive(labelBacking)
+	if rc != vllmOK {
+		return nil, fmt.Errorf("vllm-cpp: NER failed: %s", vllmLastError())
+	}
+	defer vllmNerResultFree(unsafe.Pointer(&out)) // #nosec G103 -- frees C-owned members
+
+	entities := make([]*pb.TokenClassifyEntity, 0, out.nEntities)
+	if out.nEntities > 0 && out.entities != 0 {
+		// #nosec:govet // C-owned array, valid for this call before vllmNerResultFree
+		cents := unsafe.Slice((*cNerEntity)(unsafe.Pointer(out.entities)), int(out.nEntities)) // #nosec G103 -- C-owned, copied out immediately
+		for i := range cents {
+			e := &cents[i]
+			entities = append(entities, &pb.TokenClassifyEntity{
+				EntityGroup: goString(e.label),
+				Start:       e.charStart,
+				End:         e.charEnd,
+				Score:       e.confidence,
+				Text:        goString(e.text),
+			})
+		}
+	}
+	return &pb.TokenClassifyResponse{Entities: entities}, nil
+}
+
 func (v *VllmCpp) PredictStream(opts *pb.PredictOptions, results chan string) error {
 	if v.engine == 0 {
 		close(results)

backend/go/vllm-cpp/govllmcpp.go

@@ -21,7 +21,7 @@ import (
 // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks
 // the two against each other, because a mismatch is only caught at runtime by
 // registerLib, where it takes the backend down on every load (issue #11379).
-const abiVersion = 26
+const abiVersion = 27
 
 // The ABI's tri-state toggles (enable_prefix_caching ABI v7,
 // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
@@ -252,8 +252,30 @@ var (
 	vllmVideoResultFree func(out unsafe.Pointer)
 	vllmVideoMuxArgv    func(params, outArgv, outArgc unsafe.Pointer) int32
 	vllmVideoMuxArgvFre func(argv uintptr, argc int32)
+
+	// Zero-shot NER (ABI v27, GLiNER2.5).
+	vllmGlinerNer     func(engine uintptr, text string, labels uintptr, nLabels int32, threshold float32, maxWidth int32, out unsafe.Pointer) int32
+	vllmNerResultFree func(out unsafe.Pointer)
 )
 
+// cNerEntity mirrors vllm_ner_entity. Layout matches the C struct on LP64:
+// two pointer-width fields, four int32, one float, padded to 40 bytes.
+type cNerEntity struct {
+	label      uintptr // char*
+	text       uintptr // char*
+	charStart  int32
+	charEnd    int32
+	tokenStart int32
+	tokenEnd   int32
+	confidence float32
+}
+
+// cNerResult mirrors vllm_ner_result.
+type cNerResult struct {
+	entities  uintptr // vllm_ner_entity*
+	nEntities int32
+}
+
 type libFunc struct {
 	ptr  any
 	name string
@@ -285,6 +307,8 @@ func registerLib(libName string) error {
 		{&vllmVideoResultFree, "vllm_video_result_free"},
 		{&vllmVideoMuxArgv, "vllm_video_mux_argv"},
 		{&vllmVideoMuxArgvFre, "vllm_video_mux_argv_free"},
+		{&vllmGlinerNer, "vllm_gliner_ner"},
+		{&vllmNerResultFree, "vllm_ner_result_free"},
 	} {
 		purego.RegisterLibFunc(lf.ptr, lib, lf.name)
 	}

backend/go/vllm-cpp/options.go

@@ -65,6 +65,15 @@ type loadOptions struct {
 	// MiniMax-H3 video+audio generation (ABI v12). Present only when the config
 	// carries at least one of its keys; see videoOptions.engaged.
 	video videoOptions
+	// Zero-shot NER labels (ABI v27, GLiNER2.5). GLiNER2.5 is truly zero-shot:
+	// the model ships no default labels, so the entity types to extract are
+	// supplied here from engine_args.ner_labels. When empty, a general-purpose
+	// default set is used.
+	nerLabels []string
+	// nerThreshold is the default sigmoid floor (0 = model default 0.5).
+	nerThreshold float32
+	// nerMaxWidth is the maximum span width in tokens (0 = engine default 12).
+	nerMaxWidth int32
 }
 
 // videoOptions is the MiniMax-H3 checkpoint SET plus its generation defaults.
@@ -342,6 +351,22 @@ func applyEngineArgs(lo *loadOptions, engineArgs string) {
 			if b, ok := v.(bool); ok {
 				lo.enableJumpForward = boolTriState(b)
 			}
+		case "ner_labels":
+			if arr, ok := v.([]any); ok {
+				for _, e := range arr {
+					if s, ok := e.(string); ok && s != "" {
+						lo.nerLabels = append(lo.nerLabels, s)
+					}
+				}
+			}
+		case "ner_threshold":
+			if f, ok := v.(float64); ok {
+				lo.nerThreshold = float32(f)
+			}
+		case "ner_max_width":
+			if f, ok := v.(float64); ok {
+				lo.nerMaxWidth = int32(f)
+			}
 		default:
 			if s, ok := videoScalarString(v); ok && applyVideoOption(&lo.video, k, s) {
 				continue

backend/go/vllm-cpp/vllmcpp_test.go

@@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) {
 	RunSpecs(t, "vllm-cpp suite")
 }
 
-// The Go POD mirrors must match the C struct layout of vllm.h (ABI v26)
+// The Go POD mirrors must match the C struct layout of vllm.h (ABI v27)
 // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
 // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
 var _ = Describe("C ABI struct mirrors", func() {
@@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() {
 		// VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile).
 		// Moving the pin past this without growing the mirrors below ships a
 		// backend that refuses every load at startup (issue #11379).
-		Expect(abiVersion).To(Equal(26))
+		Expect(abiVersion).To(Equal(27))
 	})
 
 	It("cModelParams matches vllm_model_params", func() {
@@ -92,6 +92,25 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(c.CompletionTokens)).To(Equal(uintptr(20)))
 		Expect(unsafe.Sizeof(c)).To(Equal(uintptr(24)))
 	})
+
+	It("cNerEntity matches vllm_ner_entity (ABI v27)", func() {
+		var e cNerEntity
+		Expect(unsafe.Offsetof(e.label)).To(Equal(uintptr(0)))
+		Expect(unsafe.Offsetof(e.text)).To(Equal(uintptr(8)))
+		Expect(unsafe.Offsetof(e.charStart)).To(Equal(uintptr(16)))
+		Expect(unsafe.Offsetof(e.charEnd)).To(Equal(uintptr(20)))
+		Expect(unsafe.Offsetof(e.tokenStart)).To(Equal(uintptr(24)))
+		Expect(unsafe.Offsetof(e.tokenEnd)).To(Equal(uintptr(28)))
+		Expect(unsafe.Offsetof(e.confidence)).To(Equal(uintptr(32)))
+		Expect(unsafe.Sizeof(e)).To(Equal(uintptr(40)))
+	})
+
+	It("cNerResult matches vllm_ner_result (ABI v27)", func() {
+		var r cNerResult
+		Expect(unsafe.Offsetof(r.entities)).To(Equal(uintptr(0)))
+		Expect(unsafe.Offsetof(r.nEntities)).To(Equal(uintptr(8)))
+		Expect(unsafe.Sizeof(r)).To(Equal(uintptr(16)))
+	})
 })
 
 // Pin/mirror skew is the failure mode this backend is most exposed to: the Go

core/config/backend_capabilities.go

@@ -342,12 +342,17 @@ var BackendCapabilities = map[string]BackendCapability{
 	//
 	// AcceptsImages is the fl2va keyframe (start_image/end_image), the same
 	// reason longcat-video declares it; the text path takes no image input.
+	//
+	// TokenClassify is possible (GLiNER2.5 zero-shot NER via vllm_gliner_ner,
+	// ABI v27), declared explicitly via known_usecases: [token_classify]. The
+	// engine refuses non-BoundaryExtractor architectures, so a chat or embedding
+	// model returns an error rather than silent garbage.
 	"vllm-cpp": {
-		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo},
-		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo},
+		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateVideo, MethodTokenClassify},
+		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseVideo, UsecaseTokenClassify},
 		DefaultUsecases:  []string{UsecaseChat},
 		AcceptsImages:    true,
-		Description:      "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation plus MiniMax-H3 video+audio generation",
+		Description:      "vllm.cpp — the LocalAI team's C++20 port of vLLM; text generation, MiniMax-H3 video+audio generation, and GLiNER2.5 zero-shot NER",
 	},
 	"vllm-omni": {
 		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodGenerateImage, MethodGenerateVideo, MethodTTS},

docs/content/features/vllm-cpp.md

@@ -133,6 +133,33 @@ engine_args:
   tool_parser: qwen3_coder
 ```
 
+## Named entity recognition (GLiNER2.5)
+
+The `vllm-cpp` backend serves [GLiNER2.5](https://huggingface.co/fastino/gliner2.5-multi-v1),
+a zero-shot NER and structured-extraction model. Point the backend at the
+safetensors directory and the backend exposes the `TokenClassify` gRPC method,
+which LocalAI maps to its standard NER API surface.
+
+Labels are supplied at inference time, not baked into the model config. Set
+them in `engine_args`:
+
+```yaml
+engine_args:
+  ner_labels: "person,organization,location,date,time,money,quantity"
+  ner_threshold: 0.5
+  ner_max_width: 12
+```
+
+`ner_labels` is a comma-separated list. When omitted, the backend falls back to
+a built-in default set (`person`, `organization`, `location`, `date`, `time`,
+`money`, `quantity`). `ner_threshold` is the sigmoid cutoff (default 0.5);
+`ner_max_width` is the maximum span length in tokens (default 12).
+
+The model runs the DeBERTa v2 encoder with disentangled attention on the host
+forward, which is the required contract for pooling models in vllm.cpp. A
+device-resident forward is tracked as a performance optimization, not a
+correctness gap.
+
 ## Beyond text generation
 
 The `vllm-cpp` backend also serves MiniMax-H3, which generates video and audio

pkg/grpc/interface.go

@@ -97,3 +97,13 @@ type AIModelRich interface {
 	PredictRich(*pb.PredictOptions) (*pb.Reply, error)
 	PredictStreamRich(*pb.PredictOptions, chan<- *pb.Reply) error
 }
+
+// ClassifyModel is an optional extension to AIModel for backends that
+// implement the TokenClassify RPC (zero-shot NER). The gRPC server
+// type-asserts to this interface; backends that do not implement it
+// fall through to the UnimplementedBackendServer default. This mirrors
+// the AIModelRich pattern: adding a method to AIModel itself would
+// break every backend, so the capability is opt-in.
+type ClassifyModel interface {
+	TokenClassify(context.Context, *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error)
+}

pkg/grpc/server.go

@@ -102,6 +102,21 @@ func (s *server) Embedding(ctx context.Context, in *pb.PredictOptions) (*pb.Embe
 	}, nil
 }
 
+func (s *server) TokenClassify(ctx context.Context, in *pb.TokenClassifyRequest) (*pb.TokenClassifyResponse, error) {
+	if err := s.checkModelIdentity(in); err != nil {
+		return nil, err
+	}
+	cm, ok := s.llm.(ClassifyModel)
+	if !ok {
+		return nil, status.Errorf(codes.Unimplemented, "method TokenClassify not implemented")
+	}
+	if s.llm.Locking() {
+		s.llm.Lock()
+		defer s.llm.Unlock()
+	}
+	return cm.TokenClassify(ctx, in)
+}
+
 func (s *server) LoadModel(ctx context.Context, in *pb.ModelOptions) (*pb.Result, error) {
 	if s.llm.Locking() {
 		s.llm.Lock()