mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-05-30 11:36:31 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(backend): rfdetr-cpp native object detection + segmentation backend (#10028)

Browse Source 
Adds a Go native gRPC backend that dlopens librfdetrcpp.so (built from
mudler/rf-detr.cpp at the pinned RFDETR_VERSION) via purego and exposes
the rfdetr.cpp inference pipeline through LocalAI's existing Detect RPC.

Supports all 5 RF-DETR detection variants (Nano/Small/Base/Medium/Large)
and 6 segmentation variants (SegNano/SegSmall/SegMedium/SegLarge/
SegXLarge/Seg2XLarge) with F32/F16/Q8_0/Q4_K quantizations. Pre-built
GGUFs ship at mudler/rfdetr-cpp-* on HuggingFace.

Detection returns Bbox + class_name + confidence; segmentation also
returns PNG-encoded per-detection masks via the rfdetr_capi accessor
functions (rfdetr_capi_get_detection_{class_id,box,score,class_name,
mask_png}).

End-to-end verified through POST /v1/detection: HTTP -> gRPC -> purego
dlopen -> rfdetr.cpp -> ggml -> response (9 detections on the detection
model, 21 detections + valid PNG masks on the seg-nano model against
the kitchen fixture).

Wiring:
  - backend/go/rfdetr-cpp/{main.go,gorfdetrcpp.go,CMakeLists.txt,
    Makefile,run.sh,package.sh,test.sh,.gitignore}
  - Top-level Makefile: BACKEND_RFDETR_CPP, docker-build target,
    .NOTPARALLEL, prepare-test-extra, test-extra
  - backend/go/rfdetr-cpp/Makefile: `test` target invoked by test-extra
  - .github/backend-matrix.yml: CPU + CUDA-12/13 + L4T CUDA-12/13
    (arm64) + HIP + Vulkan (amd64 + arm64) + SYCL f32/f16
  - backend/index.yaml: rfdetr-cpp meta anchor + latest/development
    image entries for every matrix tag-suffix
  - .github/workflows/bump_deps.yaml: RFDETR_VERSION pin tracking
    (mudler/rf-detr.cpp branch main)
  - gallery/index.yaml: 11 rfdetr-cpp-* entries (nano + 4 detection
    variants + 6 seg variants), all backed by mudler/rfdetr-cpp-*
    on HuggingFace with sha256 pinning on the F16 default
  - core/gallery/importers/rfdetr.go: GGUF auto-routing for HF imports
    (mudler/rfdetr-cpp-* repos route to rfdetr-cpp, Transformer-format
    repos stay on the Python rfdetr backend; explicit preferences.backend
    overrides both heuristics)
  - core/gallery/importers/rfdetr_test.go: table-driven coverage of the
    auto-routing + a live mudler/rfdetr-cpp-nano cross-check

scripts/changed-backends.js needs no change: the existing
Dockerfile.golang -> backend/go/${item.backend}/ branch already routes
the 9 rfdetr-cpp matrix entries to the correct backend path.

Assisted-by: Claude:claude-opus-4-7 [Claude Code]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
LocalAI [bot]
2026-05-27 18:43:57 +02:00
committed by GitHub
parent 893e69cbf8
commit 7a4ca8f60d
18 changed files with 1697 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
core/gallery/importers/rfdetr.go
View File

@@ -31,6 +31,29 @@ func repoLooksLikeRFDetr(repo string) bool {
return strings.Contains(lower, "rf-detr") || strings.Contains(lower, "rfdetr")
}
// repoHasGGUF inspects the HuggingFace file list (when available) to decide
// whether the repo ships RF-DETR weights in ggml/GGUF form — the native
// rfdetr-cpp backend's input format. Mudler's rfdetr-cpp-* repos
// (mudler/rfdetr-cpp-nano, mudler/rfdetr-cpp-base, ...) match.
func repoHasGGUF(details Details) bool {
if details.HuggingFace == nil {
return false
}
for _, f := range details.HuggingFace.Files {
if strings.HasSuffix(strings.ToLower(f.Path), ".gguf") {
return true
}
}
return false
}
func repoLooksLikeRFDetrCpp(repo string) bool {
lower := strings.ToLower(repo)
return strings.Contains(lower, "rfdetr-cpp") || strings.Contains(lower, "rf-detr-cpp") ||
strings.Contains(lower, "rfdetr.cpp") || strings.Contains(lower, "rt-detr.cpp") ||
strings.Contains(lower, "rf-detr.cpp")
}
func (i *RFDetrImporter) Match(details Details) bool {
preferences, err := details.Preferences.MarshalJSON()
if err != nil {
@@ -43,7 +66,7 @@ func (i *RFDetrImporter) Match(details Details) bool {
}
}
if b, ok := preferencesMap["backend"].(string); ok && b == "rfdetr" {
if b, ok := preferencesMap["backend"].(string); ok && (b == "rfdetr" || b == "rfdetr-cpp") {
return true
}
@@ -99,10 +122,28 @@ func (i *RFDetrImporter) Import(details Details) (gallery.ModelConfig, error) {
model = owner + "/" + repo
}
// Route GGUF-bearing repos (mudler/rfdetr-cpp-*) to the native
// rfdetr-cpp backend; HF transformer repos keep the Python rfdetr
// backend. Explicit preferences.backend overrides the heuristic.
backend := "rfdetr"
if b, ok := preferencesMap["backend"].(string); ok && b != "" {
backend = b
} else if repoHasGGUF(details) {
backend = "rfdetr-cpp"
} else if details.HuggingFace != nil {
repoName := details.HuggingFace.ModelID
if idx := strings.Index(repoName, "/"); idx >= 0 {
repoName = repoName[idx+1:]
}
if repoLooksLikeRFDetrCpp(repoName) {
backend = "rfdetr-cpp"
}
}
modelConfig := config.ModelConfig{
Name: name,
Description: description,
Backend: "rfdetr",
Backend: backend,
KnownUsecaseStrings: []string{"detection"},
PredictionOptions: schema.PredictionOptions{
BasicModelRequest: schema.BasicModelRequest{Model: model},

121
core/gallery/importers/rfdetr_test.go
View File

@@ -129,4 +129,125 @@ var _ = Describe("RFDetrImporter", func() {
Expect(modelConfig.Description).To(Equal("Custom"))
})
})
// Table-driven coverage of the GGUF auto-routing path between the
// Python rfdetr backend (HF transformer repos) and the native
// rfdetr-cpp backend (GGUF repos like mudler/rfdetr-cpp-*).
//
// Cases are kept offline-deterministic by injecting Details directly
// rather than going through DiscoverModelConfig (which would hit live HF).
// The live HF cross-check lives in its own Context below.
Context("GGUF auto-routing (offline)", func() {
hfFile := func(path string) hfapi.ModelFile {
return hfapi.ModelFile{Path: path}
}
type tc struct {
name string
uri string
modelID string
files []hfapi.ModelFile
prefs string
expectBackend string // expected `backend:` line content
rejectBackends []string
}
entries := []tc{
{
name: "GGUF repo with rfdetr-cpp prefix routes to rfdetr-cpp",
uri: "https://huggingface.co/mudler/rfdetr-cpp-nano",
modelID: "mudler/rfdetr-cpp-nano",
files: []hfapi.ModelFile{hfFile("rfdetr-nano-q8_0.gguf"), hfFile("README.md")},
prefs: "",
expectBackend: "backend: rfdetr-cpp",
},
{
name: "GGUF presence alone routes to rfdetr-cpp even when repo name lacks -cpp",
uri: "https://huggingface.co/some/rf-detr-ggml",
modelID: "some/rf-detr-ggml",
files: []hfapi.ModelFile{hfFile("rfdetr-base-f16.gguf")},
prefs: "",
expectBackend: "backend: rfdetr-cpp",
},
{
name: "transformer repo without GGUF stays on the Python rfdetr backend",
uri: "https://huggingface.co/roboflow/rf-detr-base",
modelID: "roboflow/rf-detr-base",
files: []hfapi.ModelFile{hfFile("config.json"), hfFile("pytorch_model.bin")},
prefs: "",
expectBackend: "backend: rfdetr\n",
rejectBackends: []string{"backend: rfdetr-cpp"},
},
{
name: "explicit preferences.backend=rfdetr overrides GGUF auto-detect",
uri: "https://huggingface.co/mudler/rfdetr-cpp-nano",
modelID: "mudler/rfdetr-cpp-nano",
files: []hfapi.ModelFile{hfFile("rfdetr-nano-q8_0.gguf")},
prefs: `{"backend": "rfdetr"}`,
expectBackend: "backend: rfdetr\n",
rejectBackends: []string{"backend: rfdetr-cpp"},
},
{
name: "explicit preferences.backend=rfdetr-cpp wins on non-GGUF transformer repo",
uri: "https://huggingface.co/roboflow/rf-detr-base",
modelID: "roboflow/rf-detr-base",
files: []hfapi.ModelFile{hfFile("config.json")},
prefs: `{"backend": "rfdetr-cpp"}`,
expectBackend: "backend: rfdetr-cpp",
},
{
name: "repo name with rfdetr.cpp pattern routes to rfdetr-cpp even without HF file list",
uri: "https://huggingface.co/some/rfdetr.cpp-bundle",
modelID: "some/rfdetr.cpp-bundle",
files: nil,
prefs: "",
expectBackend: "backend: rfdetr-cpp",
},
}
for _, e := range entries {
e := e // capture for closure
It(e.name, func() {
imp := &importers.RFDetrImporter{}
details := importers.Details{
URI: e.uri,
HuggingFace: &hfapi.ModelDetails{
ModelID: e.modelID,
Files: e.files,
},
}
if e.prefs != "" {
details.Preferences = json.RawMessage(e.prefs)
}
// Match must always be true for these fixtures — they're
// either preference-driven or have an rfdetr/rf-detr token.
Expect(imp.Match(details)).To(BeTrue(), fmt.Sprintf("Match should fire for %+v", details))
modelConfig, err := imp.Import(details)
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Import error: %v", err))
Expect(modelConfig.ConfigFile).To(ContainSubstring(e.expectBackend),
fmt.Sprintf("Model config: %+v", modelConfig))
for _, rej := range e.rejectBackends {
Expect(modelConfig.ConfigFile).ToNot(ContainSubstring(rej),
fmt.Sprintf("did not expect %q in: %+v", rej, modelConfig))
}
})
}
})
// Live HF cross-check: the canonical native GGUF repo for the
// rfdetr-cpp backend. Marked broad — we only assert the routing
// decision, not file lists (upstream may add quants over time).
Context("detection from HuggingFace: mudler/rfdetr-cpp-nano", func() {
It("auto-routes to the native rfdetr-cpp backend without preferences", func() {
uri := "https://huggingface.co/mudler/rfdetr-cpp-nano"
modelConfig, err := importers.DiscoverModelConfig(uri, json.RawMessage(`{}`))
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err))
Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: rfdetr-cpp"),
fmt.Sprintf("Model config: %+v", modelConfig))
Expect(modelConfig.ConfigFile).To(ContainSubstring("mudler/rfdetr-cpp-nano"))
})
})
})