From 2ba66004409caafb666172445518c014c26bac93 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 8 Sep 2026 22:59:57 +0200 Subject: [PATCH] fix(detection): avoid temporary image files (#11938) RF-DETR and Locate Anything wrote each decoded request image to the OS temporary directory. A full temporary filesystem then disabled detection, even though both native libraries already accept encoded image buffers. Pass decoded images directly to the native buffer APIs. This removes the request-time disk dependency and prevents crash-orphaned image files. Assisted-by: Codex:gpt-5 Co-authored-by: Ettore Di Giacinto --- .../golocateanythingcpp.go | 26 ++++----- .../golocateanythingcpp_unit_test.go | 54 ++++++++++++++++++ backend/go/rfdetr-cpp/gorfdetrcpp.go | 29 +++++----- .../go/rfdetr-cpp/gorfdetrcpp_unit_test.go | 56 +++++++++++++++++++ 4 files changed, 134 insertions(+), 31 deletions(-) create mode 100644 backend/go/locate-anything-cpp/golocateanythingcpp_unit_test.go create mode 100644 backend/go/rfdetr-cpp/gorfdetrcpp_unit_test.go diff --git a/backend/go/locate-anything-cpp/golocateanythingcpp.go b/backend/go/locate-anything-cpp/golocateanythingcpp.go index 25c7b80c5..678ed30aa 100644 --- a/backend/go/locate-anything-cpp/golocateanythingcpp.go +++ b/backend/go/locate-anything-cpp/golocateanythingcpp.go @@ -11,6 +11,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "unsafe" "github.com/mudler/LocalAI/pkg/grpc/base" @@ -109,30 +110,25 @@ func (r *LocateAnythingCpp) Detect(opts *pb.DetectOptions) (pb.DetectResponse, e return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: a text prompt is required (open-vocabulary detection)") } - // Decode base64 image and write to temp file. imgData, err := base64.StdEncoding.DecodeString(opts.Src) if err != nil { return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to decode base64 image: %w", err) } - - tmpFile, err := os.CreateTemp("", "locate-anything-*.img") - if err != nil { - return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to create temp file: %w", err) - } - defer func() { _ = os.Remove(tmpFile.Name()) }() - - if _, err := tmpFile.Write(imgData); err != nil { - _ = tmpFile.Close() - return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to write temp file: %w", err) - } - if err := tmpFile.Close(); err != nil { - return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: failed to close temp file: %w", err) + if len(imgData) == 0 { + return pb.DetectResponse{}, fmt.Errorf("locate-anything-cpp: decoded image is empty") } // mode 0 = hybrid (Parallel Box Decoding). The JSON return value is unused: // structured detections are read via the accessor functions. Still must // free the returned string. - jsonPtr := CapiLocatePath(r.handle, tmpFile.Name(), prompt, 0) + jsonPtr := CapiLocateBuffer( + r.handle, + uintptr(unsafe.Pointer(unsafe.SliceData(imgData))), + uintptr(len(imgData)), + prompt, + 0, + ) + runtime.KeepAlive(imgData) if jsonPtr != 0 { CapiFreeString(jsonPtr) } diff --git a/backend/go/locate-anything-cpp/golocateanythingcpp_unit_test.go b/backend/go/locate-anything-cpp/golocateanythingcpp_unit_test.go new file mode 100644 index 000000000..3fa7d09df --- /dev/null +++ b/backend/go/locate-anything-cpp/golocateanythingcpp_unit_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "encoding/base64" + "path/filepath" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("LocateAnythingCpp detection input", func() { + It("detects from memory when the temporary directory is unavailable", func() { + originalLocateBuffer := CapiLocateBuffer + originalLocatePath := CapiLocatePath + originalGetNDetections := CapiGetNDetections + defer func() { + CapiLocateBuffer = originalLocateBuffer + CapiLocatePath = originalLocatePath + CapiGetNDetections = originalGetNDetections + }() + + image := []byte("encoded-image") + var receivedData uintptr + var receivedLength uintptr + CapiLocateBuffer = func(_ uintptr, data uintptr, length uintptr, _ string, _ int32) uintptr { + receivedData = data + receivedLength = length + return 0 + } + CapiLocatePath = func(_ uintptr, _ string, _ string, _ int32) uintptr { + Fail("path-based detection must not be called") + return 0 + } + CapiGetNDetections = func(uintptr) int32 { return 0 } + GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "missing")) + + result, err := (&LocateAnythingCpp{handle: 1}).Detect(&pb.DetectOptions{ + Src: base64.StdEncoding.EncodeToString(image), + Prompt: "the object", + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(result.Detections).To(BeEmpty()) + Expect(receivedData).NotTo(BeZero()) + Expect(receivedLength).To(Equal(uintptr(len(image)))) + }) + + It("rejects an empty decoded image", func() { + _, err := (&LocateAnythingCpp{handle: 1}).Detect(&pb.DetectOptions{Prompt: "the object"}) + + Expect(err).To(MatchError("locate-anything-cpp: decoded image is empty")) + }) +}) diff --git a/backend/go/rfdetr-cpp/gorfdetrcpp.go b/backend/go/rfdetr-cpp/gorfdetrcpp.go index ae45319e1..7e507780b 100644 --- a/backend/go/rfdetr-cpp/gorfdetrcpp.go +++ b/backend/go/rfdetr-cpp/gorfdetrcpp.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strconv" "unsafe" @@ -102,24 +103,12 @@ func (r *RFDetrCpp) Detect(opts *pb.DetectOptions) (pb.DetectResponse, error) { return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: model not loaded") } - // Decode base64 image and write to temp file. imgData, err := base64.StdEncoding.DecodeString(opts.Src) if err != nil { return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to decode base64 image: %w", err) } - - tmpFile, err := os.CreateTemp("", "rfdetr-*.img") - if err != nil { - return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to create temp file: %w", err) - } - defer func() { _ = os.Remove(tmpFile.Name()) }() - - if _, err := tmpFile.Write(imgData); err != nil { - _ = tmpFile.Close() - return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to write temp file: %w", err) - } - if err := tmpFile.Close(); err != nil { - return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: failed to close temp file: %w", err) + if len(imgData) == 0 { + return pb.DetectResponse{}, fmt.Errorf("rfdetr-cpp: decoded image is empty") } threshold := opts.Threshold @@ -127,10 +116,18 @@ func (r *RFDetrCpp) Detect(opts *pb.DetectOptions) (pb.DetectResponse, error) { threshold = 0.5 } - // JSON output from detect_path is unused: we read structured detections via + // JSON output from the detection ABI is unused: we read structured detections via // the accessor functions. Still must free the returned string. var jsonPtr uintptr - rc := CapiDetectPath(r.handle, tmpFile.Name(), threshold, uint32(defaultTopK), &jsonPtr) + rc := CapiDetectBuffer( + r.handle, + uintptr(unsafe.Pointer(unsafe.SliceData(imgData))), + uintptr(len(imgData)), + threshold, + uint32(defaultTopK), + &jsonPtr, + ) + runtime.KeepAlive(imgData) if jsonPtr != 0 { CapiFreeString(jsonPtr) } diff --git a/backend/go/rfdetr-cpp/gorfdetrcpp_unit_test.go b/backend/go/rfdetr-cpp/gorfdetrcpp_unit_test.go new file mode 100644 index 000000000..266fa44b9 --- /dev/null +++ b/backend/go/rfdetr-cpp/gorfdetrcpp_unit_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "encoding/base64" + "path/filepath" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("RFDetrCpp detection input", func() { + It("detects from memory when the temporary directory is unavailable", func() { + originalDetectBuffer := CapiDetectBuffer + originalDetectPath := CapiDetectPath + originalFreeString := CapiFreeString + originalGetNDetections := CapiGetNDetections + defer func() { + CapiDetectBuffer = originalDetectBuffer + CapiDetectPath = originalDetectPath + CapiFreeString = originalFreeString + CapiGetNDetections = originalGetNDetections + }() + + image := []byte("encoded-image") + var receivedData uintptr + var receivedLength uintptr + CapiDetectBuffer = func(_ uintptr, data uintptr, length uintptr, _ float32, _ uint32, _ *uintptr) int32 { + receivedData = data + receivedLength = length + return 0 + } + CapiDetectPath = func(_ uintptr, _ string, _ float32, _ uint32, _ *uintptr) int32 { + Fail("path-based detection must not be called") + return -1 + } + CapiFreeString = func(uintptr) {} + CapiGetNDetections = func(uintptr) int32 { return 0 } + GinkgoT().Setenv("TMPDIR", filepath.Join(GinkgoT().TempDir(), "missing")) + + result, err := (&RFDetrCpp{handle: 1}).Detect(&pb.DetectOptions{ + Src: base64.StdEncoding.EncodeToString(image), + }) + + Expect(err).NotTo(HaveOccurred()) + Expect(result.Detections).To(BeEmpty()) + Expect(receivedData).NotTo(BeZero()) + Expect(receivedLength).To(Equal(uintptr(len(image)))) + }) + + It("rejects an empty decoded image", func() { + _, err := (&RFDetrCpp{handle: 1}).Detect(&pb.DetectOptions{}) + + Expect(err).To(MatchError("rfdetr-cpp: decoded image is empty")) + }) +})