mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-30 09:57:57 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): stop the backend-trace specs racing the lossy trace channel (#11146)

Browse Source 
RecordBackendTrace does a non-blocking send onto a 100-slot channel and
drops when it is full, so tracing never stalls inference. The payload
bounding specs pushed all 200 traces in one tight loop, which overruns
that channel on a loaded machine: entries are dropped for good and the
Eventually waiting for 200 can never be satisfied, no matter the timeout.
CI hit this on master at 0a8a7fbb, settling at 158/200.

Feed the traces in chunks of 50, draining after each, so the channel is
never overrun and the count stays exact. Reproduced with 60 busy loops on
a 20-core box at GOMAXPROCS=2: 0/12 runs passed before, 12/12 after.


Assisted-by: Claude Code:claude-opus-5 [Read] [Edit] [Bash]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
mudler's LocalAI [bot]
2026-07-27 22:38:23 +02:00
committed by GitHub
parent a7fa678d83
commit 05ff401de8
1 changed files with 22 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
core/http/endpoints/localai/traces_test.go
View File

@@ -96,6 +96,12 @@ var _ = Describe("Backend traces payload bounding", func() {
// 3.4 MB: every entry carries the full input text.
const heavyText = 8192
// totalTraces overfills the 100-slot trace channel on purpose (the specs
// assert on paging and on X-Total-Count), so it has to be fed in chunks -
// see the BeforeEach. traceChunk stays below that channel capacity.
const totalTraces = 200
const traceChunk = 50
BeforeEach(func() {
app = echo.New()
app.GET("/api/backend-traces", GetBackendTracesEndpoint())
@@ -104,7 +110,7 @@ var _ = Describe("Backend traces payload bounding", func() {
trace.ClearBackendTraces()
trace.InitBackendTracingIfEnabled(500, 0)
base := time.Now()
for i := range 200 {
for i := range totalTraces {
trace.RecordBackendTrace(trace.BackendTrace{
// Distinct timestamps keep the newest-first ordering (and
// therefore the offset paging) deterministic.
@@ -115,9 +121,20 @@ var _ = Describe("Backend traces payload bounding", func() {
Body: strings.Repeat("b", heavyText),
Data: map[string]any{"input_text": strings.Repeat("x", heavyText)},
})
// RecordBackendTrace is deliberately lossy: it does a non-blocking
// send and drops when the channel is full, so tracing never stalls
// inference. Pushing all 200 in one tight loop therefore overruns
// the 100-slot channel on a loaded machine and the buffer settles
// below 200 permanently - no amount of waiting recovers a dropped
// trace. Drain in chunks well under the channel capacity instead,
// which keeps the count exact regardless of scheduling.
if (i+1)%traceChunk == 0 {
Eventually(func() int { return len(trace.GetBackendTraces()) }, 30*time.Second, 10*time.Millisecond).
Should(Equal(i + 1))
}
}
// Recording is asynchronous through a channel; wait for the buffer.
Eventually(func() int { return len(trace.GetBackendTraces()) }).Should(Equal(200))
Eventually(func() int { return len(trace.GetBackendTraces()) }, 30*time.Second, 10*time.Millisecond).
Should(Equal(totalTraces))
})
AfterEach(func() {
@@ -138,7 +155,7 @@ var _ = Describe("Backend traces payload bounding", func() {
var out []map[string]any
Expect(json.Unmarshal(rec.Body.Bytes(), &out)).To(Succeed())
Expect(out).To(HaveLen(DefaultTraceListLimit))
Expect(rec.Header().Get("X-Total-Count")).To(Equal("200"))
Expect(rec.Header().Get("X-Total-Count")).To(Equal(strconv.Itoa(totalTraces)))
})
It("omits the heavy body and data fields from list entries", func() {
@@ -156,7 +173,7 @@ var _ = Describe("Backend traces payload bounding", func() {
bounded := get("/api/backend-traces").Body.Len()
unbounded := get("/api/backend-traces?limit=0&full=true").Body.Len()
Expect(unbounded).To(BeNumerically(">", 200*heavyText))
Expect(unbounded).To(BeNumerically(">", totalTraces*heavyText))
Expect(bounded).To(BeNumerically("<", unbounded/100))
})