test(studio): remove two timing races from the distributed e2e run

The circuit breaker spec used a real 150 ms cool-down. Under the race
detector on a loaded runner, the turns between opening the breaker and
the "no call goes out" check took longer than that, so the rest was
already over and a call went out. The spec now runs on a service clock
that only moves when the spec moves it.

The reply-language spec sent "/new" as soon as the previous answer
reached the chat. The answer is delivered before the turn is marked
finished, so on a slow run the command was refused as busy and the
refusal text (in the fixed language) came back instead of the new-chat
text. The spec helper now waits for the turn to end first, as a person
does after reading the answer.

Assisted-by: Claude:claude-sonnet-5-5
This commit is contained in:
Ettore Di Giacinto committed 2026-11-05 02:22:39 +00:00
1 parent 3a8414efc2
commit 63777004a5
2 files changed
+23 -5

No files matched your search

@@ -77,6 +77,19 @@ var _ = Describe("the language of the host sentences in a chat", func() {
ask := func(text string, n int) string {
ingest(dm(text))
waitFor("the answer", func() bool { return len(fake.texts("11", "")) >= n })
// The answer reaches the chat before the turn is marked finished, and a
// command that comes in that gap is refused as busy. Wait for the end of
// the turn, as a person does when they read the answer.
waitFor("the turn to end", func() bool {
chats, err := r.svc.List(ctx, alice, 50)
Expect(err).NotTo(HaveOccurred())
for _, c := range chats {
if turn(r.svc, alice, c.ID).Running {
return false
}
}
return true
})
t := fake.texts("11", "")
return t[len(t)-1]
}
@@ -9,6 +9,7 @@ import (
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
addontools "github.com/mudler/LocalAI/core/services/addons/tools"
@@ -126,6 +127,7 @@ var _ = Describe("the host's own calls to a provider", func() {
host *callHost
stored map[string]string
smu sync.Mutex
skew atomic.Int64
logs *bytes.Buffer
lmu sync.Mutex
)
@@ -174,8 +176,12 @@ var _ = Describe("the host's own calls to a provider", func() {
st, _ := store.NewJSON("")
ag, _ := agents.NewJSON("")
host = newCallHost()
skew.Store(0)
host.on("context", func(map[string]any) (string, error) { return result(map[string]any{"text": "- likes tea"}), nil })
svc, err = New(Deps{Store: st, Provider: provider.New(), StateDir: GinkgoT().TempDir(), Files: f, Agents: ag, AddonTools: host,
// Time only moves when a spec moves it: the cool-down must not end
// because a slow machine took long between two calls.
Clock: func() time.Time { return time.Unix(1_800_000_000, 0).Add(time.Duration(skew.Load())) },
Logger: slog.New(slog.NewTextHandler(writerFunc(func(p []byte) (int, error) { lmu.Lock(); defer lmu.Unlock(); return logs.Write(p) }), nil)),
SetUserSetting: func(_ context.Context, user, addon, key, value string) error {
smu.Lock()
@@ -258,7 +264,7 @@ var _ = Describe("the host's own calls to a provider", func() {
Describe("the circuit breaker", func() {
BeforeEach(func() {
svc.providerTimings.coolDown = 150 * time.Millisecond
svc.providerTimings.coolDown = time.Minute
host.on("context", func(map[string]any) (string, error) { return "", context.DeadlineExceeded })
})
@@ -277,10 +283,9 @@ var _ = Describe("the host's own calls to a provider", func() {
Expect(host.count("context")).To(Equal(ProviderFailuresToOpen), "no call goes out while it rests")
Expect(logText()).To(ContainSubstring("outcome=cool_down"))
Eventually(func() bool {
v, _ := svc.ProviderSlot(ctx, "alice", MemorySlot)
return v.Health.CoolingDown
}, "2s", "20ms").Should(BeFalse())
skew.Add(int64(time.Minute + time.Second))
v, _ = svc.ProviderSlot(ctx, "alice", MemorySlot)
Expect(v.Health.CoolingDown).To(BeFalse(), "the rest is over")
host.on("context", func(map[string]any) (string, error) { return result(map[string]any{"text": "- back"}), nil })
Expect(turn("hi").Context).To(ContainSubstring("- back"))
Expect(host.count("context")).To(Equal(ProviderFailuresToOpen + 1))