mirror of
https://github.com/mudler/LocalAI.git
synced 2026-11-06 12:33:16 -05:00
fix(chats): show the network switcher and say why execution status is unknown
The Network row of the Tools menu could never show the Off, Registries and All switcher: the host stopped setting `applies` because isolation alone does not prove network-policy enforcement, and nothing else set it. Let the provider report enforcement in its typed execution status (`networkPolicyEnforced`). The host sets `applies` only when that report is true and the status is available and isolated. The Sandbox reports it for the gVisor and KVM backends, whose guests reach the network only through the proxy that asks the host. Providers that do not report it keep the "not enforced" text. When the host cannot read the status, give the unknown state a reason (`unknown_reason`) in the API and the Tools menu: owner unreachable, not reported, stale, invalid, not selected, no answer or reported unknown. Log the class of each failure, rate limited. After a failure that a restart of the add-on owner can cause, let a long turn bind and read again, at most once every five seconds, instead of keeping the first failure for the whole turn. The Sandbox now reports "unavailable" with a reason when it started without settings, and explains a backend it does not know. When the Sandbox document skills are hidden because the Sandbox is not known to be isolated, say so in the skill list, in the turn note and in the `load_skill` error, instead of "This chat has no skills now". Assisted-by: Claude Code:claude-sonnet-5-5 Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
26 files changed
+833
-81
No files matched your search
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -331,11 +332,16 @@ var _ = Describe("execution binding owner resolver", func() {
|
||||
value := request.Observe(ctx)
|
||||
Expect(string(value.Readiness)).To(Equal("available"))
|
||||
Expect(string(value.Isolation)).To(Equal("isolated"))
|
||||
Expect(explanation()).To(BeEmpty())
|
||||
Expect(explanation().Explanation).To(BeEmpty())
|
||||
Expect(request.Observe(ctx)).To(Equal(value))
|
||||
Expect(a.studioStore.SetAddonsOff(ctx, studio.AnonymousPrincipal, claim.ConversationID, []string{"renamed-runner", "alternate-worker"})).To(Succeed())
|
||||
Expect(string(request.Observe(ctx).Isolation)).To(Equal("unknown"))
|
||||
Expect(explanation().Failure).To(Equal(studio.UnknownStale))
|
||||
Expect(broker.Isolation(ctx, studio.AnonymousPrincipal, "renamed-runner")).To(BeEmpty())
|
||||
none, selectedNone, noneDetail := broker.NewExecutionObservation(ctx, studio.AnonymousPrincipal, claim.ConversationID)
|
||||
Expect(none).To(BeNil())
|
||||
Expect(selectedNone).To(BeEmpty())
|
||||
Expect(noneDetail().Failure).To(Equal(studio.UnknownNotReported))
|
||||
Expect(a.studioStore.SetAddonsOff(ctx, studio.AnonymousPrincipal, claim.ConversationID, nil)).To(Succeed())
|
||||
fresh, selected, _ := broker.NewExecutionObservation(ctx, studio.AnonymousPrincipal, claim.ConversationID)
|
||||
Expect(fresh).NotTo(BeNil())
|
||||
@@ -439,3 +445,26 @@ var _ = Describe("workspace operation declarations", func() {
|
||||
Entry("similar prefix", []string{"workspaces.read", "workspace-other.read", "Workspace.read", "other.workspace.read"}, false),
|
||||
)
|
||||
})
|
||||
|
||||
var _ = Describe("why an execution status could not be read", func() {
|
||||
DescribeTable("names the reason", func(stage string, err error, want studio.UnknownReason) {
|
||||
Expect(executionFailureReason(stage, err)).To(Equal(want))
|
||||
},
|
||||
Entry("owner down", "read", gateway.ErrUnavailable, studio.UnknownOwnerUnreachable),
|
||||
Entry("owner not sent to", "read", gateway.ErrNotSent, studio.UnknownOwnerUnreachable),
|
||||
Entry("owner down while binding", "binding", gateway.ErrUnavailable, studio.UnknownOwnerUnreachable),
|
||||
Entry("carrier error", "read", errors.New("carrier"), studio.UnknownOwnerUnreachable),
|
||||
Entry("no status tool bound", "binding", gateway.ErrForbidden, studio.UnknownNotReported),
|
||||
Entry("binding mismatch on read", "read", gateway.ErrForbidden, studio.UnknownStale),
|
||||
Entry("binding changed on validate", "validate", gateway.ErrForbidden, studio.UnknownStale),
|
||||
Entry("payload refused", "decode", errors.New("invalid execution readiness"), studio.UnknownInvalid),
|
||||
Entry("no answer in time", "read", context.DeadlineExceeded, studio.UnknownNoAnswer),
|
||||
)
|
||||
It("logs each reason at most once a minute", func() {
|
||||
t := time.Now()
|
||||
Expect(failureLogDue("test-reason", t)).To(BeTrue())
|
||||
Expect(failureLogDue("test-reason", t.Add(failureLogEvery/2))).To(BeFalse())
|
||||
Expect(failureLogDue("other-reason", t.Add(failureLogEvery/2))).To(BeTrue())
|
||||
Expect(failureLogDue("test-reason", t.Add(failureLogEvery))).To(BeTrue())
|
||||
})
|
||||
})
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -15,67 +16,169 @@ import (
|
||||
"github.com/mudler/LocalAI/core/services/studio/executionrequest"
|
||||
"github.com/mudler/LocalAI/pkg/addons/contract"
|
||||
model "github.com/mudler/LocalAI/pkg/addons/model/executionstatus"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
|
||||
func (l lateStudioAddons) NewExecutionObservation(ctx context.Context, user, conversation string) (*executionrequest.Request, string, func() string) {
|
||||
func (l lateStudioAddons) NewExecutionObservation(ctx context.Context, user, conversation string) (*executionrequest.Request, string, func() studio.ExecutionDetail) {
|
||||
if p, ok := l.a.StudioAddonTools().(studio.ExecutionObserver); ok {
|
||||
return p.NewExecutionObservation(ctx, user, conversation)
|
||||
}
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
func (s *studioAddons) NewExecutionObservation(ctx context.Context, user, conversation string) (*executionrequest.Request, string, func() string) {
|
||||
func (s *studioAddons) NewExecutionObservation(ctx context.Context, user, conversation string) (*executionrequest.Request, string, func() studio.ExecutionDetail) {
|
||||
var mu sync.Mutex
|
||||
var detail studio.ExecutionDetail
|
||||
replied := false
|
||||
var addonID string
|
||||
// fail records why no status could be read. The first failure wins: later
|
||||
// ones are consequences of it.
|
||||
fail := func(stage string, err error) {
|
||||
reason := executionFailureReason(stage, err)
|
||||
mu.Lock()
|
||||
first := detail.Failure == ""
|
||||
if first {
|
||||
detail.Failure = reason
|
||||
}
|
||||
mu.Unlock()
|
||||
if first && !errors.Is(err, context.Canceled) {
|
||||
logExecutionFailure(stage, reason, err, addonID, conversation)
|
||||
}
|
||||
}
|
||||
current := func() studio.ExecutionDetail {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
d := detail
|
||||
if d.Failure == "" && !replied {
|
||||
d.Failure = studio.UnknownNoAnswer
|
||||
}
|
||||
return d
|
||||
}
|
||||
actor, err := s.actor(user)
|
||||
if err != nil {
|
||||
return nil, "", nil
|
||||
fail("binding", err)
|
||||
return nil, "", current
|
||||
}
|
||||
b, err := s.adapter.BindingForActor(ctx, actor, gateway.ExecutionBindingClaim{ConversationID: conversation})
|
||||
if err != nil {
|
||||
return nil, "", nil
|
||||
fail("binding", err)
|
||||
return nil, "", current
|
||||
}
|
||||
var id [24]byte
|
||||
if _, err = rand.Read(id[:]); err != nil {
|
||||
return nil, "", nil
|
||||
fail("binding", err)
|
||||
return nil, "", current
|
||||
}
|
||||
addonID = b.AddonID
|
||||
q := model.ObservationID(hex.EncodeToString(id[:]))
|
||||
// Keep actor and conversation identity explicit. IO retains the original DTO.
|
||||
mb := model.Binding{Actor: actor.UserID, Credential: actor.CredentialID, Conversation: b.ConversationID, Mode: actor.Mode, Addon: b.AddonID, Generation: strconv.FormatUint(b.Generation, 10), Incarnation: b.Incarnation, ExecutionTool: b.ExecutionTool, StatusTool: b.StatusTool}
|
||||
var mu sync.Mutex
|
||||
explanation := ""
|
||||
io := executionrequest.IO{
|
||||
Validate: func(ctx context.Context, current model.Binding) error {
|
||||
if current != mb {
|
||||
fail("validate", gateway.ErrForbidden)
|
||||
return gateway.ErrForbidden
|
||||
}
|
||||
return s.adapter.ValidateExecutionBinding(ctx, actor, b)
|
||||
if err := s.adapter.ValidateExecutionBinding(ctx, actor, b); err != nil {
|
||||
fail("validate", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Read: func(ctx context.Context, current model.Binding, observation model.ObservationID, deadline time.Time) (model.Payload, error) {
|
||||
if current != mb || observation != q {
|
||||
fail("read", gateway.ErrForbidden)
|
||||
return model.InvalidPayload(), gateway.ErrForbidden
|
||||
}
|
||||
ctx, cancel := context.WithDeadline(ctx, deadline)
|
||||
defer cancel()
|
||||
reply, err := s.adapter.ExecutionStatusRead(ctx, actor, gateway.ExecutionStatusRequest{Version: 1, Binding: b, ObservationID: string(q)})
|
||||
if err != nil {
|
||||
fail("read", err)
|
||||
return model.InvalidPayload(), err
|
||||
}
|
||||
if reply.Version != 1 || reply.Binding != b || reply.ObservationID != string(q) {
|
||||
fail("read", gateway.ErrForbidden)
|
||||
return model.InvalidPayload(), gateway.ErrForbidden
|
||||
}
|
||||
raw, err := json.Marshal(reply.Status)
|
||||
if err != nil {
|
||||
fail("decode", err)
|
||||
return model.InvalidPayload(), err
|
||||
}
|
||||
status, err := contract.DecodeExecutionStatus(raw)
|
||||
if err != nil {
|
||||
fail("decode", err)
|
||||
return model.InvalidPayload(), err
|
||||
}
|
||||
mu.Lock()
|
||||
explanation = status.Explanation
|
||||
detail.Explanation, detail.NetworkPolicyEnforced = status.Explanation, status.NetworkPolicyEnforced
|
||||
replied = true
|
||||
mu.Unlock()
|
||||
return model.ValidPayload(model.Value{Readiness: model.Readiness(status.Readiness), Isolation: model.Isolation(status.Isolation)}), nil
|
||||
},
|
||||
}
|
||||
request := executionrequest.New(ctx, mb, q, model.Eligible, io, nil)
|
||||
return request, b.AddonID, func() string { mu.Lock(); defer mu.Unlock(); return explanation }
|
||||
return request, b.AddonID, current
|
||||
}
|
||||
|
||||
// executionFailureReason maps a failure to the reason the person is told. The
|
||||
// gateway answers forbidden for every refusal, so the stage tells them apart:
|
||||
// while binding, the chat has no add-on that reports a status; later, what was
|
||||
// bound no longer matches.
|
||||
func executionFailureReason(stage string, err error) studio.UnknownReason {
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
return studio.UnknownNoAnswer
|
||||
case errors.Is(err, gateway.ErrUnavailable):
|
||||
return studio.UnknownOwnerUnreachable
|
||||
case errors.Is(err, gateway.ErrForbidden):
|
||||
if stage == "binding" {
|
||||
return studio.UnknownNotReported
|
||||
}
|
||||
return studio.UnknownStale
|
||||
case stage == "decode":
|
||||
return studio.UnknownInvalid
|
||||
}
|
||||
return studio.UnknownOwnerUnreachable
|
||||
}
|
||||
|
||||
// failureLogEvery bounds the log line per reason: a chat page asks for the
|
||||
// status often and a broken add-on would fill the log.
|
||||
const failureLogEvery = time.Minute
|
||||
|
||||
var failureLog = struct {
|
||||
sync.Mutex
|
||||
last map[studio.UnknownReason]time.Time
|
||||
}{last: map[studio.UnknownReason]time.Time{}}
|
||||
|
||||
// failureLogDue says whether a failure of this reason may be logged at now,
|
||||
// and counts it if so.
|
||||
func failureLogDue(reason studio.UnknownReason, now time.Time) bool {
|
||||
failureLog.Lock()
|
||||
defer failureLog.Unlock()
|
||||
if last, ok := failureLog.last[reason]; ok && now.Sub(last) < failureLogEvery {
|
||||
return false
|
||||
}
|
||||
failureLog.last[reason] = now
|
||||
return true
|
||||
}
|
||||
|
||||
// logExecutionFailure logs the class of a failure, never its text, which may
|
||||
// carry add-on or user data.
|
||||
func logExecutionFailure(stage string, reason studio.UnknownReason, err error, addon, conversation string) {
|
||||
if !failureLogDue(reason, time.Now()) {
|
||||
return
|
||||
}
|
||||
class := "other"
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
class = "deadline"
|
||||
case errors.Is(err, gateway.ErrUnavailable):
|
||||
class = "unavailable"
|
||||
case errors.Is(err, gateway.ErrForbidden):
|
||||
class = "forbidden"
|
||||
}
|
||||
xlog.Warn("Chats: the execution status could not be read", "stage", stage, "reason", string(reason), "class", class,
|
||||
"addon", addon, "chat", conversation[:min(len(conversation), 8)])
|
||||
}
|
||||
Reference in new issue
Block a user