LocalAI/core/services/studio/changes_test.go

package studio_test

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"

	"github.com/mudler/LocalAI/addons/slack/slacktest"
	slack "github.com/mudler/LocalAI/addons/slack/transport"
	"github.com/mudler/LocalAI/core/services/studio"
	"github.com/mudler/LocalAI/core/services/studio/agents"
	"github.com/mudler/LocalAI/core/services/studio/proposals"
	"github.com/mudler/LocalAI/core/services/studio/session"
	"github.com/mudler/LocalAI/core/services/studio/studiotest"
	"github.com/mudler/LocalAI/core/services/studio/workspace"
	"github.com/mudler/LocalAI/pkg/addons/contract"
	"github.com/mudler/LocalAI/tests/studiobot"
)

// proposeArgs is a propose_change call as a model writes it.
func proposeArgs(kind, scope, target, content, summary string) string {
	raw, _ := json.Marshal(map[string]string{"kind": kind, "scope": scope, "target": target, "content": content, "summary": summary})
	return string(raw)
}

// changeScript proposes a memory note on "remember" and reverts on "forget".
func changeScript(scope, content string) func(workspace.Actor, string) []studiotest.ToolStep {
	return func(_ workspace.Actor, text string) []studiotest.ToolStep {
		switch text {
		case "remember":
			return []studiotest.ToolStep{{Tool: "propose_change", Args: proposeArgs("memory", scope, "index.md", content, "Where the user lives")}, {Say: "ok"}}
		case "forget":
			return []studiotest.ToolStep{{Tool: "revert_change", Args: `{}`}, {Say: "ok"}}
		}
		return []studiotest.ToolStep{{Say: "ok"}}
	}
}

// grantAll is the gate a grants feature would give: every chat may.
var grantAll = proposals.GateFunc(func(context.Context, proposals.Request) error { return nil })

// changeShared is a shared rig whose service has a clock and a gate.
func changeShared(gate proposals.Gate) *shared {
	sh := newShared()
	sh.deps = func(d *studio.Deps) {
		d.Clock = sh.clock.Now
		d.ProposalGate = gate
	}
	return sh
}

// grantMemory makes the saved agent of the rig one that may propose memory.
func (c *connRig) grantMemory() { c.grantBuiltins("memory.write") }

// grantBuiltins gives the saved agent of the rig the builtin grants.
func (c *connRig) grantBuiltins(grants ...string) {
	ag, err := c.svc.SaveAgent(context.Background(), alice, agents.Agent{Name: "Support", Model: "m", Addons: []string{"office"}, Builtins: grants})
	Expect(err).NotTo(HaveOccurred())
	c.agent = ag
}

func channelFile(src contract.ChangeSource) string {
	return "Memory/channels/" + proposals.ChannelDir(src) + "/index.md"
}

func readHome(r *trigRig, rel string) (string, bool) {
	text, ok, err := r.sh.files.ReadHome(context.Background(), alice, rel, 1<<20)
	Expect(err).NotTo(HaveOccurred())
	return text, ok
}

// onlyProposal returns the one proposal of alice.
func onlyProposal(r *trigRig) contract.ChangeProposal {
	list, err := r.svc.Changes().List(context.Background(), alice)
	Expect(err).NotTo(HaveOccurred())
	Expect(list).To(HaveLen(1), "proposals = %+v", list)
	return list[0]
}

// expectProposeConfirmRevert drives the whole life of one proposal that a
// chat made: nothing is written until the owner confirms, a confirmation
// writes one change set, and a revert by the model takes it back.
func expectProposeConfirmRevert(r *trigRig, send func(string), wantKind, wantID string, file func(contract.ChangeSource) string) {
	ctx := context.Background()
	send("remember")
	until("the proposal", func() bool {
		res := r.h.ResultsFor("propose_change")
		return len(res) == 1
	})
	res := r.h.ResultsFor("propose_change")
	Expect(res[0].Err).To(BeNil(), "result = %+v", res)
	Expect(res[0].Out).To(ContainSubstring("Nothing is saved yet"))
	p := onlyProposal(r)
	Expect(p.Status).To(Equal(contract.ProposalPending))
	wantSource := p.Source
	Expect(wantSource.Kind).To(Equal(wantKind))
	Expect(wantSource.ID).To(Equal(wantID))
	Expect(wantSource.Chat).NotTo(BeEmpty())
	wantFile := file(p.Source)
	Expect(p.Target).To(Equal(wantFile))
	_, ok := readHome(r, wantFile)
	Expect(ok).To(BeFalse(), "the proposal wrote the file")

	_, err := r.svc.Changes().Confirm(ctx, "bob", p.ID)
	Expect(err).To(MatchError(proposals.ErrNotFound), "another user confirmed it")

	done, err := r.svc.Changes().Confirm(ctx, alice, p.ID)
	Expect(err).NotTo(HaveOccurred())
	Expect(done.Status).To(Equal(contract.ProposalApplied))
	Expect(done.Rev).To(BeNumerically(">", 0))
	text, ok := readHome(r, wantFile)
	Expect(ok).To(BeTrue())
	Expect(text).To(Equal("- home: Rome\n"))
	_, err = r.svc.Changes().Confirm(ctx, alice, p.ID)
	Expect(err).To(MatchError(studio.ErrChangeAnswered))

	send("forget")
	until("the revert", func() bool { return len(r.h.ResultsFor("revert_change")) == 1 })
	Expect(r.h.ResultsFor("revert_change")[0].Err).To(BeNil())
	_, ok = readHome(r, wantFile)
	Expect(ok).To(BeFalse(), "the revert left the file")
	back, _ := r.svc.Changes().Get(ctx, alice, p.ID)
	Expect(back.Status).To(Equal(contract.ProposalReverted))
	Expect(back.RevertRev).To(BeNumerically(">", back.Rev))

	events, err := r.svc.Changes().Events(ctx, alice, 20)
	Expect(err).NotTo(HaveOccurred())
	var actions []string
	for _, e := range events {
		actions = append(actions, e.Action)
		Expect(e.Source).To(Equal(wantSource))
		Expect(e.Hash).NotTo(BeEmpty())
		Expect(fmt.Sprintf("%+v", e)).NotTo(ContainSubstring("Rome"), "the audit holds content")
	}
	Expect(actions).To(Equal([]string{"revert", "confirm", "propose"}))
}

var _ = Describe("change proposals", func() {
	Describe("from a plain Studio chat", func() {
		It("stores, confirms and reverts a memory change", func() {
			r := changeShared(nil).rig(changeScript("user", "- home: Rome\n"), studio.TriggerOptions{})
			id := create(r.svc, alice)
			expectProposeConfirmRevert(r, func(t string) { sent(r.svc, alice, id, t); idle(r.svc, alice, id) }, "", "",
				func(contract.ChangeSource) string { return "Memory/index.md" })
		})

		It("refuses a target outside Memory, a profile without an agent and too much text", func() {
			r := changeShared(nil).rig(func(workspace.Actor, string) []studiotest.ToolStep { return nil }, studio.TriggerOptions{})
			ctx := context.Background()
			c := r.svc.Changes()
			in := studio.ProposeInput{User: alice, Kind: "memory", Scope: "user", Target: "index.md", Content: "x", Summary: "s"}
			bad := func(mod func(*studio.ProposeInput)) error {
				x := in
				mod(&x)
				_, err := c.Propose(ctx, x)
				return err
			}
			for name, mod := range map[string]func(*studio.ProposeInput){
				"parent":      func(x *studio.ProposeInput) { x.Target = "../../x.md" },
				"profile":     func(x *studio.ProposeInput) { x.Kind, x.Scope = "profile", "agent" },
				"big":         func(x *studio.ProposeInput) { x.Content = strings.Repeat("a", contract.MaxChangeBytes+1) },
				"unknown":     func(x *studio.ProposeInput) { x.Kind = "page" },
				"scope":       func(x *studio.ProposeInput) { x.Scope = "team" },
				"no summary":  func(x *studio.ProposeInput) { x.Summary = " " },
				"binary":      func(x *studio.ProposeInput) { x.Content = "a\x00b" },
				"agent scope": func(x *studio.ProposeInput) { x.Scope = "agent" },
			} {
				Expect(bad(mod)).To(MatchError(studio.ErrBadChange), name)
			}
			list, _ := c.List(ctx, alice)
			Expect(list).To(BeEmpty())
		})

		It("holds a profile to 4 KiB", func() {
			r := changeShared(grantAll).rig(func(workspace.Actor, string) []studiotest.ToolStep { return nil }, studio.TriggerOptions{})
			ag, err := r.svc.SaveAgent(context.Background(), alice, agents.Agent{Name: "Helper", Model: "m", Builtins: []string{"profile.write"}})
			Expect(err).NotTo(HaveOccurred())
			in := studio.ProposeInput{User: alice, Agent: ag.ID, Kind: "profile", Scope: "agent", Content: strings.Repeat("a", contract.MaxProfileBytes+1), Summary: "s"}
			_, err = r.svc.Changes().Propose(context.Background(), in)
			Expect(err).To(MatchError(studio.ErrBadChange))
			in.Content = strings.Repeat("a", contract.MaxProfileBytes)
			p, err := r.svc.Changes().Propose(context.Background(), in)
			Expect(err).NotTo(HaveOccurred())
			Expect(p.Target).To(Equal("LocalAI/agents/" + ag.ID + "/profile.md"))
		})

		It("limits 10 pending per user and 5 per hour per agent", func() {
			sh := changeShared(nil)
			r := sh.rig(func(workspace.Actor, string) []studiotest.ToolStep { return nil }, studio.TriggerOptions{})
			ctx := context.Background()
			c := r.svc.Changes()
			propose := func(user, name string) (contract.ChangeProposal, error) {
				return c.Propose(ctx, studio.ProposeInput{User: user, Kind: "memory", Scope: "user", Target: name + ".md", Content: name, Summary: name})
			}
			var made []contract.ChangeProposal
			for i := range contract.MaxChangesPerHour {
				p, err := propose(alice, fmt.Sprintf("n%d", i))
				Expect(err).NotTo(HaveOccurred())
				made = append(made, p)
			}
			_, err := propose(alice, "extra")
			Expect(err).To(MatchError(proposals.ErrRateLimited))
			_, err = propose("bob", "bobs")
			Expect(err).NotTo(HaveOccurred(), "the rate is per person")
			sh.clock.Advance(61 * time.Minute)
			for i := range contract.MaxChangesPerHour {
				p, err := propose(alice, fmt.Sprintf("m%d", i))
				Expect(err).NotTo(HaveOccurred(), "after an hour, number %d", i)
				made = append(made, p)
			}
			_, err = propose(alice, "over")
			Expect(err).To(MatchError(proposals.ErrTooManyPending), "10 are waiting")
			_, err = c.Dismiss(ctx, alice, made[0].ID)
			Expect(err).NotTo(HaveOccurred())
			_, err = c.Dismiss(ctx, alice, made[0].ID)
			Expect(err).To(MatchError(studio.ErrChangeAnswered))
		})

		It("expires a proposal after 24 hours", func() {
			sh := changeShared(nil)
			r := sh.rig(changeScript("user", "- home: Rome\n"), studio.TriggerOptions{})
			id := create(r.svc, alice)
			sent(r.svc, alice, id, "remember")
			idle(r.svc, alice, id)
			p := onlyProposal(r)
			Expect(p.Expires.Sub(p.Created)).To(Equal(24 * time.Hour))
			sh.clock.Advance(25 * time.Hour)
			_, err := r.svc.Changes().Confirm(context.Background(), alice, p.ID)
			Expect(err).To(MatchError(studio.ErrChangeExpired))
			got, _ := r.svc.Changes().Get(context.Background(), alice, p.ID)
			Expect(got.Status).To(Equal(contract.ProposalExpired))
			_, ok := readHome(r, "Memory/index.md")
			Expect(ok).To(BeFalse())
		})

		It("writes the owner's own text when the owner edits before confirming", func() {
			r := changeShared(nil).rig(changeScript("user", "- home: Rome\n"), studio.TriggerOptions{})
			ctx := context.Background()
			id := create(r.svc, alice)
			sent(r.svc, alice, id, "remember")
			idle(r.svc, alice, id)
			p := onlyProposal(r)
			_, err := r.svc.Changes().ConfirmEdited(ctx, alice, p.ID, strings.Repeat("x", 16<<10+1))
			Expect(err).To(MatchError(studio.ErrBadChange))
			got, _ := r.svc.Changes().Get(ctx, alice, p.ID)
			Expect(got.Status).To(Equal(contract.ProposalPending), "a refused edit leaves the proposal waiting")
			done, err := r.svc.Changes().ConfirmEdited(ctx, alice, p.ID, "- home: Milan\n")
			Expect(err).NotTo(HaveOccurred())
			Expect(done.Status).To(Equal(contract.ProposalApplied))
			text, _ := readHome(r, "Memory/index.md")
			Expect(text).To(Equal("- home: Milan\n"))
			_, err = r.svc.Changes().Revert(ctx, alice, p.ID)
			Expect(err).NotTo(HaveOccurred())
			_, ok := readHome(r, "Memory/index.md")
			Expect(ok).To(BeFalse())
		})

		It("refuses a stale proposal and keeps the newer file", func() {
			r := changeShared(nil).rig(changeScript("user", "- home: Rome\n"), studio.TriggerOptions{})
			ctx := context.Background()
			_, err := r.sh.files.WriteHome(ctx, alice, "Memory/index.md", "- old\n", "seed", 1024)
			Expect(err).NotTo(HaveOccurred())
			id := create(r.svc, alice)
			sent(r.svc, alice, id, "remember")
			idle(r.svc, alice, id)
			p := onlyProposal(r)
			Expect(p.BaseHash).NotTo(BeEmpty())
			_, err = r.sh.files.WriteHome(ctx, alice, "Memory/index.md", "- edited by hand\n", "edit", 1024)
			Expect(err).NotTo(HaveOccurred())
			_, err = r.svc.Changes().Confirm(ctx, alice, p.ID)
			Expect(err).To(MatchError(studio.ErrTargetChanged))
			text, _ := readHome(r, "Memory/index.md")
			Expect(text).To(Equal("- edited by hand\n"))
			got, _ := r.svc.Changes().Get(ctx, alice, p.ID)
			Expect(got.Status).To(Equal(contract.ProposalStale))
		})

		It("refuses a revert when the file was edited since", func() {
			r := changeShared(nil).rig(changeScript("user", "- home: Rome\n"), studio.TriggerOptions{})
			ctx := context.Background()
			id := create(r.svc, alice)
			sent(r.svc, alice, id, "remember")
			idle(r.svc, alice, id)
			p, err := r.svc.Changes().Confirm(ctx, alice, onlyProposal(r).ID)
			Expect(err).NotTo(HaveOccurred())
			_, err = r.sh.files.WriteHome(ctx, alice, "Memory/index.md", "- edited\n", "edit", 1024)
			Expect(err).NotTo(HaveOccurred())
			_, err = r.svc.Changes().Revert(ctx, alice, p.ID)
			Expect(err).To(MatchError(studio.ErrCannotRevert))
			got, _ := r.svc.Changes().Get(ctx, alice, p.ID)
			Expect(got.Status).To(Equal(contract.ProposalApplied), "a failed revert must leave the change applied")
		})

		It("is not offered to a sub-agent or an unattended run", func() {
			r := changeShared(nil).rig(func(_ workspace.Actor, text string) []studiotest.ToolStep {
				if text == "go" {
					return []studiotest.ToolStep{{Tool: "spawn_agent", Args: `{"task":"child job","wait":true}`}, {Say: "done"}}
				}
				return []studiotest.ToolStep{{Say: "ok"}}
			}, studio.TriggerOptions{})
			id := create(r.svc, alice)
			sent(r.svc, alice, id, "go")
			idle(r.svc, alice, id)
			Expect(r.h.ToolNames(id)).To(ContainElements("propose_change", "revert_change"))
			for _, c := range childrenOf(r.svc, alice, id) {
				Expect(r.h.ToolNames(c.ID)).NotTo(ContainElement("propose_change"))
			}
			tr := r.schedule(studio.TriggerInput{})
			r.due()
			run := r.waitEnded(tr.ID)
			Expect(r.h.ToolNames(run.ChatID)).NotTo(ContainElement("propose_change"))
			Expect(r.h.ToolNames(run.ChatID)).NotTo(ContainElement("revert_change"))
		})
	})

	Describe("from a chat that runs a saved agent", func() {
		It("is refused without the grant and allowed with it", func() {
			for name, grant := range map[string][]string{"default": nil, "granted": {"memory.write"}} {
				r := changeShared(nil).rig(changeScript("agent", "- rule\n"), studio.TriggerOptions{})
				ag, err := r.svc.SaveAgent(context.Background(), alice, agents.Agent{Name: "Helper", Model: "m", Builtins: grant})
				Expect(err).NotTo(HaveOccurred())
				rec, err := r.svc.Create(context.Background(), alice, studio.CreateInput{AgentID: ag.ID})
				Expect(err).NotTo(HaveOccurred())
				sent(r.svc, alice, rec.ID, "remember")
				idle(r.svc, alice, rec.ID)
				res := r.h.ResultsFor("propose_change")
				list, _ := r.svc.Changes().List(context.Background(), alice)
				if name == "default" {
					// Without a grant the tool does not exist in the chat.
					Expect(r.h.ToolNames(rec.ID)).NotTo(ContainElement("propose_change"))
					Expect(res).To(HaveLen(1))
					Expect(res[0].Err).To(MatchError(ContainSubstring("no tool named")))
					Expect(list).To(BeEmpty())
					continue
				}
				Expect(res).To(HaveLen(1), name)
				Expect(res[0].Err).To(BeNil())
				Expect(list).To(HaveLen(1))
				Expect(list[0].Agent).To(Equal(ag.ID))
				Expect(list[0].Target).To(Equal("Memory/agents/" + ag.ID + "/index.md"))
			}
		})

		It("cannot revert the change of another agent by its id", func() {
			var other string
			r := changeShared(grantAll).rig(func(_ workspace.Actor, text string) []studiotest.ToolStep {
				if text == "forget" {
					return []studiotest.ToolStep{{Tool: "revert_change", Args: `{"id":"` + other + `"}`}, {Say: "ok"}}
				}
				return []studiotest.ToolStep{{Say: "ok"}}
			}, studio.TriggerOptions{})
			ctx := context.Background()
			p, err := r.svc.Changes().Propose(ctx, studio.ProposeInput{User: alice, Kind: "memory", Scope: "user", Target: "a.md", Content: "x", Summary: "s"})
			Expect(err).NotTo(HaveOccurred())
			_, err = r.svc.Changes().Confirm(ctx, alice, p.ID)
			Expect(err).NotTo(HaveOccurred())
			other = p.ID
			ag, _ := r.svc.SaveAgent(ctx, alice, agents.Agent{Name: "Helper", Model: "m", Builtins: []string{"memory.write"}})
			rec, _ := r.svc.Create(ctx, alice, studio.CreateInput{AgentID: ag.ID})
			sent(r.svc, alice, rec.ID, "forget")
			idle(r.svc, alice, rec.ID)
			res := r.h.ResultsFor("revert_change")
			Expect(res).To(HaveLen(1))
			Expect(res[0].Err).To(HaveOccurred())
			got, _ := r.svc.Changes().Get(ctx, alice, p.ID)
			Expect(got.Status).To(Equal(contract.ProposalApplied))
		})
	})

	Describe("from connector chats of two platforms", func() {
		// The host has no platform code: the same spec runs over the Telegram
		// and the Slack fakes, and the proposal differs only in the binding.
		It("works the same over Telegram", func() {
			sh, bot := changeShared(grantAll), newScriptedBot()
			r := newConnRig(sh, connStore(), bot, changeScript("channel", "- home: Rome\n"), nil)
			r.grantMemory()
			b := r.bind(studio.ConnectorInput{Allow: []string{"11"}})
			n := 0
			send := func(text string) {
				n++
				bot.say(n, 11, "Ada", "ada", text)
				waitFor("the answer", func() bool { return len(bot.answers()) == n })
			}
			expectProposeConfirmRevert(r.trigRig, send, session.OriginConnector, b.ID, channelFile)
		})

		It("works the same over Slack", func() {
			sh := changeShared(grantAll)
			f := slacktest.New()
			DeferCleanup(f.Close)
			f.AddApp(slacktest.App{Bot: slackBot, AppToken: slackApp, UserID: "UBOT"})
			f.AddUser("U1", "ada", "Ada")
			rt := slack.New(&slack.API{BaseURL: f.URL, FilesHost: f.FilesHost})
			DeferCleanup(rt.Close)
			r := newConnRigWith(sh, connStore(), studiobot.NewTransport(studiobot.Slack(rt)), changeScript("channel", "- home: Rome\n"), nil)
			r.grantMemory()
			b := r.bind(studio.ConnectorInput{Kind: "slack", Token: slackBot + " " + slackApp, Allow: []string{"C1"}})
			waitFor("the Socket Mode link", func() bool { return f.Connected(slackApp) })
			posted := func() int {
				return len((&slackRig{connRig: r, slack: f}).answers())
			}
			n := 0
			send := func(text string) {
				n++
				f.Push(slackApp, slacktest.Mention("C1", "U1", "UBOT", text, ""))
				waitFor("the answer", func() bool { return posted() == n })
			}
			expectProposeConfirmRevert(r.trigRig, send, session.OriginConnector, b.ID, channelFile)
		})

		It("keeps the connector chat out of the files of its sender when it only proposes", func() {
			// A connector chat that runs an agent without a grant gets the
			// default refusal: the safe default for every platform.
			sh, bot := changeShared(nil), newScriptedBot()
			r := newConnRig(sh, connStore(), bot, changeScript("channel", "x"), nil)
			r.bind(studio.ConnectorInput{Allow: []string{"11"}})
			bot.say(1, 11, "Ada", "ada", "remember")
			waitFor("the answer", func() bool { return len(bot.answers()) == 1 })
			list, _ := r.svc.Changes().List(context.Background(), alice)
			Expect(list).To(BeEmpty())
		})
	})
})