mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-21 21:54:52 -04:00
The three NATS queue groups jobs.new, jobs.mcp-ci.new and agent.execute are gone. Dispatching work is now a row in a work_claims table, taken by one frontend replica with SELECT ... FOR UPDATE SKIP LOCKED and driven on an agent worker as a streaming control RPC over that worker's tunnel. Exactly-one delivery among competing consumers is a database problem, not a broker feature. An agent worker has no database, so it never claims; it executes what the claiming replica hands it. A claim must not outlive the replica that took it. The reap releases a claim whose owner is no longer a live replica in the instances table, on the database clock, and never asks how long the claim has been held. A job that legitimately runs for an hour on a heartbeating replica is left alone, while a claim whose owner stopped heartbeating becomes claimable again within one liveness window. A replica with no advertised address has no instances row at all, so it refuses to claim rather than have its work reaped out from under it mid-run. The settle rule is stated once, in settleClaim, and every exit path calls it. A transport failure releases the claim and never completes or discards it; only a decoded reply line completes it. That line is deliberately not cluster.IsWorkerAnswer, which accepts the stream refusals a worker's tunnel writes before any request body reaches its control server: completing on those would discard work that never ran. The terminal line is persisted before the claim is completed, so a store that refuses leaves the claim standing rather than leaving the job running for ever. That is the dropped-result defect fixed structurally rather than by retry. This also surfaces a pre-existing gap rather than causing one: no worker has ever served plain task jobs, and publishing them into an empty queue group left them running with no trace. Such a claim is now failed with a reason. Removes QueueWorkers, --agent-subject and --agent-queue, and narrows an agent worker's minted JWT by agent.execute and jobs.mcp-ci.new. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
148 lines
4.4 KiB
Go
148 lines
4.4 KiB
Go
package agents
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/mudler/LocalAI/core/services/advisorylock"
|
|
"github.com/mudler/LocalAI/core/services/jobs"
|
|
"github.com/mudler/xlog"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// SchedulerStore is the interface for the scheduler's database needs.
|
|
type SchedulerStore interface {
|
|
ListConfigs(userID string) ([]AgentConfigRecord, error)
|
|
UpdateLastRun(userID, name string) error
|
|
}
|
|
|
|
// AgentScheduler periodically checks for agents with standalone_job=true
|
|
// and enqueues a background run as a claim on the job store.
|
|
// Uses a PostgreSQL advisory lock so only one instance fires the cron.
|
|
// Same pattern as notetaker's runAgentScheduler and LocalAI's cronLeaderLoop.
|
|
type AgentScheduler struct {
|
|
db *gorm.DB
|
|
store SchedulerStore
|
|
skillProvider SkillContentProvider // optional: loads full skill info for enriching events
|
|
pollInterval time.Duration // how often to check for due agents
|
|
}
|
|
|
|
// AgentSchedulerOpt is a functional option for AgentScheduler.
|
|
type AgentSchedulerOpt func(*AgentScheduler)
|
|
|
|
// WithSchedulerSkillProvider sets the skill provider for enriching events with per-user skills.
|
|
func WithSchedulerSkillProvider(provider SkillContentProvider) AgentSchedulerOpt {
|
|
return func(s *AgentScheduler) {
|
|
s.skillProvider = provider
|
|
}
|
|
}
|
|
|
|
// NewAgentScheduler creates a new background agent scheduler.
|
|
func NewAgentScheduler(db *gorm.DB, store SchedulerStore, opts ...AgentSchedulerOpt) *AgentScheduler {
|
|
s := &AgentScheduler{
|
|
db: db,
|
|
store: store,
|
|
pollInterval: 15 * time.Second,
|
|
}
|
|
for _, opt := range opts {
|
|
opt(s)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// Start begins the scheduler loop. Blocks until ctx is cancelled.
|
|
func (s *AgentScheduler) Start(ctx context.Context) {
|
|
xlog.Info("Agent scheduler started", "pollInterval", s.pollInterval)
|
|
advisorylock.RunLeaderLoop(ctx, s.db, advisorylock.KeyAgentScheduler, s.pollInterval, func() { s.runDueAgents(ctx) })
|
|
xlog.Info("Agent scheduler stopped")
|
|
}
|
|
|
|
// runDueAgents finds all agents with standalone_job=true that are due for a run
|
|
// and enqueues a background execution claim for each.
|
|
func (s *AgentScheduler) runDueAgents(ctx context.Context) {
|
|
configs, err := s.store.ListConfigs("") // all users
|
|
if err != nil {
|
|
xlog.Error("Agent scheduler: failed to list configs", "error", err)
|
|
return
|
|
}
|
|
|
|
for _, rec := range configs {
|
|
if rec.Status != StatusActive {
|
|
continue
|
|
}
|
|
|
|
var cfg AgentConfig
|
|
if err := ParseConfigJSON(rec.ConfigJSON, &cfg); err != nil {
|
|
continue
|
|
}
|
|
|
|
if !cfg.StandaloneJob {
|
|
continue
|
|
}
|
|
|
|
// Parse the periodic run interval
|
|
interval := parseInterval(cfg.PeriodicRuns)
|
|
|
|
// Check if the agent is due
|
|
if !isDue(rec.LastRunAt, interval) {
|
|
continue
|
|
}
|
|
|
|
xlog.Info("Scheduling background agent run", "agent", rec.Name, "user", rec.UserID, "interval", interval)
|
|
|
|
// Enrich the event with config and skills so the worker needs no DB access
|
|
var skills []SkillInfo
|
|
if cfg.EnableSkills && s.skillProvider != nil {
|
|
if loaded, err := s.skillProvider(rec.UserID); err == nil {
|
|
skills = loaded
|
|
}
|
|
}
|
|
|
|
// Enqueue the background run as a claim
|
|
evt := AgentChatEvent{
|
|
AgentName: rec.Name,
|
|
UserID: rec.UserID,
|
|
MessageID: fmt.Sprintf("bg-%d", time.Now().UnixNano()),
|
|
Role: RoleSystem,
|
|
Config: &cfg,
|
|
Skills: skills,
|
|
}
|
|
if _, err := jobs.EnqueueClaim(ctx, s.db, jobs.ClaimKindAgentRun, evt); err != nil {
|
|
xlog.Error("Agent scheduler: failed to enqueue a background run", "agent", rec.Name, "error", err)
|
|
continue
|
|
}
|
|
|
|
// Update last run timestamp
|
|
if err := s.store.UpdateLastRun(rec.UserID, rec.Name); err != nil {
|
|
xlog.Warn("Agent scheduler: failed to update last run", "agent", rec.Name, "error", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// parseInterval parses a duration string like "10m", "1h", "30s".
|
|
// Returns a default of 10 minutes if empty or invalid.
|
|
func parseInterval(s string) time.Duration {
|
|
if s == "" {
|
|
return 10 * time.Minute
|
|
}
|
|
d, err := time.ParseDuration(s)
|
|
if err != nil || d <= 0 {
|
|
return 10 * time.Minute
|
|
}
|
|
return d
|
|
}
|
|
|
|
// IsDueExported is the exported version of isDue for testing.
|
|
func IsDueExported(lastRun *time.Time, interval time.Duration) bool {
|
|
return isDue(lastRun, interval)
|
|
}
|
|
|
|
// isDue checks if enough time has elapsed since lastRun for the given interval.
|
|
func isDue(lastRun *time.Time, interval time.Duration) bool {
|
|
if lastRun == nil {
|
|
return true // never run before — due now
|
|
}
|
|
return time.Since(*lastRun) >= interval
|
|
}
|