Files
LocalAI/core/services/agents/scheduler.go
Ettore Di Giacinto 06f58d0d88 feat(distributed): dispatch queued work as a claim queue
The three NATS queue groups jobs.new, jobs.mcp-ci.new and agent.execute are
gone. Dispatching work is now a row in a work_claims table, taken by one
frontend replica with SELECT ... FOR UPDATE SKIP LOCKED and driven on an agent
worker as a streaming control RPC over that worker's tunnel. Exactly-one
delivery among competing consumers is a database problem, not a broker feature.
An agent worker has no database, so it never claims; it executes what the
claiming replica hands it.

A claim must not outlive the replica that took it. The reap releases a claim
whose owner is no longer a live replica in the instances table, on the database
clock, and never asks how long the claim has been held. A job that legitimately
runs for an hour on a heartbeating replica is left alone, while a claim whose
owner stopped heartbeating becomes claimable again within one liveness window.
A replica with no advertised address has no instances row at all, so it refuses
to claim rather than have its work reaped out from under it mid-run.

The settle rule is stated once, in settleClaim, and every exit path calls it. A
transport failure releases the claim and never completes or discards it; only a
decoded reply line completes it. That line is deliberately not
cluster.IsWorkerAnswer, which accepts the stream refusals a worker's tunnel
writes before any request body reaches its control server: completing on those
would discard work that never ran.

The terminal line is persisted before the claim is completed, so a store that
refuses leaves the claim standing rather than leaving the job running for ever.
That is the dropped-result defect fixed structurally rather than by retry.

This also surfaces a pre-existing gap rather than causing one: no worker has
ever served plain task jobs, and publishing them into an empty queue group left
them running with no trace. Such a claim is now failed with a reason.

Removes QueueWorkers, --agent-subject and --agent-queue, and narrows an agent
worker's minted JWT by agent.execute and jobs.mcp-ci.new.

Assisted-by: Claude Opus 5 [claude-code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
2026-09-20 03:05:34 +00:00

148 lines
4.4 KiB
Go

package agents
import (
"context"
"fmt"
"time"
"github.com/mudler/LocalAI/core/services/advisorylock"
"github.com/mudler/LocalAI/core/services/jobs"
"github.com/mudler/xlog"
"gorm.io/gorm"
)
// SchedulerStore is the interface for the scheduler's database needs.
type SchedulerStore interface {
ListConfigs(userID string) ([]AgentConfigRecord, error)
UpdateLastRun(userID, name string) error
}
// AgentScheduler periodically checks for agents with standalone_job=true
// and enqueues a background run as a claim on the job store.
// Uses a PostgreSQL advisory lock so only one instance fires the cron.
// Same pattern as notetaker's runAgentScheduler and LocalAI's cronLeaderLoop.
type AgentScheduler struct {
db *gorm.DB
store SchedulerStore
skillProvider SkillContentProvider // optional: loads full skill info for enriching events
pollInterval time.Duration // how often to check for due agents
}
// AgentSchedulerOpt is a functional option for AgentScheduler.
type AgentSchedulerOpt func(*AgentScheduler)
// WithSchedulerSkillProvider sets the skill provider for enriching events with per-user skills.
func WithSchedulerSkillProvider(provider SkillContentProvider) AgentSchedulerOpt {
return func(s *AgentScheduler) {
s.skillProvider = provider
}
}
// NewAgentScheduler creates a new background agent scheduler.
func NewAgentScheduler(db *gorm.DB, store SchedulerStore, opts ...AgentSchedulerOpt) *AgentScheduler {
s := &AgentScheduler{
db: db,
store: store,
pollInterval: 15 * time.Second,
}
for _, opt := range opts {
opt(s)
}
return s
}
// Start begins the scheduler loop. Blocks until ctx is cancelled.
func (s *AgentScheduler) Start(ctx context.Context) {
xlog.Info("Agent scheduler started", "pollInterval", s.pollInterval)
advisorylock.RunLeaderLoop(ctx, s.db, advisorylock.KeyAgentScheduler, s.pollInterval, func() { s.runDueAgents(ctx) })
xlog.Info("Agent scheduler stopped")
}
// runDueAgents finds all agents with standalone_job=true that are due for a run
// and enqueues a background execution claim for each.
func (s *AgentScheduler) runDueAgents(ctx context.Context) {
configs, err := s.store.ListConfigs("") // all users
if err != nil {
xlog.Error("Agent scheduler: failed to list configs", "error", err)
return
}
for _, rec := range configs {
if rec.Status != StatusActive {
continue
}
var cfg AgentConfig
if err := ParseConfigJSON(rec.ConfigJSON, &cfg); err != nil {
continue
}
if !cfg.StandaloneJob {
continue
}
// Parse the periodic run interval
interval := parseInterval(cfg.PeriodicRuns)
// Check if the agent is due
if !isDue(rec.LastRunAt, interval) {
continue
}
xlog.Info("Scheduling background agent run", "agent", rec.Name, "user", rec.UserID, "interval", interval)
// Enrich the event with config and skills so the worker needs no DB access
var skills []SkillInfo
if cfg.EnableSkills && s.skillProvider != nil {
if loaded, err := s.skillProvider(rec.UserID); err == nil {
skills = loaded
}
}
// Enqueue the background run as a claim
evt := AgentChatEvent{
AgentName: rec.Name,
UserID: rec.UserID,
MessageID: fmt.Sprintf("bg-%d", time.Now().UnixNano()),
Role: RoleSystem,
Config: &cfg,
Skills: skills,
}
if _, err := jobs.EnqueueClaim(ctx, s.db, jobs.ClaimKindAgentRun, evt); err != nil {
xlog.Error("Agent scheduler: failed to enqueue a background run", "agent", rec.Name, "error", err)
continue
}
// Update last run timestamp
if err := s.store.UpdateLastRun(rec.UserID, rec.Name); err != nil {
xlog.Warn("Agent scheduler: failed to update last run", "agent", rec.Name, "error", err)
}
}
}
// parseInterval parses a duration string like "10m", "1h", "30s".
// Returns a default of 10 minutes if empty or invalid.
func parseInterval(s string) time.Duration {
if s == "" {
return 10 * time.Minute
}
d, err := time.ParseDuration(s)
if err != nil || d <= 0 {
return 10 * time.Minute
}
return d
}
// IsDueExported is the exported version of isDue for testing.
func IsDueExported(lastRun *time.Time, interval time.Duration) bool {
return isDue(lastRun, interval)
}
// isDue checks if enough time has elapsed since lastRun for the given interval.
func isDue(lastRun *time.Time, interval time.Duration) bool {
if lastRun == nil {
return true // never run before — due now
}
return time.Since(*lastRun) >= interval
}