Files
mudler's LocalAI [bot]andEttore Di Giacinto 82c191afad fix(distributed): keep model replicas config-consistent (#11664)
* docs: design configurable copy buffering

Document the context-aware copy buffer option and its validation plan.

Assisted-by: Codex:gpt-5

* docs: design durable distributed staging operations

Assisted-by: Codex:gpt-5

* docs: design distributed model config revisions

Assisted-by: Codex:GPT-5 [apply_patch] [exec_command]

* feat(config): add stable model revisions

Hash typed model configuration and effective protobuf options deterministically for distributed revision comparisons.

Assisted-by: Codex:GPT-5 [apply_patch] [exec_command]

* feat(worker): acknowledge exact model stops

Assisted-by: Codex:GPT-5 [apply_patch] [exec_command]

* feat(nodes): track model config revisions

Assisted-by: Codex:GPT-5 [apply_patch]

* fix(distributed): retry quarantined model cleanup

Stop quarantined replicas by exact process identity, retain failed cleanup as durable capped retries, and compare-and-delete only the claimed registry row. Process one sufficiently leased row at a time so multiple frontends cannot duplicate slow cleanup work.

Assisted-by: Codex:gpt-5

* fix(distributed): bind loads to config revisions

Assisted-by: Codex: GPT-5 [OpenAI Codex]

* fix(modeladmin): apply config revisions consistently

Route model edits, patches, state changes, deletion, and peer refreshes through the same revision lifecycle. Quarantine stale replicas before exact cleanup and report durable pending cleanup without failing successful config writes.

Assisted-by: Codex: GPT-5 [OpenAI Codex]

* feat(distributed): expose model config revision state

Document replica revision observability and durable cleanup behavior. Keep pending cleanup explicit in model mutation responses and verify endpoint contracts expose revision state without serialized load options.

Assisted-by: Codex:GPT-5 [OpenAI Codex]

* test(distributed): cover model revision convergence

Exercise cross-frontend quarantine, stale replay rejection, exact cleanup retry, worker re-registration, and current-generation replica convergence against the distributed PostgreSQL harness.

Assisted-by: Codex:gpt-5

* fix(distributed): pass config revision CI checks

Keep configured gallery sources out of authoritative runtime snapshots only after validating their real schema, and harden rollback snapshots against symlink races and non-regular files.

Assisted-by: Codex: GPT-5 [OpenAI Codex]

---------

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
2026-08-22 22:44:03 +02:00

77 lines
2.1 KiB
Go

package modeladmin
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/mudler/LocalAI/pkg/safefile"
)
type savedMutationFile struct {
path string
data []byte
mode os.FileMode
exists bool
}
func (s *ConfigService) withMutationRollback(paths []string, mutate func() error) error {
configs := s.Loader.GetAllModelsConfigs()
files := make([]savedMutationFile, 0, len(paths))
seen := map[string]struct{}{}
for _, path := range paths {
if _, ok := seen[path]; ok {
continue
}
seen[path] = struct{}{}
name, err := directMutationEntry(s.modelsPath(), path)
if err != nil {
return fmt.Errorf("snapshot config mutation: %w", err)
}
file := savedMutationFile{path: path}
file.data, file.mode, err = safefile.ReadRegularAt(s.modelsPath(), name)
if err == nil {
file.exists = true
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("snapshot config mutation: %w", err)
}
files = append(files, file)
}
if err := mutate(); err != nil {
var restoreErr error
for _, file := range files {
if file.exists {
restoreErr = errors.Join(restoreErr, writeFileAtomic(file.path, file.data, file.mode))
} else if removeErr := os.Remove(file.path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
restoreErr = errors.Join(restoreErr, removeErr)
}
}
s.Loader.ReplaceModelConfigs(configs)
if restoreErr != nil {
return errors.Join(err, fmt.Errorf("restore prior model configuration: %w", restoreErr))
}
return err
}
return nil
}
func directMutationEntry(modelsPath, path string) (string, error) {
root, err := filepath.Abs(modelsPath)
if err != nil {
return "", err
}
candidate, err := filepath.Abs(path)
if err != nil {
return "", err
}
rel, err := filepath.Rel(root, candidate)
if err != nil || rel == "." || rel == ".." || filepath.IsAbs(rel) || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.Dir(candidate) != root {
return "", fmt.Errorf("config path %q is not a direct entry of the configured models directory", path)
}
return filepath.Base(candidate), nil
}