mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-30 18:14:32 -04:00
mudler-agentandEttore Di Giacinto
97ad8f1d70
* fix(distributed): stage every shard of a split GGUF A split GGUF is configured by its first shard only. llama.cpp opens the other "-0000N-of-0000M.gguf" files from the same directory by name. The router staged only the configured path, so the worker received shard 1 and the load failed with "failed to load GGUF split". The router now stages the remaining shards next to the first one. A missing shard fails the load and names the file. The file count for progress and the payload size also include all shards. The payload size feeds the load deadline and the disk-headroom check. For a 111 GB model whose first shard is 10 MB, both were sized for less than 1 GB. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(distributed): stage the files a model install declares Replace the split GGUF file name matching with the model's own file list. A gallery install or an import records every file of the model in ._gallery_<name>.yaml (files:), and a config can list more under download_files:. The router now stages all of these files, not only the files that the config's path fields name. This includes the other shards of a split GGUF, which llama.cpp opens by name. The application gives the router a resolver that reads the two lists. The resolver looks up the files by model name when it stages them, so a replica that the reconciler loads from saved load options gets the same files. backend.proto does not change. A declared file that is missing on the frontend is skipped with a warning. The load deadline and the disk headroom check include the declared files. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
47 lines
1.7 KiB
Go
47 lines
1.7 KiB
Go
package gallery
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/mudler/LocalAI/pkg/utils"
|
|
"github.com/mudler/xlog"
|
|
)
|
|
|
|
// InstalledModelFiles returns the absolute paths of the files that the install
|
|
// of model name declared (the entry's `files:`), as recorded in its gallery
|
|
// file. A model config names only the file a backend opens first, while a
|
|
// backend can read more by itself (llama.cpp opens the other shards of a split
|
|
// GGUF by name), so this is the complete list of what the model needs on disk.
|
|
// It returns nil for a model that was not installed from a gallery or import.
|
|
func InstalledModelFiles(modelsPath, name string) []string {
|
|
// Model names can hold path separators; the gallery file flattens them
|
|
// the same way listModelFiles does.
|
|
rel := galleryFileName(strings.ReplaceAll(name, string(os.PathSeparator), "__"))
|
|
if err := utils.VerifyPath(rel, modelsPath); err != nil {
|
|
return nil
|
|
}
|
|
galleryFile := filepath.Join(modelsPath, rel)
|
|
if _, err := os.Stat(galleryFile); err != nil {
|
|
return nil
|
|
}
|
|
cfg, err := ReadConfigFile[ModelConfig](galleryFile)
|
|
if err != nil {
|
|
xlog.Warn("Failed to read gallery file for installed model files", "model", name, "file", galleryFile, "error", err)
|
|
return nil
|
|
}
|
|
|
|
files := make([]string, 0, len(cfg.Files))
|
|
for _, f := range cfg.Files {
|
|
// VerifyPath joins its argument onto modelsPath itself, so it must
|
|
// get the relative name; an absolute path would always pass.
|
|
if err := utils.VerifyPath(f.Filename, modelsPath); err != nil {
|
|
xlog.Warn("Ignoring declared model file outside the models path", "model", name, "file", f.Filename)
|
|
continue
|
|
}
|
|
files = append(files, filepath.Join(modelsPath, f.Filename))
|
|
}
|
|
return files
|
|
}
|