mirror of
https://github.com/mudler/LocalAI.git
synced 2026-09-29 01:25:03 -04:00
mudler-agentandEttore Di Giacinto
97ad8f1d70
* fix(distributed): stage every shard of a split GGUF A split GGUF is configured by its first shard only. llama.cpp opens the other "-0000N-of-0000M.gguf" files from the same directory by name. The router staged only the configured path, so the worker received shard 1 and the load failed with "failed to load GGUF split". The router now stages the remaining shards next to the first one. A missing shard fails the load and names the file. The file count for progress and the payload size also include all shards. The payload size feeds the load deadline and the disk-headroom check. For a 111 GB model whose first shard is 10 MB, both were sized for less than 1 GB. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(distributed): stage the files a model install declares Replace the split GGUF file name matching with the model's own file list. A gallery install or an import records every file of the model in ._gallery_<name>.yaml (files:), and a config can list more under download_files:. The router now stages all of these files, not only the files that the config's path fields name. This includes the other shards of a split GGUF, which llama.cpp opens by name. The application gives the router a resolver that reads the two lists. The resolver looks up the files by model name when it stages them, so a replica that the reconciler loads from saved load options gets the same files. backend.proto does not change. A declared file that is missing on the frontend is skipped with a warning. The load deadline and the disk headroom check include the declared files. Assisted-by: Claude:claude-opus-5-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
79 lines
2.4 KiB
Go
79 lines
2.4 KiB
Go
package nodes
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
|
"github.com/mudler/xlog"
|
|
)
|
|
|
|
// declaredExtraFiles returns the files the model's install declared that the
|
|
// path fields of opts do not already stage: neither named by a field nor
|
|
// inside a directory a field names. It must run on the local paths, before
|
|
// staging rewrites the fields to remote ones.
|
|
func (r *SmartRouter) declaredExtraFiles(trackingKey string, opts *pb.ModelOptions) []string {
|
|
if r.modelFiles == nil || opts == nil || trackingKey == "" {
|
|
return nil
|
|
}
|
|
covered := append([]string{
|
|
opts.ModelFile, opts.MMProj, opts.LoraAdapter, opts.DraftModel,
|
|
opts.CLIPModel, opts.Tokenizer, opts.AudioPath, opts.LoraBase,
|
|
}, opts.LoraAdapters...)
|
|
|
|
seen := map[string]struct{}{}
|
|
var extra []string
|
|
for _, p := range r.modelFiles(trackingKey) {
|
|
p = filepath.Clean(p)
|
|
if _, dup := seen[p]; dup || coveredByField(p, covered) {
|
|
continue
|
|
}
|
|
seen[p] = struct{}{}
|
|
extra = append(extra, p)
|
|
}
|
|
return extra
|
|
}
|
|
|
|
func coveredByField(path string, fields []string) bool {
|
|
for _, f := range fields {
|
|
if f == "" {
|
|
continue
|
|
}
|
|
f = filepath.Clean(f)
|
|
if path == f || strings.HasPrefix(path, f+string(filepath.Separator)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// existingFiles drops declared files that are not on the frontend. An install
|
|
// can declare files that are gone by load time (an archive unpacked and then
|
|
// removed, say), so a missing one is not a reason to refuse the load; the
|
|
// backend reports it if it really needed it.
|
|
func existingFiles(paths []string, nodeName, trackingKey string) []string {
|
|
out := paths[:0:0]
|
|
for _, p := range paths {
|
|
if _, err := os.Stat(p); err != nil {
|
|
xlog.Warn("Skipping staging for declared model file that is not on the frontend", "path", p, "node", nodeName, "model", trackingKey, "error", err)
|
|
continue
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// stagingPayloadBytes totals the on-disk size of everything staging uploads
|
|
// for a model: the path fields plus the declared files they do not cover. The
|
|
// first shard of a split GGUF can be a few MB of metadata while the weights
|
|
// sit in the others, so sizing the fields alone starves the load budget and
|
|
// the disk-headroom check.
|
|
func (r *SmartRouter) stagingPayloadBytes(trackingKey string, opts *pb.ModelOptions) int64 {
|
|
total := modelPayloadBytes(opts)
|
|
for _, p := range r.declaredExtraFiles(trackingKey, opts) {
|
|
total += pathBytes(p)
|
|
}
|
|
return total
|
|
}
|