mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
d7eb059f44
Gallery entries could already carry a list of alternatives, but selection was
an authored, ordered, first-match policy: every candidate declared a
`capability` string and the VRAM floors had to descend in a hand-tuned order.
That pushed hardware knowledge onto whoever edits the gallery and made ordering
load-bearing, so a reordered list silently changed what users installed.
None of it was necessary. SystemState.IsBackendCompatible already derives
hardware support from a backend name alone: it knows MLX and metal are
Darwin-only, CUDA is NVIDIA-only, ROCm AMD-only, SYCL Intel-only. Selection can
read that instead of asking authors to restate it.
Authoring is now just a list of names:
- name: qwen3.6-27b
min_memory: 4GiB
variants:
- model: qwen3.6-27b-mlx-8bit
- model: qwen3.6-27b-gguf-q8
min_memory: 28GiB
and all the intelligence moved into the selector. Given a host it drops the
variants whose backend cannot run here, drops those whose known memory
requirement exceeds what the host has, and takes the LARGEST of what is left,
because a bigger footprint is a higher quality quantization of the same model.
A variant of unknown size is kept, since nothing proves it does not fit, but it
ranks last so a proven fit always beats a guess. An explicit pin still wins
outright, and if nothing survives the entry installs its own payload: the base
always installs, this never refuses.
Available memory is VRAM when a GPU was detected and system RAM otherwise, read
through xsysinfo so a cgroup limit is honored and a container gets its own
limit rather than the node's RAM.
Capability disappears entirely, from the types, the schema and the lint. VRAM
and RAM collapse into one `min_memory`, because a model's footprint is roughly
the same wherever it lives and one figure is compared against whichever applies.
The lint rules about ordering, the capability vocabulary and floor
relationships are deleted with the hazards they described; what remains is that
every variant names an entry that exists and does not itself declare variants,
plus that any memory figure actually parses.
Assisted-by: Claude:claude-opus-4-8
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
106 lines
2.8 KiB
Go
106 lines
2.8 KiB
Go
package gallery
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
|
|
"github.com/mudler/LocalAI/core/config"
|
|
"github.com/mudler/LocalAI/pkg/modelartifacts"
|
|
"github.com/mudler/xlog"
|
|
)
|
|
|
|
type ArtifactMaterializer interface {
|
|
Ensure(context.Context, string, modelartifacts.Spec) (modelartifacts.Result, error)
|
|
}
|
|
|
|
type installOptions struct {
|
|
materializer ArtifactMaterializer
|
|
variant string
|
|
}
|
|
|
|
type InstallOption func(*installOptions)
|
|
|
|
func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption {
|
|
return func(options *installOptions) {
|
|
if materializer != nil {
|
|
options.materializer = materializer
|
|
}
|
|
}
|
|
}
|
|
|
|
// WithVariant pins a gallery entry to a specific variant by name, bypassing
|
|
// hardware-based selection. The entry's own name is a valid pin, since the
|
|
// entry is itself the last resort. Ignored for entries that declare no
|
|
// variants.
|
|
func WithVariant(variant string) InstallOption {
|
|
return func(options *installOptions) {
|
|
options.variant = variant
|
|
}
|
|
}
|
|
|
|
func applyInstallOptions(options ...InstallOption) installOptions {
|
|
result := installOptions{materializer: modelartifacts.NewDefaultManager()}
|
|
for _, option := range options {
|
|
option(&result)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func bindPrimaryArtifact(ctx context.Context, modelsPath string, typed *config.ModelConfig, configMap map[string]any, materializer ArtifactMaterializer, artifactSpec modelartifacts.Spec, inferred bool) (bool, error) {
|
|
result, err := materializer.Ensure(ctx, modelsPath, artifactSpec)
|
|
if err != nil {
|
|
if inferred {
|
|
xlog.Warn("falling back to legacy model loading after artifact materialization failed", "model", typed.Name, "error", err)
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("materialize primary model artifact: %w", err)
|
|
}
|
|
next := []modelartifacts.Spec{result.Spec}
|
|
if len(typed.Artifacts) > 1 {
|
|
next = append(next, typed.Artifacts[1:]...)
|
|
}
|
|
typed.Artifacts = next
|
|
artifactYAML, err := yaml.Marshal(typed.Artifacts)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
var artifactValue any
|
|
if err := yaml.Unmarshal(artifactYAML, &artifactValue); err != nil {
|
|
return false, err
|
|
}
|
|
configMap["artifacts"] = artifactValue
|
|
return true, nil
|
|
}
|
|
|
|
func writeModelConfigAtomic(fileName string, data []byte) error {
|
|
temporary, err := os.CreateTemp(filepath.Dir(fileName), ".model-config-*")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
temporaryName := temporary.Name()
|
|
defer func() { _ = os.Remove(temporaryName) }()
|
|
if err := temporary.Chmod(0600); err != nil {
|
|
_ = temporary.Close()
|
|
return err
|
|
}
|
|
if _, err := temporary.Write(data); err != nil {
|
|
_ = temporary.Close()
|
|
return err
|
|
}
|
|
if err := temporary.Sync(); err != nil {
|
|
_ = temporary.Close()
|
|
return err
|
|
}
|
|
if err := temporary.Close(); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Chmod(temporaryName, 0644); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(temporaryName, fileName)
|
|
}
|