diff --git a/core/config/model_config.go b/core/config/model_config.go index cacb6f1ec..c6121eb8c 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -1369,6 +1369,12 @@ func (c *ModelConfig) syncKnownUsecasesFromString() { c.KnownUsecaseStrings = append(c.KnownUsecaseStrings, k) } } + // GetAllModelConfigUsecases returns a map, and ranging one yields a random + // order per call. KnownUsecaseStrings is part of the serialized config, so + // an unsorted list gives the same file a different config revision on every + // load. In distributed mode that reads as a config change and the router + // rejects the request with ErrStaleModelConfigRevision. + slices.Sort(c.KnownUsecaseStrings) } func (c *ModelConfig) UnmarshalYAML(value *yaml.Node) error { diff --git a/core/config/model_config_revision_stability_test.go b/core/config/model_config_revision_stability_test.go new file mode 100644 index 000000000..19a0d8590 --- /dev/null +++ b/core/config/model_config_revision_stability_test.go @@ -0,0 +1,90 @@ +package config_test + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/system" +) + +// The distributed controller pins a model's replicas to its config revision and +// rejects any request carrying a different one. A revision that is not stable +// for one unchanged file on disk therefore wedges the model. +var _ = Describe("Model config revision stability", func() { + // A chat model with an mmproj derives two usecase flags, FLAG_CHAT and + // FLAG_VISION. syncKnownUsecasesFromString builds that list by ranging a + // map, so an unstable order shows up with two or more flags and stays + // hidden with one. + const multiUsecaseModel = `backend: llama-cpp +context_size: 50000 +known_usecases: + - chat +mmproj: llama-cpp/mmproj/example/mmproj.gguf +name: example +options: + - use_jinja:true + - parallel:2 +parameters: + model: llama-cpp/models/example/example.gguf +template: + use_tokenizer_template: true +` + + var ( + dir string + appConfig *config.ApplicationConfig + ) + + BeforeEach(func() { + dir = GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), []byte(multiUsecaseModel), 0o600)).To(Succeed()) + appConfig = config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + }) + + loadRevision := func() string { + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + cfg, ok := loader.GetModelConfig("example") + Expect(ok).To(BeTrue()) + revision, err := config.ModelConfigRevision(&cfg) + Expect(err).ToNot(HaveOccurred()) + return revision + } + + It("does not change when the same file is loaded repeatedly", func() { + baseline := loadRevision() + for i := 0; i < 20; i++ { + Expect(loadRevision()).To(Equal(baseline), "revision changed between two loads of one unchanged file") + } + }) + + It("orders the derived usecases deterministically", func() { + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + cfg, ok := loader.GetModelConfig("example") + Expect(ok).To(BeTrue()) + Expect(len(cfg.KnownUsecaseStrings)).To(BeNumerically(">=", 2), "fixture must derive several usecases to expose ordering") + Expect(cfg.KnownUsecaseStrings).To(Equal([]string{"FLAG_CHAT", "FLAG_VISION"})) + }) + + // The request pipeline reloads the config through LoadModelConfigFileByName, + // which applies SetDefaults a second time. That must not move the revision + // away from the one model administration publishes from the loader map. + It("survives the extra SetDefaults the request path applies", func() { + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + stored, ok := loader.GetModelConfig("example") + Expect(ok).To(BeTrue()) + adminRevision, err := config.ModelConfigRevision(&stored) + Expect(err).ToNot(HaveOccurred()) + + requestCfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(requestCfg.PersistedConfigRevision()).To(Equal(adminRevision)) + }) +}) diff --git a/core/http/middleware/request_config_revision_test.go b/core/http/middleware/request_config_revision_test.go index 372a033ea..419ae8e04 100644 --- a/core/http/middleware/request_config_revision_test.go +++ b/core/http/middleware/request_config_revision_test.go @@ -50,7 +50,11 @@ var _ = Describe("Model config revision seen by inference requests", func() { Expect(os.WriteFile( filepath.Join(modelDir, "test-model.yaml"), - []byte("name: test-model\nbackend: llama-cpp\ncontext_size: 4096\n"), + // The mmproj makes this derive several usecase flags. A single-flag + // model hides any instability in how that derived list is ordered. + []byte("name: test-model\nbackend: llama-cpp\ncontext_size: 4096\n"+ + "mmproj: llama-cpp/mmproj/test-model/mmproj.gguf\n"+ + "known_usecases:\n - chat\n"), 0o600, )).To(Succeed())