LocalAI/pkg/model/loader_test.go

package model_test

import (
	"errors"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"github.com/mudler/LocalAI/pkg/model"
	"github.com/mudler/LocalAI/pkg/system"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
)

var _ = Describe("ModelLoader", func() {
	var (
		modelLoader *model.ModelLoader
		modelPath   string
		mockModel   *model.Model
	)

	BeforeEach(func() {
		// Setup the model loader with a test directory
		modelPath = "/tmp/test_model_path"
		os.Mkdir(modelPath, 0755)

		systemState, err := system.GetSystemState(
			system.WithModelPath(modelPath),
		)
		Expect(err).ToNot(HaveOccurred())
		modelLoader = model.NewModelLoader(systemState)
	})

	AfterEach(func() {
		// Cleanup test directory
		os.RemoveAll(modelPath)
	})

	Context("NewModelLoader", func() {
		It("should create a new ModelLoader with an empty model map", func() {
			Expect(modelLoader).ToNot(BeNil())
			Expect(modelLoader.ModelPath).To(Equal(modelPath))
			Expect(modelLoader.ListLoadedModels()).To(BeEmpty())
		})
	})

	Context("ExistsInModelPath", func() {
		It("should return true if a file exists in the model path", func() {
			testFile := filepath.Join(modelPath, "test.model")
			os.Create(testFile)
			Expect(modelLoader.ExistsInModelPath("test.model")).To(BeTrue())
		})

		It("should return false if a file does not exist in the model path", func() {
			Expect(modelLoader.ExistsInModelPath("nonexistent.model")).To(BeFalse())
		})
	})

	Context("ListFilesInModelPath", func() {
		It("should list all valid model files in the model path", func() {
			os.Create(filepath.Join(modelPath, "test.model"))
			os.Create(filepath.Join(modelPath, "model.gguf"))
			os.Create(filepath.Join(modelPath, "README.md"))

			files, err := modelLoader.ListFilesInModelPath()
			Expect(err).To(BeNil())
			Expect(files).To(ContainElement("test.model"))
			Expect(files).ToNot(ContainElement("model.gguf"))
			Expect(files).ToNot(ContainElement("README.md"))
		})
	})

	Context("LoadModel", func() {
		It("should load a model and keep it in memory", func() {
			mockModel = model.NewModel("foo", "test.model", nil)
			mockModel.MarkHealthy() // skip gRPC health check (no real server)

			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				return mockModel, nil
			}

			model, err := modelLoader.LoadModel("foo", "test.model", mockLoader)
			Expect(err).To(BeNil())
			Expect(model).To(Equal(mockModel))
			Expect(modelLoader.CheckIsLoaded("foo")).To(Equal(mockModel))
		})

		It("should return an error if loading the model fails", func() {
			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				return nil, errors.New("failed to load model")
			}

			model, err := modelLoader.LoadModel("foo", "test.model", mockLoader)
			Expect(err).To(HaveOccurred())
			Expect(model).To(BeNil())
		})
	})

	Context("Remote model eviction", func() {
		It("should evict unreachable remote models from cache on health check", func() {
			// Create a remote model (process=nil) with an unreachable address
			remoteModel := model.NewModel("remote-test", "127.0.0.1:1", nil)

			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				return remoteModel, nil
			}

			_, err := modelLoader.LoadModel("remote-test", "test.model", mockLoader)
			Expect(err).To(BeNil())

			// CheckIsLoaded should detect the connection error and evict
			result := modelLoader.CheckIsLoaded("remote-test")
			Expect(result).To(BeNil(), "unreachable remote model should be evicted from cache")
		})

		It("should keep recently-healthy remote models in cache", func() {
			remoteModel := model.NewModel("healthy-remote", "127.0.0.1:1", nil)
			remoteModel.MarkHealthy() // simulate a recent successful health check

			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				return remoteModel, nil
			}

			loaded, err := modelLoader.LoadModel("healthy-remote", "test.model", mockLoader)
			Expect(err).To(BeNil())

			// Within TTL, should return the model without health check
			result := modelLoader.CheckIsLoaded("healthy-remote")
			Expect(result).To(Equal(loaded), "recently-healthy model should be returned from cache")
		})
	})

	Context("ShutdownModel", func() {
		It("should shutdown a loaded model", func() {
			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				return model.NewModel("foo", "test.model", nil), nil
			}

			_, err := modelLoader.LoadModel("foo", "test.model", mockLoader)
			Expect(err).To(BeNil())

			err = modelLoader.ShutdownModel("foo")
			Expect(err).To(BeNil())
			Expect(modelLoader.CheckIsLoaded("foo")).To(BeNil())
		})
	})

	Context("Concurrent Loading", func() {
		It("should handle concurrent requests for the same model", func() {
			var loadCount int32
			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				atomic.AddInt32(&loadCount, 1)
				time.Sleep(100 * time.Millisecond) // Simulate loading time
				m := model.NewModel(modelID, modelName, nil)
				m.MarkHealthy() // skip gRPC health check (no real server)
				return m, nil
			}

			var wg sync.WaitGroup
			results := make([]*model.Model, 5)
			errs := make([]error, 5)

			// Start 5 concurrent requests for the same model
			for i := range 5 {
				wg.Go(func() {
					results[i], errs[i] = modelLoader.LoadModel("concurrent-model", "test.model", mockLoader)
				})
			}

			wg.Wait()

			// All requests should succeed
			for i := range 5 {
				Expect(errs[i]).To(BeNil())
				Expect(results[i]).ToNot(BeNil())
			}

			// The loader should only have been called once
			Expect(atomic.LoadInt32(&loadCount)).To(Equal(int32(1)))

			// All results should be the same model instance
			for i := 1; i < 5; i++ {
				Expect(results[i]).To(Equal(results[0]))
			}
		})

		It("should handle concurrent requests for different models", func() {
			var loadCount int32
			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				atomic.AddInt32(&loadCount, 1)
				time.Sleep(50 * time.Millisecond) // Simulate loading time
				m := model.NewModel(modelID, modelName, nil)
				m.MarkHealthy() // skip gRPC health check (no real server)
				return m, nil
			}

			var wg sync.WaitGroup
			modelCount := 3

			// Start concurrent requests for different models
			for i := range modelCount {
				wg.Go(func() {
					modelID := "model-" + string(rune('A'+i))
					_, err := modelLoader.LoadModel(modelID, "test.model", mockLoader)
					Expect(err).To(BeNil())
				})
			}

			wg.Wait()

			// Each model should be loaded exactly once
			Expect(atomic.LoadInt32(&loadCount)).To(Equal(int32(modelCount)))

			// All models should be loaded
			Expect(modelLoader.CheckIsLoaded("model-A")).ToNot(BeNil())
			Expect(modelLoader.CheckIsLoaded("model-B")).ToNot(BeNil())
			Expect(modelLoader.CheckIsLoaded("model-C")).ToNot(BeNil())
		})

		It("should track loading count correctly", func() {
			loadStarted := make(chan struct{})
			loadComplete := make(chan struct{})

			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				close(loadStarted)
				<-loadComplete // Wait until we're told to complete
				return model.NewModel(modelID, modelName, nil), nil
			}

			// Start loading in background
			go func() {
				modelLoader.LoadModel("slow-model", "test.model", mockLoader)
			}()

			// Wait for loading to start
			<-loadStarted

			// Loading count should be 1
			Expect(modelLoader.GetLoadingCount()).To(Equal(1))

			// Complete the loading
			close(loadComplete)

			// Wait a bit for cleanup
			time.Sleep(50 * time.Millisecond)

			// Loading count should be back to 0
			Expect(modelLoader.GetLoadingCount()).To(Equal(0))
		})

		It("should retry loading if first attempt fails", func() {
			var attemptCount int32
			mockLoader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				count := atomic.AddInt32(&attemptCount, 1)
				if count == 1 {
					// Hold the loading slot so the second request coalesces as a
					// follower before this leader fails. That follower then gets
					// the one in-burst retry, which bypasses the failure cooldown
					// (the cooldown only gates fresh, independent load triggers).
					time.Sleep(50 * time.Millisecond)
					return nil, errors.New("first attempt fails")
				}
				return model.NewModel(modelID, modelName, nil), nil
			}

			// First goroutine will fail
			var wg sync.WaitGroup

			var err1, err2 error
			var m1, m2 *model.Model

			wg.Go(func() {
				m1, err1 = modelLoader.LoadModel("retry-model", "test.model", mockLoader)
			})

			// Give first goroutine a head start so it owns the loading slot.
			time.Sleep(10 * time.Millisecond)

			wg.Go(func() {
				m2, err2 = modelLoader.LoadModel("retry-model", "test.model", mockLoader)
			})

			wg.Wait()

			// At least one should succeed (the second attempt after retry)
			successCount := 0
			if err1 == nil && m1 != nil {
				successCount++
			}
			if err2 == nil && m2 != nil {
				successCount++
			}
			Expect(successCount).To(BeNumerically(">=", 1))
		})
	})

	Context("GetLoadingCount", func() {
		It("should return 0 when nothing is loading", func() {
			Expect(modelLoader.GetLoadingCount()).To(Equal(0))
		})
	})

	Context("LRU Eviction Retry Settings", func() {
		It("should allow updating retry settings", func() {
			modelLoader.SetLRUEvictionRetrySettings(50, 2*time.Second)
			// Settings are updated - we can verify through behavior if needed
			// For now, just verify the call doesn't panic
			Expect(modelLoader).ToNot(BeNil())
		})
	})

	Context("Load failure cooldown", func() {
		It("refuses a fresh load within the cooldown window without re-invoking the loader", func() {
			modelLoader.SetLoadFailureCooldown(60*time.Millisecond, 240*time.Millisecond)

			var loadCount int32
			failing := func(modelID, modelName, modelFile string) (*model.Model, error) {
				atomic.AddInt32(&loadCount, 1)
				return nil, errors.New("boom")
			}

			// First attempt runs the loader and fails (not a cooldown error).
			_, err := modelLoader.LoadModel("broken", "test.model", failing)
			Expect(err).To(HaveOccurred())
			var coolErr *model.ModelLoadCooldownError
			Expect(errors.As(err, &coolErr)).To(BeFalse())
			Expect(atomic.LoadInt32(&loadCount)).To(Equal(int32(1)))

			// An immediate retry is short-circuited: cooldown error, loader untouched.
			_, err = modelLoader.LoadModel("broken", "test.model", failing)
			Expect(errors.As(err, &coolErr)).To(BeTrue())
			Expect(coolErr.ModelID).To(Equal("broken"))
			Expect(coolErr.RetryAfter).To(BeNumerically(">", time.Duration(0)))
			Expect(atomic.LoadInt32(&loadCount)).To(Equal(int32(1)))

			// Once the window elapses the loader is attempted again.
			Eventually(func() int32 {
				_, _ = modelLoader.LoadModel("broken", "test.model", failing)
				return atomic.LoadInt32(&loadCount)
			}, "1s", "20ms").Should(BeNumerically(">=", 2))
		})

		It("clears the cooldown after a successful load", func() {
			modelLoader.SetLoadFailureCooldown(60*time.Millisecond, 240*time.Millisecond)

			var attempts int32
			loader := func(modelID, modelName, modelFile string) (*model.Model, error) {
				if atomic.AddInt32(&attempts, 1) == 1 {
					return nil, errors.New("boom")
				}
				m := model.NewModel(modelID, modelName, nil)
				m.MarkHealthy()
				return m, nil
			}

			_, err := modelLoader.LoadModel("flaky", "test.model", loader)
			Expect(err).To(HaveOccurred())

			// After the window, the retry succeeds and resets the failure state.
			var m *model.Model
			Eventually(func() error {
				m, err = modelLoader.LoadModel("flaky", "test.model", loader)
				return err
			}, "1s", "20ms").Should(Succeed())
			Expect(m).ToNot(BeNil())

			// A subsequent load returns the cached model, never a cooldown error.
			m2, err := modelLoader.LoadModel("flaky", "test.model", loader)
			Expect(err).ToNot(HaveOccurred())
			Expect(m2).To(Equal(m))
		})

		It("grows the cooldown on consecutive failures", func() {
			modelLoader.SetLoadFailureCooldown(50*time.Millisecond, 10*time.Second)
			failing := func(modelID, modelName, modelFile string) (*model.Model, error) {
				return nil, errors.New("boom")
			}

			// Failure 1, then read its cooldown.
			_, err := modelLoader.LoadModel("bad", "test.model", failing)
			Expect(err).To(HaveOccurred())
			_, err = modelLoader.LoadModel("bad", "test.model", failing)
			var c1 *model.ModelLoadCooldownError
			Expect(errors.As(err, &c1)).To(BeTrue())

			// Wait out the first window, trigger failure 2, read its (larger) cooldown.
			time.Sleep(70 * time.Millisecond)
			_, err = modelLoader.LoadModel("bad", "test.model", failing)
			Expect(err).To(HaveOccurred())
			_, err = modelLoader.LoadModel("bad", "test.model", failing)
			var c2 *model.ModelLoadCooldownError
			Expect(errors.As(err, &c2)).To(BeTrue())

			Expect(c2.RetryAfter).To(BeNumerically(">", c1.RetryAfter))
		})
	})
})