Add LowVRAM option parameter (#642)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

api/config.go

@@ -35,6 +35,7 @@ type Config struct {
 	NGPULayers     int               `yaml:"gpu_layers"`
 	MMap           bool              `yaml:"mmap"`
 	MMlock         bool              `yaml:"mmlock"`
+	LowVRAM        bool              `yaml:"low_vram"`
 
 	TensorSplit           string `yaml:"tensor_split"`
 	MainGPU               string `yaml:"main_gpu"`

api/prediction.go

@@ -48,6 +48,10 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
 		llamaOpts = append(llamaOpts, llama.SetNBatch(512))
 	}
 
+	if c.LowVRAM {
+		llamaOpts = append(llamaOpts, llama.EnabelLowVRAM)
+	}
+
 	return llamaOpts
 }
 

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/swaggo/swag v1.16.1
 	github.com/tmc/langchaingo v0.0.0-20230616220619-1b3da4433944
 	github.com/urfave/cli/v2 v2.25.7
-	github.com/valyala/fasthttp v1.47.0
+	github.com/valyala/fasthttp v1.48.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )

go.sum

@@ -163,6 +163,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
 github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
+github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc=
+github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=